You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by GitBox <gi...@apache.org> on 2018/03/14 09:25:35 UTC
[GitHub] XiaoZYang closed pull request #1219: Issue 1069: Provide a setting in consumer configuration to specify where to start consuming messages
XiaoZYang closed pull request #1219: Issue 1069: Provide a setting in consumer configuration to specify where to start consuming messages
URL: https://github.com/apache/incubator-pulsar/pull/1219
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/.gitignore b/.gitignore
index aa6ed821b..570c6f3f6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -23,6 +23,9 @@ pulsar-broker/src/test/resources/log4j2.yaml
*.iml
*.iws
+# Vscode
+.vscode/
+
# Mac
.DS_Store
diff --git a/.test-infra/jenkins/common_job_properties.groovy b/.test-infra/jenkins/common_job_properties.groovy
new file mode 100644
index 000000000..c9e48fd81
--- /dev/null
+++ b/.test-infra/jenkins/common_job_properties.groovy
@@ -0,0 +1,257 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+// Contains functions that help build Jenkins projects. Functions typically set
+// common properties that are shared among all Jenkins projects.
+// Code in this directory should conform to the Groovy style guide.
+// http://groovy-lang.org/style-guide.html
+class common_job_properties {
+
+ // Sets common top-level job properties for website repository jobs.
+ static void setTopLevelWebsiteJobProperties(context,
+ String branch = 'master') {
+ // GitHub project.
+ context.properties {
+ githubProjectUrl('https://jenkins@github.com/apache/incubator-pulsar/')
+ }
+
+ setTopLevelJobProperties(
+ context,
+ 'https://gitbox.apache.org/repos/asf/incubator-pulsar.git',
+ branch,
+ 'git-websites',
+ 30)
+ }
+
+ // Sets common top-level job properties for main repository jobs.
+ static void setTopLevelMainJobProperties(context,
+ String branch = 'master',
+ String jdkVersion = 'JDK 1.8 (latest)',
+ int timeout = 200,
+ String jenkinsExecutorLabel = 'ubuntu') {
+ // GitHub project.
+ context.properties {
+ githubProjectUrl('https://github.com/apache/incubator-pulsar/')
+ }
+
+
+ setTopLevelJobProperties(
+ context,
+ 'https://github.com/apache/incubator-pulsar.git',
+ branch,
+ jenkinsExecutorLabel,
+ timeout)
+ }
+
+ // Sets common top-level job properties. Accessed through one of the above
+ // methods to protect jobs from internal details of param defaults.
+ private static void setTopLevelJobProperties(context,
+ String scmUrl,
+ String defaultBranch,
+ String jenkinsExecutorLabel,
+ int defaultTimeout,
+ String jdkVersion = 'JDK 1.8 (latest)') {
+ // Set JDK version.
+ context.jdk(jdkVersion)
+
+ // Restrict this project to run only on Jenkins executors as specified
+ context.label(jenkinsExecutorLabel)
+
+ // Discard old builds. Build records are only kept up to this number of days.
+ context.logRotator {
+ daysToKeep(14)
+ }
+
+ // Source code management.
+ context.scm {
+ git {
+ remote {
+ url(scmUrl)
+ refspec('+refs/heads/*:refs/remotes/origin/* ' +
+ '+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*')
+ }
+ branch('${sha1}')
+ extensions {
+ cleanAfterCheckout()
+ }
+ }
+ }
+
+ context.parameters {
+ // This is a recommended setup if you want to run the job manually. The
+ // ${sha1} parameter needs to be provided, and defaults to the main branch.
+ stringParam(
+ 'sha1',
+ defaultBranch,
+ 'Commit id or refname (eg: origin/pr/9/head) you want to build.')
+ }
+
+ context.wrappers {
+ // Abort the build if it's stuck for more minutes than specified.
+ timeout {
+ absolute(defaultTimeout)
+ abortBuild()
+ }
+ }
+ }
+
+ // Sets the pull request build trigger. Accessed through precommit methods
+ // below to insulate callers from internal parameter defaults.
+ private static void setPullRequestBuildTrigger(context,
+ String commitStatusContext,
+ String successComment = '--none--',
+ String prTriggerPhrase = '') {
+ context.triggers {
+ githubPullRequest {
+ admins(['asfbot'])
+ useGitHubHooks()
+ orgWhitelist(['apache'])
+ allowMembersOfWhitelistedOrgsAsAdmin()
+ permitAll()
+ // prTriggerPhrase is the argument which gets set when we want to allow
+ // post-commit builds to run against pending pull requests. This block
+ // overrides the default trigger phrase with the new one. Setting this
+ // will disable automatic invocation of this build; the phrase will be
+ // required to start it.
+ if (prTriggerPhrase) {
+ triggerPhrase(prTriggerPhrase)
+ onlyTriggerPhrase()
+ }
+
+ extensions {
+ commitStatus {
+ // This is the name that will show up in the GitHub pull request UI
+ // for this Jenkins project.
+ delegate.context("Jenkins: " + commitStatusContext)
+ }
+
+ /*
+ This section is disabled, because of jenkinsci/ghprb-plugin#417 issue.
+ For the time being, an equivalent configure section below is added.
+
+ // Comment messages after build completes.
+ buildStatus {
+ completedStatus('SUCCESS', successComment)
+ completedStatus('FAILURE', '--none--')
+ completedStatus('ERROR', '--none--')
+ }
+ */
+ }
+ }
+ }
+
+ // Comment messages after build completes.
+ context.configure {
+ def messages = it / triggers / 'org.jenkinsci.plugins.ghprb.GhprbTrigger' / extensions / 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildStatus' / messages
+ messages << 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildResultMessage' {
+ message(successComment)
+ result('SUCCESS')
+ }
+ messages << 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildResultMessage' {
+ message('--none--')
+ result('ERROR')
+ }
+ messages << 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildResultMessage' {
+ message('--none--')
+ result('FAILURE')
+ }
+ }
+ }
+
+ // Sets common config for Maven jobs.
+ static void setMavenConfig(context, mavenInstallation='Maven 3.5.0', mavenOpts='-Xmx4096m -Xms2048m') {
+ context.mavenInstallation(mavenInstallation)
+ context.mavenOpts('-Dorg.slf4j.simpleLogger.showDateTime=true')
+ context.mavenOpts('-Dorg.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd\\\'T\\\'HH:mm:ss.SSS')
+ // The -XX:+TieredCompilation -XX:TieredStopAtLevel=1 JVM options enable
+ // tiered compilation to make the JVM startup times faster during the tests.
+ context.mavenOpts('-XX:+TieredCompilation')
+ context.mavenOpts('-XX:TieredStopAtLevel=1')
+ context.mavenOpts(mavenOpts)
+ context.rootPOM('pom.xml')
+ // Use a repository local to the workspace for better isolation of jobs.
+ context.localRepository(LocalRepositoryLocation.LOCAL_TO_WORKSPACE)
+ // Disable archiving the built artifacts by default, as this is slow and flaky.
+ // We can usually recreate them easily, and we can also opt-in individual jobs
+ // to artifact archiving.
+ if (context.metaClass.respondsTo(context, 'archivingDisabled', boolean)) {
+ context.archivingDisabled(true)
+ }
+ }
+
+ // Sets common config for PreCommit jobs.
+ static void setPreCommit(context,
+ String commitStatusName,
+ String successComment = '--none--') {
+ // Set pull request build trigger.
+ setPullRequestBuildTrigger(context, commitStatusName, successComment)
+ }
+
+ // Enable triggering postcommit runs against pull requests. Users can comment the trigger phrase
+ // specified in the postcommit job and have the job run against their PR to run
+ // tests not in the presubmit suite for additional confidence.
+ static void enablePhraseTriggeringFromPullRequest(context,
+ String commitStatusName,
+ String prTriggerPhrase) {
+ setPullRequestBuildTrigger(
+ context,
+ commitStatusName,
+ '--none--',
+ prTriggerPhrase)
+ }
+
+ // Sets common config for PostCommit jobs.
+ static void setPostCommit(context,
+ String buildSchedule = '0 */6 * * *',
+ boolean triggerEveryPush = true,
+ String notifyAddress = 'dev@pulsar.incubator.apache.org',
+ boolean emailIndividuals = true) {
+ // Set build triggers
+ context.triggers {
+ // By default runs every 6 hours.
+ cron(buildSchedule)
+ if (triggerEveryPush) {
+ githubPush()
+ }
+ }
+
+ context.publishers {
+ // Notify an email address for each failed build (defaults to commits@).
+ mailer(notifyAddress, false, emailIndividuals)
+ }
+ }
+
+ // Sets common config for Website PostCommit jobs.
+ static void setWebsitePostCommit(context,
+ String buildSchedule = 'H 1 * * *',
+ String notifyAddress = 'dev@pulsar.incubator.apache.org',
+ boolean emailIndividuals = true) {
+ // Set build triggers
+ context.triggers {
+ // By default runs every 6 hours.
+ scm(buildSchedule)
+ githubPush()
+ }
+
+ context.publishers {
+ // Notify an email address for each failed build (defaults to commits@).
+ mailer(notifyAddress, false, emailIndividuals)
+ }
+ }
+
+}
diff --git a/.test-infra/jenkins/job_pulsar_release_nightly_snapshot.groovy b/.test-infra/jenkins/job_pulsar_release_nightly_snapshot.groovy
new file mode 100644
index 000000000..e99605ff5
--- /dev/null
+++ b/.test-infra/jenkins/job_pulsar_release_nightly_snapshot.groovy
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import common_job_properties
+
+// This job deploys a snapshot of latest master to artifactory nightly
+mavenJob('pulsar_release_nightly_snapshot') {
+ description('runs a `mvn clean deploy` of the nightly snapshot for pulsar.')
+
+ // Set common parameters.
+ common_job_properties.setTopLevelMainJobProperties(delegate)
+
+ // Sets that this is a PostCommit job.
+ common_job_properties.setPostCommit(
+ delegate,
+ 'H 12 * * *',
+ false)
+
+ // Allows triggering this build against pull requests.
+ common_job_properties.enablePhraseTriggeringFromPullRequest(
+ delegate,
+ 'Release Snapshot',
+ '/release-snapshot')
+
+ // Set maven parameters.
+ common_job_properties.setMavenConfig(delegate)
+
+ // Maven build project.
+ goals('clean package deploy -DskipTests')
+}
diff --git a/.test-infra/jenkins/job_seed.groovy b/.test-infra/jenkins/job_seed.groovy
new file mode 100644
index 000000000..b13ea81dc
--- /dev/null
+++ b/.test-infra/jenkins/job_seed.groovy
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import common_job_properties
+
+// Defines the seed job, which creates or updates all other Jenkins projects.
+job('pulsar-seed') {
+ description('Automatically configures all Apache BookKeeper Jenkins projects based' +
+ ' on Jenkins DSL groovy files checked into the code repository.')
+
+ // Set common parameters.
+ common_job_properties.setTopLevelMainJobProperties(delegate)
+
+ // This is a post-commit job that runs once per day, not for every push.
+ common_job_properties.setPostCommit(
+ delegate,
+ 'H 6 * * *',
+ false,
+ 'dev@pulsar.incubator.apache.org')
+
+ // Allows triggering this build against pull requests.
+ common_job_properties.enablePhraseTriggeringFromPullRequest(
+ delegate,
+ 'Seed Job',
+ '/seed')
+
+ steps {
+ dsl {
+ // A list or a glob of other groovy files to process.
+ external('.test-infra/jenkins/job_*.groovy')
+
+ // If a job is removed from the script, delete it
+ removeAction('DELETE')
+ }
+ }
+}
diff --git a/all/pom.xml b/all/pom.xml
index 0c02d51ef..736870054 100644
--- a/all/pom.xml
+++ b/all/pom.xml
@@ -104,6 +104,24 @@
<artifactId>pulsar-zookeeper</artifactId>
<version>${project.version}</version>
</dependency>
+
+ <!-- function examples -->
+ <dependency>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-functions-api-examples</artifactId>
+ <version>${project.version}</version>
+ <!-- make sure the api examples are compiled before assembly -->
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- runtime-all -->
+ <dependency>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-functions-runtime-all</artifactId>
+ <version>${project.version}</version>
+ <!-- make sure the api examples are compiled before assembly -->
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<build>
diff --git a/all/src/assemble/bin.xml b/all/src/assemble/bin.xml
index b295bab9e..e74b68405 100644
--- a/all/src/assemble/bin.xml
+++ b/all/src/assemble/bin.xml
@@ -43,6 +43,10 @@
<fileSet>
<directory>${basedir}/licenses</directory>
</fileSet>
+ <fileSet>
+ <directory>${basedir}/../pulsar-functions/runtime/target/python-instance</directory>
+ <outputDirectory>instances/python-instance</outputDirectory>
+ </fileSet>
</fileSets>
<files>
<file>
@@ -68,6 +72,16 @@
<outputDirectory>.</outputDirectory>
<fileMode>644</fileMode>
</file>
+ <file>
+ <source>${basedir}/../pulsar-functions/runtime-all/target/java-instance.jar</source>
+ <destName>java-instance.jar</destName>
+ <outputDirectory>instances</outputDirectory>
+ </file>
+ <file>
+ <source>${basedir}/../pulsar-functions/java-examples/target/pulsar-functions-api-examples.jar</source>
+ <destName>api-examples.jar</destName>
+ <outputDirectory>examples</outputDirectory>
+ </file>
</files>
<dependencySets>
<dependencySet>
diff --git a/bin/pulsar b/bin/pulsar
index ed7e988c8..0bd2228ae 100755
--- a/bin/pulsar
+++ b/bin/pulsar
@@ -31,6 +31,14 @@ DEFAULT_STANDALONE_CONF=$PULSAR_HOME/conf/standalone.conf
DEFAULT_WEBSOCKET_CONF=$PULSAR_HOME/conf/websocket.conf
DEFAULT_LOG_CONF=$PULSAR_HOME/conf/log4j2.yaml
+# functions related variables
+FUNCTIONS_HOME=$PULSAR_HOME/pulsar-functions
+DEFAULT_WORKER_CONF=$PULSAR_HOME/conf/functions_worker.yml
+DEFAULT_JAVA_INSTANCE_JAR=$PULSAR_HOME/instances/java-instance.jar
+JAVA_INSTANCE_JAR=${PULSAR_JAVA_INSTANCE_JAR:-"${DEFAULT_JAVA_INSTANCE_JAR}"}
+DEFAULT_PY_INSTANCE_FILE=$PULSAR_HOME/instances/python-instance/python_instance_main.py
+PY_INSTANCE_FILE=${PULSAR_PY_INSTANCE_FILE:-"${DEFAULT_PY_INSTANCE_FILE}"}
+
if [ -f "$PULSAR_HOME/conf/pulsar_env.sh" ]
then
. "$PULSAR_HOME/conf/pulsar_env.sh"
@@ -63,6 +71,34 @@ elif [ -e "$BUILT_JAR" ]; then
PULSAR_JAR=$BUILT_JAR
fi
+#
+# find the instance locations for pulsar-functions
+#
+
+# find the java instance location
+if [ ! -f "${JAVA_INSTANCE_JAR}" ]; then
+ # didn't find a released jar, then search the built jar
+ BUILT_JAVA_INSTANCE_JAR="${FUNCTIONS_HOME}/runtime-all/target/java-instance.jar"
+ if [ -z "${BUILT_JAVA_INSTANCE_JAR}" ]; then
+ echo "\nCouldn't find pulsar-functions java instance jar.";
+ echo "Make sure you've run 'mvn package'\n";
+ exit 1;
+ fi
+ JAVA_INSTANCE_JAR=${BUILT_JAVA_INSTANCE_JAR}
+fi
+
+# find the python instance location
+if [ ! -f "${PY_INSTANCE_FILE}" ]; then
+ # didn't find a released python instance, then search the built python instance
+ BUILT_PY_INSTANCE_FILE="${FUNCTIONS_HOME}/runtime/target/python-instance/python_instance_main.py"
+ if [ -z "${BUILT_PY_INSTANCE_FILE}" ]; then
+ echo "\nCouldn't find pulsar-functions python instance.";
+ echo "Make sure you've run 'mvn package'\n";
+ exit 1;
+ fi
+ PY_INSTANCE_FILE=${BUILT_PY_INSTANCE_FILE}
+fi
+
pulsar_help() {
cat <<EOF
Usage: pulsar <command>
@@ -74,6 +110,7 @@ where command is one of:
discovery Run a discovery server
proxy Run a pulsar proxy
websocket Run a web socket proxy server
+ functions-worker Run a functions worker server
standalone Run a broker server with local bookies and local zookeeper
compact-topic Run compaction against a topic
@@ -93,6 +130,7 @@ Environment variables:
PULSAR_DISCOVERY_CONF Configuration file for discovery service (default: $DEFAULT_DISCOVERY_CONF)
PULSAR_WEBSOCKET_CONF Configuration file for websocket proxy (default: $DEFAULT_WEBSOCKET_CONF)
PULSAR_PROXY_CONF Configuration file for Pulsar proxy (default: $DEFAULT_PROXY_CONF)
+ PULSAR_WORKER_CONF Configuration file for functions worker (default: $DEFAULT_WORKER_CONF)
PULSAR_STANDALONE_CONF Configuration file for standalone (default: $DEFAULT_STANDALONE_CONF)
PULSAR_EXTRA_OPTS Extra options to be passed to the jvm
PULSAR_EXTRA_CLASSPATH Add extra paths to the pulsar classpath
@@ -115,7 +153,7 @@ add_maven_deps_to_classpath() {
f="${PULSAR_HOME}/all/target/classpath.txt"
if [ ! -f "${f}" ]
then
- ${MVN} -f "${PULSAR_HOME}/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null
+ ${MVN} -f "${PULSAR_HOME}/pom.xml" dependency:build-classpath -DincludeScope=compile -Dmdep.outputFile="${f}" &> /dev/null
fi
PULSAR_CLASSPATH=${CLASSPATH}:`cat "${f}"`
}
@@ -136,6 +174,10 @@ fi
COMMAND=$1
shift
+if [ -z "$PULSAR_WORKER_CONF" ]; then
+ PULSAR_WORKER_CONF=$DEFAULT_WORKER_CONF
+fi
+
if [ -z "$PULSAR_BROKER_CONF" ]; then
PULSAR_BROKER_CONF=$DEFAULT_BROKER_CONF
fi
@@ -186,12 +228,22 @@ OPTS="-cp $PULSAR_CLASSPATH $OPTS"
OPTS="$OPTS $PULSAR_EXTRA_OPTS"
# log directory & file
-PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"Console"}
PULSAR_LOG_DIR=${PULSAR_LOG_DIR:-"$PULSAR_HOME/logs"}
+PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"RoutingAppender"}
+PULSAR_LOG_LEVEL=${PULSAR_LOG_LEVEL:-"info"}
+PULSAR_ROUTING_APPENDER_DEFAULT=${PULSAR_ROUTING_APPENDER_DEFAULT:-"Console"}
#Configure log configuration system properties
OPTS="$OPTS -Dpulsar.log.appender=$PULSAR_LOG_APPENDER"
OPTS="$OPTS -Dpulsar.log.dir=$PULSAR_LOG_DIR"
+OPTS="$OPTS -Dpulsar.log.level=$PULSAR_LOG_LEVEL"
+OPTS="$OPTS -Dpulsar.routing.appender.default=$PULSAR_ROUTING_APPENDER_DEFAULT"
+
+# Functions related logging
+OPTS="$OPTS -Dpulsar.functions.process.container.log.dir=$PULSAR_LOG_DIR"
+# instance
+OPTS="$OPTS -Dpulsar.functions.java.instance.jar=${JAVA_INSTANCE_JAR}"
+OPTS="$OPTS -Dpulsar.functions.python.instance.file=${PY_INSTANCE_FILE}"
#Change to PULSAR_HOME to support relative paths
cd "$PULSAR_HOME"
@@ -218,6 +270,9 @@ elif [ $COMMAND == "proxy" ]; then
elif [ $COMMAND == "websocket" ]; then
PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-websocket.log"}
exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.websocket.service.WebSocketServiceStarter $PULSAR_WEBSOCKET_CONF $@
+elif [ $COMMAND == "functions-worker" ]; then
+ PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-functions-worker.log"}
+ exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.functions.worker.FunctionWorkerStarter -c $PULSAR_WORKER_CONF $@
elif [ $COMMAND == "standalone" ]; then
PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-standalone.log"}
exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.PulsarStandaloneStarter --config $PULSAR_STANDALONE_CONF $@
diff --git a/bin/pulsar-admin b/bin/pulsar-admin
index 8de438a78..c8b41da45 100755
--- a/bin/pulsar-admin
+++ b/bin/pulsar-admin
@@ -24,6 +24,13 @@ PULSAR_HOME=`cd $BINDIR/..;pwd`
DEFAULT_CLIENT_CONF=$PULSAR_HOME/conf/client.conf
DEFAULT_LOG_CONF=$PULSAR_HOME/conf/log4j2.yaml
+# functions related variables
+FUNCTIONS_HOME=$PULSAR_HOME/pulsar-functions
+DEFAULT_JAVA_INSTANCE_JAR=$PULSAR_HOME/instances/java-instance.jar
+JAVA_INSTANCE_JAR=${PULSAR_JAVA_INSTANCE_JAR:-"${DEFAULT_JAVA_INSTANCE_JAR}"}
+DEFAULT_PY_INSTANCE_FILE=$PULSAR_HOME/instances/python-instance/python_instance_main.py
+PY_INSTANCE_FILE=${PULSAR_PY_INSTANCE_FILE:-"${DEFAULT_PY_INSTANCE_FILE}"}
+
if [ -f "$PULSAR_HOME/conf/pulsar_tools_env.sh" ]
then
. "$PULSAR_HOME/conf/pulsar_tools_env.sh"
@@ -41,7 +48,7 @@ else
fi
# exclude tests jar
-RELEASE_JAR=`ls $PULSAR_HOME/pulsar-*.jar 2> /dev/null | grep -v tests | tail -1`
+RELEASE_JAR=`ls $PULSAR_HOME/pulsar-*.jar 2> /dev/null | grep -v tests | tail -1`
if [ $? == 0 ]; then
PULSAR_JAR=$RELEASE_JAR
fi
@@ -61,20 +68,20 @@ add_maven_deps_to_classpath() {
if [ "$MAVEN_HOME" != "" ]; then
MVN=${MAVEN_HOME}/bin/mvn
fi
-
+
# Need to generate classpath from maven pom. This is costly so generate it
# and cache it. Save the file into our target dir so a mvn clean will get
# clean it up and force us create a new one.
f="${PULSAR_HOME}/all/target/classpath.txt"
if [ ! -f "${f}" ]
then
- ${MVN} -f "${PULSAR_HOME}/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null
+ ${MVN} -f "${PULSAR_HOME}/pom.xml" dependency:build-classpath -DincludeScope=compile -Dmdep.outputFile="${f}" &> /dev/null
fi
PULSAR_CLASSPATH=${CLASSPATH}:`cat "${f}"`
}
if [ -d "$PULSAR_HOME/lib" ]; then
- PULSAR_CLASSPATH="$PULSAR_CLASSPATH:$PULSAR_HOME/lib/*"
+ PULSAR_CLASSPATH="$PULSAR_CLASSPATH:$PULSAR_HOME/lib/*"
else
add_maven_deps_to_classpath
fi
@@ -97,11 +104,45 @@ OPTS="$OPTS $PULSAR_EXTRA_OPTS"
# log directory & file
PULSAR_LOG_DIR=${PULSAR_LOG_DIR:-"$PULSAR_HOME/logs"}
-PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"Console"}
+PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"RoutingAppender"}
+PULSAR_LOG_LEVEL=${PULSAR_LOG_LEVEL:-"info"}
+PULSAR_ROUTING_APPENDER_DEFAULT=${PULSAR_ROUTING_APPENDER_DEFAULT:-"Console"}
#Configure log configuration system properties
-OPTS="$OPTS -Dpulsar.log.dir=$PULSAR_LOG_DIR"
OPTS="$OPTS -Dpulsar.log.appender=$PULSAR_LOG_APPENDER"
+OPTS="$OPTS -Dpulsar.log.dir=$PULSAR_LOG_DIR"
+OPTS="$OPTS -Dpulsar.log.level=$PULSAR_LOG_LEVEL"
+OPTS="$OPTS -Dpulsar.routing.appender.default=$PULSAR_ROUTING_APPENDER_DEFAULT"
+
+# find the java instance location
+if [ ! -f "${JAVA_INSTANCE_JAR}" ]; then
+ # didn't find a released jar, then search the built jar
+ BUILT_JAVA_INSTANCE_JAR="${FUNCTIONS_HOME}/runtime-all/target/java-instance.jar"
+ if [ -f "${BUILT_JAVA_INSTANCE_JAR}" ]; then
+ JAVA_INSTANCE_JAR=${BUILT_JAVA_INSTANCE_JAR}
+ else
+ echo "\nCouldn't find pulsar java instance jar.";
+ echo "Make sure you've run 'mvn package'\n";
+ exit 1;
+ fi
+fi
+
+# find the python instance location
+if [ ! -f "${PY_INSTANCE_FILE}" ]; then
+ # didn't find a released python instance, then search the built python instance
+ BUILT_PY_INSTANCE_FILE="${FUNCTIONS_HOME}/runtime/target/python-instance/python_instance_main.py"
+ if [ -f "${BUILT_PY_INSTANCE_FILE}" ]; then
+ PY_INSTANCE_FILE=${BUILT_PY_INSTANCE_FILE}
+ else
+ echo "\nCouldn't find pulsar python instance.";
+ echo "Make sure you've run 'mvn package'\n";
+ exit 1;
+ fi
+fi
+
+# functions
+OPTS="$OPTS -Dpulsar.functions.java.instance.jar=${JAVA_INSTANCE_JAR}"
+OPTS="$OPTS -Dpulsar.functions.python.instance.file=${PY_INSTANCE_FILE}"
#Change to PULSAR_HOME to support relative paths
cd "$PULSAR_HOME"
diff --git a/buildtools/pom.xml b/buildtools/pom.xml
index ae217e4da..d83bb2c90 100644
--- a/buildtools/pom.xml
+++ b/buildtools/pom.xml
@@ -22,6 +22,12 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache</groupId>
+ <artifactId>apache</artifactId>
+ <version>18</version>
+ </parent>
+
<groupId>org.apache.pulsar</groupId>
<artifactId>buildtools</artifactId>
<version>2.0.0-incubating-SNAPSHOT</version>
@@ -34,6 +40,21 @@
<artifactId>testng</artifactId>
<version>6.13.1</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ <version>2.10.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <version>2.10.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <version>2.10.0</version>
+ </dependency>
</dependencies>
<build>
diff --git a/buildtools/src/main/resources/log4j2.xml b/buildtools/src/main/resources/log4j2.xml
new file mode 100644
index 000000000..e7329532c
--- /dev/null
+++ b/buildtools/src/main/resources/log4j2.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<Configuration status="INFO">
+ <Appenders>
+ <Console name="Console" target="SYSTEM_OUT">
+ <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t:%C@%L] %-5level %logger{36} - %msg%n" />
+ </Console>
+ </Appenders>
+ <Loggers>
+ <Root level="warn">
+ <AppenderRef ref="Console" />
+ </Root>
+ <Logger name="org.apache.pulsar" level="info"/>
+ </Loggers>
+</Configuration>
diff --git a/conf/broker.conf b/conf/broker.conf
index 93489e5cd..59a19d7b8 100644
--- a/conf/broker.conf
+++ b/conf/broker.conf
@@ -285,6 +285,10 @@ managedLedgerDefaultWriteQuorum=2
# Number of guaranteed copies (acks to wait before write is complete)
managedLedgerDefaultAckQuorum=2
+# Default type of checksum to use when writing to BookKeeper. Default is "CRC32"
+# Other possible options are "CRC32C" (which is faster), "MAC" or "DUMMY" (no checksum).
+managedLedgerDigestType=CRC32
+
# Amount of memory to use for caching data payload in managed ledger. This memory
# is allocated from JVM direct memory and it's shared across all the topics
# running in the same broker
@@ -440,3 +444,9 @@ webSocketConnectionsPerBroker=8
# Enable topic level metrics
exposeTopicLevelMetricsInPrometheus=true
+
+### --- Functions --- ###
+
+# Enable Functions Worker Service in Broker
+functionsWorkerEnabled=false
+
diff --git a/conf/example.yml b/conf/example.yml
new file mode 100644
index 000000000..5c052730d
--- /dev/null
+++ b/conf/example.yml
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+tenant: "test"
+namespace: "test-namespace"
+name: "example"
+className: "org.apache.pulsar.functions.api.examples.ExclamationFunction"
+inputs: ["persistent://sample/standalone/ns1/test_src"]
+userConfig:
+ "PublishTopic" : "persistent://sample/standalone/ns1/test_result"
+
+output: "persistent://sample/standalone/ns1/test_result"
+autoAck: true
+parallelism: 1
diff --git a/conf/functions_worker.yml b/conf/functions_worker.yml
new file mode 100644
index 000000000..daaeae9a9
--- /dev/null
+++ b/conf/functions_worker.yml
@@ -0,0 +1,47 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+workerId: standalone
+workerHostname: localhost
+workerPort: 6750
+functionMetadataTopicName: metadata
+functionMetadataSnapshotsTopicPath: snapshots
+clusterCoordinationTopicName: coordinate
+pulsarFunctionsNamespace: sample/standalone/functions
+pulsarServiceUrl: pulsar://localhost:6650
+pulsarWebServiceUrl: http://localhost:8080
+numFunctionPackageReplicas: 1
+downloadDirectory: /tmp/pulsar_functions
+metricsConfig:
+ metricsSinkClassName: org.apache.pulsar.functions.metrics.sink.PrometheusSink
+ metricsCollectionInterval: 30
+ metricsSinkConfig:
+ path: /metrics
+ port: 9099
+#threadContainerFactory:
+# threadGroupName: "Thread Function Container Group"
+processContainerFactory:
+ logDirectory:
+
+schedulerClassName: "org.apache.pulsar.functions.worker.scheduler.RoundRobinScheduler"
+functionAssignmentTopicName: "assignments"
+failureCheckFreqMs: 30000
+rescheduleTimeoutMs: 60000
+initialBrokerReconnectMaxRetries: 60
+assignmentWriteMaxRetries: 60
diff --git a/conf/log4j2.yaml b/conf/log4j2.yaml
index 13b07adc9..cd0ec880b 100644
--- a/conf/log4j2.yaml
+++ b/conf/log4j2.yaml
@@ -23,6 +23,21 @@ Configuration:
monitorInterval: 30
name: pulsar
+ Properties:
+ Property:
+ - name: "pulsar.log.dir"
+ value: "logs"
+ - name: "pulsar.log.file"
+ value: "pulsar.log"
+ - name: "pulsar.log.appender"
+ value: "RoutingAppender"
+ - name: "pulsar.log.level"
+ value: "info"
+ - name: "pulsar.routing.appender.default"
+ value: "Console"
+ - name: "bk.log.level"
+ value: "info"
+
# Example: logger-filter script
Scripts:
ScriptFile:
@@ -32,19 +47,19 @@ Configuration:
charset: UTF-8
Appenders:
-
- # Console
+
+ # Console
Console:
name: Console
target: SYSTEM_OUT
PatternLayout:
Pattern: "%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"
-
- # Rolling file appender configuration
+
+ # Rolling file appender configuration
RollingFile:
name: RollingFile
fileName: "${sys:pulsar.log.dir}/${sys:pulsar.log.file}"
- filePattern: "/${sys:pulsar.log.file}-%d{MM-dd-yyyy}-%i.log.gz"
+ filePattern: "${sys:pulsar.log.dir}/${sys:pulsar.log.file}-%d{MM-dd-yyyy}-%i.log.gz"
immediateFlush: false
PatternLayout:
Pattern: "%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"
@@ -54,7 +69,7 @@ Configuration:
modulate: true
SizeBasedTriggeringPolicy:
size: 1 GB
- # Trigger every day at midnight that also scan
+ # Trigger every day at midnight that also scan
# roll-over strategy that deletes older file
CronTriggeringPolicy:
schedule: "0 0 0 * * ?"
@@ -67,16 +82,101 @@ Configuration:
glob: "*/${sys:pulsar.log.file}*log.gz"
IfLastModified:
age: 30d
-
+
+ # Rolling file appender configuration for bk
+ RollingRandomAccessFile:
+ name: BkRollingFile
+ fileName: "${sys:pulsar.log.dir}/${sys:pulsar.log.file}.bk"
+ filePattern: "${sys:pulsar.log.dir}/${sys:pulsar.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz"
+ immediateFlush: true
+ PatternLayout:
+ Pattern: "%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"
+ Policies:
+ TimeBasedTriggeringPolicy:
+ interval: 1
+ modulate: true
+ SizeBasedTriggeringPolicy:
+ size: 1 GB
+ # Trigger every day at midnight that also scan
+ # roll-over strategy that deletes older file
+ CronTriggeringPolicy:
+ schedule: "0 0 0 * * ?"
+ # Delete file older than 30days
+ DefaultRolloverStrategy:
+ Delete:
+ basePath: ${sys:pulsar.log.dir}
+ maxDepth: 2
+ IfFileName:
+ glob: "*/${sys:pulsar.log.file}.bk*log.gz"
+ IfLastModified:
+ age: 30d
+
+ # Routing
+ Routing:
+ name: RoutingAppender
+ Routes:
+ pattern: "$${ctx:function}"
+ Route:
+ -
+ Routing:
+ name: InstanceRoutingAppender
+ Routes:
+ pattern: "$${ctx:instance}"
+ Route:
+ -
+ RollingFile:
+ name: "Rolling-${ctx:function}"
+ fileName : "${sys:pulsar.log.dir}/functions/${ctx:function}/function.log"
+ filePattern : "${sys:pulsar.log.dir}/functions/${ctx:function}-%d{MM-dd-yyyy}-%i.log.gz"
+ PatternLayout:
+ Pattern: "%d{ABSOLUTE} %level{length=5} [%thread] [instance: %X{instance}] %logger{1} - %msg%n"
+ Policies:
+ TimeBasedTriggeringPolicy:
+ interval: 1
+ modulate: true
+ SizeBasedTriggeringPolicy:
+ size: "20MB"
+ # Trigger every day at midnight that also scan
+ # roll-over strategy that deletes older file
+ CronTriggeringPolicy:
+ schedule: "0 0 0 * * ?"
+ # Delete file older than 30days
+ DefaultRolloverStrategy:
+ Delete:
+ basePath: ${sys:pulsar.log.dir}
+ maxDepth: 2
+ IfFileName:
+ glob: "*/${sys:pulsar.log.file}*log.gz"
+ IfLastModified:
+ age: 30d
+ - ref: "${sys:pulsar.routing.appender.default}"
+ key: "${ctx:function}"
+ - ref: "${sys:pulsar.routing.appender.default}"
+ key: "${ctx:function}"
Loggers:
+
+ Logger:
+ name: org.apache.bookkeeper
+ level: "${sys:bk.log.level}"
+ additivity: false
+ AppenderRef:
+ - ref: BkRollingFile
+
+ Logger:
+ name: org.apache.distributedlog
+ level: "${sys:bk.log.level}"
+ additivity: false
+ AppenderRef:
+ - ref: BkRollingFile
# Default root logger configuration
Root:
level: info
additivity: false
AppenderRef:
- ref: "${sys:pulsar.log.appender}"
+ - ref: "${sys:pulsar.log.appender}"
+ level: "${sys:pulsar.log.level}"
# Logger to inject filter script
# Logger:
@@ -90,4 +190,4 @@ Configuration:
# onMisMatch: DENY
# ScriptRef:
# ref: filter.js
-
\ No newline at end of file
+
diff --git a/conf/standalone.conf b/conf/standalone.conf
index de0406492..dc6f1b743 100644
--- a/conf/standalone.conf
+++ b/conf/standalone.conf
@@ -250,6 +250,10 @@ managedLedgerDefaultWriteQuorum=1
# Number of guaranteed copies (acks to wait before write is complete)
managedLedgerDefaultAckQuorum=1
+# Default type of checksum to use when writing to BookKeeper. Default is "CRC32"
+# Other possible options are "CRC32C" (which is faster), "MAC" or "DUMMY" (no checksum).
+managedLedgerDigestType=CRC32
+
# Amount of memory to use for caching data payload in managed ledger. This memory
# is allocated from JVM direct memory and it's shared across all the topics
# running in the same broker
diff --git a/faq.md b/faq.md
deleted file mode 100644
index 1ff70938b..000000000
--- a/faq.md
+++ /dev/null
@@ -1,273 +0,0 @@
-# Frequently Asked Questions
-- Getting Started
-- Concepts and Design
-- Usage and Configuration
-
----
-
-## Getting Started
-
-### What is the minimum requirements for Apache Pulsar ?
-You need 3 kind of clusters: bookie, broker, zookeeper. But if not have enough resource, it's ok to run them on same machine.
-
----
-
-## Concepts and Design
-
-### Is ack tied to subscription?
-Yes, ack is tied to a particular subscription.
-
-### Where should I look into to tweak load balancing ?
-There are few parameters to look at :
-1. The topic assignments to brokers are done in terms of “bundles”, that is in group of topic
-2. Topics are matched to bundles by hashing on the name
-3. Effectively, a bundle is a hash-range where topics falls into
-4. Initially the default is to have 4 “bundles” for a namespace
-5. When the traffic increases on a given bundle, it will be split in 2 and reassigned to a different broker
-6. There are some adjustable thresholds that can be used to control when the split happens, based on number of topics/partitions, messages in/out, bytes in/out, etc..
-7. It’s also possible to specify a higher number of bundles when creating a namepsac
-8. There are the load-manager threshold that control when a broker should offload some of the bundles to other brokers
-
-### What is the lifecycle of subscription?
-Once it’s created, it retains all messages published after that (minus explicit TTL). Subscriptions can be dropped by explicitly unsubscribing (in `Consumer` API) or through the REST/CLI .
-
-### What is a bundle?
-In Pulsar, "namespaces" are the administrative unit: you can configure most options on a namespace and they will be applied on the topics contained on the namespace. It gives the convenience of doing settings and operations on a group of topics rather than doing it once per topic.
-
-In general, the pattern is to use a namespace for each user application. So a single user/tenant, can create multiple namespaces to manage its own applications.
-
-When it comes to topics, we need a way to assign topics to brokers, control the load and move them if a broker becomes overloaded. Rather that doing this operations per each single topic (ownership, load-monitoring, assigning), we do it in bundles, or "groups of topics".
-
-In practical words, the number of bundles determines "into how many brokers can I spread the topics for a given namespace".
-
-From the client API or implementation, there's no concept of bundles, clients will lookup the topics that want to publish/consumer individually.
-
-On the broker side, the namespace is broken down into multiple bundles, and each bundle can be assigned to a different broker. Effectively, bundles are the "unit of assignment" for topics into brokers and this is what the load-manager uses to track the traffic and decide where to place "bundles" and whether to offload them to other brokers.
-
-A bundle is represented by a hash-range. The 32-bit hash space is initially divided equally into the requested bundles. Topics are matched to a bundle by hashing on the topic name.
-
-Default number of bundles is configured in `broker.conf`: `defaultNumberOfNamespaceBundles=4`
-
-When the traffic increases on a given bundle, it will be split in 2 and reassigned to a different broker.
-
-Enable auto-split: `loadBalancerAutoBundleSplitEnable=true` trigger unload and reassignment after splitting: `loadBalancerAutoUnloadSplitsEnable=true`.
-
-If is expected to have a high traffic on a particular namespace, it's a good practice to specify a higher number of bundles when creating the namespace: `bin/pulsar-admin namespaces create $NS --bundles 64`. This will avoid the initial auto-adjustment phase.
-
-All the thresholds for the auto-splitting can be configured in `broker.conf`, eg: number of topics/partitions, messages in/out, bytes in/out, etc...
-
-### How the design deals with isolation between tenants, which concepts enable that and up to what extent, how huge difference can exist between tenants so that impact on each other is noticeable via degraded latency.
-The isolation between tenants (and topics of same tenant) happens at many different points. I'll start from the bottom up.
-
-#### Storage
-You're probably familiar with BookKeeper, but of the main strength is that each bookie can efficiently serve many different ledger (segments of topic data). We tested with 100s of thousand per single node.
-
-This is because there is a single journal (on its own device) where all the write operations gets appended and then the entries are periodically flushed in background on the storage device.
-
-This gives isolation between writes and reads in a bookie. You can read as fast as you can, maxing out the IO on the storage device, but your write throughput and latency are going to be unaffected.
-
-#### Broker
-Everything in the broker happens asynchronously. The amount of memory that is used is also capped per broker.
-
-Whenever the broker is marked as overloaded, traffic can be quickly shifted (manually or without intervention) to less loaded brokers. LoadManager component in brokers is dedicated to that.
-
-There are several points of flow control:
-- On the producer side, there are limits on the in-flight message for broker bookies, that will slow down users trying to publish faster that the system can absorb
-- On the consumer side, it's possible to throttle the delivery to a certain rate
-
-#### Quotas
-Can configure different storage quotas for different tenants/namespaces and take different actions when the quotas are filled up (block producer, give exception, drop older messages).
-
-#### Broker level isolation
-There is the option to isolate certain tenants/namespaces to a particular set of broker. Typically the reason for using that was to experiment with different configurations, debugging and quickly react to unexpected situations.
-
-For example, a particular user might be triggering a bad behavior in the broker that can impact performance for other tenants.
-
-In this case, the particular user can be "isolated" a subset of brokers that will not serve any other traffic, until a proper fix that correctly handles the condition can be deployed.
-
-This is a lightweight option of having multiple clusters for different users, since most of the other parts are still shared (ZK, BK,...).
-
-
-### Is there "regex" topic in Pulsar?
-There is regex subscription coming up in Pulsar 2.0. See [PIP-13](https://github.com/apache/incubator-pulsar/wiki/PIP-13:-Subscribe-to-topics-represented-by-regular-expressions).
-
-### Does Pulsar have, or plan to have, a concept of log compaction where only the latest message with the same key will be kept ?
-Yes, see [PIP-9](https://github.com/ivankelly/incubator-pulsar-wiki/pull/1/files) for more details.
-
-### When I use an exclusive subscription to a partitioned topic, is the subscription attached to the "whole topic" or to a "topic partition"?
-On a partitioned topic, you can use all the 3 supported subscription types (exclusive, failover, shared), same as with non partitioned topics.
-The “subscription” concept is roughly similar to a “consumer-group” in Kafka. You can have multiple of them in the same topic, with different names.
-
-If you use “exclusive”, a consumer will try to consume from all partitions, or fail if any partition is already being consumer.
-
-The mode similar to Kafka is “failover” subscription. In this case, you have 1 active consumer per partition, the active/stand-by decision is made at the partition level, and Pulsar will make sure to spread the partition assignments evenly across consumer.
-
-### What is the proxy component?
-It’s a component that was introduced recently. Essentially it’s a stateless proxy that speaks that Pulsar binary protocol. The motivation is to avoid (or overcome the impossibility) of direct connection between clients and brokers.
-
----
-
-## Usage and Configuration
-### Can I manually change the number of bundles after creating namespaces?
-Yes, you can split a given bundle manually.
-
-### Is the producer kafka wrapper thread-safe?
-The producer wrapper should be thread-safe.
-
-### Can I just remove a subscription?
-Yes, you can use the cli tool `bin/pulsar-admin persistent unsubscribe $TOPIC -s $SUBSCRIPTION`.
-
-### How are subscription modes set? Can I create new subscriptions over the WebSocket API?
-Yes, you can set most of the producer/consumer configuration option in websocket, by passing them as HTTP query parameters like:
-`ws://localhost:8080/ws/consumer/persistent/sample/standalone/ns1/my-topic/my-sub?subscriptionType=Shared`
-
-see [the doc](http://pulsar.apache.org/docs/latest/clients/WebSocket/#RunningtheWebSocketservice-1fhsvp).
-
-### Is there any sort of order of operations or best practices on the upgrade procedure for a geo-replicated Pulsar cluster?
-In general, updating the Pulsar brokers is an easy operation, since the brokers don't have local state. The typical rollout is a rolling upgrade, either doing 1 broker at a time or some percentage of them in parallel.
-
-There are not complicated requirements to upgrade geo-replicated clusters, since we take particular care in ensuring backward and forward compatibility.
-
-Both the client and the brokers are reporting their own protocol version and they're able to disable newer features if the other side doesn't support them yet.
-
-Additionally, when making metadata breaking format changes (if the need arises), we make sure to spread the changes along at least 2 releases.
-
-This is to always allow the possibility to downgrade a running cluster to a previous version, in case any server problem is identified in production.
-
-So, one release will understand the new format while the next one will actually start using it.
-
-### Since Pulsar has configurable retention per namespace, can I set a "forever" value, ie., always keep all data in the namespaces?
-So, retention applies to "consumed" messages. Ones, for which the consumer has already acknowledged the processing. By default, retention is 0, so it means data is deleted as soon as all consumers acknowledge. You can set retention to delay the retention.
-
-That also means, that data is kept forever, by default, if the consumers are not acknowledging.
-
-There is no currently "infinite" retention, other than setting to very high value.
-
-### How can a consumer "replay" a topic from the beginning, ie., where can I set an offset for the consumer?
-1. Use admin API (or CLI tool):
- - Reset to a specific point in time (3h ago)
- - Reset to a message id
-2. You can use the client API `seek`.
-
-### When create a consumer, does this affect other consumers ?
-The key is that you should use different subscriptions for each consumer. Each subscription is completely independent from others.
-
-### The default when creating a consumer, is it to "tail" from "now" on the topic, or from the "last acknowledged" or something else?
-So when you spin up a consumer, it will try to subscribe to the topic, if the subscription doesn't exist, a new one will be created, and it will be positioned at the end of the topic ("now").
-
-Once you reconnect, the subscription will still be there and it will be positioned on the last acknowledged messages from the previous session.
-
-### I want some produce lock, i.e., to pessimistically or optimistically lock a specified topic so only one producer can write at a time and all further producers know they have to reprocess data before trying again to write a topic.
-To ensure only one producer is connected, you just need to use the same "producerName", the broker will ensure that no 2 producers with same name are publishing on a given topic.
-
-### I tested the performance using PerformanceProducer between two server node with 10,000Mbits NIC(and I tested tcp throughput can be larger than 1GB/s). I saw that the max msg throughput is around 1000,000 msg/s when using little msg_size(such as 64/128Bytes), when I increased the msg_size to 1028 or larger , then the msg/s will decreased sharply to 150,000msg/s, and both has max throughput around 1600Mbit/s, which is far from 1GB/s. And I'm curious that the throughput between producer and broker why can't excess 1600Mbit/s ? It seems that the Producer executor only use one thread, is this the reason？Then I start two producer client jvm, the throughput increased not much, just about little beyond 1600Mbit/s. Any other reasons?
-Most probably, when increasing the payload size, you're reaching the disk max write rate on a single bookie.
-
-There are few tricks that can be used to increase throughput (other than just partitioning)
-
-1. Enable striping in BK, by setting ensemble to bigger than write quorum. E.g. e=5 w=2 a=2. Write 2 copies of each message but stripe them across 5 bookies
-
-2. If there are already multiple topics/partitions, you can try to configure the bookies with multiple journals (e.g. 4). This should increase the throughput when the journal is on SSDs, since the controller has multiple IO queues and can efficiently sustain multiple threads each doing sequential writes
-
-- Option (1) you just configure it on a given pulsar namespace, look at "namespaces set-persistence" command
-
-- Option (2) needs to be configured in bookies
-
-### Is there any work on a Mesos Framework for Pulsar/Bookkeeper this point? Would this be useful?
-We don’t have anything ready available for Mesos/DCOS though there should be nothing preventing it
-
-It would surely be useful.
-
-
-### Is there an HDFS like interface?
-Not for Pulsar.There was some work in BK / DistributedLog community to have it but not at the messaging layer.
-
-### Where can I find information about `receiveAsync` parameters? In particular, is there a timeout as in `receive`?
-There’s no other info about `receiveAsync()`. The method doesn’t take any parameters. Currently there’s no timeout on it. You can always set a timeout on the `CompletableFuture` itself, but the problem is how to cancel the future and avoid “getting” the message.
-
-What’s your use case for timeout on the `receiveAsync()`? Could that be achieved more easily by using the `MessageListener`?
-
-### Why do we choose to use bookkeeper to store consumer offset instead of zookeeper? I mean what's the benefits?
-ZooKeeper is a “consensus” system that while it exposes a key/value interface is not meant to support a large volume of writes per second.
-
-ZK is not an “horizontally scalable” system, because every node receive every transaction and keeps the whole data set. Effectively, ZK is based on a single “log” that is replicated consistently across the participants.
-
-The max throughput we have observed on a well configured ZK on good hardware was around ~10K writes/s. If you want to do more than that, you would have to shard it..
-
-To store consumers cursor positions, we need to write potentially a large number of updates per second. Typically we persist the cursor every 1 second, though the rate is configurable and if you want to reduce the amount of potential duplicates, you can increase the persistent frequency.
-
-With BookKeeper it’s very efficient to have a large throughput across a huge number of different “logs”. In our case, we use 1 log per cursor, and it becomes feasible to persist every single cursor update.
-
-### I'm facing some issue using `.receiveAsync` that it seems to be related with `UnAckedMessageTracker` and `PartitionedConsumerImpl`. We are consuming messages with `receiveAsync`, doing instant `acknowledgeAsync` when message is received, after that the process will delay the next execution of itself. In such scenario we are consuming a lot more messages (repeated) than the num of messages produced. We are using Partitioned topics with setAckTimeout 30 seconds and I believe this issue could be related with `PartitionedConsumerImpl` because the same test in a non-partitioned topic does not generate any repeated message.
-PartitionedConsumer is composed of a set of regular consumers, one per partition. To have a single `receive()` abstraction, messages from all partitions are then pushed into a shared queue.
-
-The thing is that the unacked message tracker works at the partition level.So when the timeout happens, it’s able to request redelivery for the messages and clear them from the queue when that happens,
-but if the messages were already pushed into the shared queue, the “clearing” part will not happen.
-
-- the only quick workaround that I can think of is to increase the “ack-timeout” to a level in which timeout doesn’t occur in processing
-- another option would be to reduce the receiver queue size, so that less messages are sitting in the queue
-
-### Can I use bookkeeper newer v3 wire protocol in Pulsar? How can I enable it?
-The answer is currently not, because we force the broker to use v2 protocol and that's not configurable at the moment.
-
-### Is "kubernetes/generic/proxy.yaml" meant to be used whenever we want to expose a Pulsar broker outside the Kubernetes cluster?
-Yes, the “proxy” is an additional component to deploy a stateless proxy frontend that can be exposed through a load balancer and that doesn’t require direct connectivity to the actual brokers. No need to use it from within Kubernetes cluster. Also in some cases it’s simpler to have expose the brokers through `NodePort` or `ClusterIp` for other outside producer/consumers.
-
-### Is there a way of having both authentication and the Pulsar dashboard working at same time?
-The key is that with authorization, the stats collector needs to access the APIs that require the credentials. That’s not a problem for stats collected through Prometheus but it is for the “Pulsar dashboard” which is where the per-topic stats are shown. I think that should be quite easy to fix.
-
-### How can I know when I've reached the end of the stream during replay?
-There is no direct way because messages can still be published in the topic, and relying on the `readNext(timeout)` is not precise because the client might be temporarily disconnected from broker in that moment.
-
-One option is to use `publishTimestamp` of messages. When you start replaying you can check current "now", then you replay util you hit a message with timestamp >= now.
-
-Another option is to "terminate" the topic. Once a topic is "terminated", no more message can be published on the topic, a reader/consumer can check the `hasReachedEndOfTopic()` condition to know when that happened.
-
-A final option is to check the topic stats. This is a tiny bit involved, because it requires the admin client (or using REST) to get the stats for the topic and checking the "backlog". If the backlog is 0, it means we've hit the end.
-
-### How can I prevent an inactive topic to be deleted under any circumstance? I want to set no time or space limit for a certain namespace.
-There’s not currently an option for “infinite” (though it sounds a good idea! maybe we could use `-1` for that). The only option now is to use INT_MAX for `retentionTimeInMinutes` and LONG_MAX for `retentionSizeInMB`. It’s not “infinite” but 4085 years of retention should probably be enough!
-
-### Is there a profiling option in Pulsar, so that we can breakdown the time costed in every stage? For instance, message A stay in queue 1ms, bk writing time 2ms(interval between sending to bk and receiving ack from bk) and so on.
-There are latency stats at different stages. In the client (eg: reported every 1min in info logs).
-In the broker: accessible through the broker metrics, and finally in bookies where there are several different latency metrics.
-
-In broker there’s just the write latency on BK, because there is no other queuing involved in the write path.
-
-### How can I have multiple readers that each get all the messages from a topic from the beginning concurrently? I.e., no round-robin, no exclusivity
-you can create reader with `MessageId.earliest`
-
-
-### Does broker validate if a property exists or not when producer/consumer connects ?
-yes, broker performs auth&auth while creating producer/consumer and this information presents under namespace policies.. so, if auth is enabled then broker does validation
-
-### From what I’ve seen so far, it seems that I’d instead want to do a partitioned topic when I want a firehose/mix of data, and shuffle that firehose in to specific topics per entity when I’d have more discrete consumers. Is that accurate?
-Precisely, you can use either approach, and even combine them, depending on what is more convenient for the use case. The general traits to choose one or the other are:
-
-- Partitions -> Maintain a single “logical” topic but scale throughput to multiple machines. Also, ability to consume in order for a “partition” of the keys. In general, consumers are assigned a partition (and thus a subset of keys) without specifying anything.
-
-- Multiple topics -> When each topic represent some concrete existing “concept” in the application and it is “finite” (eg: using a topic per each user when the number of users is unbound and can be in the 100s of millions it’s not a good idea), within 10s or 100s of thousands. Having multiple topics makes it easier for a consumer to consume a specific portion of messages.
-
-### For subscribing to a large number of topics like that, would i need to call `subscribe` for each one individually, or is there some sort of wildcard capability?
-Currently you can only subscribe individually, (though you can automate it by getting the list of topics and going through it), but we’re working on the wildcard subscribe and we’re targeting that for next release.
-
-### Hi, is the difference between a consumer and a reader documented somewhere?
-Main difference: a reader can be used when manually managing the offset/messageId, rather than relying on Pulsar to keep track of it with the acknowledgments
-- consumer -> managed subscriptions with acks and auto retention
-- reader -> always specify start message id on creation
-
-
-### Hey, question on routing mode for partitioned topics. What is the default configuration and what is used in the Kafka adaptor?
-The default is to use the hash of the key on a message. If the message has no key, the producer will use a “default” partition (picks 1 random partition and use it for all the messages it publishes).
-
-This is to maintain the same ordering guarantee when no partitions are there: per-producer ordering.
-
-The same applies when using the Kafka wrapper.
-
-### I'm setting up bookies on AWS d2.4xlarge instances (16 cores, 122G memory, 12x2TB raid-0 hd). Do you have any recommendation for memory configuration for this kind of setup? For configurations like java heap, direct memory and dbStorage_writeCacheMaxSizeMb, dbStorage_readAheadCacheMaxSizeMb, dbStorage_rocksDB_blockCacheSize. BTW, I'm going to use journalSyncData=false since we cannot recover machines when they shutdown. So no fsync is required for every message.
-Since the VM has lot of RAM you can increase a lot from the defaults and leave the rest page cache. For JVM heap I'd say ~24g. WriteCacheMaxSize and ReadAheadCacheMaxSize are both coming from JVM direct memory. I'd say to start with 16g @ 16g. For rocksdb block cache, which is allocated in JNI so it's completely out of JVM configuration, ideally you want to cache most of the indexes. I'd say 4gb should be enough to index all the data in the 24Tb storage space.
-
-### When there are multiple consumers for a topic, the broker reads once from bookies and send them to all consumers with some buffer? or go get from bookies all the time for each consumers ?
-In general, all dispatching is done directly by broker memory. We only read from bookies when consumer are falling behind.
-
-
diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml
index e8f4daecb..b4f0017fa 100644
--- a/managed-ledger/pom.xml
+++ b/managed-ledger/pom.xml
@@ -36,35 +36,6 @@
<dependency>
<groupId>org.apache.bookkeeper</groupId>
<artifactId>bookkeeper-server-shaded</artifactId>
- <version>${bookkeeper.version}</version>
- <exclusions>
- <exclusion>
- <artifactId>slf4j-log4j12</artifactId>
- <groupId>org.slf4j</groupId>
- </exclusion>
- <exclusion>
- <artifactId>log4j</artifactId>
- <groupId>log4j</groupId>
- </exclusion>
- <exclusion>
- <groupId>org.jboss.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.bookkeeper</groupId>
- <artifactId>bookkeeper-server</artifactId>
- <version>${bookkeeper.version}</version>
- <classifier>tests</classifier>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java
index 9149bb9f9..8dbb846d1 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java
@@ -26,6 +26,7 @@
import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteLedgerCallback;
import org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback;
import org.apache.bookkeeper.mledger.AsyncCallbacks.TerminateCallback;
+import org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.InitialPosition;
/**
* A ManagedLedger it's a superset of a BookKeeper ledger concept.
@@ -132,11 +133,28 @@
*
* @param name
* the name associated with the ManagedCursor
+ * @param initializeOnLatest
+ * the flag tell the method wthether it should intialize the cursor at latest position or not.
* @return the ManagedCursor
* @throws ManagedLedgerException
*/
ManagedCursor openCursor(String name) throws InterruptedException, ManagedLedgerException;
+ /**
+ * Open a ManagedCursor in this ManagedLedger.
+ * <p>
+ * If the cursors doesn't exist, a new one will be created and its position will be at the end of the ManagedLedger.
+ *
+ * @param name
+ * the name associated with the ManagedCursor
+ * @param initialPosition
+ * the cursor will be set at lastest position or not when first created
+ * default is <b>true</b>
+ * @return the ManagedCursor
+ * @throws ManagedLedgerException
+ */
+ public ManagedCursor openCursor(String name, InitialPosition initialPosition) throws InterruptedException, ManagedLedgerException;
+
/**
* Creates a new cursor whose metadata is not backed by durable storage. A caller can treat the non-durable cursor
* exactly like a normal cursor, with the only difference in that after restart it will not remember which entries
@@ -194,7 +212,23 @@
void asyncOpenCursor(String name, OpenCursorCallback callback, Object ctx);
/**
- * Get a list of all the cursors reading from this ManagedLedger.
+ * Open a ManagedCursor asynchronously.
+ *
+ * @see #openCursor(String)
+ * @param name
+ * the name associated with the ManagedCursor
+ * @param initialPosition
+ * the cursor will be set at lastest position or not when first created
+ * default is <b>true</b>
+ * @param callback
+ * callback object
+ * @param ctx
+ * opaque context
+ */
+ public void asyncOpenCursor(String name, InitialPosition initialPosition, OpenCursorCallback callback, Object ctx);
+
+ /**
+ * Get a list of all the cursors reading from this ManagedLedger
*
* @return a list of cursors
*/
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java
index bbac80a03..ed4ccb354 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java
@@ -51,7 +51,7 @@
private long retentionSizeInMB = 0;
private boolean autoSkipNonRecoverableData;
- private DigestType digestType = DigestType.MAC;
+ private DigestType digestType = DigestType.CRC32C;
private byte[] password = "".getBytes(Charsets.UTF_8);
/**
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
index ac030e52b..bbdae32ae 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
@@ -78,6 +78,7 @@
import org.apache.bookkeeper.mledger.util.Pair;
import org.apache.pulsar.common.api.Commands;
import org.apache.pulsar.common.api.proto.PulsarApi.MessageMetadata;
+import org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.InitialPosition;
import org.apache.pulsar.common.util.collections.ConcurrentLongHashMap;
import org.apache.pulsar.common.util.collections.GrowableArrayBlockingQueue;
import org.slf4j.Logger;
@@ -546,7 +547,12 @@ public synchronized void asyncAddEntry(ByteBuf buffer, AddEntryCallback callback
}
@Override
- public ManagedCursor openCursor(String cursorName) throws InterruptedException, ManagedLedgerException {
+ public ManagedCursor openCursor(String cursorName) throws InterruptedException, ManagedLedgerException{
+ return openCursor(cursorName, InitialPosition.Latest);
+ }
+
+ @Override
+ public ManagedCursor openCursor(String cursorName, InitialPosition initialPosition) throws InterruptedException, ManagedLedgerException {
final CountDownLatch counter = new CountDownLatch(1);
class Result {
ManagedCursor cursor = null;
@@ -554,7 +560,7 @@ public ManagedCursor openCursor(String cursorName) throws InterruptedException,
}
final Result result = new Result();
- asyncOpenCursor(cursorName, new OpenCursorCallback() {
+ asyncOpenCursor(cursorName, initialPosition, new OpenCursorCallback() {
@Override
public void openCursorComplete(ManagedCursor cursor, Object ctx) {
result.cursor = cursor;
@@ -582,9 +588,12 @@ public void openCursorFailed(ManagedLedgerException exception, Object ctx) {
}
@Override
- public synchronized void asyncOpenCursor(final String cursorName, final OpenCursorCallback callback,
- final Object ctx) {
+ public synchronized void asyncOpenCursor(final String cursorName, final OpenCursorCallback callback, Object ctx){
+ this.asyncOpenCursor(cursorName, InitialPosition.Latest, callback, ctx);
+ }
+ @Override
+ public synchronized void asyncOpenCursor(final String cursorName, final InitialPosition initialPosition, final OpenCursorCallback callback, final Object ctx){
try {
checkManagedLedgerIsOpen();
checkFenced();
@@ -624,8 +633,8 @@ public void operationComplete() {
log.info("[{}] Opened new cursor: {}", name, cursor);
cursor.setActive();
// Update the ack position (ignoring entries that were written while the cursor was being created)
- cursor.initializeCursorPosition(getLastPositionAndCounter());
-
+ cursor.initializeCursorPosition(initialPosition == InitialPosition.Latest ? getLastPositionAndCounter() : getFirstPositionAndCounter());
+
synchronized (this) {
cursors.add(cursor);
uninitializedCursors.remove(cursorName).complete(cursor);
@@ -2030,6 +2039,22 @@ PositionImpl getMarkDeletePositionOfSlowestConsumer() {
return Pair.create(pos, count);
}
+ /**
+ * Get the first position written in the managed ledger, alongside with the associated counter
+ */
+ Pair<PositionImpl, Long> getFirstPositionAndCounter() {
+ PositionImpl pos;
+ long count;
+ Pair<PositionImpl, Long> lastPositionAndCounter;
+
+ do {
+ pos = getFirstPosition();
+ lastPositionAndCounter = getLastPositionAndCounter();
+ count = lastPositionAndCounter.second - getNumberOfEntries(Range.openClosed(pos, lastPositionAndCounter.first));
+ } while (pos.compareTo(getFirstPosition()) != 0 && lastPositionAndCounter.first.compareTo(getLastPosition()) != 0);
+ return Pair.create(pos, count);
+ }
+
public void activateCursor(ManagedCursor cursor) {
if (activeCursors.get(cursor.getName()) == null) {
activeCursors.add(cursor);
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java
index ee49810d5..312b8949c 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java
@@ -2515,7 +2515,7 @@ public void operationFailed(MetaStoreException e) {
// verify cursor-ledger's last entry has individual-deleted positions
final CountDownLatch latch = new CountDownLatch(1);
final AtomicInteger individualDeletedMessagesCount = new AtomicInteger(0);
- bkc.asyncOpenLedger(c1.getCursorLedger(), DigestType.MAC, "".getBytes(), (rc, lh, ctx) -> {
+ bkc.asyncOpenLedger(c1.getCursorLedger(), DigestType.CRC32C, "".getBytes(), (rc, lh, ctx) -> {
if (rc == BKException.Code.OK) {
long lastEntry = lh.getLastAddConfirmed();
lh.asyncReadEntries(lastEntry, lastEntry, (rc1, lh1, seq, ctx1) -> {
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java
index 59d35d69b..3035c286a 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java
@@ -345,7 +345,7 @@ public void ledgerFencedByAutoReplication() throws Exception {
PositionImpl p1 = (PositionImpl) ledger.addEntry("entry-1".getBytes());
// Trigger the closure of the data ledger
- bkc.openLedger(p1.getLedgerId(), DigestType.MAC, new byte[] {});
+ bkc.openLedger(p1.getLedgerId(), DigestType.CRC32C, new byte[] {});
ledger.addEntry("entry-2".getBytes());
@@ -502,4 +502,37 @@ public void addFailed(ManagedLedgerException exception, Object ctx) {
factory.shutdown();
}
+ @Test
+ public void testChangeCrcType() throws Exception {
+ ManagedLedgerFactoryImpl factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
+ ManagedLedgerConfig config = new ManagedLedgerConfig();
+ config.setEnsembleSize(2).setAckQuorumSize(2).setMetadataEnsembleSize(2);
+ config.setDigestType(DigestType.CRC32);
+ ManagedLedger ledger = factory.open("my_test_ledger", config);
+ ManagedCursor c1 = ledger.openCursor("c1");
+
+ ledger.addEntry("entry-0".getBytes());
+ ledger.addEntry("entry-1".getBytes());
+ ledger.addEntry("entry-2".getBytes());
+
+ ledger.close();
+
+ config.setDigestType(DigestType.CRC32C);
+ ledger = factory.open("my_test_ledger", config);
+ c1 = ledger.openCursor("c1");
+
+ ledger.addEntry("entry-3".getBytes());
+
+ assertEquals(c1.getNumberOfEntries(), 4);
+ assertEquals(c1.getNumberOfEntriesInBacklog(), 4);
+
+ List<Entry> entries = c1.readEntries(4);
+ assertEquals(entries.size(), 4);
+ for (int i = 0; i < 4; i++) {
+ assertEquals(new String(entries.get(i).getData()), "entry-" + i);
+ }
+
+ factory.shutdown();
+ }
+
}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java
index 460e5227d..e8c995c87 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java
@@ -21,6 +21,7 @@
import static org.testng.Assert.*;
import com.google.common.base.Charsets;
+import com.google.common.collect.Range;
import com.google.common.collect.Sets;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.PooledByteBufAllocator;
@@ -66,6 +67,7 @@
import org.apache.bookkeeper.mledger.util.Pair;
import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
import org.apache.pulsar.common.api.ByteBufPair;
+import org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.InitialPosition;
import org.apache.pulsar.common.api.proto.PulsarApi.MessageMetadata;
import org.apache.pulsar.common.util.protobuf.ByteBufCodedOutputStream;
import org.apache.zookeeper.CreateMode;
@@ -2152,4 +2154,34 @@ public ByteBuf getMessageWithMetadata(byte[] data) throws IOException {
return ByteBufPair.coalesce(ByteBufPair.get(headers, payload));
}
+ @Test
+ public void testOpenCursorOnLatestAndEarliest() throws Exception {
+ final int MAX_ENTRY_PER_LEDGER = 2;
+ ManagedLedgerConfig config = new ManagedLedgerConfig().setMaxEntriesPerLedger(MAX_ENTRY_PER_LEDGER);
+ ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("lastest_earliest_ledger", config);
+ // ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("lastest_earliest_ledger");
+ final int totalInsertedEntries = 20;
+ for (int i = 0; i < totalInsertedEntries; i++) {
+ String content = "entry" + i; // 5 bytes
+ ledger.addEntry(content.getBytes());
+ }
+ // Open Cursor also adds cursor into activeCursor-container
+ ManagedCursor latestCursor = ledger.openCursor("c1", InitialPosition.Latest);
+ ManagedCursor earliestCursor = ledger.openCursor("c2", InitialPosition.Earliest);
+
+ // Since getReadPosition returns the next position, we decrease the entryId by 1
+ PositionImpl p1 = (PositionImpl) latestCursor.getReadPosition();
+ PositionImpl p2 = (PositionImpl) earliestCursor.getReadPosition();
+
+ Pair<PositionImpl, Long> latestPositionAndCounter = ledger.getLastPositionAndCounter();
+ Pair<PositionImpl, Long> earliestPositionAndCounter = ledger.getFirstPositionAndCounter();
+
+ assertEquals(latestPositionAndCounter.first.getNext(), p1);
+ assertEquals(earliestPositionAndCounter.first.getNext(), p2);
+
+ assertEquals(latestPositionAndCounter.second.longValue(), totalInsertedEntries);
+ assertEquals(earliestPositionAndCounter.second.longValue(), totalInsertedEntries - earliestCursor.getNumberOfEntriesInBacklog());
+
+ ledger.close();
+ }
}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java b/managed-ledger/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java
index 685dd92fa..1680f71c5 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java
@@ -139,6 +139,7 @@ protected void stopZKCluster() throws Exception {
protected void startBKCluster() throws Exception {
baseClientConf.setZkServers(zkUtil.getZooKeeperConnectString());
baseClientConf.setUseV2WireProtocol(true);
+ baseClientConf.setEnableDigestTypeAutodetection(true);
if (numBookies > 0) {
bkc = new BookKeeperTestClient(baseClientConf);
}
diff --git a/pom.xml b/pom.xml
index c3b1406d3..2472cc125 100644
--- a/pom.xml
+++ b/pom.xml
@@ -83,8 +83,9 @@ flexible messaging model and an intuitive client API.</description>
<module>pulsar-client-shaded</module>
<module>pulsar-client-admin</module>
<module>pulsar-client-admin-shaded</module>
+ <module>pulsar-client-admin-shaded-for-functions</module>
<module>pulsar-client-tools</module>
- <module>pulsar-client-tools-shaded</module>
+ <module>pulsar-client-tools-test</module>
<module>pulsar-websocket</module>
<module>pulsar-proxy</module>
<module>pulsar-discovery-service</module>
@@ -99,6 +100,9 @@ flexible messaging model and an intuitive client API.</description>
<module>all</module>
<module>docker</module>
<module>tests</module>
+ <module>pulsar-log4j2-appender</module>
+ <!-- functions related modules -->
+ <module>pulsar-functions</module>
</modules>
<issueManagement>
@@ -126,6 +130,16 @@ flexible messaging model and an intuitive client API.</description>
<jackson.version>2.8.4</jackson.version>
<puppycrawl.checkstyle.version>6.19</puppycrawl.checkstyle.version>
<dockerfile-maven.version>1.3.7</dockerfile-maven.version>
+ <typetools.version>0.5.0</typetools.version>
+ <protobuf2.version>2.4.1</protobuf2.version>
+ <protobuf3.version>3.5.1</protobuf3.version>
+ <grpc.version>1.5.0</grpc.version>
+ <protoc-gen-grpc-java.version>1.0.0</protoc-gen-grpc-java.version>
+ <gson.version>2.8.2</gson.version>
+ <sketches.version>0.6.0</sketches.version>
+
+ <!-- test dependencies -->
+ <disruptor.version>3.4.0</disruptor.version>
</properties>
<dependencyManagement>
@@ -218,6 +232,32 @@ flexible messaging model and an intuitive client API.</description>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.apache.bookkeeper</groupId>
+ <artifactId>bookkeeper-server-shaded</artifactId>
+ <version>${bookkeeper.version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>log4j</artifactId>
+ <groupId>log4j</groupId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.jboss.netty</groupId>
+ <artifactId>netty</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.bookkeeper</groupId>
+ <artifactId>stream-storage-java-client</artifactId>
+ <version>${bookkeeper.version}</version>
+ </dependency>
+
<dependency>
<groupId>org.apache.bookkeeper</groupId>
<artifactId>bookkeeper-bookkeeper-stats-api</artifactId>
@@ -280,14 +320,14 @@ flexible messaging model and an intuitive client API.</description>
<dependency>
<groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- <version>3.10.1.Final</version>
+ <artifactId>netty-tcnative-boringssl-static</artifactId>
+ <version>2.0.7.Final</version>
</dependency>
<dependency>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- <version>2.8.0</version>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ <version>3.10.1.Final</version>
</dependency>
<dependency>
@@ -362,6 +402,33 @@ flexible messaging model and an intuitive client API.</description>
<version>${log4j2.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ <type>test-jar</type>
+ <version>${log4j2.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <type>test-jar</type>
+ <version>${log4j2.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ <type>test-jar</type>
+ <version>${log4j2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <type>test-jar</type>
+ <version>${log4j2.version}</version>
+ </dependency>
+
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
@@ -599,6 +666,45 @@ flexible messaging model and an intuitive client API.</description>
<artifactId>bcpkix-jdk15on</artifactId>
<version>${bouncycastle.version}</version>
</dependency>
+
+ <dependency>
+ <groupId>net.jodah</groupId>
+ <artifactId>typetools</artifactId>
+ <version>${typetools.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-all</artifactId>
+ <version>${grpc.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>${gson.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.yahoo.datasketches</groupId>
+ <artifactId>sketches-core</artifactId>
+ <version>${sketches.version}</version>
+ </dependency>
+
+ <!-- use shaded dependency util pulsar bump zookeeper version to 3.5 -->
+ <dependency>
+ <groupId>org.apache.distributedlog</groupId>
+ <artifactId>distributedlog-core-shaded</artifactId>
+ <version>${bookkeeper.version}</version>
+ </dependency>
+
+ <!-- test dependencies -->
+ <dependency>
+ <groupId>com.lmax</groupId>
+ <artifactId>disruptor</artifactId>
+ <version>${disruptor.version}</version>
+ </dependency>
+
</dependencies>
</dependencyManagement>
@@ -611,22 +717,6 @@ flexible messaging model and an intuitive client API.</description>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-api</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j-impl</artifactId>
- <scope>test</scope>
- </dependency>
-
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
@@ -657,6 +747,28 @@ flexible messaging model and an intuitive client API.</description>
<version>1.16.20</version>
<scope>provided</scope>
</dependency>
+
+ <dependency>
+ <!-- We use MockedBookKeeper in many unit tests -->
+ <groupId>org.apache.bookkeeper</groupId>
+ <artifactId>bookkeeper-server-tests-shaded</artifactId>
+ <version>${bookkeeper.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>log4j</artifactId>
+ <groupId>log4j</groupId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.jboss.netty</groupId>
+ <artifactId>netty</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
</dependencies>
<build>
@@ -690,6 +802,7 @@ flexible messaging model and an intuitive client API.</description>
<configuration>
<argLine> -Xmx2G -XX:MaxDirectMemorySize=8G
-Dio.netty.leakDetectionLevel=advanced
+ -Dlog4j.configurationFile=log4j2.xml
</argLine>
<reuseForks>false</reuseForks>
<forkCount>1</forkCount>
@@ -866,7 +979,7 @@ flexible messaging model and an intuitive client API.</description>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
- <version>2.4.2</version>
+ <version>3.1.0</version>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index 8aa2f821f..f851f7dd9 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -24,6 +24,7 @@
import java.util.Properties;
import java.util.Set;
+import org.apache.bookkeeper.client.BookKeeper.DigestType;
import org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider;
import org.apache.pulsar.common.configuration.FieldContext;
import org.apache.pulsar.common.configuration.PulsarConfiguration;
@@ -201,7 +202,7 @@
// Specify the tls cipher the broker will use to negotiate during TLS Handshake.
// Example:- [TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256]
private Set<String> tlsCiphers = Sets.newTreeSet();
-
+
/***** --- Authentication --- ****/
// Enable authentication
private boolean authenticationEnabled = false;
@@ -222,7 +223,7 @@
private Set<String> proxyRoles = Sets.newTreeSet();
// If this flag is set then the broker authenticates the original Auth data
- // else it just accepts the originalPrincipal and authorizes it (if required).
+ // else it just accepts the originalPrincipal and authorizes it (if required).
private boolean authenticateOriginalAuthData = false;
// Allow wildcard matching in authorization
@@ -236,7 +237,7 @@
private String brokerClientAuthenticationParameters = "";
// Path for the trusted TLS certificate file for outgoing connection to a server (broker)
private String brokerClientTrustCertsFilePath = "";
-
+
// When this parameter is not empty, unauthenticated users perform as anonymousUserRole
private String anonymousUserRole = null;
@@ -278,6 +279,11 @@
// Number of guaranteed copies (acks to wait before write is complete)
@FieldContext(minValue = 1)
private int managedLedgerDefaultAckQuorum = 1;
+
+ // Default type of checksum to use when writing to BookKeeper. Default is "CRC32"
+ // Other possible options are "CRC32C" (which is faster), "MAC" or "DUMMY" (no checksum).
+ private DigestType managedLedgerDigestType = DigestType.CRC32;
+
// Max number of bookies to use when creating a ledger
@FieldContext(minValue = 1)
private int managedLedgerMaxEnsembleSize = 5;
@@ -415,6 +421,8 @@
@FieldContext(dynamic = true)
private boolean preferLaterVersions = false;
+ private String schemaRegistryStorageClassName = "org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorageFactory";
+
/**** --- WebSocket --- ****/
// Number of IO threads in Pulsar Client used in WebSocket proxy
private int webSocketNumIoThreads = Runtime.getRuntime().availableProcessors();
@@ -425,6 +433,9 @@
// If true, export topic level metrics otherwise namespace level
private boolean exposeTopicLevelMetricsInPrometheus = true;
+ /**** --- Functions --- ****/
+ private boolean functionsWorkerEnabled = false;
+
public String getZookeeperServers() {
return zookeeperServers;
}
@@ -859,15 +870,15 @@ public void setAuthorizationEnabled(boolean authorizationEnabled) {
public Set<String> getSuperUserRoles() {
return superUserRoles;
}
-
+
public Set<String> getProxyRoles() {
return proxyRoles;
}
-
+
public void setProxyRoles(Set<String> proxyRoles) {
this.proxyRoles = proxyRoles;
}
-
+
public boolean getAuthorizationAllowWildcardsMatching() {
return authorizationAllowWildcardsMatching;
}
@@ -903,7 +914,7 @@ public String getBrokerClientTrustCertsFilePath() {
public void setBrokerClientTrustCertsFilePath(String brokerClientTrustCertsFilePath) {
this.brokerClientTrustCertsFilePath = brokerClientTrustCertsFilePath;
}
-
+
public String getAnonymousUserRole() {
return anonymousUserRole;
}
@@ -1026,6 +1037,14 @@ public void setManagedLedgerDefaultAckQuorum(int managedLedgerDefaultAckQuorum)
this.managedLedgerDefaultAckQuorum = managedLedgerDefaultAckQuorum;
}
+ public DigestType getManagedLedgerDigestType() {
+ return managedLedgerDigestType;
+ }
+
+ public void setManagedLedgerDigestType(DigestType managedLedgerDigestType) {
+ this.managedLedgerDigestType = managedLedgerDigestType;
+ }
+
public int getManagedLedgerMaxEnsembleSize() {
return managedLedgerMaxEnsembleSize;
}
@@ -1446,7 +1465,15 @@ public boolean exposeTopicLevelMetricsInPrometheus() {
public void setExposeTopicLevelMetricsInPrometheus(boolean exposeTopicLevelMetricsInPrometheus) {
this.exposeTopicLevelMetricsInPrometheus = exposeTopicLevelMetricsInPrometheus;
}
-
+
+ public String getSchemaRegistryStorageClassName() {
+ return schemaRegistryStorageClassName;
+ }
+
+ public void setSchemaRegistryStorageClassName(String className) {
+ schemaRegistryStorageClassName = className;
+ }
+
public boolean authenticateOriginalAuthData() {
return authenticateOriginalAuthData;
}
@@ -1454,7 +1481,7 @@ public boolean authenticateOriginalAuthData() {
public void setAuthenticateOriginalAuthData(boolean authenticateOriginalAuthData) {
this.authenticateOriginalAuthData = authenticateOriginalAuthData;
}
-
+
public Set<String> getTlsProtocols() {
return tlsProtocols;
}
@@ -1470,4 +1497,14 @@ public void setTlsProtocols(Set<String> tlsProtocols) {
public void setTlsCiphers(Set<String> tlsCiphers) {
this.tlsCiphers = tlsCiphers;
}
+
+ /**** --- Function ---- ****/
+
+ public void setFunctionsWorkerEnabled(boolean enabled) {
+ this.functionsWorkerEnabled = enabled;
+ }
+
+ public boolean isFunctionsWorkerEnabled() {
+ return functionsWorkerEnabled;
+ }
}
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasic.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasic.java
index f2f2f6588..faa8718a0 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasic.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasic.java
@@ -24,6 +24,8 @@
import org.apache.commons.lang3.StringUtils;
import org.apache.pulsar.broker.ServiceConfiguration;
+import lombok.Cleanup;
+
import javax.naming.AuthenticationException;
import java.io.BufferedReader;
import java.io.File;
@@ -49,7 +51,8 @@ public void initialize(ServiceConfiguration config) throws IOException {
} else if (!confFile.isFile()) {
throw new IOException("The path is not a file");
}
- BufferedReader reader = new BufferedReader(new FileReader(confFile));
+
+ @Cleanup BufferedReader reader = new BufferedReader(new FileReader(confFile));
users = new HashMap<>();
for (String line : reader.lines().toArray(s -> new String[s])) {
List<String> splitLine = Arrays.asList(line.split(":"));
@@ -58,7 +61,6 @@ public void initialize(ServiceConfiguration config) throws IOException {
}
users.put(splitLine.get(0), splitLine.get(1));
}
- reader.close();
}
@Override
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java
index f38482d3e..76e98ccfd 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java
@@ -21,28 +21,21 @@
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.pulsar.zookeeper.ZooKeeperCache.cacheTimeOutInSec;
-import java.util.Map;
-import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import javax.servlet.http.HttpServletRequest;
import org.apache.commons.lang3.StringUtils;
import org.apache.pulsar.broker.PulsarServerException;
import org.apache.pulsar.broker.ServiceConfiguration;
import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
import org.apache.pulsar.broker.cache.ConfigurationCacheService;
-import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.naming.NamespaceName;
+import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.policies.data.AuthAction;
import org.apache.pulsar.common.util.FutureUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Maps;
-
/**
* Authorization service that manages pluggable authorization provider and authorize requests accordingly.
*
diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/common/configuration/PulsarConfigurationLoaderTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/common/configuration/PulsarConfigurationLoaderTest.java
index 5ff95dc44..038fb4dec 100644
--- a/pulsar-broker-common/src/test/java/org/apache/pulsar/common/configuration/PulsarConfigurationLoaderTest.java
+++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/common/configuration/PulsarConfigurationLoaderTest.java
@@ -32,6 +32,7 @@
import java.io.PrintWriter;
import java.util.Properties;
+import org.apache.bookkeeper.client.BookKeeper.DigestType;
import org.apache.pulsar.broker.ServiceConfiguration;
import org.testng.annotations.Test;
@@ -100,6 +101,7 @@ public void testPulsarConfiguraitonLoadingStream() throws Exception {
printWriter.println("superUserRoles=appid1,appid2");
printWriter.println("brokerServicePort=7777");
printWriter.println("managedLedgerDefaultMarkDeleteRateLimit=5.0");
+ printWriter.println("managedLedgerDigestType=CRC32C");
printWriter.close();
testConfigFile.deleteOnExit();
InputStream stream = new FileInputStream(testConfigFile);
@@ -111,6 +113,7 @@ public void testPulsarConfiguraitonLoadingStream() throws Exception {
assertEquals(serviceConfig.getClusterName(), "usc");
assertEquals(serviceConfig.getBrokerClientAuthenticationParameters(), "role:my-role");
assertEquals(serviceConfig.getBrokerServicePort(), 7777);
+ assertEquals(serviceConfig.getManagedLedgerDigestType(), DigestType.CRC32C);
}
@Test
diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml
index b5c60564b..8681cad71 100644
--- a/pulsar-broker/pom.xml
+++ b/pulsar-broker/pom.xml
@@ -54,6 +54,24 @@
<artifactId>netty-all</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <version>${protobuf2.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>pulsar-common</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.bookkeeper</groupId>
+ <artifactId>circe-checksum</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>pulsar-client-original</artifactId>
@@ -119,6 +137,16 @@
<scope>test</scope>
</dependency>
+ <!-- functions related dependencies (begin) -->
+
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>pulsar-functions-worker</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <!-- functions related dependencies (end) -->
+
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
@@ -236,21 +264,6 @@
<version>${project.version}</version>
<scope>test</scope>
</dependency>
-
- <dependency>
- <groupId>org.apache.bookkeeper</groupId>
- <artifactId>bookkeeper-server</artifactId>
- <version>${bookkeeper.version}</version>
- <classifier>tests</classifier>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
</dependencies>
<build>
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarBrokerStarter.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarBrokerStarter.java
index 40f8d59ca..42d90e271 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarBrokerStarter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarBrokerStarter.java
@@ -34,6 +34,7 @@
import java.net.MalformedURLException;
import java.nio.file.Paths;
import java.util.Arrays;
+import java.util.Optional;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.proto.BookieServer;
import org.apache.bookkeeper.replication.AutoRecoveryMain;
@@ -42,6 +43,9 @@
import org.apache.commons.configuration.ConfigurationException;
import org.apache.pulsar.broker.PulsarService;
import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.ServiceConfigurationUtils;
+import org.apache.pulsar.functions.worker.WorkerConfig;
+import org.apache.pulsar.functions.worker.WorkerService;
import org.aspectj.weaver.loadtime.Agent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -72,6 +76,12 @@ private static ServiceConfiguration loadConfig(String configFile) throws Excepti
@Parameter(names = {"-bc", "--bookie-conf"}, description = "Configuration file for Bookie")
private String bookieConfigFile = Paths.get("").toAbsolutePath().normalize().toString() + "/conf/bookkeeper.conf";
+ @Parameter(names = {"-rfw", "--run-functions-worker"}, description = "Run functions worker with Broker")
+ private boolean runFunctionsWorker = false;
+
+ @Parameter(names = {"-fwc", "--functions-worker-conf"}, description = "Configuration file for Functions Worker")
+ private String fnWorkerConfigFile = Paths.get("").toAbsolutePath().normalize().toString() + "/conf/functions_worker.yml";
+
@Parameter(names = {"-h", "--help"}, description = "Show this help message")
private boolean help = false;
}
@@ -103,6 +113,7 @@ private static boolean argsContains(String[] args, String arg) {
private final AutoRecoveryMain autoRecoveryMain;
private final StatsProvider bookieStatsProvider;
private final ServerConfiguration bookieConfig;
+ private final WorkerService functionsWorkerService;
BrokerStarter(String[] args) throws Exception{
StarterArguments starterArguments = new StarterArguments();
@@ -116,15 +127,40 @@ private static boolean argsContains(String[] args, String arg) {
System.exit(-1);
}
- // init broker config and pulsar service
+ // init broker config
if (isBlank(starterArguments.brokerConfigFile)) {
jcommander.usage();
throw new IllegalArgumentException("Need to specify a configuration file for broker");
} else {
brokerConfig = loadConfig(starterArguments.brokerConfigFile);
- pulsarService = new PulsarService(brokerConfig);
}
+ // init functions worker
+ if (starterArguments.runFunctionsWorker || brokerConfig.isFunctionsWorkerEnabled()) {
+ WorkerConfig workerConfig;
+ if (isBlank(starterArguments.fnWorkerConfigFile)) {
+ workerConfig = new WorkerConfig();
+ } else {
+ workerConfig = WorkerConfig.load(starterArguments.fnWorkerConfigFile);
+ }
+ // worker talks to local broker
+ workerConfig.setPulsarServiceUrl("pulsar://127.0.0.1:" + brokerConfig.getBrokerServicePort());
+ workerConfig.setPulsarWebServiceUrl("http://127.0.0.1:" + brokerConfig.getWebServicePort());
+ String hostname = ServiceConfigurationUtils.getDefaultOrConfiguredAddress(
+ brokerConfig.getAdvertisedAddress());
+ workerConfig.setWorkerHostname(hostname);
+ workerConfig.setWorkerId(
+ "c-" + brokerConfig.getClusterName()
+ + "-fw-" + hostname
+ + "-" + workerConfig.getWorkerPort());
+ functionsWorkerService = new WorkerService(workerConfig);
+ } else {
+ functionsWorkerService = null;
+ }
+
+ // init pulsar service
+ pulsarService = new PulsarService(brokerConfig, Optional.ofNullable(functionsWorkerService));
+
// if no argument to run bookie in cmd line, read from pulsar config
if (!argsContains(args, "-rb") && !argsContains(args, "--run-bookie")) {
checkState(starterArguments.runBookie == false,
@@ -189,6 +225,16 @@ public void start() throws Exception {
pulsarService.start();
log.info("PulsarService started.");
+
+ // after broker is started, start the functions worker
+ if (null != functionsWorkerService) {
+ try {
+ functionsWorkerService.start();
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ throw ie;
+ }
+ }
}
public void join() throws InterruptedException {
@@ -203,6 +249,11 @@ public void join() throws InterruptedException {
}
public void shutdown() {
+ if (null != functionsWorkerService) {
+ functionsWorkerService.stop();
+ log.info("Shut down functions worker service successfully.");
+ }
+
pulsarService.getShutdownService().run();
log.info("Shut down broker service successfully.");
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandaloneStarter.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandaloneStarter.java
index 314eacf65..6c8dc14dc 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandaloneStarter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandaloneStarter.java
@@ -23,6 +23,8 @@
import java.io.FileInputStream;
import java.net.URL;
+import java.nio.file.Paths;
+import java.util.Optional;
import org.apache.pulsar.broker.PulsarService;
import org.apache.pulsar.broker.ServiceConfiguration;
import org.apache.pulsar.broker.ServiceConfigurationUtils;
@@ -31,6 +33,8 @@
import org.apache.pulsar.common.configuration.PulsarConfigurationLoader;
import org.apache.pulsar.common.policies.data.ClusterData;
import org.apache.pulsar.common.policies.data.PropertyAdmin;
+import org.apache.pulsar.functions.worker.WorkerConfig;
+import org.apache.pulsar.functions.worker.WorkerService;
import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble;
import org.aspectj.weaver.loadtime.Agent;
import org.slf4j.Logger;
@@ -48,6 +52,7 @@
PulsarAdmin admin;
LocalBookkeeperEnsemble bkEnsemble;
ServiceConfiguration config;
+ WorkerService fnWorkerService;
@Parameter(names = { "-c", "--config" }, description = "Configuration file path", required = true)
private String configFile;
@@ -76,6 +81,12 @@
@Parameter(names = { "--only-broker" }, description = "Only start Pulsar broker service (no ZK, BK)")
private boolean onlyBroker = false;
+ @Parameter(names = {"-nfw", "--no-functions-worker"}, description = "Run functions worker with Broker")
+ private boolean noFunctionsWorker = false;
+
+ @Parameter(names = {"-fwc", "--functions-worker-conf"}, description = "Configuration file for Functions Worker")
+ private String fnWorkerConfigFile = Paths.get("").toAbsolutePath().normalize().toString() + "/conf/functions_worker.yml";
+
@Parameter(names = { "-a", "--advertised-address" }, description = "Standalone broker advertised address")
private String advertisedAddress = null;
@@ -127,6 +138,10 @@ public PulsarStandaloneStarter(String[] args) throws Exception {
Runtime.getRuntime().addShutdownHook(new Thread() {
public void run() {
try {
+ if (fnWorkerService != null) {
+ fnWorkerService.stop();
+ }
+
if (broker != null) {
broker.close();
}
@@ -162,8 +177,29 @@ void start() throws Exception {
// load aspectj-weaver agent for instrumentation
AgentLoader.loadAgentClass(Agent.class.getName(), null);
+ // initialize the functions worker
+ if (!noFunctionsWorker) {
+ WorkerConfig workerConfig;
+ if (isBlank(fnWorkerConfigFile)) {
+ workerConfig = new WorkerConfig();
+ } else {
+ workerConfig = WorkerConfig.load(fnWorkerConfigFile);
+ }
+ // worker talks to local broker
+ workerConfig.setPulsarServiceUrl("pulsar://127.0.0.1:" + config.getBrokerServicePort());
+ workerConfig.setPulsarWebServiceUrl("http://127.0.0.1:" + config.getWebServicePort());
+ String hostname = ServiceConfigurationUtils.getDefaultOrConfiguredAddress(
+ config.getAdvertisedAddress());
+ workerConfig.setWorkerHostname(hostname);
+ workerConfig.setWorkerId(
+ "c-" + config.getClusterName()
+ + "-fw-" + hostname
+ + "-" + workerConfig.getWorkerPort());
+ fnWorkerService = new WorkerService(workerConfig);
+ }
+
// Start Broker
- broker = new PulsarService(config);
+ broker = new PulsarService(config, Optional.ofNullable(fnWorkerService));
broker.start();
// Create a sample namespace
@@ -203,6 +239,10 @@ void start() throws Exception {
log.info(e.getMessage());
}
+ if (null != fnWorkerService) {
+ fnWorkerService.start();
+ }
+
log.debug("--- setup completed ---");
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/BookKeeperClientFactoryImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/BookKeeperClientFactoryImpl.java
index 954d4ef89..0677800c6 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/BookKeeperClientFactoryImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/BookKeeperClientFactoryImpl.java
@@ -53,6 +53,7 @@ public BookKeeper create(ServiceConfiguration conf, ZooKeeper zkClient) throws I
bkConf.setSpeculativeReadTimeout(conf.getBookkeeperClientSpeculativeReadTimeoutInMillis());
bkConf.setNumChannelsPerBookie(16);
bkConf.setUseV2WireProtocol(true);
+ bkConf.setEnableDigestTypeAutodetection(true);
bkConf.setLedgerManagerFactoryClassName(HierarchicalLedgerManagerFactory.class.getName());
if (conf.isBookkeeperClientHealthCheckEnabled()) {
bkConf.enableBookieHealthCheck();
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
index 9b2343337..51014a7e1 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
@@ -20,10 +20,14 @@
import static org.apache.pulsar.broker.cache.ConfigurationCacheService.POLICIES;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import io.netty.util.concurrent.DefaultThreadFactory;
import java.io.IOException;
import java.net.URL;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
@@ -33,7 +37,6 @@
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Supplier;
-
import org.apache.bookkeeper.client.BookKeeper;
import org.apache.bookkeeper.common.util.OrderedScheduler;
import org.apache.bookkeeper.mledger.ManagedLedgerFactory;
@@ -51,6 +54,7 @@
import org.apache.pulsar.broker.namespace.NamespaceService;
import org.apache.pulsar.broker.service.BrokerService;
import org.apache.pulsar.broker.service.Topic;
+import org.apache.pulsar.broker.service.schema.SchemaRegistryService;
import org.apache.pulsar.broker.stats.MetricsGenerator;
import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsServlet;
import org.apache.pulsar.broker.web.WebService;
@@ -62,6 +66,7 @@
import org.apache.pulsar.common.naming.NamespaceName;
import org.apache.pulsar.common.policies.data.ClusterData;
import org.apache.pulsar.common.util.FutureUtil;
+import org.apache.pulsar.functions.worker.WorkerService;
import org.apache.pulsar.utils.PulsarBrokerVersionStringUtils;
import org.apache.pulsar.websocket.WebSocketConsumerServlet;
import org.apache.pulsar.websocket.WebSocketProducerServlet;
@@ -78,11 +83,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-import io.netty.util.concurrent.DefaultThreadFactory;
-
/**
* Main class for Pulsar broker service
*/
@@ -121,6 +121,8 @@
private final String brokerServiceUrl;
private final String brokerServiceUrlTls;
private final String brokerVersion;
+ private SchemaRegistryService schemaRegistryService = null;
+ private final Optional<WorkerService> functionWorkerService;
private final MessagingServiceShutdownHook shutdownService;
@@ -136,6 +138,10 @@
private final Condition isClosedCondition = mutex.newCondition();
public PulsarService(ServiceConfiguration config) {
+ this(config, Optional.empty());
+ }
+
+ public PulsarService(ServiceConfiguration config, Optional<WorkerService> functionWorkerService) {
// Validate correctness of configuration
PulsarConfigurationLoader.isComplete(config);
@@ -151,6 +157,7 @@ public PulsarService(ServiceConfiguration config) {
this.shutdownService = new MessagingServiceShutdownHook(this);
this.loadManagerExecutor = Executors
.newSingleThreadScheduledExecutor(new DefaultThreadFactory("pulsar-load-manager"));
+ this.functionWorkerService = functionWorkerService;
}
/**
@@ -225,6 +232,10 @@ public void close() throws PulsarServerException {
loadManager.stop();
}
+ if (schemaRegistryService != null) {
+ schemaRegistryService.close();
+ }
+
state = State.Closed;
} catch (Exception e) {
@@ -351,6 +362,8 @@ public synchronized void brokerIsAFollowerNow() {
this.metricsGenerator = new MetricsGenerator(this);
+ schemaRegistryService = SchemaRegistryService.create(this);
+
state = State.Started;
acquireSLANamespace();
@@ -543,6 +556,10 @@ public NamespaceService getNamespaceService() {
return this.nsservice;
}
+ public WorkerService getWorkerService() {
+ return functionWorkerService.orElse(null);
+ }
+
/**
* Get a reference of the current <code>BrokerService</code> instance associated with the current
* <code>PulsarService</code> instance.
@@ -689,4 +706,8 @@ public String getBrokerServiceUrlTls() {
public String getBrokerVersion() {
return brokerVersion;
}
+
+ public SchemaRegistryService getSchemaRegistryService() {
+ return schemaRegistryService;
+ }
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java
index 527f455db..1e7764083 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java
@@ -276,7 +276,9 @@ protected void validateTopicName(String property, String cluster, String namespa
*/
protected void validateBrokerName(String broker) throws MalformedURLException {
String brokerUrl = String.format("http://%s", broker);
- if (!pulsar().getWebServiceAddress().equals(brokerUrl)) {
+ String brokerUrlTls = String.format("https://%s", broker);
+ if (!pulsar().getWebServiceAddress().equals(brokerUrl)
+ && !pulsar().getWebServiceAddressTls().equals(brokerUrlTls)) {
String[] parts = broker.split(":");
checkArgument(parts.length == 2, "Invalid broker url %s", broker);
String host = parts[0];
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java
new file mode 100644
index 000000000..7080474ec
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.admin.impl;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.function.Supplier;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import org.apache.pulsar.broker.admin.AdminResource;
+import org.apache.pulsar.functions.worker.WorkerService;
+import org.apache.pulsar.functions.worker.rest.api.FunctionsImpl;
+import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
+import org.glassfish.jersey.media.multipart.FormDataParam;
+
+public class FunctionsBase extends AdminResource implements Supplier<WorkerService> {
+
+ private final FunctionsImpl functions;
+
+ public FunctionsBase() {
+ this.functions = new FunctionsImpl(this);
+ }
+
+ @Override
+ public WorkerService get() {
+ return pulsar().getWorkerService();
+ }
+
+ @POST
+ @Path("/{tenant}/{namespace}/{functionName}")
+ @Consumes(MediaType.MULTIPART_FORM_DATA)
+ public Response registerFunction(final @PathParam("tenant") String tenant,
+ final @PathParam("namespace") String namespace,
+ final @PathParam("functionName") String functionName,
+ final @FormDataParam("data") InputStream uploadedInputStream,
+ final @FormDataParam("data") FormDataContentDisposition fileDetail,
+ final @FormDataParam("functionConfig") String functionConfigJson) {
+
+ return functions.registerFunction(
+ tenant, namespace, functionName, uploadedInputStream, fileDetail, functionConfigJson);
+
+ }
+
+ @PUT
+ @Path("/{tenant}/{namespace}/{functionName}")
+ @Consumes(MediaType.MULTIPART_FORM_DATA)
+ public Response updateFunction(final @PathParam("tenant") String tenant,
+ final @PathParam("namespace") String namespace,
+ final @PathParam("functionName") String functionName,
+ final @FormDataParam("data") InputStream uploadedInputStream,
+ final @FormDataParam("data") FormDataContentDisposition fileDetail,
+ final @FormDataParam("functionConfig") String functionConfigJson) {
+
+ return functions.updateFunction(
+ tenant, namespace, functionName, uploadedInputStream, fileDetail, functionConfigJson);
+
+ }
+
+
+ @DELETE
+ @Path("/{tenant}/{namespace}/{functionName}")
+ public Response deregisterFunction(final @PathParam("tenant") String tenant,
+ final @PathParam("namespace") String namespace,
+ final @PathParam("functionName") String functionName) {
+ return functions.deregisterFunction(
+ tenant, namespace, functionName);
+ }
+
+ @GET
+ @Path("/{tenant}/{namespace}/{functionName}")
+ public Response getFunctionInfo(final @PathParam("tenant") String tenant,
+ final @PathParam("namespace") String namespace,
+ final @PathParam("functionName") String functionName)
+ throws IOException {
+ return functions.getFunctionInfo(
+ tenant, namespace, functionName);
+ }
+
+ @GET
+ @Path("/{tenant}/{namespace}/{functionName}/{instanceId}/status")
+ public Response getFunctionInstanceStatus(final @PathParam("tenant") String tenant,
+ final @PathParam("namespace") String namespace,
+ final @PathParam("functionName") String functionName,
+ final @PathParam("instanceId") String instanceId) throws IOException {
+ return functions.getFunctionInstanceStatus(
+ tenant, namespace, functionName, instanceId);
+ }
+
+ @GET
+ @Path("/{tenant}/{namespace}/{functionName}/status")
+ public Response getFunctionStatus(final @PathParam("tenant") String tenant,
+ final @PathParam("namespace") String namespace,
+ final @PathParam("functionName") String functionName) throws IOException {
+ return functions.getFunctionStatus(
+ tenant, namespace, functionName);
+ }
+
+ @GET
+ @Path("/{tenant}/{namespace}")
+ public Response listFunctions(final @PathParam("tenant") String tenant,
+ final @PathParam("namespace") String namespace) {
+ return functions.listFunctions(
+ tenant, namespace);
+
+ }
+
+ @GET
+ @Path("/cluster")
+ public Response getCluster() {
+ return functions.getCluster();
+ }
+
+ @GET
+ @Path("/assignments")
+ public Response getAssignments() {
+ return functions.getAssignments();
+ }
+
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
index 5ecca2409..5f5b75861 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
@@ -1074,7 +1074,7 @@ protected RetentionPolicies internalGetRetention() {
private boolean checkQuotas(Policies policies, RetentionPolicies retention) {
Map<BacklogQuota.BacklogQuotaType, BacklogQuota> backlog_quota_map = policies.backlog_quota_map;
- if (backlog_quota_map.isEmpty() || retention.getRetentionSizeInMB() == 0) {
+ if (backlog_quota_map.isEmpty() || retention.getRetentionSizeInMB() == 0 || retention.getRetentionSizeInMB() == -1) {
return true;
}
BacklogQuota quota = backlog_quota_map.get(BacklogQuotaType.destination_storage);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
index 9386df5ae..937ed27f1 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
@@ -75,6 +75,7 @@
import org.apache.pulsar.common.api.Commands;
import org.apache.pulsar.common.api.proto.PulsarApi.KeyValue;
import org.apache.pulsar.common.api.proto.PulsarApi.MessageMetadata;
+import org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.InitialPosition;
import org.apache.pulsar.common.compression.CompressionCodec;
import org.apache.pulsar.common.compression.CompressionCodecProvider;
import org.apache.pulsar.common.naming.TopicDomain;
@@ -820,7 +821,7 @@ protected void internalCreateSubscription(String subscriptionName, MessageIdImpl
}
PersistentSubscription subscription = (PersistentSubscription) topic
- .createSubscription(subscriptionName).get();
+ .createSubscription(subscriptionName, InitialPosition.Latest).get();
subscription.resetCursor(PositionImpl.get(messageId.getLedgerId(), messageId.getEntryId())).get();
log.info("[{}][{}] Successfully created subscription {} at message id {}", clientAppId(),
topicName, subscriptionName, messageId);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Functions.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Functions.java
new file mode 100644
index 000000000..174073219
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Functions.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.admin.v1;
+
+import io.swagger.annotations.Api;
+import javax.ws.rs.Path;
+import org.apache.pulsar.broker.admin.impl.FunctionsBase;
+
+@Path("/functions")
+@Api(value = "/functions", description = "Functions admin apis", tags = "functions")
+public class Functions extends FunctionsBase {
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java
index 8e105424c..a973b0729 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java
@@ -39,6 +39,7 @@
import org.apache.pulsar.broker.admin.impl.PersistentTopicsBase;
import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.InitialPosition;
import org.apache.pulsar.client.impl.MessageIdImpl;
import org.apache.pulsar.common.partition.PartitionedTopicMetadata;
import org.apache.pulsar.common.policies.data.AuthAction;
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Functions.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Functions.java
new file mode 100644
index 000000000..cc6867765
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Functions.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.admin.v2;
+
+import com.wordnik.swagger.annotations.Api;
+import javax.ws.rs.Path;
+import org.apache.pulsar.broker.admin.impl.FunctionsBase;
+
+@Path("/functions")
+@Api(value = "/functions", description = "Functions admin apis", tags = "functions")
+public class Functions extends FunctionsBase {
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java
index 19190088d..d3d15d926 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java
@@ -47,7 +47,7 @@
protected volatile ProducerImpl producer;
protected final int producerQueueSize;
- protected final ProducerBuilder producerBuilder;
+ protected final ProducerBuilder<byte[]> producerBuilder;
protected final Backoff backOff = new Backoff(100, TimeUnit.MILLISECONDS, 1, TimeUnit.MINUTES, 0 ,TimeUnit.MILLISECONDS);
@@ -80,7 +80,7 @@ public AbstractReplicator(String topicName, String replicatorPrefix, String loca
STATE_UPDATER.set(this, State.Stopped);
}
- protected abstract void readEntries(org.apache.pulsar.client.api.Producer producer);
+ protected abstract void readEntries(org.apache.pulsar.client.api.Producer<byte[]> producer);
protected abstract Position getReplicatorReadPosition();
@@ -231,7 +231,7 @@ public static String getReplicatorName(String replicatorPrefix, String cluster)
* Therefore, replicator can't be started on root-partition topic which can internally create multiple partitioned
* producers.
*
- * @param topicName
+ * @param topic
* @param brokerService
*/
private void validatePartitionedTopic(String topic, BrokerService brokerService) throws NamingException {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
index 575f1ec3d..577fab32e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
@@ -78,10 +78,12 @@
import org.apache.pulsar.broker.zookeeper.aspectj.ClientCnxnAspect;
import org.apache.pulsar.broker.zookeeper.aspectj.ClientCnxnAspect.EventListner;
import org.apache.pulsar.broker.zookeeper.aspectj.ClientCnxnAspect.EventType;
-import org.apache.pulsar.client.api.ClientConfiguration;
+import org.apache.pulsar.client.api.ClientBuilder;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.PulsarClientException;
+import org.apache.pulsar.client.impl.ClientBuilderImpl;
import org.apache.pulsar.client.impl.PulsarClientImpl;
+import org.apache.pulsar.client.impl.conf.ClientConfigurationData;
import org.apache.pulsar.common.configuration.FieldContext;
import org.apache.pulsar.common.naming.NamespaceBundle;
import org.apache.pulsar.common.naming.NamespaceBundleFactory;
@@ -498,27 +500,29 @@ public PulsarClient getReplicationClient(String cluster) {
String path = PulsarWebResource.path("clusters", cluster);
ClusterData data = this.pulsar.getConfigurationCache().clustersCache().get(path)
.orElseThrow(() -> new KeeperException.NoNodeException(path));
- ClientConfiguration configuration = new ClientConfiguration();
- configuration.setUseTcpNoDelay(false);
- configuration.setConnectionsPerBroker(pulsar.getConfiguration().getReplicationConnectionsPerBroker());
- configuration.setStatsInterval(0, TimeUnit.SECONDS);
+ ClientBuilder clientBuilder = PulsarClient.builder()
+ .enableTcpNoDelay(false)
+ .connectionsPerBroker(pulsar.getConfiguration().getReplicationConnectionsPerBroker())
+ .statsInterval(0, TimeUnit.SECONDS);
if (pulsar.getConfiguration().isAuthenticationEnabled()) {
- configuration.setAuthentication(pulsar.getConfiguration().getBrokerClientAuthenticationPlugin(),
+ clientBuilder.authentication(pulsar.getConfiguration().getBrokerClientAuthenticationPlugin(),
pulsar.getConfiguration().getBrokerClientAuthenticationParameters());
}
- String clusterUrl = null;
if (pulsar.getConfiguration().isReplicationTlsEnabled()) {
- clusterUrl = isNotBlank(data.getBrokerServiceUrlTls()) ? data.getBrokerServiceUrlTls()
- : data.getServiceUrlTls();
- configuration.setUseTls(true);
- configuration.setTlsTrustCertsFilePath(pulsar.getConfiguration().getBrokerClientTrustCertsFilePath());
- configuration
- .setTlsAllowInsecureConnection(pulsar.getConfiguration().isTlsAllowInsecureConnection());
+ clientBuilder
+ .serviceUrl(isNotBlank(data.getBrokerServiceUrlTls()) ? data.getBrokerServiceUrlTls()
+ : data.getServiceUrlTls())
+ .enableTls(true)
+ .tlsTrustCertsFilePath(pulsar.getConfiguration().getBrokerClientTrustCertsFilePath())
+ .allowTlsInsecureConnection(pulsar.getConfiguration().isTlsAllowInsecureConnection());
} else {
- clusterUrl = isNotBlank(data.getBrokerServiceUrl()) ? data.getBrokerServiceUrl()
- : data.getServiceUrl();
+ clientBuilder.serviceUrl(
+ isNotBlank(data.getBrokerServiceUrl()) ? data.getBrokerServiceUrl() : data.getServiceUrl());
}
- return new PulsarClientImpl(clusterUrl, configuration, this.workerGroup);
+
+ // Share all the IO threads across broker and client connections
+ ClientConfigurationData conf = ((ClientBuilderImpl) clientBuilder).getClientConfigurationData();
+ return new PulsarClientImpl(conf, workerGroup);
} catch (Exception e) {
throw new RuntimeException(e);
}
@@ -670,7 +674,7 @@ public void openLedgerFailed(ManagedLedgerException exception, Object ctx) {
managedLedgerConfig.setWriteQuorumSize(persistencePolicies.getBookkeeperWriteQuorum());
managedLedgerConfig.setAckQuorumSize(persistencePolicies.getBookkeeperAckQuorum());
managedLedgerConfig.setThrottleMarkDelete(persistencePolicies.getManagedLedgerMaxMarkDeleteRate());
- managedLedgerConfig.setDigestType(DigestType.CRC32);
+ managedLedgerConfig.setDigestType(serviceConfig.getManagedLedgerDigestType());
managedLedgerConfig.setMaxUnackedRangesToPersist(serviceConfig.getManagedLedgerMaxUnackedRangesToPersist());
managedLedgerConfig.setMaxUnackedRangesToPersistInZk(serviceConfig.getManagedLedgerMaxUnackedRangesToPersistInZooKeeper());
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java
index 9268ac7af..35bc3151b 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java
@@ -23,12 +23,16 @@
import static org.apache.pulsar.common.api.Commands.hasChecksum;
import static org.apache.pulsar.common.api.Commands.readChecksum;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.MoreObjects;
+import io.netty.buffer.ByteBuf;
+import io.netty.util.Recycler;
+import io.netty.util.Recycler.Handle;
import java.util.Collections;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.atomic.AtomicLongFieldUpdater;
-
import org.apache.bookkeeper.mledger.util.Rate;
import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
import org.apache.pulsar.broker.service.BrokerServiceException.TopicClosedException;
@@ -42,17 +46,11 @@
import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.policies.data.NonPersistentPublisherStats;
import org.apache.pulsar.common.policies.data.PublisherStats;
+import org.apache.pulsar.common.schema.SchemaVersion;
import org.apache.pulsar.common.util.DateFormatter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.MoreObjects;
-
-import io.netty.buffer.ByteBuf;
-import io.netty.util.Recycler;
-import io.netty.util.Recycler.Handle;
-
/**
* Represents a currently connected producer
*/
@@ -82,8 +80,10 @@
private final Map<String, String> metadata;
+ private final SchemaVersion schemaVersion;
+
public Producer(Topic topic, ServerCnx cnx, long producerId, String producerName, String appId,
- boolean isEncrypted, Map<String, String> metadata) {
+ boolean isEncrypted, Map<String, String> metadata, SchemaVersion schemaVersion) {
this.topic = topic;
this.cnx = cnx;
this.producerId = producerId;
@@ -110,6 +110,7 @@ public Producer(Topic topic, ServerCnx cnx, long producerId, String producerName
this.remoteCluster = isRemote ? producerName.split("\\.")[2] : null;
this.isEncrypted = isEncrypted;
+ this.schemaVersion = schemaVersion;
}
@Override
@@ -492,6 +493,10 @@ public void checkEncryption() {
}
}
+ public SchemaVersion getSchemaVersion() {
+ return schemaVersion;
+ }
+
private static final Logger log = LoggerFactory.getLogger(Producer.class);
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
index a386ece9a..353977c40 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
@@ -25,6 +25,12 @@
import static org.apache.pulsar.common.api.Commands.newLookupErrorResponse;
import static org.apache.pulsar.common.api.proto.PulsarApi.ProtocolVersion.v5;
+import com.google.protobuf.GeneratedMessageLite;
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.ChannelHandler;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelOption;
+import io.netty.handler.ssl.SslHandler;
import java.net.SocketAddress;
import java.util.List;
import java.util.Map;
@@ -32,6 +38,7 @@
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
import javax.naming.AuthenticationException;
import javax.net.ssl.SSLSession;
import org.apache.bookkeeper.mledger.Position;
@@ -52,6 +59,7 @@
import org.apache.pulsar.common.api.CommandUtils;
import org.apache.pulsar.common.api.Commands;
import org.apache.pulsar.common.api.PulsarHandler;
+import org.apache.pulsar.common.api.proto.PulsarApi;
import org.apache.pulsar.common.api.proto.PulsarApi.CommandAck;
import org.apache.pulsar.common.api.proto.PulsarApi.CommandCloseConsumer;
import org.apache.pulsar.common.api.proto.PulsarApi.CommandCloseProducer;
@@ -68,30 +76,25 @@
import org.apache.pulsar.common.api.proto.PulsarApi.CommandSeek;
import org.apache.pulsar.common.api.proto.PulsarApi.CommandSend;
import org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe;
+import org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.InitialPosition;
import org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.SubType;
import org.apache.pulsar.common.api.proto.PulsarApi.CommandUnsubscribe;
import org.apache.pulsar.common.api.proto.PulsarApi.MessageIdData;
import org.apache.pulsar.common.api.proto.PulsarApi.MessageMetadata;
import org.apache.pulsar.common.api.proto.PulsarApi.ProtocolVersion;
import org.apache.pulsar.common.api.proto.PulsarApi.ServerError;
-import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.naming.Metadata;
import org.apache.pulsar.common.naming.NamespaceName;
+import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.policies.data.BacklogQuota;
import org.apache.pulsar.common.policies.data.ConsumerStats;
+import org.apache.pulsar.common.schema.SchemaData;
+import org.apache.pulsar.common.schema.SchemaType;
+import org.apache.pulsar.common.schema.SchemaVersion;
import org.apache.pulsar.common.util.collections.ConcurrentLongHashMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Sets;
-import com.google.protobuf.GeneratedMessageLite;
-
-import io.netty.buffer.ByteBuf;
-import io.netty.channel.ChannelHandler;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.ChannelOption;
-import io.netty.handler.ssl.SslHandler;
-
public class ServerCnx extends PulsarHandler {
private final BrokerService service;
private final ConcurrentLongHashMap<CompletableFuture<Producer>> producers;
@@ -557,6 +560,7 @@ protected void handleSubscribe(final CommandSubscribe subscribe) {
final int priorityLevel = subscribe.hasPriorityLevel() ? subscribe.getPriorityLevel() : 0;
final boolean readCompacted = subscribe.getReadCompacted();
final Map<String, String> metadata = CommandUtils.metadataFromCommand(subscribe);
+ final InitialPosition initialPosition = subscribe.getInitialPosition();
CompletableFuture<Boolean> isProxyAuthorizedFuture;
if (service.isAuthorizationEnabled() && originalPrincipal != null) {
@@ -620,7 +624,7 @@ protected void handleSubscribe(final CommandSubscribe subscribe) {
service.getTopic(topicName.toString())
.thenCompose(topic -> topic.subscribe(ServerCnx.this, subscriptionName, consumerId,
subType, priorityLevel, consumerName, isDurable,
- startMessageId, metadata, readCompacted))
+ startMessageId, metadata, readCompacted, initialPosition))
.thenAccept(consumer -> {
if (consumerFuture.complete(consumer)) {
log.info("[{}] Created subscription on topic {} / {}", remoteAddress, topicName,
@@ -697,6 +701,36 @@ protected void handleSubscribe(final CommandSubscribe subscribe) {
});
}
+ private static SchemaType getType(PulsarApi.Schema.Type protocolType) {
+ switch (protocolType) {
+ case Json:
+ return SchemaType.JSON;
+ case Avro:
+ return SchemaType.AVRO;
+ case Thrift:
+ return SchemaType.THRIFT;
+ case Protobuf:
+ return SchemaType.PROTOBUF;
+ default:
+ return SchemaType.NONE;
+ }
+ }
+
+ private SchemaData getSchema(PulsarApi.Schema protocolSchema) {
+ return SchemaData.builder()
+ .data(protocolSchema.getSchemaData().toByteArray())
+ .isDeleted(false)
+ .timestamp(System.currentTimeMillis())
+ .user(originalPrincipal)
+ .type(getType(protocolSchema.getType()))
+ .props(protocolSchema.getPropertiesList().stream().collect(
+ Collectors.toMap(
+ PulsarApi.KeyValue::getKey,
+ PulsarApi.KeyValue::getValue
+ )
+ )).build();
+ }
+
@Override
protected void handleProducer(final CommandProducer cmdProducer) {
checkArgument(state == State.Connected);
@@ -752,7 +786,8 @@ protected void handleProducer(final CommandProducer cmdProducer) {
Producer producer = existingProducerFuture.getNow(null);
log.info("[{}] Producer with the same id is already created: {}", remoteAddress,
producer);
- ctx.writeAndFlush(Commands.newProducerSuccess(requestId, producer.getProducerName()));
+ ctx.writeAndFlush(Commands.newProducerSuccess(requestId, producer.getProducerName(),
+ producer.getSchemaVersion()));
return null;
} else {
// There was an early request to create a producer with
@@ -805,41 +840,56 @@ protected void handleProducer(final CommandProducer cmdProducer) {
disableTcpNoDelayIfNeeded(topicName.toString(), producerName);
- Producer producer = new Producer(topic, ServerCnx.this, producerId, producerName, authRole,
- isEncrypted, metadata);
+ CompletableFuture<SchemaVersion> schemaVersionFuture;
+ if (cmdProducer.hasSchema()) {
+ schemaVersionFuture = topic.addSchema(getSchema(cmdProducer.getSchema()));
+ } else {
+ schemaVersionFuture = CompletableFuture.completedFuture(SchemaVersion.Empty);
+ }
+
+ schemaVersionFuture.exceptionally(exception -> {
+ ctx.writeAndFlush(Commands.newError(requestId, ServerError.UnknownError, exception.getMessage()));
+ producers.remove(producerId, producerFuture);
+ return null;
+ });
+
+ schemaVersionFuture.thenAccept(schemaVersion -> {
+ Producer producer = new Producer(topic, ServerCnx.this, producerId, producerName, authRole,
+ isEncrypted, metadata, schemaVersion);
- try {
- topic.addProducer(producer);
+ try {
+ topic.addProducer(producer);
- if (isActive()) {
- if (producerFuture.complete(producer)) {
- log.info("[{}] Created new producer: {}", remoteAddress, producer);
- ctx.writeAndFlush(Commands.newProducerSuccess(requestId, producerName,
- producer.getLastSequenceId()));
- return;
+ if (isActive()) {
+ if (producerFuture.complete(producer)) {
+ log.info("[{}] Created new producer: {}", remoteAddress, producer);
+ ctx.writeAndFlush(Commands.newProducerSuccess(requestId, producerName,
+ producer.getLastSequenceId(), producer.getSchemaVersion()));
+ return;
+ } else {
+ // The producer's future was completed before by
+ // a close command
+ producer.closeNow();
+ log.info("[{}] Cleared producer created after timeout on client side {}",
+ remoteAddress, producer);
+ }
} else {
- // The producer's future was completed before by
- // a close command
producer.closeNow();
- log.info("[{}] Cleared producer created after timeout on client side {}",
- remoteAddress, producer);
- }
- } else {
- producer.closeNow();
- log.info("[{}] Cleared producer created after connection was closed: {}",
+ log.info("[{}] Cleared producer created after connection was closed: {}",
remoteAddress, producer);
- producerFuture.completeExceptionally(
+ producerFuture.completeExceptionally(
new IllegalStateException("Producer created after connection was closed"));
- }
- } catch (BrokerServiceException ise) {
- log.error("[{}] Failed to add producer to topic {}: {}", remoteAddress, topicName,
+ }
+ } catch (BrokerServiceException ise) {
+ log.error("[{}] Failed to add producer to topic {}: {}", remoteAddress, topicName,
ise.getMessage());
- ctx.writeAndFlush(Commands.newError(requestId,
+ ctx.writeAndFlush(Commands.newError(requestId,
BrokerServiceException.getClientErrorCode(ise), ise.getMessage()));
- producerFuture.completeExceptionally(ise);
- }
+ producerFuture.completeExceptionally(ise);
+ }
- producers.remove(producerId, producerFuture);
+ producers.remove(producerId, producerFuture);
+ });
}).exceptionally(exception -> {
Throwable cause = exception.getCause();
if (!(cause instanceof ServiceUnitNotReadyException)) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java
index 80aed778b..24ceea548 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java
@@ -18,25 +18,27 @@
*/
package org.apache.pulsar.broker.service;
+import io.netty.buffer.ByteBuf;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
-
import org.apache.bookkeeper.mledger.Position;
import org.apache.pulsar.broker.stats.ClusterReplicationMetrics;
import org.apache.pulsar.broker.stats.NamespaceStats;
import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.InitialPosition;
import org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.SubType;
+import org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.InitialPosition;
import org.apache.pulsar.common.policies.data.BacklogQuota;
import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats;
import org.apache.pulsar.common.policies.data.PersistentTopicStats;
import org.apache.pulsar.common.policies.data.Policies;
+import org.apache.pulsar.common.schema.SchemaData;
+import org.apache.pulsar.common.schema.SchemaVersion;
import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap;
import org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet;
import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats;
import org.apache.pulsar.utils.StatsOutputStream;
-import io.netty.buffer.ByteBuf;
-
public interface Topic {
public interface PublishContext {
@@ -80,9 +82,9 @@ default long getOriginalSequenceId() {
CompletableFuture<Consumer> subscribe(ServerCnx cnx, String subscriptionName, long consumerId, SubType subType,
int priorityLevel, String consumerName, boolean isDurable, MessageId startMessageId,
- Map<String, String> metadata, boolean readCompacted);
+ Map<String, String> metadata, boolean readCompacted, InitialPosition initialPosition);
- CompletableFuture<Subscription> createSubscription(String subscriptionName);
+ CompletableFuture<Subscription> createSubscription(String subscriptionName, InitialPosition initialPosition);
CompletableFuture<Void> unsubscribe(String subName);
@@ -125,4 +127,6 @@ void updateRates(NamespaceStats nsStats, NamespaceBundleStats currentBundleStats
PersistentTopicInternalStats getInternalStats();
Position getLastMessageId();
+
+ CompletableFuture<SchemaVersion> addSchema(SchemaData schema);
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java
index 74af8b646..42b99adac 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java
@@ -59,7 +59,7 @@ public NonPersistentReplicator(NonPersistentTopic topic, String localCluster, St
}
@Override
- protected void readEntries(Producer producer) {
+ protected void readEntries(Producer<byte[]> producer) {
this.producer = (ProducerImpl) producer;
if (STATE_UPDATER.compareAndSet(this, State.Starting, State.Started)) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java
index ef355f0ea..56b340787 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java
@@ -22,6 +22,13 @@
import static org.apache.bookkeeper.mledger.impl.EntryCacheManager.create;
import static org.apache.pulsar.broker.cache.ConfigurationCacheService.POLICIES;
+import com.carrotsearch.hppc.ObjectObjectHashMap;
+import com.google.common.base.MoreObjects;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import io.netty.buffer.ByteBuf;
+import io.netty.util.concurrent.FastThreadLocal;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -32,7 +39,6 @@
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLongFieldUpdater;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-
import org.apache.bookkeeper.common.util.OrderedScheduler;
import org.apache.bookkeeper.mledger.Entry;
import org.apache.bookkeeper.mledger.Position;
@@ -58,6 +64,7 @@
import org.apache.pulsar.broker.stats.ClusterReplicationMetrics;
import org.apache.pulsar.broker.stats.NamespaceStats;
import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.InitialPosition;
import org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.SubType;
import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.policies.data.BacklogQuota;
@@ -70,6 +77,8 @@
import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats.CursorStats;
import org.apache.pulsar.common.policies.data.Policies;
import org.apache.pulsar.common.policies.data.PublisherStats;
+import org.apache.pulsar.common.schema.SchemaData;
+import org.apache.pulsar.common.schema.SchemaVersion;
import org.apache.pulsar.common.util.FutureUtil;
import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap;
import org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet;
@@ -79,15 +88,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.carrotsearch.hppc.ObjectObjectHashMap;
-import com.google.common.base.MoreObjects;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-
-import io.netty.buffer.ByteBuf;
-import io.netty.util.concurrent.FastThreadLocal;
-
public class NonPersistentTopic implements Topic {
private final String topic;
@@ -323,7 +323,7 @@ public void removeProducer(Producer producer) {
@Override
public CompletableFuture<Consumer> subscribe(final ServerCnx cnx, String subscriptionName, long consumerId,
SubType subType, int priorityLevel, String consumerName, boolean isDurable, MessageId startMessageId,
- Map<String, String> metadata, boolean readCompacted) {
+ Map<String, String> metadata, boolean readCompacted, InitialPosition initialPosition) {
final CompletableFuture<Consumer> future = new CompletableFuture<>();
@@ -397,7 +397,7 @@ public void removeProducer(Producer producer) {
}
@Override
- public CompletableFuture<Subscription> createSubscription(String subscriptionName) {
+ public CompletableFuture<Subscription> createSubscription(String subscriptionName, InitialPosition initialPosition) {
return CompletableFuture.completedFuture(new NonPersistentSubscription(this, subscriptionName));
}
@@ -973,7 +973,14 @@ public void markBatchMessagePublished() {
this.hasBatchMessagePublished = true;
}
-
-
private static final Logger log = LoggerFactory.getLogger(NonPersistentTopic.class);
+
+ @Override
+ public CompletableFuture<SchemaVersion> addSchema(SchemaData schema) {
+ String base = TopicName.get(getName()).getPartitionedTopicName();
+ String id = TopicName.get(base).getSchemaName();
+ return brokerService.pulsar()
+ .getSchemaRegistryService()
+ .putSchemaIfAbsent(id, schema);
+ }
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java
index e7e50b8cb..498f0d89c 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java
@@ -259,7 +259,7 @@ public void openCursorFailed(ManagedLedgerException exception, Object ctx) {
future.completeExceptionally(exception);
}
- }, null);
+ }, null, true);
return future;
} else {
// Nothing to do, we are in the correct state
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java
index 52ed4b453..99804df5a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java
@@ -105,7 +105,7 @@ public PersistentReplicator(PersistentTopic topic, ManagedCursor cursor, String
}
@Override
- protected void readEntries(org.apache.pulsar.client.api.Producer producer) {
+ protected void readEntries(org.apache.pulsar.client.api.Producer<byte[]> producer) {
// Rewind the cursor to be sure to read again all non-acked messages sent while restarting
cursor.rewind();
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
index 62017f0f6..a6aefc299 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
@@ -80,6 +80,7 @@
import org.apache.pulsar.client.impl.BatchMessageIdImpl;
import org.apache.pulsar.client.impl.MessageIdImpl;
import org.apache.pulsar.client.impl.MessageImpl;
+import org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.InitialPosition;
import org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.SubType;
import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.policies.data.BacklogQuota;
@@ -92,6 +93,8 @@
import org.apache.pulsar.common.policies.data.PublisherStats;
import org.apache.pulsar.common.policies.data.ReplicatorStats;
import org.apache.pulsar.common.policies.data.SubscriptionStats;
+import org.apache.pulsar.common.schema.SchemaData;
+import org.apache.pulsar.common.schema.SchemaVersion;
import org.apache.pulsar.common.util.Codec;
import org.apache.pulsar.common.util.DateFormatter;
import org.apache.pulsar.common.util.FutureUtil;
@@ -440,7 +443,7 @@ public void removeProducer(Producer producer) {
@Override
public CompletableFuture<Consumer> subscribe(final ServerCnx cnx, String subscriptionName, long consumerId,
SubType subType, int priorityLevel, String consumerName, boolean isDurable, MessageId startMessageId,
- Map<String, String> metadata, boolean readCompacted) {
+ Map<String, String> metadata, boolean readCompacted, InitialPosition initialPosition) {
final CompletableFuture<Consumer> future = new CompletableFuture<>();
@@ -488,7 +491,7 @@ public void removeProducer(Producer producer) {
}
CompletableFuture<? extends Subscription> subscriptionFuture = isDurable ? //
- getDurableSubscription(subscriptionName) //
+ getDurableSubscription(subscriptionName, initialPosition) //
: getNonDurableSubscription(subscriptionName, startMessageId);
int maxUnackedMessages = isDurable ? brokerService.pulsar().getConfiguration().getMaxUnackedMessagesPerConsumer() :0;
@@ -531,15 +534,15 @@ public void removeProducer(Producer producer) {
return future;
}
- private CompletableFuture<Subscription> getDurableSubscription(String subscriptionName) {
+ private CompletableFuture<Subscription> getDurableSubscription(String subscriptionName, InitialPosition initialPosition) {
CompletableFuture<Subscription> subscriptionFuture = new CompletableFuture<>();
- ledger.asyncOpenCursor(Codec.encode(subscriptionName), new OpenCursorCallback() {
+ ledger.asyncOpenCursor(Codec.encode(subscriptionName), initialPosition, new OpenCursorCallback() {
@Override
public void openCursorComplete(ManagedCursor cursor, Object ctx) {
if (log.isDebugEnabled()) {
log.debug("[{}][{}] Opened cursor", topic, subscriptionName);
}
-
+
subscriptionFuture.complete(subscriptions.computeIfAbsent(subscriptionName,
name -> createPersistentSubscription(subscriptionName, cursor)));
}
@@ -596,8 +599,8 @@ public void openCursorFailed(ManagedLedgerException exception, Object ctx) {
@SuppressWarnings("unchecked")
@Override
- public CompletableFuture<Subscription> createSubscription(String subscriptionName) {
- return getDurableSubscription(subscriptionName);
+ public CompletableFuture<Subscription> createSubscription(String subscriptionName, InitialPosition initialPosition) {
+ return getDurableSubscription(subscriptionName, initialPosition);
}
/**
@@ -1601,4 +1604,13 @@ public Position getLastMessageId() {
}
private static final Logger log = LoggerFactory.getLogger(PersistentTopic.class);
+
+ @Override
+ public CompletableFuture<SchemaVersion> addSchema(SchemaData schema) {
+ String base = TopicName.get(getName()).getPartitionedTopicName();
+ String id = TopicName.get(base).getSchemaName();
+ return brokerService.pulsar()
+ .getSchemaRegistryService()
+ .putSchemaIfAbsent(id, schema);
+ }
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/DefaultSchemaRegistryService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/DefaultSchemaRegistryService.java
new file mode 100644
index 000000000..db3b9f7a9
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/DefaultSchemaRegistryService.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.service.schema;
+
+import static java.util.concurrent.CompletableFuture.completedFuture;
+
+import java.util.concurrent.CompletableFuture;
+import org.apache.pulsar.common.schema.SchemaData;
+import org.apache.pulsar.common.schema.SchemaVersion;
+
+public class DefaultSchemaRegistryService implements SchemaRegistryService {
+ @Override
+ public CompletableFuture<SchemaAndMetadata> getSchema(String schemaId) {
+ return completedFuture(null);
+ }
+
+ @Override
+ public CompletableFuture<SchemaAndMetadata> getSchema(String schemaId, SchemaVersion version) {
+ return completedFuture(null);
+ }
+
+ @Override
+ public CompletableFuture<SchemaVersion> putSchemaIfAbsent(String schemaId, SchemaData schema) {
+ return completedFuture(null);
+ }
+
+ @Override
+ public CompletableFuture<SchemaVersion> deleteSchema(String schemaId, String user) {
+ return completedFuture(null);
+ }
+
+ @Override
+ public SchemaVersion versionFromBytes(byte[] version) {
+ return null;
+ }
+
+ @Override
+ public void close() throws Exception {
+
+ }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistry.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistry.java
new file mode 100644
index 000000000..4dfbd6d99
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistry.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.service.schema;
+
+import com.google.common.base.MoreObjects;
+import java.util.Objects;
+import java.util.concurrent.CompletableFuture;
+import org.apache.pulsar.common.schema.SchemaData;
+import org.apache.pulsar.common.schema.SchemaVersion;
+
+public interface SchemaRegistry extends AutoCloseable {
+
+ CompletableFuture<SchemaAndMetadata> getSchema(String schemaId);
+
+ CompletableFuture<SchemaAndMetadata> getSchema(String schemaId, SchemaVersion version);
+
+ CompletableFuture<SchemaVersion> putSchemaIfAbsent(String schemaId, SchemaData schema);
+
+ CompletableFuture<SchemaVersion> deleteSchema(String schemaId, String user);
+
+ SchemaVersion versionFromBytes(byte[] version);
+
+ class SchemaAndMetadata {
+ public final String id;
+ public final SchemaData schema;
+ public final SchemaVersion version;
+
+ SchemaAndMetadata(String id, SchemaData schema, SchemaVersion version) {
+ this.id = id;
+ this.schema = schema;
+ this.version = version;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ SchemaAndMetadata that = (SchemaAndMetadata) o;
+ return version == that.version &&
+ Objects.equals(id, that.id) &&
+ Objects.equals(schema, that.schema);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(id, schema, version);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("id", id)
+ .add("schema", schema)
+ .add("version", version)
+ .toString();
+ }
+ }
+
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryService.java
new file mode 100644
index 000000000..69e736481
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryService.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.service.schema;
+
+import java.lang.reflect.Method;
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.ServiceConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public interface SchemaRegistryService extends SchemaRegistry {
+ String CreateMethodName = "create";
+ Logger log = LoggerFactory.getLogger(SchemaRegistryService.class);
+
+ static SchemaRegistryService create(PulsarService pulsar) {
+ try {
+ ServiceConfiguration config = pulsar.getConfiguration();
+ final Class<?> storageClass = Class.forName(config.getSchemaRegistryStorageClassName());
+ Object factoryInstance = storageClass.newInstance();
+ Method createMethod = storageClass.getMethod(CreateMethodName, PulsarService.class);
+ SchemaStorage schemaStorage = (SchemaStorage) createMethod.invoke(factoryInstance, pulsar);
+ return new SchemaRegistryServiceImpl(schemaStorage);
+ } catch (Exception e) {
+ log.warn("Error when trying to create scehema registry storage: {}", e);
+ }
+ return new DefaultSchemaRegistryService();
+ }
+
+ void close() throws Exception;
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java
new file mode 100644
index 000000000..1aec03950
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java
@@ -0,0 +1,190 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.service.schema;
+
+import static java.util.Objects.isNull;
+import static java.util.concurrent.CompletableFuture.completedFuture;
+import static org.apache.pulsar.broker.service.schema.SchemaRegistryServiceImpl.Functions.toPairs;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.InvalidProtocolBufferException;
+import java.time.Clock;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import javax.validation.constraints.NotNull;
+import org.apache.pulsar.broker.service.schema.proto.SchemaRegistryFormat;
+import org.apache.pulsar.common.schema.SchemaData;
+import org.apache.pulsar.common.schema.SchemaType;
+import org.apache.pulsar.common.schema.SchemaVersion;
+
+public class SchemaRegistryServiceImpl implements SchemaRegistryService {
+ private final SchemaStorage schemaStorage;
+ private final Clock clock;
+
+ @VisibleForTesting
+ SchemaRegistryServiceImpl(SchemaStorage schemaStorage, Clock clock) {
+ this.schemaStorage = schemaStorage;
+ this.clock = clock;
+ }
+
+ @VisibleForTesting
+ SchemaRegistryServiceImpl(SchemaStorage schemaStorage) {
+ this(schemaStorage, Clock.systemUTC());
+ }
+
+ @Override
+ @NotNull
+ public CompletableFuture<SchemaAndMetadata> getSchema(String schemaId) {
+ return getSchema(schemaId, SchemaVersion.Latest);
+ }
+
+ @Override
+ @NotNull
+ public CompletableFuture<SchemaAndMetadata> getSchema(String schemaId, SchemaVersion version) {
+ return schemaStorage.get(schemaId, version).thenCompose(stored -> {
+ if (isNull(stored)) {
+ return completedFuture(null);
+ } else {
+ return Functions.bytesToSchemaInfo(stored.data)
+ .thenApply(Functions::schemaInfoToSchema)
+ .thenApply(schema -> new SchemaAndMetadata(schemaId, schema, stored.version));
+ }
+ }
+ );
+ }
+
+ @Override
+ @NotNull
+ public CompletableFuture<SchemaVersion> putSchemaIfAbsent(String schemaId, SchemaData schema) {
+ SchemaRegistryFormat.SchemaInfo info = SchemaRegistryFormat.SchemaInfo.newBuilder()
+ .setType(Functions.convertFromDomainType(schema.getType()))
+ .setSchema(ByteString.copyFrom(schema.getData()))
+ .setSchemaId(schemaId)
+ .setUser(schema.getUser())
+ .setDeleted(false)
+ .setTimestamp(clock.millis())
+ .addAllProps(toPairs(schema.getProps()))
+ .build();
+ return schemaStorage.put(schemaId, info.toByteArray());
+ }
+
+ @Override
+ @NotNull
+ public CompletableFuture<SchemaVersion> deleteSchema(String schemaId, String user) {
+ byte[] deletedEntry = deleted(schemaId, user).toByteArray();
+ return schemaStorage.put(schemaId, deletedEntry);
+ }
+
+ @Override
+ public SchemaVersion versionFromBytes(byte[] version) {
+ return schemaStorage.versionFromBytes(version);
+ }
+
+ @Override
+ public void close() throws Exception {
+ schemaStorage.close();
+ }
+
+ private SchemaRegistryFormat.SchemaInfo deleted(String schemaId, String user) {
+ return SchemaRegistryFormat.SchemaInfo.newBuilder()
+ .setSchemaId(schemaId)
+ .setType(SchemaRegistryFormat.SchemaInfo.SchemaType.NONE)
+ .setSchema(ByteString.EMPTY)
+ .setUser(user)
+ .setDeleted(true)
+ .setTimestamp(clock.millis())
+ .build();
+ }
+
+ interface Functions {
+ static SchemaType convertToDomainType(SchemaRegistryFormat.SchemaInfo.SchemaType type) {
+ switch (type) {
+ case AVRO:
+ return SchemaType.AVRO;
+ case JSON:
+ return SchemaType.JSON;
+ case PROTO:
+ return SchemaType.PROTOBUF;
+ case THRIFT:
+ return SchemaType.THRIFT;
+ default:
+ return SchemaType.NONE;
+ }
+ }
+
+ static SchemaRegistryFormat.SchemaInfo.SchemaType convertFromDomainType(SchemaType type) {
+ switch (type) {
+ case AVRO:
+ return SchemaRegistryFormat.SchemaInfo.SchemaType.AVRO;
+ case JSON:
+ return SchemaRegistryFormat.SchemaInfo.SchemaType.JSON;
+ case THRIFT:
+ return SchemaRegistryFormat.SchemaInfo.SchemaType.THRIFT;
+ case PROTOBUF:
+ return SchemaRegistryFormat.SchemaInfo.SchemaType.PROTO;
+ default:
+ return SchemaRegistryFormat.SchemaInfo.SchemaType.NONE;
+ }
+ }
+
+ static Map<String, String> toMap(List<SchemaRegistryFormat.SchemaInfo.KeyValuePair> pairs) {
+ Map<String, String> map = new HashMap<>();
+ for (SchemaRegistryFormat.SchemaInfo.KeyValuePair pair : pairs) {
+ map.put(pair.getKey(), pair.getValue());
+ }
+ return map;
+ }
+
+ static List<SchemaRegistryFormat.SchemaInfo.KeyValuePair> toPairs(Map<String, String> map) {
+ List<SchemaRegistryFormat.SchemaInfo.KeyValuePair> pairs = new ArrayList<>(map.size());
+ for (Map.Entry<String, String> entry : map.entrySet()) {
+ SchemaRegistryFormat.SchemaInfo.KeyValuePair.Builder builder =
+ SchemaRegistryFormat.SchemaInfo.KeyValuePair.newBuilder();
+ pairs.add(builder.setKey(entry.getKey()).setValue(entry.getValue()).build());
+ }
+ return pairs;
+ }
+
+ static SchemaData schemaInfoToSchema(SchemaRegistryFormat.SchemaInfo info) {
+ return SchemaData.builder()
+ .user(info.getUser())
+ .type(convertToDomainType(info.getType()))
+ .data(info.getSchema().toByteArray())
+ .isDeleted(info.getDeleted())
+ .props(toMap(info.getPropsList()))
+ .build();
+ }
+
+ static CompletableFuture<SchemaRegistryFormat.SchemaInfo> bytesToSchemaInfo(byte[] bytes) {
+ CompletableFuture<SchemaRegistryFormat.SchemaInfo> future;
+ try {
+ future = completedFuture(SchemaRegistryFormat.SchemaInfo.parseFrom(bytes));
+ } catch (InvalidProtocolBufferException e) {
+ future = new CompletableFuture<>();
+ future.completeExceptionally(e);
+ }
+ return future;
+ }
+ }
+
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaStorage.java
new file mode 100644
index 000000000..4d0d8af18
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaStorage.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.service.schema;
+
+import java.util.concurrent.CompletableFuture;
+import org.apache.pulsar.common.schema.SchemaVersion;
+
+public interface SchemaStorage {
+
+ CompletableFuture<SchemaVersion> put(String key, byte[] value);
+
+ CompletableFuture<StoredSchema> get(String key, SchemaVersion version);
+
+ CompletableFuture<SchemaVersion> delete(String key);
+
+ SchemaVersion versionFromBytes(byte[] version);
+
+ void close() throws Exception;
+
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaStorageFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaStorageFactory.java
new file mode 100644
index 000000000..c4cff34b3
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaStorageFactory.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.service.schema;
+
+import javax.validation.constraints.NotNull;
+import org.apache.pulsar.broker.PulsarService;
+
+public interface SchemaStorageFactory {
+ @NotNull
+ SchemaStorage create(PulsarService pulsar) throws Exception;
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/StoredSchema.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/StoredSchema.java
new file mode 100644
index 000000000..f28a70797
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/StoredSchema.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.service.schema;
+
+import com.google.common.base.MoreObjects;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Objects;
+import org.apache.pulsar.common.schema.SchemaVersion;
+
+public class StoredSchema {
+ public final byte[] data;
+ public final SchemaVersion version;
+ public final Map<String, String> metadata;
+
+ public StoredSchema(byte[] data, SchemaVersion version, Map<String, String> metadata) {
+ this.data = data;
+ this.version = version;
+ this.metadata = metadata;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ StoredSchema that = (StoredSchema) o;
+ return Arrays.equals(data, that.data) &&
+ Objects.equals(version, that.version) &&
+ Objects.equals(metadata, that.metadata);
+ }
+
+ @Override
+ public int hashCode() {
+
+ int result = Objects.hash(version, metadata);
+ result = 31 * result + Arrays.hashCode(data);
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("data", data)
+ .add("version", version)
+ .add("metadata", metadata)
+ .toString();
+ }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/proto/SchemaRegistryFormat.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/proto/SchemaRegistryFormat.java
new file mode 100644
index 000000000..39227313c
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/proto/SchemaRegistryFormat.java
@@ -0,0 +1,1373 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+// Generated by the protocol buffer compiler. DO NOT EDIT!
+// source: src/main/proto/SchemaRegistryFormat.proto
+
+package org.apache.pulsar.broker.service.schema.proto;
+
+public final class SchemaRegistryFormat {
+ private SchemaRegistryFormat() {}
+ public static void registerAllExtensions(
+ com.google.protobuf.ExtensionRegistryLite registry) {
+ }
+ public interface SchemaInfoOrBuilder
+ extends com.google.protobuf.MessageLiteOrBuilder {
+
+ // required string schema_id = 1;
+ boolean hasSchemaId();
+ String getSchemaId();
+
+ // required string user = 2;
+ boolean hasUser();
+ String getUser();
+
+ // required .pulsar.schema.SchemaInfo.SchemaType type = 3;
+ boolean hasType();
+ SchemaRegistryFormat.SchemaInfo.SchemaType getType();
+
+ // required bytes schema = 4;
+ boolean hasSchema();
+ com.google.protobuf.ByteString getSchema();
+
+ // required int64 timestamp = 5;
+ boolean hasTimestamp();
+ long getTimestamp();
+
+ // required bool deleted = 6;
+ boolean hasDeleted();
+ boolean getDeleted();
+
+ // repeated .pulsar.schema.SchemaInfo.KeyValuePair props = 7;
+ java.util.List<SchemaRegistryFormat.SchemaInfo.KeyValuePair>
+ getPropsList();
+ SchemaRegistryFormat.SchemaInfo.KeyValuePair getProps(int index);
+ int getPropsCount();
+ }
+ public static final class SchemaInfo extends
+ com.google.protobuf.GeneratedMessageLite
+ implements SchemaInfoOrBuilder {
+ // Use SchemaInfo.newBuilder() to construct.
+ private SchemaInfo(Builder builder) {
+ super(builder);
+ }
+ private SchemaInfo(boolean noInit) {}
+
+ private static final SchemaInfo defaultInstance;
+ public static SchemaInfo getDefaultInstance() {
+ return defaultInstance;
+ }
+
+ public SchemaInfo getDefaultInstanceForType() {
+ return defaultInstance;
+ }
+
+ public enum SchemaType
+ implements com.google.protobuf.Internal.EnumLite {
+ NONE(0, 1),
+ THRIFT(1, 2),
+ AVRO(2, 3),
+ JSON(3, 4),
+ PROTO(4, 5),
+ ;
+
+ public static final int NONE_VALUE = 1;
+ public static final int THRIFT_VALUE = 2;
+ public static final int AVRO_VALUE = 3;
+ public static final int JSON_VALUE = 4;
+ public static final int PROTO_VALUE = 5;
+
+
+ public final int getNumber() { return value; }
+
+ public static SchemaType valueOf(int value) {
+ switch (value) {
+ case 1: return NONE;
+ case 2: return THRIFT;
+ case 3: return AVRO;
+ case 4: return JSON;
+ case 5: return PROTO;
+ default: return null;
+ }
+ }
+
+ public static com.google.protobuf.Internal.EnumLiteMap<SchemaType>
+ internalGetValueMap() {
+ return internalValueMap;
+ }
+ private static com.google.protobuf.Internal.EnumLiteMap<SchemaType>
+ internalValueMap =
+ new com.google.protobuf.Internal.EnumLiteMap<SchemaType>() {
+ public SchemaType findValueByNumber(int number) {
+ return SchemaType.valueOf(number);
+ }
+ };
+
+ private final int value;
+
+ private SchemaType(int index, int value) {
+ this.value = value;
+ }
+
+ // @@protoc_insertion_point(enum_scope:pulsar.schema.SchemaInfo.SchemaType)
+ }
+
+ public interface KeyValuePairOrBuilder
+ extends com.google.protobuf.MessageLiteOrBuilder {
+
+ // required string key = 1;
+ boolean hasKey();
+ String getKey();
+
+ // required string value = 2;
+ boolean hasValue();
+ String getValue();
+ }
+ public static final class KeyValuePair extends
+ com.google.protobuf.GeneratedMessageLite
+ implements KeyValuePairOrBuilder {
+ // Use KeyValuePair.newBuilder() to construct.
+ private KeyValuePair(Builder builder) {
+ super(builder);
+ }
+ private KeyValuePair(boolean noInit) {}
+
+ private static final KeyValuePair defaultInstance;
+ public static KeyValuePair getDefaultInstance() {
+ return defaultInstance;
+ }
+
+ public KeyValuePair getDefaultInstanceForType() {
+ return defaultInstance;
+ }
+
+ private int bitField0_;
+ // required string key = 1;
+ public static final int KEY_FIELD_NUMBER = 1;
+ private java.lang.Object key_;
+ public boolean hasKey() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ public String getKey() {
+ java.lang.Object ref = key_;
+ if (ref instanceof String) {
+ return (String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ String s = bs.toStringUtf8();
+ if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+ key_ = s;
+ }
+ return s;
+ }
+ }
+ private com.google.protobuf.ByteString getKeyBytes() {
+ java.lang.Object ref = key_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+ key_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
+ // required string value = 2;
+ public static final int VALUE_FIELD_NUMBER = 2;
+ private java.lang.Object value_;
+ public boolean hasValue() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ public String getValue() {
+ java.lang.Object ref = value_;
+ if (ref instanceof String) {
+ return (String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ String s = bs.toStringUtf8();
+ if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+ value_ = s;
+ }
+ return s;
+ }
+ }
+ private com.google.protobuf.ByteString getValueBytes() {
+ java.lang.Object ref = value_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+ value_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
+ private void initFields() {
+ key_ = "";
+ value_ = "";
+ }
+ private byte memoizedIsInitialized = -1;
+ public final boolean isInitialized() {
+ byte isInitialized = memoizedIsInitialized;
+ if (isInitialized != -1) return isInitialized == 1;
+
+ if (!hasKey()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasValue()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ memoizedIsInitialized = 1;
+ return true;
+ }
+
+ public void writeTo(com.google.protobuf.CodedOutputStream output)
+ throws java.io.IOException {
+ getSerializedSize();
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ output.writeBytes(1, getKeyBytes());
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ output.writeBytes(2, getValueBytes());
+ }
+ }
+
+ private int memoizedSerializedSize = -1;
+ public int getSerializedSize() {
+ int size = memoizedSerializedSize;
+ if (size != -1) return size;
+
+ size = 0;
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(1, getKeyBytes());
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(2, getValueBytes());
+ }
+ memoizedSerializedSize = size;
+ return size;
+ }
+
+ private static final long serialVersionUID = 0L;
+ @java.lang.Override
+ protected java.lang.Object writeReplace()
+ throws java.io.ObjectStreamException {
+ return super.writeReplace();
+ }
+
+ public static SchemaRegistryFormat.SchemaInfo.KeyValuePair parseFrom(
+ com.google.protobuf.ByteString data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return newBuilder().mergeFrom(data).buildParsed();
+ }
+ public static SchemaRegistryFormat.SchemaInfo.KeyValuePair parseFrom(
+ com.google.protobuf.ByteString data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return newBuilder().mergeFrom(data, extensionRegistry)
+ .buildParsed();
+ }
+ public static SchemaRegistryFormat.SchemaInfo.KeyValuePair parseFrom(byte[] data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return newBuilder().mergeFrom(data).buildParsed();
+ }
+ public static SchemaRegistryFormat.SchemaInfo.KeyValuePair parseFrom(
+ byte[] data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return newBuilder().mergeFrom(data, extensionRegistry)
+ .buildParsed();
+ }
+ public static SchemaRegistryFormat.SchemaInfo.KeyValuePair parseFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return newBuilder().mergeFrom(input).buildParsed();
+ }
+ public static SchemaRegistryFormat.SchemaInfo.KeyValuePair parseFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return newBuilder().mergeFrom(input, extensionRegistry)
+ .buildParsed();
+ }
+ public static SchemaRegistryFormat.SchemaInfo.KeyValuePair parseDelimitedFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ Builder builder = newBuilder();
+ if (builder.mergeDelimitedFrom(input)) {
+ return builder.buildParsed();
+ } else {
+ return null;
+ }
+ }
+ public static SchemaRegistryFormat.SchemaInfo.KeyValuePair parseDelimitedFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ Builder builder = newBuilder();
+ if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+ return builder.buildParsed();
+ } else {
+ return null;
+ }
+ }
+ public static SchemaRegistryFormat.SchemaInfo.KeyValuePair parseFrom(
+ com.google.protobuf.CodedInputStream input)
+ throws java.io.IOException {
+ return newBuilder().mergeFrom(input).buildParsed();
+ }
+ public static SchemaRegistryFormat.SchemaInfo.KeyValuePair parseFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return newBuilder().mergeFrom(input, extensionRegistry)
+ .buildParsed();
+ }
+
+ public static Builder newBuilder() { return Builder.create(); }
+ public Builder newBuilderForType() { return newBuilder(); }
+ public static Builder newBuilder(SchemaRegistryFormat.SchemaInfo.KeyValuePair prototype) {
+ return newBuilder().mergeFrom(prototype);
+ }
+ public Builder toBuilder() { return newBuilder(this); }
+
+ public static final class Builder extends
+ com.google.protobuf.GeneratedMessageLite.Builder<
+ SchemaRegistryFormat.SchemaInfo.KeyValuePair, Builder>
+ implements SchemaRegistryFormat.SchemaInfo.KeyValuePairOrBuilder {
+ // Construct using org.apache.pulsar.broker.service.schema.proto.SchemaRegistryFormat.SchemaInfo.KeyValuePair.newBuilder()
+ private Builder() {
+ maybeForceBuilderInitialization();
+ }
+
+ private void maybeForceBuilderInitialization() {
+ }
+ private static Builder create() {
+ return new Builder();
+ }
+
+ public Builder clear() {
+ super.clear();
+ key_ = "";
+ bitField0_ = (bitField0_ & ~0x00000001);
+ value_ = "";
+ bitField0_ = (bitField0_ & ~0x00000002);
+ return this;
+ }
+
+ public Builder clone() {
+ return create().mergeFrom(buildPartial());
+ }
+
+ public SchemaRegistryFormat.SchemaInfo.KeyValuePair getDefaultInstanceForType() {
+ return SchemaRegistryFormat.SchemaInfo.KeyValuePair.getDefaultInstance();
+ }
+
+ public SchemaRegistryFormat.SchemaInfo.KeyValuePair build() {
+ SchemaRegistryFormat.SchemaInfo.KeyValuePair result = buildPartial();
+ if (!result.isInitialized()) {
+ throw newUninitializedMessageException(result);
+ }
+ return result;
+ }
+
+ private SchemaRegistryFormat.SchemaInfo.KeyValuePair buildParsed()
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ SchemaRegistryFormat.SchemaInfo.KeyValuePair result = buildPartial();
+ if (!result.isInitialized()) {
+ throw newUninitializedMessageException(
+ result).asInvalidProtocolBufferException();
+ }
+ return result;
+ }
+
+ public SchemaRegistryFormat.SchemaInfo.KeyValuePair buildPartial() {
+ SchemaRegistryFormat.SchemaInfo.KeyValuePair result = new SchemaRegistryFormat.SchemaInfo.KeyValuePair(this);
+ int from_bitField0_ = bitField0_;
+ int to_bitField0_ = 0;
+ if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+ to_bitField0_ |= 0x00000001;
+ }
+ result.key_ = key_;
+ if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+ to_bitField0_ |= 0x00000002;
+ }
+ result.value_ = value_;
+ result.bitField0_ = to_bitField0_;
+ return result;
+ }
+
+ public Builder mergeFrom(SchemaRegistryFormat.SchemaInfo.KeyValuePair other) {
+ if (other == SchemaRegistryFormat.SchemaInfo.KeyValuePair.getDefaultInstance()) return this;
+ if (other.hasKey()) {
+ setKey(other.getKey());
+ }
+ if (other.hasValue()) {
+ setValue(other.getValue());
+ }
+ return this;
+ }
+
+ public final boolean isInitialized() {
+ if (!hasKey()) {
+
+ return false;
+ }
+ if (!hasValue()) {
+
+ return false;
+ }
+ return true;
+ }
+
+ public Builder mergeFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ while (true) {
+ int tag = input.readTag();
+ switch (tag) {
+ case 0:
+
+ return this;
+ default: {
+ if (!parseUnknownField(input, extensionRegistry, tag)) {
+
+ return this;
+ }
+ break;
+ }
+ case 10: {
+ bitField0_ |= 0x00000001;
+ key_ = input.readBytes();
+ break;
+ }
+ case 18: {
+ bitField0_ |= 0x00000002;
+ value_ = input.readBytes();
+ break;
+ }
+ }
+ }
+ }
+
+ private int bitField0_;
+
+ // required string key = 1;
+ private java.lang.Object key_ = "";
+ public boolean hasKey() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ public String getKey() {
+ java.lang.Object ref = key_;
+ if (!(ref instanceof String)) {
+ String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+ key_ = s;
+ return s;
+ } else {
+ return (String) ref;
+ }
+ }
+ public Builder setKey(String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000001;
+ key_ = value;
+
+ return this;
+ }
+ public Builder clearKey() {
+ bitField0_ = (bitField0_ & ~0x00000001);
+ key_ = getDefaultInstance().getKey();
+
+ return this;
+ }
+ void setKey(com.google.protobuf.ByteString value) {
+ bitField0_ |= 0x00000001;
+ key_ = value;
+
+ }
+
+ // required string value = 2;
+ private java.lang.Object value_ = "";
+ public boolean hasValue() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ public String getValue() {
+ java.lang.Object ref = value_;
+ if (!(ref instanceof String)) {
+ String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+ value_ = s;
+ return s;
+ } else {
+ return (String) ref;
+ }
+ }
+ public Builder setValue(String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000002;
+ value_ = value;
+
+ return this;
+ }
+ public Builder clearValue() {
+ bitField0_ = (bitField0_ & ~0x00000002);
+ value_ = getDefaultInstance().getValue();
+
+ return this;
+ }
+ void setValue(com.google.protobuf.ByteString value) {
+ bitField0_ |= 0x00000002;
+ value_ = value;
+
+ }
+
+ // @@protoc_insertion_point(builder_scope:pulsar.schema.SchemaInfo.KeyValuePair)
+ }
+
+ static {
+ defaultInstance = new KeyValuePair(true);
+ defaultInstance.initFields();
+ }
+
+ // @@protoc_insertion_point(class_scope:pulsar.schema.SchemaInfo.KeyValuePair)
+ }
+
+ private int bitField0_;
+ // required string schema_id = 1;
+ public static final int SCHEMA_ID_FIELD_NUMBER = 1;
+ private java.lang.Object schemaId_;
+ public boolean hasSchemaId() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ public String getSchemaId() {
+ java.lang.Object ref = schemaId_;
+ if (ref instanceof String) {
+ return (String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ String s = bs.toStringUtf8();
+ if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+ schemaId_ = s;
+ }
+ return s;
+ }
+ }
+ private com.google.protobuf.ByteString getSchemaIdBytes() {
+ java.lang.Object ref = schemaId_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+ schemaId_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
+ // required string user = 2;
+ public static final int USER_FIELD_NUMBER = 2;
+ private java.lang.Object user_;
+ public boolean hasUser() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ public String getUser() {
+ java.lang.Object ref = user_;
+ if (ref instanceof String) {
+ return (String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ String s = bs.toStringUtf8();
+ if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+ user_ = s;
+ }
+ return s;
+ }
+ }
+ private com.google.protobuf.ByteString getUserBytes() {
+ java.lang.Object ref = user_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+ user_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
+ // required .pulsar.schema.SchemaInfo.SchemaType type = 3;
+ public static final int TYPE_FIELD_NUMBER = 3;
+ private SchemaRegistryFormat.SchemaInfo.SchemaType type_;
+ public boolean hasType() {
+ return ((bitField0_ & 0x00000004) == 0x00000004);
+ }
+ public SchemaRegistryFormat.SchemaInfo.SchemaType getType() {
+ return type_;
+ }
+
+ // required bytes schema = 4;
+ public static final int SCHEMA_FIELD_NUMBER = 4;
+ private com.google.protobuf.ByteString schema_;
+ public boolean hasSchema() {
+ return ((bitField0_ & 0x00000008) == 0x00000008);
+ }
+ public com.google.protobuf.ByteString getSchema() {
+ return schema_;
+ }
+
+ // required int64 timestamp = 5;
+ public static final int TIMESTAMP_FIELD_NUMBER = 5;
+ private long timestamp_;
+ public boolean hasTimestamp() {
+ return ((bitField0_ & 0x00000010) == 0x00000010);
+ }
+ public long getTimestamp() {
+ return timestamp_;
+ }
+
+ // required bool deleted = 6;
+ public static final int DELETED_FIELD_NUMBER = 6;
+ private boolean deleted_;
+ public boolean hasDeleted() {
+ return ((bitField0_ & 0x00000020) == 0x00000020);
+ }
+ public boolean getDeleted() {
+ return deleted_;
+ }
+
+ // repeated .pulsar.schema.SchemaInfo.KeyValuePair props = 7;
+ public static final int PROPS_FIELD_NUMBER = 7;
+ private java.util.List<SchemaRegistryFormat.SchemaInfo.KeyValuePair> props_;
+ public java.util.List<SchemaRegistryFormat.SchemaInfo.KeyValuePair> getPropsList() {
+ return props_;
+ }
+ public java.util.List<? extends SchemaRegistryFormat.SchemaInfo.KeyValuePairOrBuilder>
+ getPropsOrBuilderList() {
+ return props_;
+ }
+ public int getPropsCount() {
+ return props_.size();
+ }
+ public SchemaRegistryFormat.SchemaInfo.KeyValuePair getProps(int index) {
+ return props_.get(index);
+ }
+ public SchemaRegistryFormat.SchemaInfo.KeyValuePairOrBuilder getPropsOrBuilder(
+ int index) {
+ return props_.get(index);
+ }
+
+ private void initFields() {
+ schemaId_ = "";
+ user_ = "";
+ type_ = SchemaRegistryFormat.SchemaInfo.SchemaType.NONE;
+ schema_ = com.google.protobuf.ByteString.EMPTY;
+ timestamp_ = 0L;
+ deleted_ = false;
+ props_ = java.util.Collections.emptyList();
+ }
+ private byte memoizedIsInitialized = -1;
+ public final boolean isInitialized() {
+ byte isInitialized = memoizedIsInitialized;
+ if (isInitialized != -1) return isInitialized == 1;
+
+ if (!hasSchemaId()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasUser()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasType()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasSchema()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasTimestamp()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasDeleted()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ for (int i = 0; i < getPropsCount(); i++) {
+ if (!getProps(i).isInitialized()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ }
+ memoizedIsInitialized = 1;
+ return true;
+ }
+
+ public void writeTo(com.google.protobuf.CodedOutputStream output)
+ throws java.io.IOException {
+ getSerializedSize();
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ output.writeBytes(1, getSchemaIdBytes());
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ output.writeBytes(2, getUserBytes());
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ output.writeEnum(3, type_.getNumber());
+ }
+ if (((bitField0_ & 0x00000008) == 0x00000008)) {
+ output.writeBytes(4, schema_);
+ }
+ if (((bitField0_ & 0x00000010) == 0x00000010)) {
+ output.writeInt64(5, timestamp_);
+ }
+ if (((bitField0_ & 0x00000020) == 0x00000020)) {
+ output.writeBool(6, deleted_);
+ }
+ for (int i = 0; i < props_.size(); i++) {
+ output.writeMessage(7, props_.get(i));
+ }
+ }
+
+ private int memoizedSerializedSize = -1;
+ public int getSerializedSize() {
+ int size = memoizedSerializedSize;
+ if (size != -1) return size;
+
+ size = 0;
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(1, getSchemaIdBytes());
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(2, getUserBytes());
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeEnumSize(3, type_.getNumber());
+ }
+ if (((bitField0_ & 0x00000008) == 0x00000008)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(4, schema_);
+ }
+ if (((bitField0_ & 0x00000010) == 0x00000010)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeInt64Size(5, timestamp_);
+ }
+ if (((bitField0_ & 0x00000020) == 0x00000020)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBoolSize(6, deleted_);
+ }
+ for (int i = 0; i < props_.size(); i++) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeMessageSize(7, props_.get(i));
+ }
+ memoizedSerializedSize = size;
+ return size;
+ }
+
+ private static final long serialVersionUID = 0L;
+ @java.lang.Override
+ protected java.lang.Object writeReplace()
+ throws java.io.ObjectStreamException {
+ return super.writeReplace();
+ }
+
+ public static SchemaRegistryFormat.SchemaInfo parseFrom(
+ com.google.protobuf.ByteString data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return newBuilder().mergeFrom(data).buildParsed();
+ }
+ public static SchemaRegistryFormat.SchemaInfo parseFrom(
+ com.google.protobuf.ByteString data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return newBuilder().mergeFrom(data, extensionRegistry)
+ .buildParsed();
+ }
+ public static SchemaRegistryFormat.SchemaInfo parseFrom(byte[] data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return newBuilder().mergeFrom(data).buildParsed();
+ }
+ public static SchemaRegistryFormat.SchemaInfo parseFrom(
+ byte[] data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return newBuilder().mergeFrom(data, extensionRegistry)
+ .buildParsed();
+ }
+ public static SchemaRegistryFormat.SchemaInfo parseFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return newBuilder().mergeFrom(input).buildParsed();
+ }
+ public static SchemaRegistryFormat.SchemaInfo parseFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return newBuilder().mergeFrom(input, extensionRegistry)
+ .buildParsed();
+ }
+ public static SchemaRegistryFormat.SchemaInfo parseDelimitedFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ Builder builder = newBuilder();
+ if (builder.mergeDelimitedFrom(input)) {
+ return builder.buildParsed();
+ } else {
+ return null;
+ }
+ }
+ public static SchemaRegistryFormat.SchemaInfo parseDelimitedFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ Builder builder = newBuilder();
+ if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+ return builder.buildParsed();
+ } else {
+ return null;
+ }
+ }
+ public static SchemaRegistryFormat.SchemaInfo parseFrom(
+ com.google.protobuf.CodedInputStream input)
+ throws java.io.IOException {
+ return newBuilder().mergeFrom(input).buildParsed();
+ }
+ public static SchemaRegistryFormat.SchemaInfo parseFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return newBuilder().mergeFrom(input, extensionRegistry)
+ .buildParsed();
+ }
+
+ public static Builder newBuilder() { return Builder.create(); }
+ public Builder newBuilderForType() { return newBuilder(); }
+ public static Builder newBuilder(SchemaRegistryFormat.SchemaInfo prototype) {
+ return newBuilder().mergeFrom(prototype);
+ }
+ public Builder toBuilder() { return newBuilder(this); }
+
+ public static final class Builder extends
+ com.google.protobuf.GeneratedMessageLite.Builder<
+ SchemaRegistryFormat.SchemaInfo, Builder>
+ implements SchemaRegistryFormat.SchemaInfoOrBuilder {
+ // Construct using org.apache.pulsar.broker.service.schema.proto.SchemaRegistryFormat.SchemaInfo.newBuilder()
+ private Builder() {
+ maybeForceBuilderInitialization();
+ }
+
+ private void maybeForceBuilderInitialization() {
+ }
+ private static Builder create() {
+ return new Builder();
+ }
+
+ public Builder clear() {
+ super.clear();
+ schemaId_ = "";
+ bitField0_ = (bitField0_ & ~0x00000001);
+ user_ = "";
+ bitField0_ = (bitField0_ & ~0x00000002);
+ type_ = SchemaRegistryFormat.SchemaInfo.SchemaType.NONE;
+ bitField0_ = (bitField0_ & ~0x00000004);
+ schema_ = com.google.protobuf.ByteString.EMPTY;
+ bitField0_ = (bitField0_ & ~0x00000008);
+ timestamp_ = 0L;
+ bitField0_ = (bitField0_ & ~0x00000010);
+ deleted_ = false;
+ bitField0_ = (bitField0_ & ~0x00000020);
+ props_ = java.util.Collections.emptyList();
+ bitField0_ = (bitField0_ & ~0x00000040);
+ return this;
+ }
+
+ public Builder clone() {
+ return create().mergeFrom(buildPartial());
+ }
+
+ public SchemaRegistryFormat.SchemaInfo getDefaultInstanceForType() {
+ return SchemaRegistryFormat.SchemaInfo.getDefaultInstance();
+ }
+
+ public SchemaRegistryFormat.SchemaInfo build() {
+ SchemaRegistryFormat.SchemaInfo result = buildPartial();
+ if (!result.isInitialized()) {
+ throw newUninitializedMessageException(result);
+ }
+ return result;
+ }
+
+ private SchemaRegistryFormat.SchemaInfo buildParsed()
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ SchemaRegistryFormat.SchemaInfo result = buildPartial();
+ if (!result.isInitialized()) {
+ throw newUninitializedMessageException(
+ result).asInvalidProtocolBufferException();
+ }
+ return result;
+ }
+
+ public SchemaRegistryFormat.SchemaInfo buildPartial() {
+ SchemaRegistryFormat.SchemaInfo result = new SchemaRegistryFormat.SchemaInfo(this);
+ int from_bitField0_ = bitField0_;
+ int to_bitField0_ = 0;
+ if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+ to_bitField0_ |= 0x00000001;
+ }
+ result.schemaId_ = schemaId_;
+ if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+ to_bitField0_ |= 0x00000002;
+ }
+ result.user_ = user_;
+ if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+ to_bitField0_ |= 0x00000004;
+ }
+ result.type_ = type_;
+ if (((from_bitField0_ & 0x00000008) == 0x00000008)) {
+ to_bitField0_ |= 0x00000008;
+ }
+ result.schema_ = schema_;
+ if (((from_bitField0_ & 0x00000010) == 0x00000010)) {
+ to_bitField0_ |= 0x00000010;
+ }
+ result.timestamp_ = timestamp_;
+ if (((from_bitField0_ & 0x00000020) == 0x00000020)) {
+ to_bitField0_ |= 0x00000020;
+ }
+ result.deleted_ = deleted_;
+ if (((bitField0_ & 0x00000040) == 0x00000040)) {
+ props_ = java.util.Collections.unmodifiableList(props_);
+ bitField0_ = (bitField0_ & ~0x00000040);
+ }
+ result.props_ = props_;
+ result.bitField0_ = to_bitField0_;
+ return result;
+ }
+
+ public Builder mergeFrom(SchemaRegistryFormat.SchemaInfo other) {
+ if (other == SchemaRegistryFormat.SchemaInfo.getDefaultInstance()) return this;
+ if (other.hasSchemaId()) {
+ setSchemaId(other.getSchemaId());
+ }
+ if (other.hasUser()) {
+ setUser(other.getUser());
+ }
+ if (other.hasType()) {
+ setType(other.getType());
+ }
+ if (other.hasSchema()) {
+ setSchema(other.getSchema());
+ }
+ if (other.hasTimestamp()) {
+ setTimestamp(other.getTimestamp());
+ }
+ if (other.hasDeleted()) {
+ setDeleted(other.getDeleted());
+ }
+ if (!other.props_.isEmpty()) {
+ if (props_.isEmpty()) {
+ props_ = other.props_;
+ bitField0_ = (bitField0_ & ~0x00000040);
+ } else {
+ ensurePropsIsMutable();
+ props_.addAll(other.props_);
+ }
+
+ }
+ return this;
+ }
+
+ public final boolean isInitialized() {
+ if (!hasSchemaId()) {
+
+ return false;
+ }
+ if (!hasUser()) {
+
+ return false;
+ }
+ if (!hasType()) {
+
+ return false;
+ }
+ if (!hasSchema()) {
+
+ return false;
+ }
+ if (!hasTimestamp()) {
+
+ return false;
+ }
+ if (!hasDeleted()) {
+
+ return false;
+ }
+ for (int i = 0; i < getPropsCount(); i++) {
+ if (!getProps(i).isInitialized()) {
+
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public Builder mergeFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ while (true) {
+ int tag = input.readTag();
+ switch (tag) {
+ case 0:
+
+ return this;
+ default: {
+ if (!parseUnknownField(input, extensionRegistry, tag)) {
+
+ return this;
+ }
+ break;
+ }
+ case 10: {
+ bitField0_ |= 0x00000001;
+ schemaId_ = input.readBytes();
+ break;
+ }
+ case 18: {
+ bitField0_ |= 0x00000002;
+ user_ = input.readBytes();
+ break;
+ }
+ case 24: {
+ int rawValue = input.readEnum();
+ SchemaRegistryFormat.SchemaInfo.SchemaType value = SchemaRegistryFormat.SchemaInfo.SchemaType.valueOf(rawValue);
+ if (value != null) {
+ bitField0_ |= 0x00000004;
+ type_ = value;
+ }
+ break;
+ }
+ case 34: {
+ bitField0_ |= 0x00000008;
+ schema_ = input.readBytes();
+ break;
+ }
+ case 40: {
+ bitField0_ |= 0x00000010;
+ timestamp_ = input.readInt64();
+ break;
+ }
+ case 48: {
+ bitField0_ |= 0x00000020;
+ deleted_ = input.readBool();
+ break;
+ }
+ case 58: {
+ SchemaRegistryFormat.SchemaInfo.KeyValuePair.Builder subBuilder = SchemaRegistryFormat.SchemaInfo.KeyValuePair.newBuilder();
+ input.readMessage(subBuilder, extensionRegistry);
+ addProps(subBuilder.buildPartial());
+ break;
+ }
+ }
+ }
+ }
+
+ private int bitField0_;
+
+ // required string schema_id = 1;
+ private java.lang.Object schemaId_ = "";
+ public boolean hasSchemaId() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ public String getSchemaId() {
+ java.lang.Object ref = schemaId_;
+ if (!(ref instanceof String)) {
+ String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+ schemaId_ = s;
+ return s;
+ } else {
+ return (String) ref;
+ }
+ }
+ public Builder setSchemaId(String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000001;
+ schemaId_ = value;
+
+ return this;
+ }
+ public Builder clearSchemaId() {
+ bitField0_ = (bitField0_ & ~0x00000001);
+ schemaId_ = getDefaultInstance().getSchemaId();
+
+ return this;
+ }
+ void setSchemaId(com.google.protobuf.ByteString value) {
+ bitField0_ |= 0x00000001;
+ schemaId_ = value;
+
+ }
+
+ // required string user = 2;
+ private java.lang.Object user_ = "";
+ public boolean hasUser() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ public String getUser() {
+ java.lang.Object ref = user_;
+ if (!(ref instanceof String)) {
+ String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+ user_ = s;
+ return s;
+ } else {
+ return (String) ref;
+ }
+ }
+ public Builder setUser(String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000002;
+ user_ = value;
+
+ return this;
+ }
+ public Builder clearUser() {
+ bitField0_ = (bitField0_ & ~0x00000002);
+ user_ = getDefaultInstance().getUser();
+
+ return this;
+ }
+ void setUser(com.google.protobuf.ByteString value) {
+ bitField0_ |= 0x00000002;
+ user_ = value;
+
+ }
+
+ // required .pulsar.schema.SchemaInfo.SchemaType type = 3;
+ private SchemaRegistryFormat.SchemaInfo.SchemaType type_ = SchemaRegistryFormat.SchemaInfo.SchemaType.NONE;
+ public boolean hasType() {
+ return ((bitField0_ & 0x00000004) == 0x00000004);
+ }
+ public SchemaRegistryFormat.SchemaInfo.SchemaType getType() {
+ return type_;
+ }
+ public Builder setType(SchemaRegistryFormat.SchemaInfo.SchemaType value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000004;
+ type_ = value;
+
+ return this;
+ }
+ public Builder clearType() {
+ bitField0_ = (bitField0_ & ~0x00000004);
+ type_ = SchemaRegistryFormat.SchemaInfo.SchemaType.NONE;
+
+ return this;
+ }
+
+ // required bytes schema = 4;
+ private com.google.protobuf.ByteString schema_ = com.google.protobuf.ByteString.EMPTY;
+ public boolean hasSchema() {
+ return ((bitField0_ & 0x00000008) == 0x00000008);
+ }
+ public com.google.protobuf.ByteString getSchema() {
+ return schema_;
+ }
+ public Builder setSchema(com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000008;
+ schema_ = value;
+
+ return this;
+ }
+ public Builder clearSchema() {
+ bitField0_ = (bitField0_ & ~0x00000008);
+ schema_ = getDefaultInstance().getSchema();
+
+ return this;
+ }
+
+ // required int64 timestamp = 5;
+ private long timestamp_ ;
+ public boolean hasTimestamp() {
+ return ((bitField0_ & 0x00000010) == 0x00000010);
+ }
+ public long getTimestamp() {
+ return timestamp_;
+ }
+ public Builder setTimestamp(long value) {
+ bitField0_ |= 0x00000010;
+ timestamp_ = value;
+
+ return this;
+ }
+ public Builder clearTimestamp() {
+ bitField0_ = (bitField0_ & ~0x00000010);
+ timestamp_ = 0L;
+
+ return this;
+ }
+
+ // required bool deleted = 6;
+ private boolean deleted_ ;
+ public boolean hasDeleted() {
+ return ((bitField0_ & 0x00000020) == 0x00000020);
+ }
+ public boolean getDeleted() {
+ return deleted_;
+ }
+ public Builder setDeleted(boolean value) {
+ bitField0_ |= 0x00000020;
+ deleted_ = value;
+
+ return this;
+ }
+ public Builder clearDeleted() {
+ bitField0_ = (bitField0_ & ~0x00000020);
+ deleted_ = false;
+
+ return this;
+ }
+
+ // repeated .pulsar.schema.SchemaInfo.KeyValuePair props = 7;
+ private java.util.List<SchemaRegistryFormat.SchemaInfo.KeyValuePair> props_ =
+ java.util.Collections.emptyList();
+ private void ensurePropsIsMutable() {
+ if (!((bitField0_ & 0x00000040) == 0x00000040)) {
+ props_ = new java.util.ArrayList<SchemaRegistryFormat.SchemaInfo.KeyValuePair>(props_);
+ bitField0_ |= 0x00000040;
+ }
+ }
+
+ public java.util.List<SchemaRegistryFormat.SchemaInfo.KeyValuePair> getPropsList() {
+ return java.util.Collections.unmodifiableList(props_);
+ }
+ public int getPropsCount() {
+ return props_.size();
+ }
+ public SchemaRegistryFormat.SchemaInfo.KeyValuePair getProps(int index) {
+ return props_.get(index);
+ }
+ public Builder setProps(
+ int index, SchemaRegistryFormat.SchemaInfo.KeyValuePair value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ ensurePropsIsMutable();
+ props_.set(index, value);
+
+ return this;
+ }
+ public Builder setProps(
+ int index, SchemaRegistryFormat.SchemaInfo.KeyValuePair.Builder builderForValue) {
+ ensurePropsIsMutable();
+ props_.set(index, builderForValue.build());
+
+ return this;
+ }
+ public Builder addProps(SchemaRegistryFormat.SchemaInfo.KeyValuePair value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ ensurePropsIsMutable();
+ props_.add(value);
+
+ return this;
+ }
+ public Builder addProps(
+ int index, SchemaRegistryFormat.SchemaInfo.KeyValuePair value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ ensurePropsIsMutable();
+ props_.add(index, value);
+
+ return this;
+ }
+ public Builder addProps(
+ SchemaRegistryFormat.SchemaInfo.KeyValuePair.Builder builderForValue) {
+ ensurePropsIsMutable();
+ props_.add(builderForValue.build());
+
+ return this;
+ }
+ public Builder addProps(
+ int index, SchemaRegistryFormat.SchemaInfo.KeyValuePair.Builder builderForValue) {
+ ensurePropsIsMutable();
+ props_.add(index, builderForValue.build());
+
+ return this;
+ }
+ public Builder addAllProps(
+ java.lang.Iterable<? extends SchemaRegistryFormat.SchemaInfo.KeyValuePair> values) {
+ ensurePropsIsMutable();
+ super.addAll(values, props_);
+
+ return this;
+ }
+ public Builder clearProps() {
+ props_ = java.util.Collections.emptyList();
+ bitField0_ = (bitField0_ & ~0x00000040);
+
+ return this;
+ }
+ public Builder removeProps(int index) {
+ ensurePropsIsMutable();
+ props_.remove(index);
+
+ return this;
+ }
+
+ // @@protoc_insertion_point(builder_scope:pulsar.schema.SchemaInfo)
+ }
+
+ static {
+ defaultInstance = new SchemaInfo(true);
+ defaultInstance.initFields();
+ }
+
+ // @@protoc_insertion_point(class_scope:pulsar.schema.SchemaInfo)
+ }
+
+
+ static {
+ }
+
+ // @@protoc_insertion_point(outer_class_scope)
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java
index 20ff44b92..02c8b1a87 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java
@@ -49,6 +49,7 @@
import org.eclipse.jetty.util.resource.Resource;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.eclipse.jetty.util.thread.ExecutorThreadPool;
+import org.glassfish.jersey.media.multipart.MultiPartFeature;
import org.glassfish.jersey.server.ResourceConfig;
import org.glassfish.jersey.servlet.ServletContainer;
import org.slf4j.Logger;
@@ -120,6 +121,7 @@ public void addRestResources(String basePath, String javaPackages, boolean requi
ResourceConfig config = new ResourceConfig();
config.packages("jersey.config.server.provider.packages", javaPackages);
config.register(provider);
+ config.register(MultiPartFeature.class);
ServletHolder servletHolder = new ServletHolder(new ServletContainer(config));
servletHolder.setAsyncSupported(true);
addServlet(basePath, servletHolder, requiresAuthentication, attributeMap);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/api/RawReader.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/api/RawReader.java
index 15b03bee3..f9d297f48 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/client/api/RawReader.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/api/RawReader.java
@@ -20,8 +20,6 @@
import java.util.Map;
import java.util.concurrent.CompletableFuture;
-import org.apache.pulsar.client.api.MessageId;
-import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.impl.PulsarClientImpl;
import org.apache.pulsar.client.impl.RawReaderImpl;
@@ -32,12 +30,20 @@
/**
* Create a raw reader for a topic.
*/
+
public static CompletableFuture<RawReader> create(PulsarClient client, String topic, String subscription) {
- CompletableFuture<Consumer> future = new CompletableFuture<>();
+ CompletableFuture<Consumer<byte[]>> future = new CompletableFuture<>();
RawReader r = new RawReaderImpl((PulsarClientImpl)client, topic, subscription, future);
return future.thenCompose((consumer) -> r.seekAsync(MessageId.earliest)).thenApply((ignore) -> r);
}
+ /**
+ * Get the topic for the reader
+ *
+ * @return topic for the reader
+ */
+ String getTopic();
+
/**
* Seek to a location in the topic. After the seek, the first message read will be the one with
* with the specified message ID.
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java
index a048a5bb2..61a1fc267 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java
@@ -30,6 +30,7 @@
import org.apache.pulsar.client.api.MessageId;
import org.apache.pulsar.client.api.RawMessage;
import org.apache.pulsar.client.api.RawReader;
+import org.apache.pulsar.client.api.Schema;
import org.apache.pulsar.client.api.SubscriptionType;
import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData;
import org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType;
@@ -43,12 +44,12 @@
public class RawReaderImpl implements RawReader {
final static int DEFAULT_RECEIVER_QUEUE_SIZE = 1000;
- private final ConsumerConfigurationData consumerConfiguration;
+ private final ConsumerConfigurationData<byte[]> consumerConfiguration;
private RawConsumerImpl consumer;
public RawReaderImpl(PulsarClientImpl client, String topic, String subscription,
- CompletableFuture<Consumer> consumerFuture) {
- consumerConfiguration = new ConsumerConfigurationData();
+ CompletableFuture<Consumer<byte[]>> consumerFuture) {
+ consumerConfiguration = new ConsumerConfigurationData<>();
consumerConfiguration.getTopicNames().add(topic);
consumerConfiguration.setSubscriptionName(subscription);
consumerConfiguration.setSubscriptionType(SubscriptionType.Exclusive);
@@ -58,6 +59,12 @@ public RawReaderImpl(PulsarClientImpl client, String topic, String subscription,
consumerFuture);
}
+ @Override
+ public String getTopic() {
+ return consumerConfiguration.getTopicNames().stream()
+ .findFirst().orElse(null);
+ }
+
@Override
public CompletableFuture<Void> seekAsync(MessageId messageId) {
return consumer.seekAsync(messageId);
@@ -83,14 +90,19 @@ public RawReaderImpl(PulsarClientImpl client, String topic, String subscription,
return consumer.getLastMessageIdAsync();
}
- static class RawConsumerImpl extends ConsumerImpl {
+ @Override
+ public String toString() {
+ return "RawReader(topic=" + getTopic() + ")";
+ }
+
+ static class RawConsumerImpl extends ConsumerImpl<byte[]> {
final BlockingQueue<RawMessageAndCnx> incomingRawMessages;
final Queue<CompletableFuture<RawMessage>> pendingRawReceives;
RawConsumerImpl(PulsarClientImpl client, ConsumerConfigurationData conf,
- CompletableFuture<Consumer> consumerFuture) {
+ CompletableFuture<Consumer<byte[]>> consumerFuture) {
super(client, conf.getSingleTopic(), conf, client.externalExecutorProvider().getExecutor(), -1,
- consumerFuture, SubscriptionMode.Durable, MessageId.earliest);
+ consumerFuture, SubscriptionMode.Durable, MessageId.earliest, Schema.IDENTITY);
incomingRawMessages = new GrowableArrayBlockingQueue<>();
pendingRawReceives = new ConcurrentLinkedQueue<>();
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java
index e70381edb..b1378b648 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java
@@ -133,11 +133,11 @@ private static void findStartPointLoop(PositionImpl p, long start, long end,
CompletableFuture.allOf(startEntry, middleEntry, endEntry).thenRun(
() -> {
- if (comparePositionAndMessageId(p, startEntry.join()) < 0) {
+ if (comparePositionAndMessageId(p, startEntry.join()) <= 0) {
promise.complete(start);
- } else if (comparePositionAndMessageId(p, middleEntry.join()) < 0) {
+ } else if (comparePositionAndMessageId(p, middleEntry.join()) <= 0) {
findStartPointLoop(p, start, midpoint, promise, cache);
- } else if (comparePositionAndMessageId(p, endEntry.join()) < 0) {
+ } else if (comparePositionAndMessageId(p, endEntry.join()) <= 0) {
findStartPointLoop(p, midpoint + 1, end, promise, cache);
} else {
promise.complete(NEWER_THAN_COMPACTED);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java
index 7ae788c70..3e2d25969 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java
@@ -83,6 +83,7 @@ public TwoPhaseCompactor(ServiceConfiguration conf,
if (exception != null) {
loopPromise.completeExceptionally(exception);
} else {
+ log.info("Commencing phase one of compaction for {}, reading to {}", reader, lastMessageId);
phaseOneLoop(reader, Optional.empty(), lastMessageId, latestForKey, loopPromise);
}
});
@@ -136,8 +137,12 @@ private void scheduleTimeout(CompletableFuture<RawMessage> future) {
private CompletableFuture<Long> phaseTwo(RawReader reader, MessageId from, MessageId to,
Map<String,MessageId> latestForKey, BookKeeper bk) {
- return createLedger(bk).thenCompose(
- (ledger) -> phaseTwoSeekThenLoop(reader, from, to, latestForKey, bk, ledger));
+
+ return createLedger(bk).thenCompose((ledger) -> {
+ log.info("Commencing phase two of compaction for {}, from {} to {}, compacting {} keys to ledger {}",
+ reader, from, to, latestForKey.size(), ledger.getId());
+ return phaseTwoSeekThenLoop(reader, from, to, latestForKey, bk, ledger);
+ });
}
private CompletableFuture<Long> phaseTwoSeekThenLoop(RawReader reader, MessageId from, MessageId to,
diff --git a/pulsar-broker/src/main/proto/SchemaRegistryFormat.proto b/pulsar-broker/src/main/proto/SchemaRegistryFormat.proto
new file mode 100644
index 000000000..e497eaffc
--- /dev/null
+++ b/pulsar-broker/src/main/proto/SchemaRegistryFormat.proto
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+syntax = "proto2";
+
+package pulsar.schema;
+option java_package = "org.apache.pulsar.broker.service.schema.proto";
+option optimize_for = LITE_RUNTIME;
+
+message SchemaInfo {
+ enum SchemaType {
+ NONE = 1;
+ THRIFT = 2;
+ AVRO = 3;
+ JSON = 4;
+ PROTO = 5;
+ }
+ message KeyValuePair {
+ required string key = 1;
+ required string value = 2;
+ }
+ required string schema_id = 1;
+ required string user = 2;
+ required SchemaType type = 3;
+ required bytes schema = 4;
+ required int64 timestamp = 5;
+ required bool deleted = 6;
+
+ repeated KeyValuePair props = 7;
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java
index 7e098cf09..90d6691ce 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java
@@ -18,6 +18,7 @@
*/
package org.apache.pulsar.broker.admin;
+import static org.mockito.Mockito.spy;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNotEquals;
@@ -57,20 +58,19 @@
import org.apache.pulsar.client.admin.internal.PropertiesImpl;
import org.apache.pulsar.client.api.ClientConfiguration;
import org.apache.pulsar.client.api.Consumer;
-import org.apache.pulsar.client.api.ConsumerConfiguration;
+import org.apache.pulsar.client.api.ConsumerBuilder;
import org.apache.pulsar.client.api.Message;
+import org.apache.pulsar.client.api.MessageRoutingMode;
import org.apache.pulsar.client.api.Producer;
-import org.apache.pulsar.client.api.ProducerConfiguration;
-import org.apache.pulsar.client.api.ProducerConfiguration.MessageRoutingMode;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.SubscriptionType;
import org.apache.pulsar.common.lookup.data.LookupData;
-import org.apache.pulsar.common.naming.TopicDomain;
-import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.naming.NamespaceBundle;
import org.apache.pulsar.common.naming.NamespaceBundleFactory;
import org.apache.pulsar.common.naming.NamespaceBundles;
import org.apache.pulsar.common.naming.NamespaceName;
+import org.apache.pulsar.common.naming.TopicDomain;
+import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.partition.PartitionedTopicMetadata;
import org.apache.pulsar.common.policies.data.AuthAction;
import org.apache.pulsar.common.policies.data.AutoFailoverPolicyData;
@@ -110,10 +110,14 @@
private static final Logger LOG = LoggerFactory.getLogger(AdminApiTest.class);
+ private final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/certificate/server.crt";
+ private final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/certificate/server.key";
+
private MockedPulsarService mockPulsarSetup;
private PulsarService otherPulsar;
+ private PulsarAdmin adminTls;
private PulsarAdmin otheradmin;
private NamespaceBundleFactory bundleFactory;
@@ -122,10 +126,19 @@
@Override
public void setup() throws Exception {
conf.setLoadBalancerEnabled(true);
+ conf.setTlsEnabled(true);
+ conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
+ conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
+
super.internalSetup();
bundleFactory = new NamespaceBundleFactory(pulsar, Hashing.crc32());
+ ClientConfiguration clientConf = new ClientConfiguration();
+ clientConf.setUseTls(true);
+ clientConf.setTlsTrustCertsFilePath(TLS_SERVER_CERT_FILE_PATH);
+ adminTls = spy(new PulsarAdmin(brokerUrlTls, clientConf));
+
// create otherbroker to test redirect on calls that need
// namespace ownership
mockPulsarSetup = new MockedPulsarService(this.conf);
@@ -143,6 +156,7 @@ public void setup() throws Exception {
@AfterMethod
@Override
public void cleanup() throws Exception {
+ adminTls.close();
super.internalCleanup();
mockPulsarSetup.cleanup();
}
@@ -164,8 +178,7 @@ public void cleanup() throws Exception {
@DataProvider(name = "topicType")
public Object[][] topicTypeProvider() {
- return new Object[][] { { TopicDomain.persistent.value() },
- { TopicDomain.non_persistent.value() } };
+ return new Object[][] { { TopicDomain.persistent.value() }, { TopicDomain.non_persistent.value() } };
}
@Test
@@ -382,6 +395,12 @@ public void brokers() throws Exception {
}
}
+ String[] parts = list.get(0).split(":");
+ Assert.assertEquals(parts.length, 2);
+ Map<String, NamespaceOwnershipStatus> nsMap2 = adminTls.brokers().getOwnedNamespaces("use",
+ String.format("%s:%d", parts[0], BROKER_WEBSERVICE_PORT_TLS));
+ Assert.assertEquals(nsMap2.size(), 1);
+
admin.namespaces().deleteNamespace("prop-xyz/use/ns1");
admin.clusters().deleteCluster("use");
// "test" cluster is part of config-default cluster and it's znode gets created when PulsarService creates
@@ -443,12 +462,14 @@ public void testUpdateDynamicConfigurationWithZkWatch() throws Exception {
/**
* Verifies broker sets watch on dynamic-configuration map even with invalid init json data
+ *
* <pre>
* 1. Set invalid json at dynamic-config znode
* 2. Broker fails to deserialize znode content but sets the watch on znode
* 3. Update znode with valid json map
* 4. Broker should get watch and update the dynamic-config map
* </pre>
+ *
* @throws Exception
*/
@Test
@@ -485,6 +506,7 @@ public void testInvalidDynamicConfigContentInZK() throws Exception {
* 3. trigger watch and listener
* 4. verify that config is updated
* </pre>
+ *
* @throws Exception
*/
@Test
@@ -611,7 +633,7 @@ public void namespaces() throws PulsarAdminException, PulsarServerException, Exc
assertEquals(admin.namespaces().getPersistence("prop-xyz/use/ns1"), new PersistencePolicies(3, 2, 1, 10.0));
// Force topic creation and namespace being loaded
- Producer producer = pulsarClient.createProducer("persistent://prop-xyz/use/ns1/my-topic");
+ Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://prop-xyz/use/ns1/my-topic").create();
producer.close();
admin.persistentTopics().delete("persistent://prop-xyz/use/ns1/my-topic");
@@ -644,7 +666,7 @@ public void namespaces() throws PulsarAdminException, PulsarServerException, Exc
}
// Force topic creation and namespace being loaded
- producer = pulsarClient.createProducer("persistent://prop-xyz/use/ns2/my-topic");
+ producer = pulsarClient.newProducer().topic("persistent://prop-xyz/use/ns2/my-topic").create();
producer.close();
admin.persistentTopics().delete("persistent://prop-xyz/use/ns2/my-topic");
@@ -664,12 +686,10 @@ public void persistentTopics(String topicName) throws Exception {
// create consumer and subscription
URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
- ClientConfiguration clientConf = new ClientConfiguration();
- clientConf.setStatsInterval(0, TimeUnit.SECONDS);
- PulsarClient client = PulsarClient.create(pulsarUrl.toString(), clientConf);
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
- Consumer consumer = client.subscribe(persistentTopicName, "my-sub", conf);
+ PulsarClient client = PulsarClient.builder().serviceUrl(pulsarUrl.toString()).statsInterval(0, TimeUnit.SECONDS)
+ .build();
+ Consumer<byte[]> consumer = client.newConsumer().topic(persistentTopicName).subscriptionName("my-sub")
+ .subscriptionType(SubscriptionType.Exclusive).subscribe();
assertEquals(admin.persistentTopics().getSubscriptions(persistentTopicName), Lists.newArrayList("my-sub"));
@@ -684,7 +704,7 @@ public void persistentTopics(String topicName) throws Exception {
PersistentTopicInternalStats internalStats = admin.persistentTopics().getInternalStats(persistentTopicName);
assertEquals(internalStats.cursors.keySet(), Sets.newTreeSet(Lists.newArrayList("my-sub")));
- List<Message> messages = admin.persistentTopics().peekMessages(persistentTopicName, "my-sub", 3);
+ List<Message<byte[]>> messages = admin.persistentTopics().peekMessages(persistentTopicName, "my-sub", 3);
assertEquals(messages.size(), 3);
for (int i = 0; i < 3; i++) {
String expectedMessage = "message-" + i;
@@ -737,8 +757,8 @@ public void partitionedTopics(String topicName) throws Exception {
assertEquals(admin.persistentTopics().getPartitionedTopicList("prop-xyz/use/ns1"), Lists.newArrayList());
final String partitionedTopicName = "persistent://prop-xyz/use/ns1/" + topicName;
admin.persistentTopics().createPartitionedTopic(partitionedTopicName, 4);
- assertEquals(admin.persistentTopics().getPartitionedTopicList("prop-xyz/use/ns1"), Lists.newArrayList(partitionedTopicName));
-
+ assertEquals(admin.persistentTopics().getPartitionedTopicList("prop-xyz/use/ns1"),
+ Lists.newArrayList(partitionedTopicName));
assertEquals(admin.persistentTopics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 4);
@@ -752,12 +772,10 @@ public void partitionedTopics(String topicName) throws Exception {
// create consumer and subscription
URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
- ClientConfiguration clientConf = new ClientConfiguration();
- clientConf.setStatsInterval(0, TimeUnit.SECONDS);
- PulsarClient client = PulsarClient.create(pulsarUrl.toString(), clientConf);
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
- Consumer consumer = client.subscribe(partitionedTopicName, "my-sub", conf);
+ PulsarClient client = PulsarClient.builder().serviceUrl(pulsarUrl.toString()).statsInterval(0, TimeUnit.SECONDS)
+ .build();
+ Consumer<byte[]> consumer = client.newConsumer().topic(partitionedTopicName).subscriptionName("my-sub")
+ .subscriptionType(SubscriptionType.Exclusive).subscribe();
assertEquals(admin.persistentTopics().getSubscriptions(partitionedTopicName), Lists.newArrayList("my-sub"));
@@ -770,7 +788,8 @@ public void partitionedTopics(String topicName) throws Exception {
fail(e.getMessage());
}
- Consumer consumer1 = client.subscribe(partitionedTopicName, "my-sub-1", conf);
+ Consumer<byte[]> consumer1 = client.newConsumer().topic(partitionedTopicName).subscriptionName("my-sub-1")
+ .subscribe();
assertEquals(Sets.newHashSet(admin.persistentTopics().getSubscriptions(partitionedTopicName)),
Sets.newHashSet("my-sub", "my-sub-1"));
@@ -779,9 +798,8 @@ public void partitionedTopics(String topicName) throws Exception {
admin.persistentTopics().deleteSubscription(partitionedTopicName, "my-sub-1");
assertEquals(admin.persistentTopics().getSubscriptions(partitionedTopicName), Lists.newArrayList("my-sub"));
- ProducerConfiguration prodConf = new ProducerConfiguration();
- prodConf.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
- Producer producer = client.createProducer(partitionedTopicName, prodConf);
+ Producer<byte[]> producer = client.newProducer().topic(partitionedTopicName)
+ .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();
for (int i = 0; i < 10; i++) {
String message = "message-" + i;
@@ -835,7 +853,7 @@ public void partitionedTopics(String topicName) throws Exception {
} catch (ConflictException ce) {
}
- producer = client.createProducer(partitionedTopicName);
+ producer = client.newProducer().topic(partitionedTopicName).create();
topics = admin.persistentTopics().getList("prop-xyz/use/ns1");
assertEquals(topics.size(), 4);
@@ -893,7 +911,7 @@ public void testNamespaceSplitBundle() throws Exception {
// Force to create a topic
final String namespace = "prop-xyz/use/ns1";
final String topicName = (new StringBuilder("persistent://")).append(namespace).append("/ds2").toString();
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
producer.send("message".getBytes());
publishMessagesOnPersistentTopic(topicName, 0);
assertEquals(admin.persistentTopics().getList(namespace), Lists.newArrayList(topicName));
@@ -924,14 +942,13 @@ public void testNamespaceUnloadBundle() throws Exception {
Lists.newArrayList("persistent://prop-xyz/use/ns1/ds2"));
// create consumer and subscription
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
- Consumer consumer = pulsarClient.subscribe("persistent://prop-xyz/use/ns1/ds2", "my-sub", conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://prop-xyz/use/ns1/ds2")
+ .subscriptionName("my-sub").subscribe();
assertEquals(admin.persistentTopics().getSubscriptions("persistent://prop-xyz/use/ns1/ds2"),
Lists.newArrayList("my-sub"));
// Create producer
- Producer producer = pulsarClient.createProducer("persistent://prop-xyz/use/ns1/ds2");
+ Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://prop-xyz/use/ns1/ds2").create();
for (int i = 0; i < 10; i++) {
String message = "message-" + i;
producer.send(message.getBytes());
@@ -982,13 +999,14 @@ public void testNamespaceBundleUnload(Integer numBundles) throws Exception {
Lists.newArrayList("persistent://prop-xyz/use/ns1-bundles/ds2"));
// create consumer and subscription
- ConsumerConfiguration conf = new ConsumerConfiguration();
- Consumer consumer = pulsarClient.subscribe("persistent://prop-xyz/use/ns1-bundles/ds2", "my-sub", conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://prop-xyz/use/ns1-bundles/ds2")
+ .subscriptionName("my-sub").subscribe();
assertEquals(admin.persistentTopics().getSubscriptions("persistent://prop-xyz/use/ns1-bundles/ds2"),
Lists.newArrayList("my-sub"));
// Create producer
- Producer producer = pulsarClient.createProducer("persistent://prop-xyz/use/ns1-bundles/ds2");
+ Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://prop-xyz/use/ns1-bundles/ds2")
+ .create();
for (int i = 0; i < 10; i++) {
String message = "message-" + i;
producer.send(message.getBytes());
@@ -1028,14 +1046,20 @@ public void testClearBacklogOnNamespace(Integer numBundles) throws Exception {
admin.namespaces().createNamespace("prop-xyz/use/ns1-bundles", numBundles);
// create consumer and subscription
- pulsarClient.subscribe("persistent://prop-xyz/use/ns1-bundles/ds2", "my-sub");
- pulsarClient.subscribe("persistent://prop-xyz/use/ns1-bundles/ds2", "my-sub-1");
- pulsarClient.subscribe("persistent://prop-xyz/use/ns1-bundles/ds2", "my-sub-2");
- pulsarClient.subscribe("persistent://prop-xyz/use/ns1-bundles/ds1", "my-sub");
- pulsarClient.subscribe("persistent://prop-xyz/use/ns1-bundles/ds1", "my-sub-1");
+ pulsarClient.newConsumer().topic("persistent://prop-xyz/use/ns1-bundles/ds2").subscriptionName("my-sub")
+ .subscribe();
+ pulsarClient.newConsumer().topic("persistent://prop-xyz/use/ns1-bundles/ds2").subscriptionName("my-sub-1")
+ .subscribe();
+ pulsarClient.newConsumer().topic("persistent://prop-xyz/use/ns1-bundles/ds2").subscriptionName("my-sub-2")
+ .subscribe();
+ pulsarClient.newConsumer().topic("persistent://prop-xyz/use/ns1-bundles/ds1").subscriptionName("my-sub")
+ .subscribe();
+ pulsarClient.newConsumer().topic("persistent://prop-xyz/use/ns1-bundles/ds1").subscriptionName("my-sub-1")
+ .subscribe();
// Create producer
- Producer producer = pulsarClient.createProducer("persistent://prop-xyz/use/ns1-bundles/ds2");
+ Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://prop-xyz/use/ns1-bundles/ds2")
+ .create();
for (int i = 0; i < 10; i++) {
String message = "message-" + i;
producer.send(message.getBytes());
@@ -1044,7 +1068,8 @@ public void testClearBacklogOnNamespace(Integer numBundles) throws Exception {
producer.close();
// Create producer
- Producer producer1 = pulsarClient.createProducer("persistent://prop-xyz/use/ns1-bundles/ds1");
+ Producer<byte[]> producer1 = pulsarClient.newProducer().topic("persistent://prop-xyz/use/ns1-bundles/ds1")
+ .create();
for (int i = 0; i < 10; i++) {
String message = "message-" + i;
producer1.send(message.getBytes());
@@ -1082,11 +1107,16 @@ public void testUnsubscribeOnNamespace(Integer numBundles) throws Exception {
admin.namespaces().createNamespace("prop-xyz/use/ns1-bundles", numBundles);
// create consumer and subscription
- Consumer consumer1 = pulsarClient.subscribe("persistent://prop-xyz/use/ns1-bundles/ds2", "my-sub");
- Consumer consumer2 = pulsarClient.subscribe("persistent://prop-xyz/use/ns1-bundles/ds2", "my-sub-1");
- /* Consumer consumer3 = */ pulsarClient.subscribe("persistent://prop-xyz/use/ns1-bundles/ds2", "my-sub-2");
- Consumer consumer4 = pulsarClient.subscribe("persistent://prop-xyz/use/ns1-bundles/ds1", "my-sub");
- Consumer consumer5 = pulsarClient.subscribe("persistent://prop-xyz/use/ns1-bundles/ds1", "my-sub-1");
+ Consumer<byte[]> consumer1 = pulsarClient.newConsumer().topic("persistent://prop-xyz/use/ns1-bundles/ds2")
+ .subscriptionName("my-sub").subscribe();
+ Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic("persistent://prop-xyz/use/ns1-bundles/ds2")
+ .subscriptionName("my-sub-1").subscribe();
+ /* Consumer consumer3 = */ pulsarClient.newConsumer().topic("persistent://prop-xyz/use/ns1-bundles/ds2")
+ .subscriptionName("my-sub-2").subscribe();
+ Consumer<byte[]> consumer4 = pulsarClient.newConsumer().topic("persistent://prop-xyz/use/ns1-bundles/ds1")
+ .subscriptionName("my-sub").subscribe();
+ Consumer<byte[]> consumer5 = pulsarClient.newConsumer().topic("persistent://prop-xyz/use/ns1-bundles/ds1")
+ .subscriptionName("my-sub-1").subscribe();
try {
admin.namespaces().unsubscribeNamespace("prop-xyz/use/ns1-bundles", "my-sub");
@@ -1132,7 +1162,7 @@ private void publishMessagesOnPersistentTopic(String topicName, int messages) th
}
private void publishMessagesOnPersistentTopic(String topicName, int messages, int startIdx) throws Exception {
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
for (int i = startIdx; i < (messages + startIdx); i++) {
String message = "message-" + i;
@@ -1177,7 +1207,7 @@ public void statsOnNonExistingTopics() throws Exception {
@Test
public void testDeleteFailedReturnCode() throws Exception {
String topicName = "persistent://prop-xyz/use/ns1/my-topic";
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
try {
admin.persistentTopics().delete(topicName);
@@ -1188,7 +1218,7 @@ public void testDeleteFailedReturnCode() throws Exception {
producer.close();
- Consumer consumer = pulsarClient.subscribe(topicName, "sub");
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("sub").subscribe();
try {
admin.persistentTopics().delete(topicName);
@@ -1257,9 +1287,8 @@ public void persistentTopicsCursorReset(String topicName) throws Exception {
topicName = "persistent://prop-xyz/use/ns1/" + topicName;
// create consumer and subscription
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
- Consumer consumer = pulsarClient.subscribe(topicName, "my-sub", conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub")
+ .subscriptionType(SubscriptionType.Exclusive).subscribe();
assertEquals(admin.persistentTopics().getSubscriptions(topicName), Lists.newArrayList("my-sub"));
@@ -1271,11 +1300,11 @@ public void persistentTopicsCursorReset(String topicName) throws Exception {
publishMessagesOnPersistentTopic(topicName, 5, 5);
- List<Message> messages = admin.persistentTopics().peekMessages(topicName, "my-sub", 10);
+ List<Message<byte[]>> messages = admin.persistentTopics().peekMessages(topicName, "my-sub", 10);
assertEquals(messages.size(), 10);
for (int i = 0; i < 10; i++) {
- Message message = consumer.receive();
+ Message<byte[]> message = consumer.receive();
consumer.acknowledge(message);
}
// messages should still be available due to retention
@@ -1285,7 +1314,7 @@ public void persistentTopicsCursorReset(String topicName) throws Exception {
int receivedAfterReset = 0;
for (int i = 4; i < 10; i++) {
- Message message = consumer.receive();
+ Message<byte[]> message = consumer.receive();
consumer.acknowledge(message);
++receivedAfterReset;
String expected = "message-" + i;
@@ -1309,9 +1338,8 @@ public void persistentTopicsCursorResetAfterReset(String topicName) throws Excep
topicName = "persistent://prop-xyz/use/ns1/" + topicName;
// create consumer and subscription
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
- Consumer consumer = pulsarClient.subscribe(topicName, "my-sub", conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub")
+ .subscriptionType(SubscriptionType.Exclusive).subscribe();
assertEquals(admin.persistentTopics().getSubscriptions(topicName), Lists.newArrayList("my-sub"));
@@ -1327,14 +1355,14 @@ public void persistentTopicsCursorResetAfterReset(String topicName) throws Excep
publishMessagesOnPersistentTopic(topicName, 2, 8);
- List<Message> messages = admin.persistentTopics().peekMessages(topicName, "my-sub", 10);
+ List<Message<byte[]>> messages = admin.persistentTopics().peekMessages(topicName, "my-sub", 10);
assertEquals(messages.size(), 10);
messages.forEach(message -> {
LOG.info("Peeked message: {}", new String(message.getData()));
});
for (int i = 0; i < 10; i++) {
- Message message = consumer.receive();
+ Message<byte[]> message = consumer.receive();
consumer.acknowledge(message);
}
@@ -1344,7 +1372,7 @@ public void persistentTopicsCursorResetAfterReset(String topicName) throws Excep
// Should received messages from 4-9
for (int i = 4; i < 10; i++) {
- Message message = consumer.receive();
+ Message<byte[]> message = consumer.receive();
consumer.acknowledge(message);
++receivedAfterReset;
String expected = "message-" + i;
@@ -1358,7 +1386,7 @@ public void persistentTopicsCursorResetAfterReset(String topicName) throws Excep
// Should received messages from 7-9
for (int i = 7; i < 10; i++) {
- Message message = consumer.receive();
+ Message<byte[]> message = consumer.receive();
consumer.acknowledge(message);
++receivedAfterReset;
String expected = "message-" + i;
@@ -1381,9 +1409,8 @@ public void partitionedTopicsCursorReset(String topicName) throws Exception {
admin.persistentTopics().createPartitionedTopic(topicName, 4);
// create consumer and subscription
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
- Consumer consumer = pulsarClient.subscribe(topicName, "my-sub", conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub")
+ .subscriptionType(SubscriptionType.Exclusive).subscribe();
List<String> topics = admin.persistentTopics().getList("prop-xyz/use/ns1");
assertEquals(topics.size(), 4);
@@ -1397,7 +1424,7 @@ public void partitionedTopicsCursorReset(String topicName) throws Exception {
publishMessagesOnPersistentTopic(topicName, 5, 5);
for (int i = 0; i < 10; i++) {
- Message message = consumer.receive();
+ Message<byte[]> message = consumer.receive();
consumer.acknowledge(message);
}
// messages should still be available due to retention
@@ -1407,7 +1434,7 @@ public void partitionedTopicsCursorReset(String topicName) throws Exception {
Set<String> expectedMessages = Sets.newHashSet();
Set<String> receivedMessages = Sets.newHashSet();
for (int i = 4; i < 10; i++) {
- Message message = consumer.receive();
+ Message<byte[]> message = consumer.receive();
consumer.acknowledge(message);
expectedMessages.add("message-" + i);
receivedMessages.add(new String(message.getData()));
@@ -1434,22 +1461,20 @@ public void persistentTopicsInvalidCursorReset() throws Exception {
// create consumer and subscription
URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
- ClientConfiguration clientConf = new ClientConfiguration();
- clientConf.setStatsInterval(0, TimeUnit.SECONDS);
- PulsarClient client = PulsarClient.create(pulsarUrl.toString(), clientConf);
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
- Consumer consumer = client.subscribe(topicName, "my-sub", conf);
+ PulsarClient client = PulsarClient.builder().serviceUrl(pulsarUrl.toString()).statsInterval(0, TimeUnit.SECONDS)
+ .build();
+ Consumer<byte[]> consumer = client.newConsumer().topic(topicName).subscriptionName("my-sub")
+ .subscriptionType(SubscriptionType.Exclusive).subscribe();
assertEquals(admin.persistentTopics().getSubscriptions(topicName), Lists.newArrayList("my-sub"));
publishMessagesOnPersistentTopic(topicName, 10);
- List<Message> messages = admin.persistentTopics().peekMessages(topicName, "my-sub", 10);
+ List<Message<byte[]>> messages = admin.persistentTopics().peekMessages(topicName, "my-sub", 10);
assertEquals(messages.size(), 10);
for (int i = 0; i < 10; i++) {
- Message message = consumer.receive();
+ Message<byte[]> message = consumer.receive();
consumer.acknowledge(message);
}
// use invalid timestamp
@@ -1461,7 +1486,7 @@ public void persistentTopicsInvalidCursorReset() throws Exception {
}
admin.persistentTopics().resetCursor(topicName, "my-sub", System.currentTimeMillis() + 90000);
- consumer = client.subscribe(topicName, "my-sub", conf);
+ consumer = client.newConsumer().topic(topicName).subscriptionName("my-sub").subscribe();
consumer.close();
client.close();
@@ -1500,6 +1525,7 @@ public void testObjectWithUnknowProperties() {
* 3. expire message on sub-1 : backlog for sub-1 must be 0
* 4. expire message on all subscriptions: backlog for all subscription must be 0
* </pre>
+ *
* @throws Exception
*/
@Test
@@ -1512,14 +1538,13 @@ public void testPersistentTopicsExpireMessages() throws Exception {
// create consumer and subscription
URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
- ClientConfiguration clientConf = new ClientConfiguration();
- clientConf.setStatsInterval(0, TimeUnit.SECONDS);
- PulsarClient client = PulsarClient.create(pulsarUrl.toString(), clientConf);
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Shared);
- Consumer consumer1 = client.subscribe("persistent://prop-xyz/use/ns1/ds2", "my-sub1", conf);
- Consumer consumer2 = client.subscribe("persistent://prop-xyz/use/ns1/ds2", "my-sub2", conf);
- Consumer consumer3 = client.subscribe("persistent://prop-xyz/use/ns1/ds2", "my-sub3", conf);
+ PulsarClient client = PulsarClient.builder().serviceUrl(pulsarUrl.toString()).statsInterval(0, TimeUnit.SECONDS)
+ .build();
+ ConsumerBuilder<byte[]> consumerBuilder = client.newConsumer().topic("persistent://prop-xyz/use/ns1/ds2")
+ .subscriptionType(SubscriptionType.Shared);
+ Consumer<byte[]> consumer1 = consumerBuilder.clone().subscriptionName("my-sub1").subscribe();
+ Consumer<byte[]> consumer2 = consumerBuilder.clone().subscriptionName("my-sub2").subscribe();
+ Consumer<byte[]> consumer3 = consumerBuilder.clone().subscriptionName("my-sub3").subscribe();
assertEquals(admin.persistentTopics().getSubscriptions("persistent://prop-xyz/use/ns1/ds2").size(), 3);
@@ -1565,16 +1590,13 @@ public void testPersistentTopicExpireMessageOnParitionTopic() throws Exception {
// create consumer and subscription
URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
- ClientConfiguration clientConf = new ClientConfiguration();
- clientConf.setStatsInterval(0, TimeUnit.SECONDS);
- PulsarClient client = PulsarClient.create(pulsarUrl.toString(), clientConf);
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
- Consumer consumer = client.subscribe("persistent://prop-xyz/use/ns1/ds1", "my-sub", conf);
-
- ProducerConfiguration prodConf = new ProducerConfiguration();
- prodConf.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
- Producer producer = client.createProducer("persistent://prop-xyz/use/ns1/ds1", prodConf);
+ PulsarClient client = PulsarClient.builder().serviceUrl(pulsarUrl.toString()).statsInterval(0, TimeUnit.SECONDS)
+ .build();
+ Consumer<byte[]> consumer = client.newConsumer().topic("persistent://prop-xyz/use/ns1/ds1")
+ .subscriptionName("my-sub").subscribe();
+
+ Producer<byte[]> producer = client.newProducer().topic("persistent://prop-xyz/use/ns1/ds1")
+ .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();
for (int i = 0; i < 10; i++) {
String message = "message-" + i;
producer.send(message.getBytes());
@@ -1624,7 +1646,7 @@ public void testPulsarAdminForUriAndUrlEncoding(String topicName) throws Excepti
final int numOfPartitions = 4;
admin.persistentTopics().createPartitionedTopic(topic1, numOfPartitions);
// Create a consumer to get stats on this topic
- pulsarClient.subscribe(topic1, "my-subscriber-name", new ConsumerConfiguration());
+ pulsarClient.newConsumer().topic(topic1).subscriptionName("my-subscriber-name").subscribe();
PersistentTopicsImpl persistent = (PersistentTopicsImpl) admin.persistentTopics();
Field field = PersistentTopicsImpl.class.getDeclaredField("persistentTopics");
@@ -1753,8 +1775,7 @@ public void testTopicBundleRangeLookup() throws PulsarAdminException, PulsarServ
// (1) create a topic
final String topicName = "persistent://prop-xyz/use/getBundleNs/topic1";
String bundleRange = admin.lookups().getBundleRange(topicName);
- assertEquals(bundleRange,
- pulsar.getNamespaceService().getBundle(TopicName.get(topicName)).getBundleRange());
+ assertEquals(bundleRange, pulsar.getNamespaceService().getBundle(TopicName.get(topicName)).getBundleRange());
}
}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest2.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest2.java
index 711d12565..dbc36f393 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest2.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest2.java
@@ -44,13 +44,10 @@
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.admin.PulsarAdminException;
import org.apache.pulsar.client.admin.PulsarAdminException.PreconditionFailedException;
-import org.apache.pulsar.client.api.ClientConfiguration;
import org.apache.pulsar.client.api.Consumer;
-import org.apache.pulsar.client.api.ConsumerConfiguration;
import org.apache.pulsar.client.api.Message;
+import org.apache.pulsar.client.api.MessageRoutingMode;
import org.apache.pulsar.client.api.Producer;
-import org.apache.pulsar.client.api.ProducerConfiguration;
-import org.apache.pulsar.client.api.ProducerConfiguration.MessageRoutingMode;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.SubscriptionType;
import org.apache.pulsar.client.impl.MessageIdImpl;
@@ -65,8 +62,6 @@
import org.apache.pulsar.common.policies.data.PersistentTopicStats;
import org.apache.pulsar.common.policies.data.PropertyAdmin;
import org.apache.pulsar.common.policies.data.RetentionPolicies;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.DataProvider;
@@ -75,13 +70,11 @@
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
-import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.retryStrategically;
public class AdminApiTest2 extends MockedPulsarServiceBaseTest {
private MockedPulsarService mockPulsarSetup;
-
@BeforeMethod
@Override
public void setup() throws Exception {
@@ -109,8 +102,7 @@ public void cleanup() throws Exception {
@DataProvider(name = "topicType")
public Object[][] topicTypeProvider() {
- return new Object[][] { { TopicDomain.persistent.value() },
- { TopicDomain.non_persistent.value() } };
+ return new Object[][] { { TopicDomain.persistent.value() }, { TopicDomain.non_persistent.value() } };
}
@DataProvider(name = "namespaceNames")
@@ -137,8 +129,8 @@ public void cleanup() throws Exception {
@Test
public void testIncrementPartitionsOfTopic() throws Exception {
final String topicName = "increment-partitionedTopic";
- final String subName1 = topicName + "-my-sub 1";
- final String subName2 = topicName + "-my-sub 2";
+ final String subName1 = topicName + "-my-sub-1";
+ final String subName2 = topicName + "-my-sub-2";
final int startPartitions = 4;
final int newPartitions = 8;
final String partitionedTopicName = "persistent://prop-xyz/use/ns1/" + topicName;
@@ -151,12 +143,12 @@ public void testIncrementPartitionsOfTopic() throws Exception {
startPartitions);
// create consumer and subscriptions : check subscriptions
- PulsarClient client = PulsarClient.create(pulsarUrl.toString());
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Shared);
- Consumer consumer1 = client.subscribe(partitionedTopicName, subName1, conf);
+ PulsarClient client = PulsarClient.builder().serviceUrl(pulsarUrl.toString()).build();
+ Consumer<byte[]> consumer1 = client.newConsumer().topic(partitionedTopicName).subscriptionName(subName1)
+ .subscriptionType(SubscriptionType.Shared).subscribe();
assertEquals(admin.persistentTopics().getSubscriptions(partitionedTopicName), Lists.newArrayList(subName1));
- Consumer consumer2 = client.subscribe(partitionedTopicName, subName2, conf);
+ Consumer<byte[]> consumer2 = client.newConsumer().topic(partitionedTopicName).subscriptionName(subName2)
+ .subscriptionType(SubscriptionType.Shared).subscribe();
assertEquals(Sets.newHashSet(admin.persistentTopics().getSubscriptions(partitionedTopicName)),
Sets.newHashSet(subName1, subName2));
@@ -172,9 +164,8 @@ public void testIncrementPartitionsOfTopic() throws Exception {
.toString();
// (3) produce messages to all partitions including newly created partitions (RoundRobin)
- ProducerConfiguration prodConf = new ProducerConfiguration();
- prodConf.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
- Producer producer = client.createProducer(partitionedTopicName, prodConf);
+ Producer<byte[]> producer = client.newProducer().topic(partitionedTopicName)
+ .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();
final int totalMessages = newPartitions * 2;
for (int i = 0; i < totalMessages; i++) {
String message = "message-" + i;
@@ -184,7 +175,8 @@ public void testIncrementPartitionsOfTopic() throws Exception {
// (4) verify existing subscription has not lost any message: create new consumer with sub-2: it will load all
// newly created partition topics
consumer2.close();
- consumer2 = client.subscribe(partitionedTopicName, subName2, conf);
+ consumer2 = client.newConsumer().topic(partitionedTopicName).subscriptionName(subName2)
+ .subscriptionType(SubscriptionType.Shared).subscribe();
// sometime: mockZk fails to refresh ml-cache: so, invalidate the cache to get fresh data
pulsar.getLocalZkCacheService().managedLedgerListCache().clearTree();
assertEquals(Sets.newHashSet(admin.persistentTopics().getSubscriptions(newPartitionTopicName)),
@@ -222,10 +214,9 @@ public void testIncrementPartitionsOfTopic() throws Exception {
consumer2.close();
}
-
/**
- * verifies admin api command for non-persistent topic.
- * It verifies: partitioned-topic, stats
+ * verifies admin api command for non-persistent topic. It verifies: partitioned-topic, stats
+ *
* @throws Exception
*/
@Test
@@ -238,12 +229,10 @@ public void nonPersistentTopics() throws Exception {
// create consumer and subscription
URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
- ClientConfiguration clientConf = new ClientConfiguration();
- clientConf.setStatsInterval(0, TimeUnit.SECONDS);
- PulsarClient client = PulsarClient.create(pulsarUrl.toString(), clientConf);
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
- Consumer consumer = client.subscribe(persistentTopicName, "my-sub", conf);
+ PulsarClient client = PulsarClient.builder().serviceUrl(pulsarUrl.toString()).statsInterval(0, TimeUnit.SECONDS)
+ .build();
+ Consumer<byte[]> consumer = client.newConsumer().topic(persistentTopicName).subscriptionName("my-sub")
+ .subscribe();
publishMessagesOnTopic("non-persistent://prop-xyz/use/ns1/" + topicName, 10, 0);
@@ -270,7 +259,7 @@ public void nonPersistentTopics() throws Exception {
}
private void publishMessagesOnTopic(String topicName, int messages, int startIdx) throws Exception {
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
for (int i = startIdx; i < (messages + startIdx); i++) {
String message = "message-" + i;
@@ -333,8 +322,8 @@ public void testUpdatePersistencePolicyUpdateManagedCursor() throws Exception {
admin.namespaces().setPersistence(namespace, new PersistencePolicies(3, 3, 3, 50.0));
assertEquals(admin.namespaces().getPersistence(namespace), new PersistencePolicies(3, 3, 3, 50.0));
- Producer producer = pulsarClient.createProducer(topicName);
- Consumer consumer = pulsarClient.subscribe(topicName, "my-sub");
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").subscribe();
PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) topic.getManagedLedger();
@@ -344,7 +333,7 @@ public void testUpdatePersistencePolicyUpdateManagedCursor() throws Exception {
final int newEnsembleSize = 5;
admin.namespaces().setPersistence(namespace, new PersistencePolicies(newEnsembleSize, 3, 3, newThrottleRate));
- retryStrategically((test) -> managedLedger.getConfig().getEnsembleSize() != newEnsembleSize
+ retryStrategically((test) -> managedLedger.getConfig().getEnsembleSize() == newEnsembleSize
&& cursor.getThrottleMarkDelete() != newThrottleRate, 5, 200);
// (1) verify cursor.markDelete has been updated
@@ -370,7 +359,7 @@ public void testUnloadTopic(final String topicType) throws Exception {
admin.namespaces().createNamespace(namespace);
// create a topic by creating a producer
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
producer.close();
Topic topic = pulsar.getBrokerService().getTopicReference(topicName);
@@ -384,7 +373,7 @@ public void testUnloadTopic(final String topicType) throws Exception {
assertNull(topic);
// recreation of producer will load the topic again
- producer = pulsarClient.createProducer(topicName);
+ producer = pulsarClient.newProducer().topic(topicName).create();
topic = pulsar.getBrokerService().getTopicReference(topicName);
assertNotNull(topic);
// unload the topic
@@ -431,18 +420,17 @@ public void testResetCursorOnPosition(String namespaceName) throws Exception {
admin.namespaces().setRetention("prop-xyz/use/ns1", new RetentionPolicies(10, 10));
// create consumer and subscription
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Shared);
- Consumer consumer = pulsarClient.subscribe(topicName, "my-sub", conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub")
+ .subscriptionType(SubscriptionType.Shared).subscribe();
assertEquals(admin.persistentTopics().getSubscriptions(topicName), Lists.newArrayList("my-sub"));
publishMessagesOnPersistentTopic(topicName, totalProducedMessages, 0);
- List<Message> messages = admin.persistentTopics().peekMessages(topicName, "my-sub", 10);
+ List<Message<byte[]>> messages = admin.persistentTopics().peekMessages(topicName, "my-sub", 10);
assertEquals(messages.size(), 10);
- Message message = null;
+ Message<byte[]> message = null;
MessageIdImpl resetMessageId = null;
int resetPositionId = 10;
for (int i = 0; i < 20; i++) {
@@ -461,7 +449,8 @@ public void testResetCursorOnPosition(String namespaceName) throws Exception {
// reset position at resetMessageId
admin.persistentTopics().resetCursor(topicName, "my-sub", messageId);
- consumer = pulsarClient.subscribe(topicName, "my-sub", conf);
+ consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub")
+ .subscriptionType(SubscriptionType.Shared).subscribe();
MessageIdImpl msgId2 = (MessageIdImpl) consumer.receive(1, TimeUnit.SECONDS).getMessageId();
assertEquals(resetMessageId, msgId2);
@@ -506,7 +495,7 @@ public void testResetCursorOnPosition(String namespaceName) throws Exception {
}
private void publishMessagesOnPersistentTopic(String topicName, int messages, int startIdx) throws Exception {
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
for (int i = startIdx; i < (messages + startIdx); i++) {
String message = "message-" + i;
@@ -719,7 +708,7 @@ public void testNonPersistentTopics() throws Exception {
Set<String> topicNames = Sets.newHashSet();
for (int i = 0; i < totalTopics; i++) {
topicNames.add(topicName + i);
- Producer producer = pulsarClient.createProducer(topicName + i);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName + i).create();
producer.close();
}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/CreateSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/CreateSubscriptionTest.java
index c6f277d84..052116f00 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/CreateSubscriptionTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/CreateSubscriptionTest.java
@@ -65,7 +65,7 @@ public void createSubscriptionSingleTopic() throws Exception {
assertEquals(admin.persistentTopics().getSubscriptions(topic), Lists.newArrayList("sub-1"));
- Producer p1 = pulsarClient.createProducer(topic);
+ Producer<byte[]> p1 = pulsarClient.newProducer().topic(topic).create();
p1.send("test-1".getBytes());
p1.send("test-2".getBytes());
MessageId m3 = p1.send("test-3".getBytes());
@@ -98,8 +98,7 @@ public void createSubscriptionOnPartitionedTopic() throws Exception {
}
for (int i = 0; i < 10; i++) {
- assertEquals(
- admin.persistentTopics().getSubscriptions(TopicName.get(topic).getPartition(i).toString()),
+ assertEquals(admin.persistentTopics().getSubscriptions(TopicName.get(topic).getPartition(i).toString()),
Lists.newArrayList("sub-1"));
}
}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/IncrementPartitionsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/IncrementPartitionsTest.java
index e001ba0bf..60f204494 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/IncrementPartitionsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/IncrementPartitionsTest.java
@@ -80,15 +80,14 @@ public void testIncrementPartitionsOfTopic() throws Exception {
admin.persistentTopics().createPartitionedTopic(partitionedTopicName, 10);
assertEquals(admin.persistentTopics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 10);
- Consumer consumer = pulsarClient.subscribe(partitionedTopicName, "sub-1");
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(partitionedTopicName).subscriptionName("sub-1")
+ .subscribe();
admin.persistentTopics().updatePartitionedTopic(partitionedTopicName, 20);
assertEquals(admin.persistentTopics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 20);
- assertEquals(
- admin.persistentTopics()
- .getSubscriptions(TopicName.get(partitionedTopicName).getPartition(15).toString()),
- Lists.newArrayList("sub-1"));
+ assertEquals(admin.persistentTopics().getSubscriptions(
+ TopicName.get(partitionedTopicName).getPartition(15).toString()), Lists.newArrayList("sub-1"));
consumer.close();
}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java
index d1895599a..02055ac5b 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java
@@ -43,6 +43,7 @@
import org.apache.pulsar.broker.namespace.NamespaceService;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.api.Authentication;
+import org.apache.pulsar.client.api.Message;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.zookeeper.ZooKeeperClientFactory;
import org.apache.pulsar.zookeeper.ZookeeperClientFactoryImpl;
@@ -103,24 +104,20 @@ protected void resetConfig() {
protected final void internalSetup() throws Exception {
init();
- org.apache.pulsar.client.api.ClientConfiguration clientConf = new org.apache.pulsar.client.api.ClientConfiguration();
- clientConf.setStatsInterval(0, TimeUnit.SECONDS);
lookupUrl = new URI(brokerUrl.toString());
if (isTcpLookup) {
lookupUrl = new URI("pulsar://localhost:" + BROKER_PORT);
}
- pulsarClient = PulsarClient.create(lookupUrl.toString(), clientConf);
+ pulsarClient = PulsarClient.builder().serviceUrl(lookupUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();
}
protected final void internalSetupForStatsTest() throws Exception {
init();
- org.apache.pulsar.client.api.ClientConfiguration clientConf = new org.apache.pulsar.client.api.ClientConfiguration();
- clientConf.setStatsInterval(1, TimeUnit.SECONDS);
String lookupUrl = brokerUrl.toString();
if (isTcpLookup) {
lookupUrl = new URI("pulsar://localhost:" + BROKER_PORT).toString();
}
- pulsarClient = PulsarClient.create(lookupUrl, clientConf);
+ pulsarClient = PulsarClient.builder().serviceUrl(lookupUrl.toString()).statsInterval(1, TimeUnit.SECONDS).build();
}
protected final void init() throws Exception {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AntiAffinityNamespaceGroupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AntiAffinityNamespaceGroupTest.java
index e7161b2eb..f43b24e7d 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AntiAffinityNamespaceGroupTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AntiAffinityNamespaceGroupTest.java
@@ -24,7 +24,6 @@
import static org.testng.Assert.assertTrue;
import java.lang.reflect.Field;
-import java.net.InetAddress;
import java.net.URL;
import java.util.Map;
import java.util.Set;
@@ -511,8 +510,9 @@ public void testLoadSheddingWithAntiAffinityNamespace() throws Exception {
admin1.namespaces().setNamespaceAntiAffinityGroup(ns, namespaceAntiAffinityGroup);
}
- PulsarClient pulsarClient = PulsarClient.create(pulsar1.getWebServiceAddress());
- Producer producer = pulsarClient.createProducer("persistent://" + namespace + "0/my-topic1");
+ PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(pulsar1.getWebServiceAddress()).build();
+ Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://" + namespace + "0/my-topic1")
+ .create();
ModularLoadManagerImpl loadManager = (ModularLoadManagerImpl) ((ModularLoadManagerWrapper) pulsar1
.getLoadManager().get()).getLoadManager();
@@ -527,6 +527,7 @@ public void testLoadSheddingWithAntiAffinityNamespace() throws Exception {
private boolean isLoadManagerUpdatedDomainCache(ModularLoadManagerImpl loadManager) throws Exception {
Field mapField = ModularLoadManagerImpl.class.getDeclaredField("brokerToFailureDomainMap");
mapField.setAccessible(true);
+ @SuppressWarnings("unchecked")
Map<String, String> map = (Map<String, String>) mapField.get(loadManager);
return !map.isEmpty();
}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadBalancerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadBalancerTest.java
index 9ebba2212..e0745c44d 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadBalancerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadBalancerTest.java
@@ -195,6 +195,7 @@ private LeaderBroker loopUntilLeaderChanges(LeaderElectionService les, LeaderBro
* those load reports can be deserialized and are in valid format tests if the rankings are populated from the load
* reports are not, both broker will have zero rank
*/
+ @SuppressWarnings("unchecked")
@Test
public void testLoadReportsWrittenOnZK() throws Exception {
ZooKeeper zkc = bkEnsemble.getZkClient();
@@ -289,6 +290,7 @@ public void testUpdateLoadReportAndCheckUpdatedRanking() throws Exception {
Field ranking = ((SimpleLoadManagerImpl) pulsar.getLoadManager().get()).getClass()
.getDeclaredField("sortedRankings");
ranking.setAccessible(true);
+ @SuppressWarnings("unchecked")
AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRanking = (AtomicReference<Map<Long, Set<ResourceUnit>>>) ranking
.get(pulsar.getLoadManager().get());
return sortedRanking;
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerImplTest.java
index 7d36d391f..0502d7837 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerImplTest.java
@@ -23,8 +23,8 @@
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotEquals;
import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotEquals;
import static org.testng.Assert.assertTrue;
import java.lang.reflect.Field;
@@ -76,7 +76,6 @@
import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@@ -616,5 +615,7 @@ public void testOwnBrokerZnodeByMultipleBroker() throws Exception {
} catch (PulsarServerException e) {
//Ok.
}
+
+ pulsar.close();
}
}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java
index e26fe34c3..1771d3d07 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java
@@ -234,7 +234,7 @@ public void testBasicBrokerSelection() throws Exception {
}
- private void setObjectField(Class objClass, Object objInstance, String fieldName, Object newValue)
+ private void setObjectField(Class<?> objClass, Object objInstance, String fieldName, Object newValue)
throws Exception {
Field field = objClass.getDeclaredField(fieldName);
field.setAccessible(true);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java
index 659361c18..066693e34 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java
@@ -45,8 +45,6 @@
import org.apache.bookkeeper.util.ZkUtils;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.tuple.Pair;
-import org.apache.pulsar.broker.PulsarService;
-import org.apache.pulsar.broker.cache.LocalZooKeeperCacheService;
import org.apache.pulsar.broker.loadbalance.LoadManager;
import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl;
import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerWrapper;
@@ -55,19 +53,16 @@
import org.apache.pulsar.broker.service.Topic;
import org.apache.pulsar.broker.service.persistent.PersistentTopic;
import org.apache.pulsar.client.api.Consumer;
-import org.apache.pulsar.client.api.ConsumerConfiguration;
-import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.naming.NamespaceBundle;
import org.apache.pulsar.common.naming.NamespaceBundleFactory;
import org.apache.pulsar.common.naming.NamespaceBundles;
import org.apache.pulsar.common.naming.NamespaceName;
-import org.apache.pulsar.common.policies.data.LocalPolicies;
+import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.policies.data.Policies;
import org.apache.pulsar.common.util.ObjectMapperFactory;
import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap;
import org.apache.pulsar.policies.data.loadbalancer.LoadReport;
import org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData;
-import org.apache.pulsar.zookeeper.ZooKeeperDataCache;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.data.Stat;
@@ -267,8 +262,8 @@ public void testremoveOwnershipNamespaceBundle() throws Exception {
public void testUnloadNamespaceBundleFailure() throws Exception {
final String topicName = "persistent://my-property/use/my-ns/my-topic1";
- ConsumerConfiguration conf = new ConsumerConfiguration();
- Consumer consumer = pulsarClient.subscribe(topicName, "my-subscriber-name", conf);
+ pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name").subscribe();
+
ConcurrentOpenHashMap<String, CompletableFuture<Topic>> topics = pulsar.getBrokerService().getTopics();
Topic spyTopic = spy(topics.get(topicName).get());
topics.clear();
@@ -300,15 +295,15 @@ public void testUnloadNamespaceBundleFailure() throws Exception {
/**
* It verifies that unloading bundle will timeout and will not hung even if one of the topic-unloading stuck.
- *
+ *
* @throws Exception
*/
@Test(timeOut = 6000)
public void testUnloadNamespaceBundleWithStuckTopic() throws Exception {
final String topicName = "persistent://my-property/use/my-ns/my-topic1";
- ConsumerConfiguration conf = new ConsumerConfiguration();
- Consumer consumer = pulsarClient.subscribe(topicName, "my-subscriber-name", conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name")
+ .subscribe();
ConcurrentOpenHashMap<String, CompletableFuture<Topic>> topics = pulsar.getBrokerService().getTopics();
Topic spyTopic = spy(topics.get(topicName).get());
topics.clear();
@@ -335,7 +330,7 @@ public void testUnloadNamespaceBundleWithStuckTopic() throws Exception {
}
consumer.close();
}
-
+
/**
* <pre>
* It verifies that namespace service deserialize the load-report based on load-manager which active.
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceUnloadingTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceUnloadingTest.java
index bcd6b76d5..1da95640c 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceUnloadingTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceUnloadingTest.java
@@ -53,7 +53,8 @@ public void testUnloadNotLoadedNamespace() throws Exception {
public void testUnloadPartiallyLoadedNamespace() throws Exception {
admin.namespaces().createNamespace("prop/use/ns-test-2", 16);
- Producer producer = pulsarClient.createProducer("persistent://prop/use/ns-test-2/my-topic");
+ Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://prop/use/ns-test-2/my-topic")
+ .create();
assertTrue(admin.namespaces().getNamespaces("prop", "use").contains("prop/use/ns-test-2"));
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AddMissingPatchVersionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AddMissingPatchVersionTest.java
index bcc589a48..14b5dd488 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AddMissingPatchVersionTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AddMissingPatchVersionTest.java
@@ -18,7 +18,6 @@
*/
package org.apache.pulsar.broker.service;
-import org.apache.pulsar.broker.PulsarService;
import org.apache.pulsar.utils.PulsarBrokerVersionStringUtils;
import org.testng.Assert;
import org.testng.annotations.Test;
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java
index f1761d831..80a870c9f 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java
@@ -31,11 +31,9 @@
import org.apache.pulsar.broker.ServiceConfiguration;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.api.Authentication;
-import org.apache.pulsar.client.api.ClientConfiguration;
import org.apache.pulsar.client.api.Consumer;
import org.apache.pulsar.client.api.Message;
import org.apache.pulsar.client.api.Producer;
-import org.apache.pulsar.client.api.ProducerConfiguration;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.common.policies.data.BacklogQuota;
@@ -129,18 +127,17 @@ public void testConsumerBacklogEviction() throws Exception {
assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/ns-quota"), Maps.newTreeMap());
admin.namespaces().setBacklogQuota("prop/usc/ns-quota",
new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.consumer_backlog_eviction));
- ClientConfiguration clientConf = new ClientConfiguration();
- clientConf.setStatsInterval(0, TimeUnit.SECONDS);
- PulsarClient client = PulsarClient.create(adminUrl.toString(), clientConf);
+ PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS)
+ .build();
final String topic1 = "persistent://prop/usc/ns-quota/topic1";
final String subName1 = "c1";
final String subName2 = "c2";
final int numMsgs = 20;
- Consumer consumer1 = client.subscribe(topic1, subName1);
- Consumer consumer2 = client.subscribe(topic1, subName2);
- org.apache.pulsar.client.api.Producer producer = client.createProducer(topic1);
+ Consumer<byte[]> consumer1 = client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe();
+ Consumer<byte[]> consumer2 = client.newConsumer().topic(topic1).subscriptionName(subName2).subscribe();
+ org.apache.pulsar.client.api.Producer<byte[]> producer = client.newProducer().topic(topic1).create();
byte[] content = new byte[1024];
for (int i = 0; i < numMsgs; i++) {
producer.send(content);
@@ -161,16 +158,16 @@ public void testConsumerBacklogEvictionWithAck() throws Exception {
assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/ns-quota"), Maps.newTreeMap());
admin.namespaces().setBacklogQuota("prop/usc/ns-quota",
new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.consumer_backlog_eviction));
- PulsarClient client = PulsarClient.create(adminUrl.toString());
+ PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).build();
final String topic1 = "persistent://prop/usc/ns-quota/topic11";
final String subName1 = "c11";
final String subName2 = "c21";
final int numMsgs = 20;
- Consumer consumer1 = client.subscribe(topic1, subName1);
- Consumer consumer2 = client.subscribe(topic1, subName2);
- org.apache.pulsar.client.api.Producer producer = client.createProducer(topic1);
+ Consumer<byte[]> consumer1 = client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe();
+ Consumer<byte[]> consumer2 = client.newConsumer().topic(topic1).subscriptionName(subName2).subscribe();
+ org.apache.pulsar.client.api.Producer<byte[]> producer = client.newProducer().topic(topic1).create();
byte[] content = new byte[1024];
for (int i = 0; i < numMsgs; i++) {
producer.send(content);
@@ -201,18 +198,19 @@ public void testConcurrentAckAndEviction() throws Exception {
final CyclicBarrier barrier = new CyclicBarrier(2);
final CountDownLatch counter = new CountDownLatch(2);
final AtomicBoolean gotException = new AtomicBoolean(false);
- ClientConfiguration clientConf = new ClientConfiguration();
- clientConf.setStatsInterval(0, TimeUnit.SECONDS);
- PulsarClient client = PulsarClient.create(adminUrl.toString(), clientConf);
- PulsarClient client2 = PulsarClient.create(adminUrl.toString(), clientConf);
- Consumer consumer1 = client2.subscribe(topic1, subName1);
- Consumer consumer2 = client2.subscribe(topic1, subName2);
+ PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS)
+ .build();
+ PulsarClient client2 = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS)
+ .build();
+ Consumer<byte[]> consumer1 = client2.newConsumer().topic(topic1).subscriptionName(subName1).subscribe();
+ Consumer<byte[]> consumer2 = client2.newConsumer().topic(topic1).subscriptionName(subName2).subscribe();
Thread producerThread = new Thread() {
public void run() {
try {
barrier.await();
- final org.apache.pulsar.client.api.Producer producer = client.createProducer(topic1);
+ org.apache.pulsar.client.api.Producer<byte[]> producer = client.newProducer().topic(topic1)
+ .create();
byte[] content = new byte[1024];
for (int i = 0; i < numMsgs; i++) {
producer.send(content);
@@ -272,19 +270,20 @@ public void testNoEviction() throws Exception {
final CyclicBarrier barrier = new CyclicBarrier(2);
final CountDownLatch counter = new CountDownLatch(2);
final AtomicBoolean gotException = new AtomicBoolean(false);
- final ClientConfiguration clientConf = new ClientConfiguration();
- clientConf.setStatsInterval(0, TimeUnit.SECONDS);
- final PulsarClient client = PulsarClient.create(adminUrl.toString(), clientConf);
+ final PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString())
+ .statsInterval(0, TimeUnit.SECONDS).build();
- final Consumer consumer1 = client.subscribe(topic1, subName1);
- final Consumer consumer2 = client.subscribe(topic1, subName2);
- final PulsarClient client2 = PulsarClient.create(adminUrl.toString(), clientConf);
+ final Consumer<byte[]> consumer1 = client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe();
+ final Consumer<byte[]> consumer2 = client.newConsumer().topic(topic1).subscriptionName(subName2).subscribe();
+ final PulsarClient client2 = PulsarClient.builder().serviceUrl(adminUrl.toString())
+ .statsInterval(0, TimeUnit.SECONDS).build();
Thread producerThread = new Thread() {
public void run() {
try {
barrier.await();
- org.apache.pulsar.client.api.Producer producer = client2.createProducer(topic1);
+ org.apache.pulsar.client.api.Producer<byte[]> producer = client2.newProducer().topic(topic1)
+ .create();
byte[] content = new byte[1024];
for (int i = 0; i < numMsgs; i++) {
producer.send(content);
@@ -336,20 +335,22 @@ public void testEvictionMulti() throws Exception {
final CyclicBarrier barrier = new CyclicBarrier(4);
final CountDownLatch counter = new CountDownLatch(4);
final AtomicBoolean gotException = new AtomicBoolean(false);
- final ClientConfiguration clientConf = new ClientConfiguration();
- clientConf.setStatsInterval(0, TimeUnit.SECONDS);
- final PulsarClient client = PulsarClient.create(adminUrl.toString(), clientConf);
+ final PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString())
+ .statsInterval(0, TimeUnit.SECONDS).build();
- final Consumer consumer1 = client.subscribe(topic1, subName1);
- final Consumer consumer2 = client.subscribe(topic1, subName2);
- final PulsarClient client3 = PulsarClient.create(adminUrl.toString(), clientConf);
- final PulsarClient client2 = PulsarClient.create(adminUrl.toString(), clientConf);
+ final Consumer<byte[]> consumer1 = client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe();
+ final Consumer<byte[]> consumer2 = client.newConsumer().topic(topic1).subscriptionName(subName2).subscribe();
+ final PulsarClient client3 = PulsarClient.builder().serviceUrl(adminUrl.toString())
+ .statsInterval(0, TimeUnit.SECONDS).build();
+ final PulsarClient client2 = PulsarClient.builder().serviceUrl(adminUrl.toString())
+ .statsInterval(0, TimeUnit.SECONDS).build();
Thread producerThread1 = new Thread() {
public void run() {
try {
barrier.await();
- org.apache.pulsar.client.api.Producer producer = client2.createProducer(topic1);
+ org.apache.pulsar.client.api.Producer<byte[]> producer = client2.newProducer().topic(topic1)
+ .create();
byte[] content = new byte[1024];
for (int i = 0; i < numMsgs; i++) {
producer.send(content);
@@ -367,7 +368,8 @@ public void run() {
public void run() {
try {
barrier.await();
- org.apache.pulsar.client.api.Producer producer = client3.createProducer(topic1);
+ org.apache.pulsar.client.api.Producer<byte[]> producer = client3.newProducer().topic(topic1)
+ .create();
byte[] content = new byte[1024];
for (int i = 0; i < numMsgs; i++) {
producer.send(content);
@@ -432,19 +434,16 @@ public void testAheadProducerOnHold() throws Exception {
assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/quotahold"), Maps.newTreeMap());
admin.namespaces().setBacklogQuota("prop/usc/quotahold",
new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.producer_request_hold));
- final ClientConfiguration clientConf = new ClientConfiguration();
- clientConf.setStatsInterval(0, TimeUnit.SECONDS);
- final PulsarClient client = PulsarClient.create(adminUrl.toString(), clientConf);
+ final PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString())
+ .statsInterval(0, TimeUnit.SECONDS).build();
final String topic1 = "persistent://prop/usc/quotahold/hold";
final String subName1 = "c1hold";
final int numMsgs = 10;
- Consumer consumer = client.subscribe(topic1, subName1);
- ProducerConfiguration producerConfiguration = new ProducerConfiguration();
- producerConfiguration.setSendTimeout(2, TimeUnit.SECONDS);
+ Consumer<byte[]> consumer = client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe();
byte[] content = new byte[1024];
- Producer producer = client.createProducer(topic1, producerConfiguration);
+ Producer<byte[]> producer = client.newProducer().topic(topic1).sendTimeout(2, TimeUnit.SECONDS).create();
for (int i = 0; i <= numMsgs; i++) {
try {
producer.send(content);
@@ -473,19 +472,16 @@ public void testAheadProducerOnHoldTimeout() throws Exception {
assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/quotahold"), Maps.newTreeMap());
admin.namespaces().setBacklogQuota("prop/usc/quotahold",
new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.producer_request_hold));
- final ClientConfiguration clientConf = new ClientConfiguration();
- clientConf.setStatsInterval(0, TimeUnit.SECONDS);
- final PulsarClient client = PulsarClient.create(adminUrl.toString(), clientConf);
+ final PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString())
+ .statsInterval(0, TimeUnit.SECONDS).build();
final String topic1 = "persistent://prop/usc/quotahold/holdtimeout";
final String subName1 = "c1holdtimeout";
boolean gotException = false;
- client.subscribe(topic1, subName1);
- ProducerConfiguration producerConfiguration = new ProducerConfiguration();
- producerConfiguration.setSendTimeout(2, TimeUnit.SECONDS);
+ client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe();
byte[] content = new byte[1024];
- Producer producer = client.createProducer(topic1, producerConfiguration);
+ Producer<byte[]> producer = client.newProducer().topic(topic1).sendTimeout(2, TimeUnit.SECONDS).create();
for (int i = 0; i < 10; i++) {
producer.send(content);
}
@@ -510,19 +506,16 @@ public void testProducerException() throws Exception {
assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/quotahold"), Maps.newTreeMap());
admin.namespaces().setBacklogQuota("prop/usc/quotahold",
new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.producer_exception));
- final ClientConfiguration clientConf = new ClientConfiguration();
- clientConf.setStatsInterval(0, TimeUnit.SECONDS);
- final PulsarClient client = PulsarClient.create(adminUrl.toString(), clientConf);
+ final PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString())
+ .statsInterval(0, TimeUnit.SECONDS).build();
final String topic1 = "persistent://prop/usc/quotahold/except";
final String subName1 = "c1except";
boolean gotException = false;
- client.subscribe(topic1, subName1);
- ProducerConfiguration producerConfiguration = new ProducerConfiguration();
- producerConfiguration.setSendTimeout(2, TimeUnit.SECONDS);
+ client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe();
byte[] content = new byte[1024];
- Producer producer = client.createProducer(topic1, producerConfiguration);
+ Producer<byte[]> producer = client.newProducer().topic(topic1).sendTimeout(2, TimeUnit.SECONDS).create();
for (int i = 0; i < 10; i++) {
producer.send(content);
}
@@ -549,19 +542,16 @@ public void testProducerExceptionAndThenUnblock() throws Exception {
assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/quotahold"), Maps.newTreeMap());
admin.namespaces().setBacklogQuota("prop/usc/quotahold",
new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.producer_exception));
- final ClientConfiguration clientConf = new ClientConfiguration();
- clientConf.setStatsInterval(0, TimeUnit.SECONDS);
- final PulsarClient client = PulsarClient.create(adminUrl.toString(), clientConf);
+ final PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString())
+ .statsInterval(0, TimeUnit.SECONDS).build();
final String topic1 = "persistent://prop/usc/quotahold/exceptandunblock";
final String subName1 = "c1except";
boolean gotException = false;
- Consumer consumer = client.subscribe(topic1, subName1);
- ProducerConfiguration producerConfiguration = new ProducerConfiguration();
- producerConfiguration.setSendTimeout(2, TimeUnit.SECONDS);
+ Consumer<byte[]> consumer = client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe();
byte[] content = new byte[1024];
- Producer producer = client.createProducer(topic1, producerConfiguration);
+ Producer<byte[]> producer = client.newProducer().topic(topic1).sendTimeout(2, TimeUnit.SECONDS).create();
for (int i = 0; i < 10; i++) {
producer.send(content);
}
@@ -586,7 +576,7 @@ public void testProducerExceptionAndThenUnblock() throws Exception {
int backlog = (int) stats.subscriptions.get(subName1).msgBacklog;
for (int i = 0; i < backlog; i++) {
- Message msg = consumer.receive();
+ Message<?> msg = consumer.receive();
consumer.acknowledge(msg);
}
Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageTest.java
index b34e7d32e..2d7c7aea4 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageTest.java
@@ -37,12 +37,10 @@
import org.apache.pulsar.broker.service.persistent.PersistentTopic;
import org.apache.pulsar.client.api.CompressionType;
import org.apache.pulsar.client.api.Consumer;
-import org.apache.pulsar.client.api.ConsumerConfiguration;
import org.apache.pulsar.client.api.Message;
import org.apache.pulsar.client.api.MessageBuilder;
import org.apache.pulsar.client.api.MessageId;
import org.apache.pulsar.client.api.Producer;
-import org.apache.pulsar.client.api.ProducerConfiguration;
import org.apache.pulsar.client.api.SubscriptionType;
import org.apache.pulsar.client.impl.ConsumerImpl;
import org.apache.pulsar.common.util.FutureUtil;
@@ -82,20 +80,18 @@ public void testSimpleBatchProducerWithFixedBatchSize(CompressionType compressio
final String topicName = "persistent://prop/use/ns-abc/testSimpleBatchProducerWithFixedBatchSize";
final String subscriptionName = "sub-1" + compressionType.toString();
- Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
+ .subscribe();
consumer.close();
- ProducerConfiguration producerConf = new ProducerConfiguration();
- producerConf.setCompressionType(compressionType);
- producerConf.setBatchingMaxPublishDelay(5000, TimeUnit.MILLISECONDS);
- producerConf.setBatchingMaxMessages(numMsgsInBatch);
- producerConf.setBatchingEnabled(true);
- Producer producer = pulsarClient.createProducer(topicName, producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).compressionType(compressionType)
+ .batchingMaxPublishDelay(5, TimeUnit.SECONDS).batchingMaxMessages(numMsgsInBatch).enableBatching(true)
+ .create();
List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
for (int i = 0; i < numMsgs; i++) {
byte[] message = ("my-message-" + i).getBytes();
- Message msg = MessageBuilder.create().setContent(message).build();
+ Message<byte[]> msg = MessageBuilder.create().setContent(message).build();
sendFutureList.add(producer.sendAsync(msg));
}
FutureUtil.waitForAll(sendFutureList).get();
@@ -107,10 +103,10 @@ public void testSimpleBatchProducerWithFixedBatchSize(CompressionType compressio
// we expect 2 messages in the backlog since we sent 50 messages with the batch size set to 25. We have set the
// batch time high enough for it to not affect the number of messages in the batch
assertEquals(topic.getSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 2);
- consumer = pulsarClient.subscribe(topicName, subscriptionName);
+ consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).subscribe();
for (int i = 0; i < numMsgs; i++) {
- Message msg = consumer.receive(5, TimeUnit.SECONDS);
+ Message<byte[]> msg = consumer.receive(5, TimeUnit.SECONDS);
assertNotNull(msg);
String receivedMessage = new String(msg.getData());
String expectedMessage = "my-message-" + i;
@@ -127,14 +123,12 @@ public void testSimpleBatchProducerWithFixedBatchTime(CompressionType compressio
final String topicName = "persistent://prop/use/ns-abc/testSimpleBatchProducerWithFixedBatchTime";
final String subscriptionName = "time-sub-1" + compressionType.toString();
- Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
+ .subscribe();
consumer.close();
- ProducerConfiguration producerConf = new ProducerConfiguration();
- producerConf.setCompressionType(compressionType);
- producerConf.setBatchingMaxPublishDelay(10, TimeUnit.MILLISECONDS);
- producerConf.setBatchingEnabled(true);
- Producer producer = pulsarClient.createProducer(topicName, producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).compressionType(compressionType)
+ .batchingMaxPublishDelay(10, TimeUnit.MILLISECONDS).enableBatching(true).create();
Random random = new Random();
List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
@@ -142,7 +136,7 @@ public void testSimpleBatchProducerWithFixedBatchTime(CompressionType compressio
// put a random sleep from 0 to 3 ms
Thread.sleep(random.nextInt(4));
byte[] message = ("msg-" + i).getBytes();
- Message msg = MessageBuilder.create().setContent(message).build();
+ Message<byte[]> msg = MessageBuilder.create().setContent(message).build();
sendFutureList.add(producer.sendAsync(msg));
}
FutureUtil.waitForAll(sendFutureList).get();
@@ -164,15 +158,13 @@ public void testSimpleBatchProducerWithFixedBatchSizeAndTime(CompressionType com
final String topicName = "persistent://prop/use/ns-abc/testSimpleBatchProducerWithFixedBatchSizeAndTime";
final String subscriptionName = "time-size-sub-1" + compressionType.toString();
- Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
+ .subscribe();
consumer.close();
- ProducerConfiguration producerConf = new ProducerConfiguration();
- producerConf.setBatchingMaxPublishDelay(10, TimeUnit.MILLISECONDS);
- producerConf.setBatchingMaxMessages(5);
- producerConf.setCompressionType(compressionType);
- producerConf.setBatchingEnabled(true);
- Producer producer = pulsarClient.createProducer(topicName, producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
+ .batchingMaxPublishDelay(10, TimeUnit.MILLISECONDS).batchingMaxMessages(5)
+ .compressionType(compressionType).enableBatching(true).create();
Random random = new Random();
List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
@@ -180,7 +172,7 @@ public void testSimpleBatchProducerWithFixedBatchSizeAndTime(CompressionType com
// put a random sleep from 0 to 3 ms
Thread.sleep(random.nextInt(4));
byte[] message = ("msg-" + i).getBytes();
- Message msg = MessageBuilder.create().setContent(message).build();
+ Message<byte[]> msg = MessageBuilder.create().setContent(message).build();
sendFutureList.add(producer.sendAsync(msg));
}
FutureUtil.waitForAll(sendFutureList).get();
@@ -203,31 +195,29 @@ public void testBatchProducerWithLargeMessage(CompressionType compressionType) t
final String topicName = "persistent://prop/use/finance/testBatchProducerWithLargeMessage";
final String subscriptionName = "large-message-sub-1" + compressionType.toString();
- Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
+ .subscribe();
consumer.close();
- ProducerConfiguration producerConf = new ProducerConfiguration();
- producerConf.setCompressionType(compressionType);
- producerConf.setBatchingMaxPublishDelay(5000, TimeUnit.MILLISECONDS);
- producerConf.setBatchingMaxMessages(numMsgsInBatch);
- producerConf.setBatchingEnabled(true);
- Producer producer = pulsarClient.createProducer(topicName, producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).compressionType(compressionType)
+ .batchingMaxPublishDelay(5, TimeUnit.SECONDS).batchingMaxMessages(numMsgsInBatch).enableBatching(true)
+ .create();
List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
for (int i = 0; i < numMsgs; i++) {
if (i == 25) {
// send a large message
byte[] largeMessage = new byte[128 * 1024 + 4];
- Message msg = MessageBuilder.create().setContent(largeMessage).build();
+ Message<byte[]> msg = MessageBuilder.create().setContent(largeMessage).build();
sendFutureList.add(producer.sendAsync(msg));
} else {
byte[] message = ("msg-" + i).getBytes();
- Message msg = MessageBuilder.create().setContent(message).build();
+ Message<byte[]> msg = MessageBuilder.create().setContent(message).build();
sendFutureList.add(producer.sendAsync(msg));
}
}
byte[] message = ("msg-" + "last").getBytes();
- Message lastMsg = MessageBuilder.create().setContent(message).build();
+ Message<byte[]> lastMsg = MessageBuilder.create().setContent(message).build();
sendFutureList.add(producer.sendAsync(lastMsg));
FutureUtil.waitForAll(sendFutureList).get();
@@ -239,10 +229,10 @@ public void testBatchProducerWithLargeMessage(CompressionType compressionType) t
// we expect 3 messages in the backlog since the large message in the middle should
// close out the batch and be sent in a batch of its own
assertEquals(topic.getSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 3);
- consumer = pulsarClient.subscribe(topicName, subscriptionName);
+ consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).subscribe();
for (int i = 0; i <= numMsgs; i++) {
- Message msg = consumer.receive(5, TimeUnit.SECONDS);
+ Message<byte[]> msg = consumer.receive(5, TimeUnit.SECONDS);
assertNotNull(msg);
LOG.info("received msg - {}", msg.getData().toString());
consumer.acknowledge(msg);
@@ -260,20 +250,18 @@ public void testSimpleBatchProducerConsumer(CompressionType compressionType) thr
final String topicName = "persistent://prop/use/ns-abc/testSimpleBatchProducerConsumer";
final String subscriptionName = "pc-sub-1" + compressionType.toString();
- Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
+ .subscribe();
consumer.close();
- ProducerConfiguration producerConf = new ProducerConfiguration();
- producerConf.setCompressionType(compressionType);
- producerConf.setBatchingMaxPublishDelay(5000, TimeUnit.MILLISECONDS);
- producerConf.setBatchingMaxMessages(numMsgsInBatch);
- producerConf.setBatchingEnabled(true);
- Producer producer = pulsarClient.createProducer(topicName, producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).compressionType(compressionType)
+ .batchingMaxPublishDelay(5, TimeUnit.SECONDS).batchingMaxMessages(numMsgsInBatch).enableBatching(true)
+ .create();
List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
for (int i = 0; i < numMsgs; i++) {
byte[] message = ("msg-" + i).getBytes();
- Message msg = MessageBuilder.create().setContent(message).build();
+ Message<byte[]> msg = MessageBuilder.create().setContent(message).build();
sendFutureList.add(producer.sendAsync(msg));
}
FutureUtil.waitForAll(sendFutureList).get();
@@ -282,13 +270,12 @@ public void testSimpleBatchProducerConsumer(CompressionType compressionType) thr
rolloverPerIntervalStats();
assertTrue(topic.getProducers().values().iterator().next().getStats().msgRateIn > 0.0);
- assertEquals(topic.getSubscription(subscriptionName).getNumberOfEntriesInBacklog(),
- numMsgs / numMsgsInBatch);
- consumer = pulsarClient.subscribe(topicName, subscriptionName);
+ assertEquals(topic.getSubscription(subscriptionName).getNumberOfEntriesInBacklog(), numMsgs / numMsgsInBatch);
+ consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).subscribe();
- Message lastunackedMsg = null;
+ Message<byte[]> lastunackedMsg = null;
for (int i = 0; i < numMsgs; i++) {
- Message msg = consumer.receive(5, TimeUnit.SECONDS);
+ Message<byte[]> msg = consumer.receive(5, TimeUnit.SECONDS);
assertNotNull(msg);
if (i % 2 == 0) {
consumer.acknowledgeCumulative(msg);
@@ -312,18 +299,17 @@ public void testSimpleBatchSyncProducerWithFixedBatchSize() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/testSimpleBatchSyncProducerWithFixedBatchSize";
final String subscriptionName = "syncsub-1";
- Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
+ .subscribe();
consumer.close();
- ProducerConfiguration producerConf = new ProducerConfiguration();
- producerConf.setBatchingMaxPublishDelay(1000, TimeUnit.MILLISECONDS);
- producerConf.setBatchingMaxMessages(numMsgsInBatch);
- producerConf.setBatchingEnabled(true);
- Producer producer = pulsarClient.createProducer(topicName, producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
+ .batchingMaxPublishDelay(1, TimeUnit.SECONDS).batchingMaxMessages(numMsgsInBatch).enableBatching(true)
+ .create();
for (int i = 0; i < numMsgs; i++) {
byte[] message = ("my-message-" + i).getBytes();
- Message msg = MessageBuilder.create().setContent(message).build();
+ Message<byte[]> msg = MessageBuilder.create().setContent(message).build();
producer.send(msg);
}
@@ -334,10 +320,10 @@ public void testSimpleBatchSyncProducerWithFixedBatchSize() throws Exception {
// we expect 10 messages in the backlog since we sent 10 messages with the batch size set to 5.
// However, we are using synchronous send and so each message will go as an individual message
assertEquals(topic.getSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 10);
- consumer = pulsarClient.subscribe(topicName, subscriptionName);
+ consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).subscribe();
for (int i = 0; i < numMsgs; i++) {
- Message msg = consumer.receive(5, TimeUnit.SECONDS);
+ Message<byte[]> msg = consumer.receive(5, TimeUnit.SECONDS);
assertNotNull(msg);
String receivedMessage = new String(msg.getData());
String expectedMessage = "my-message-" + i;
@@ -356,20 +342,18 @@ public void testSimpleBatchProducerConsumer1kMessages() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/testSimpleBatchProducerConsumer1kMessages";
final String subscriptionName = "pc1k-sub-1";
- Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
+ .subscribe();
consumer.close();
- ProducerConfiguration producerConf = new ProducerConfiguration();
- producerConf.setMaxPendingMessages(numMsgs + 1);
- producerConf.setBatchingMaxPublishDelay(30000, TimeUnit.MILLISECONDS);
- producerConf.setBatchingMaxMessages(numMsgsInBatch);
- producerConf.setBatchingEnabled(true);
- Producer producer = pulsarClient.createProducer(topicName, producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).maxPendingMessages(numMsgs + 1)
+ .batchingMaxPublishDelay(30, TimeUnit.SECONDS).batchingMaxMessages(numMsgsInBatch).enableBatching(true)
+ .create();
List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
for (int i = 0; i < numMsgs; i++) {
byte[] message = ("msg-" + i).getBytes();
- Message msg = MessageBuilder.create().setContent(message).build();
+ Message<byte[]> msg = MessageBuilder.create().setContent(message).build();
sendFutureList.add(producer.sendAsync(msg));
}
FutureUtil.waitForAll(sendFutureList).get();
@@ -391,13 +375,12 @@ public void testSimpleBatchProducerConsumer1kMessages() throws Exception {
Thread.sleep(5000);
LOG.info("[{}] checking backlog stats..");
rolloverPerIntervalStats();
- assertEquals(topic.getSubscription(subscriptionName).getNumberOfEntriesInBacklog(),
- numMsgs / numMsgsInBatch);
- consumer = pulsarClient.subscribe(topicName, subscriptionName);
+ assertEquals(topic.getSubscription(subscriptionName).getNumberOfEntriesInBacklog(), numMsgs / numMsgsInBatch);
+ consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).subscribe();
- Message lastunackedMsg = null;
+ Message<byte[]> lastunackedMsg = null;
for (int i = 0; i < numMsgs; i++) {
- Message msg = consumer.receive(1, TimeUnit.SECONDS);
+ Message<byte[]> msg = consumer.receive(1, TimeUnit.SECONDS);
assertNotNull(msg);
lastunackedMsg = msg;
}
@@ -423,19 +406,18 @@ public void testOutOfOrderAcksForBatchMessage() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/testOutOfOrderAcksForBatchMessage";
final String subscriptionName = "oooack-sub-1";
- Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
+ .subscribe();
consumer.close();
- ProducerConfiguration producerConf = new ProducerConfiguration();
- producerConf.setBatchingMaxPublishDelay(5000, TimeUnit.MILLISECONDS);
- producerConf.setBatchingMaxMessages(numMsgsInBatch);
- producerConf.setBatchingEnabled(true);
- Producer producer = pulsarClient.createProducer(topicName, producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
+ .batchingMaxPublishDelay(5, TimeUnit.SECONDS).batchingMaxMessages(numMsgsInBatch).enableBatching(true)
+ .create();
List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
for (int i = 0; i < numMsgs; i++) {
byte[] message = ("msg-" + i).getBytes();
- Message msg = MessageBuilder.create().setContent(message).build();
+ Message<byte[]> msg = MessageBuilder.create().setContent(message).build();
sendFutureList.add(producer.sendAsync(msg));
}
FutureUtil.waitForAll(sendFutureList).get();
@@ -443,16 +425,15 @@ public void testOutOfOrderAcksForBatchMessage() throws Exception {
PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
rolloverPerIntervalStats();
- assertEquals(topic.getSubscription(subscriptionName).getNumberOfEntriesInBacklog(),
- numMsgs / numMsgsInBatch);
- consumer = pulsarClient.subscribe(topicName, subscriptionName);
+ assertEquals(topic.getSubscription(subscriptionName).getNumberOfEntriesInBacklog(), numMsgs / numMsgsInBatch);
+ consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).subscribe();
Set<Integer> individualAcks = new HashSet<>();
for (int i = 15; i < 20; i++) {
individualAcks.add(i);
}
- Message lastunackedMsg = null;
+ Message<byte[]> lastunackedMsg = null;
for (int i = 0; i < numMsgs; i++) {
- Message msg = consumer.receive(5, TimeUnit.SECONDS);
+ Message<byte[]> msg = consumer.receive(5, TimeUnit.SECONDS);
LOG.info("received message {}", String.valueOf(msg.getData()));
assertNotNull(msg);
if (i == 8) {
@@ -491,28 +472,27 @@ public void testNonBatchCumulativeAckAfterBatchPublish() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/testNonBatchCumulativeAckAfterBatchPublish";
final String subscriptionName = "nbcaabp-sub-1";
- Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
+ .subscribe();
consumer.close();
- ProducerConfiguration producerConf = new ProducerConfiguration();
- producerConf.setBatchingMaxPublishDelay(5000, TimeUnit.MILLISECONDS);
- producerConf.setBatchingMaxMessages(numMsgsInBatch);
- producerConf.setBatchingEnabled(true);
- Producer producer = pulsarClient.createProducer(topicName, producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
+ .batchingMaxPublishDelay(5, TimeUnit.SECONDS).batchingMaxMessages(numMsgsInBatch).enableBatching(true)
+ .create();
// create producer to publish non batch messages
- Producer noBatchProducer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> noBatchProducer = pulsarClient.newProducer().topic(topicName).create();
List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
for (int i = 0; i < numMsgs; i++) {
byte[] message = ("msg-" + i).getBytes();
- Message msg = MessageBuilder.create().setContent(message).build();
+ Message<byte[]> msg = MessageBuilder.create().setContent(message).build();
sendFutureList.add(producer.sendAsync(msg));
}
FutureUtil.waitForAll(sendFutureList).get();
sendFutureList.clear();
byte[] nobatchmsg = ("nobatch").getBytes();
- Message nmsg = MessageBuilder.create().setContent(nobatchmsg).build();
+ Message<byte[]> nmsg = MessageBuilder.create().setContent(nobatchmsg).build();
noBatchProducer.sendAsync(nmsg).get();
PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
@@ -520,11 +500,11 @@ public void testNonBatchCumulativeAckAfterBatchPublish() throws Exception {
rolloverPerIntervalStats();
assertTrue(topic.getProducers().values().iterator().next().getStats().msgRateIn > 0.0);
assertEquals(topic.getSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 2);
- consumer = pulsarClient.subscribe(topicName, subscriptionName);
+ consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).subscribe();
- Message lastunackedMsg = null;
+ Message<byte[]> lastunackedMsg = null;
for (int i = 0; i <= numMsgs; i++) {
- Message msg = consumer.receive(5, TimeUnit.SECONDS);
+ Message<byte[]> msg = consumer.receive(5, TimeUnit.SECONDS);
assertNotNull(msg);
lastunackedMsg = msg;
}
@@ -534,7 +514,7 @@ public void testNonBatchCumulativeAckAfterBatchPublish() throws Exception {
Thread.sleep(100);
rolloverPerIntervalStats();
assertEquals(topic.getSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 0);
- assertTrue(((ConsumerImpl) consumer).isBatchingAckTrackerEmpty());
+ assertTrue(((ConsumerImpl<byte[]>) consumer).isBatchingAckTrackerEmpty());
consumer.close();
producer.close();
noBatchProducer.close();
@@ -547,21 +527,20 @@ public void testBatchAndNonBatchCumulativeAcks() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/testBatchAndNonBatchCumulativeAcks";
final String subscriptionName = "bnb-sub-1";
- Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
+ .subscribe();
consumer.close();
- ProducerConfiguration producerConf = new ProducerConfiguration();
- producerConf.setBatchingMaxPublishDelay(5000, TimeUnit.MILLISECONDS);
- producerConf.setBatchingMaxMessages(numMsgsInBatch);
- producerConf.setBatchingEnabled(true);
- Producer producer = pulsarClient.createProducer(topicName, producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
+ .batchingMaxPublishDelay(5, TimeUnit.SECONDS).batchingMaxMessages(numMsgsInBatch).enableBatching(true)
+ .create();
// create producer to publish non batch messages
- Producer noBatchProducer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> noBatchProducer = pulsarClient.newProducer().topic(topicName).create();
List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
for (int i = 0; i < numMsgs / 2; i++) {
byte[] message = ("msg-" + i).getBytes();
- Message msg = MessageBuilder.create().setContent(message).build();
+ Message<byte[]> msg = MessageBuilder.create().setContent(message).build();
sendFutureList.add(producer.sendAsync(msg));
byte[] nobatchmsg = ("nobatch-" + i).getBytes();
msg = MessageBuilder.create().setContent(nobatchmsg).build();
@@ -575,11 +554,11 @@ public void testBatchAndNonBatchCumulativeAcks() throws Exception {
assertTrue(topic.getProducers().values().iterator().next().getStats().msgRateIn > 0.0);
assertEquals(topic.getSubscription(subscriptionName).getNumberOfEntriesInBacklog(),
(numMsgs / 2) / numMsgsInBatch + numMsgs / 2);
- consumer = pulsarClient.subscribe(topicName, subscriptionName);
+ consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).subscribe();
- Message lastunackedMsg = null;
+ Message<byte[]> lastunackedMsg = null;
for (int i = 0; i < numMsgs; i++) {
- Message msg = consumer.receive(5, TimeUnit.SECONDS);
+ Message<byte[]> msg = consumer.receive(5, TimeUnit.SECONDS);
assertNotNull(msg);
LOG.info("[{}] got message position{} data {}", subscriptionName, msg.getMessageId(),
String.valueOf(msg.getData()));
@@ -595,15 +574,15 @@ public void testBatchAndNonBatchCumulativeAcks() throws Exception {
}
Thread.sleep(100);
assertEquals(topic.getSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 0);
- assertTrue(((ConsumerImpl) consumer).isBatchingAckTrackerEmpty());
+ assertTrue(((ConsumerImpl<byte[]>) consumer).isBatchingAckTrackerEmpty());
consumer.close();
producer.close();
noBatchProducer.close();
}
/**
- * Verifies batch-message acking is thread-safe
- *
+ * Verifies batch-message acking is thread-safe
+ *
* @throws Exception
*/
@Test(timeOut = 3000)
@@ -612,28 +591,26 @@ public void testConcurrentBatchMessageAck() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/testConcurrentAck";
final String subscriptionName = "sub-1";
- ConsumerConfiguration consConf = new ConsumerConfiguration();
- consConf.setSubscriptionType(SubscriptionType.Shared);
- Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName, consConf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
+ .subscriptionType(SubscriptionType.Shared).subscribe();
consumer.close();
- ProducerConfiguration producerConf = new ProducerConfiguration();
- producerConf.setBatchingMaxPublishDelay(5000, TimeUnit.MILLISECONDS);
- producerConf.setBatchingMaxMessages(numMsgs);
- producerConf.setBatchingEnabled(true);
- Producer producer = pulsarClient.createProducer(topicName, producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
+ .batchingMaxPublishDelay(5, TimeUnit.SECONDS).batchingMaxMessages(numMsgs).enableBatching(true)
+ .create();
List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
for (int i = 0; i < numMsgs; i++) {
byte[] message = ("my-message-" + i).getBytes();
- Message msg = MessageBuilder.create().setContent(message).build();
+ Message<byte[]> msg = MessageBuilder.create().setContent(message).build();
sendFutureList.add(producer.sendAsync(msg));
}
FutureUtil.waitForAll(sendFutureList).get();
PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
- final Consumer myConsumer = pulsarClient.subscribe(topicName, subscriptionName, consConf);
+ final Consumer<byte[]> myConsumer = pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName(subscriptionName).subscriptionType(SubscriptionType.Shared).subscribe();
// assertEquals(dispatcher.getTotalUnackedMessages(), 1);
ExecutorService executor = Executors.newFixedThreadPool(10);
@@ -642,7 +619,7 @@ public void testConcurrentBatchMessageAck() throws Exception {
for (int i = 0; i < numMsgs; i++) {
executor.submit(() -> {
try {
- Message msg = myConsumer.receive(1, TimeUnit.SECONDS);
+ Message<byte[]> msg = myConsumer.receive(1, TimeUnit.SECONDS);
myConsumer.acknowledge(msg);
} catch (Exception e) {
failed.set(false);
@@ -662,6 +639,6 @@ public void testConcurrentBatchMessageAck() throws Exception {
myConsumer.close();
producer.close();
}
-
+
private static final Logger LOG = LoggerFactory.getLogger(BatchMessageTest.class);
}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTests.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTests.java
index d688b7fb3..7f640824b 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTests.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTests.java
@@ -30,7 +30,6 @@
import org.apache.bookkeeper.client.BookKeeper;
import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
-import org.apache.bookkeeper.mledger.impl.EntryCache;
import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl;
import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl;
import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl;
@@ -41,13 +40,10 @@
import org.apache.pulsar.broker.service.persistent.PersistentTopic;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.api.Authentication;
-import org.apache.pulsar.client.api.ClientConfiguration;
import org.apache.pulsar.client.api.Consumer;
-import org.apache.pulsar.client.api.ConsumerConfiguration;
import org.apache.pulsar.client.api.Message;
import org.apache.pulsar.client.api.Producer;
import org.apache.pulsar.client.api.PulsarClient;
-import org.apache.pulsar.client.impl.MessageImpl;
import org.apache.pulsar.common.policies.data.ClusterData;
import org.apache.pulsar.common.policies.data.PropertyAdmin;
import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble;
@@ -126,7 +122,7 @@ void shutdown() throws Exception {
/**
* It verifies that broker deletes cursor-ledger when broker-crashes without closing topic gracefully
- *
+ *
* <pre>
* 1. Create topic : publish/consume-ack msgs to update new cursor-ledger
* 2. Verify cursor-ledger is created and ledger-znode present
@@ -135,16 +131,15 @@ void shutdown() throws Exception {
* 5. Topic is recovered from old-ledger and broker deletes the old ledger
* 6. verify znode of old-ledger is deleted
* </pre>
- *
+ *
* @throws Exception
*/
@Test
public void testCrashBrokerWithoutCursorLedgerLeak() throws Exception {
ZooKeeper zk = bkEnsemble.getZkClient();
- ClientConfiguration clientConf = new ClientConfiguration();
- clientConf.setStatsInterval(0, TimeUnit.SECONDS);
- PulsarClient client = PulsarClient.create(adminUrl.toString(), clientConf);
+ PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS)
+ .build();
final String ns1 = "prop/usc/crash-broker";
@@ -154,13 +149,14 @@ public void testCrashBrokerWithoutCursorLedgerLeak() throws Exception {
// (1) create topic
// publish and ack messages so, cursor can create cursor-ledger and update metadata
- Consumer consumer = client.subscribe(topic1, "my-subscriber-name");
- Producer producer = client.createProducer(topic1);
+ Consumer<byte[]> consumer = client.newConsumer().topic(topic1).subscriptionName("my-subscriber-name")
+ .subscribe();
+ Producer<byte[]> producer = client.newProducer().topic(topic1).create();
for (int i = 0; i < 10; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
- Message msg = null;
+ Message<byte[]> msg = null;
for (int i = 0; i < 10; i++) {
msg = consumer.receive(1, TimeUnit.SECONDS);
consumer.acknowledge(msg);
@@ -189,8 +185,8 @@ public void testCrashBrokerWithoutCursorLedgerLeak() throws Exception {
// (4) Recreate topic
// publish and ack messages so, cursor can create cursor-ledger and update metadata
- consumer = client.subscribe(topic1, "my-subscriber-name");
- producer = client.createProducer(topic1);
+ consumer = client.newConsumer().topic(topic1).subscriptionName("my-subscriber-name").subscribe();
+ producer = client.newProducer().topic(topic1).create();
for (int i = 0; i < 10; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
@@ -219,23 +215,22 @@ public void testCrashBrokerWithoutCursorLedgerLeak() throws Exception {
/**
* It verifies broker-configuration using which broker can skip non-recoverable data-ledgers.
- *
+ *
* <pre>
* 1. publish messages in 5 data-ledgers each with 20 entries under managed-ledger
* 2. delete first 4 data-ledgers
* 3. consumer will fail to consume any message as first data-ledger is non-recoverable
* 4. enable dynamic config to skip non-recoverable data-ledgers
* 5. consumer will be able to consume 20 messages from last non-deleted ledger
- *
+ *
* </pre>
- *
+ *
* @throws Exception
*/
@Test(timeOut = 6000)
public void testSkipCorruptDataLedger() throws Exception {
- ClientConfiguration clientConf = new ClientConfiguration();
- clientConf.setStatsInterval(0, TimeUnit.SECONDS);
- PulsarClient client = PulsarClient.create(adminUrl.toString(), clientConf);
+ PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS)
+ .build();
final String ns1 = "prop/usc/crash-broker";
final int totalMessages = 100;
@@ -247,9 +242,8 @@ public void testSkipCorruptDataLedger() throws Exception {
final String topic1 = "persistent://" + ns1 + "/my-topic";
// Create subscription
- ConsumerConfiguration consumerConfig = new ConsumerConfiguration();
- consumerConfig.setReceiverQueueSize(5);
- Consumer consumer = client.subscribe(topic1, "my-subscriber-name", consumerConfig);
+ Consumer<byte[]> consumer = client.newConsumer().topic(topic1).subscriptionName("my-subscriber-name")
+ .receiverQueueSize(5).subscribe();
PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topic1).get();
ManagedLedgerImpl ml = (ManagedLedgerImpl) topic.getManagedLedger();
@@ -267,7 +261,7 @@ public void testSkipCorruptDataLedger() throws Exception {
BookKeeper bookKeeper = (BookKeeper) bookKeeperField.get(ml);
// (1) publish messages in 5 data-ledgers each with 20 entries under managed-ledger
- Producer producer = client.createProducer(topic1);
+ Producer<byte[]> producer = client.newProducer().topic(topic1).create();
for (int i = 0; i < totalMessages; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
@@ -304,9 +298,9 @@ public void testSkipCorruptDataLedger() throws Exception {
ledgers.clear();
// (3) consumer will fail to consume any message as first data-ledger is non-recoverable
- Message msg = null;
+ Message<byte[]> msg = null;
// start consuming message
- consumer = client.subscribe(topic1, "my-subscriber-name");
+ consumer = client.newConsumer().topic(topic1).subscriptionName("my-subscriber-name").subscribe();
msg = consumer.receive(1, TimeUnit.SECONDS);
Assert.assertNull(msg);
consumer.close();
@@ -317,7 +311,7 @@ public void testSkipCorruptDataLedger() throws Exception {
retryStrategically((test) -> config.isAutoSkipNonRecoverableData(), 5, 100);
// (5) consumer will be able to consume 20 messages from last non-deleted ledger
- consumer = client.subscribe(topic1, "my-subscriber-name");
+ consumer = client.newConsumer().topic(topic1).subscriptionName("my-subscriber-name").subscribe();
for (int i = 0; i < entriesPerLedger; i++) {
msg = consumer.receive(5, TimeUnit.SECONDS);
System.out.println(i);
@@ -329,6 +323,6 @@ public void testSkipCorruptDataLedger() throws Exception {
client.close();
}
-
+
private static final Logger LOG = LoggerFactory.getLogger(BrokerBkEnsemblesTests.class);
}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java
index f0c28ab2a..c8dc63a85 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java
@@ -55,16 +55,14 @@
import org.apache.pulsar.broker.service.persistent.PersistentTopic;
import org.apache.pulsar.client.admin.BrokerStats;
import org.apache.pulsar.client.api.Authentication;
-import org.apache.pulsar.client.api.ClientConfiguration;
import org.apache.pulsar.client.api.Consumer;
-import org.apache.pulsar.client.api.ConsumerConfiguration;
import org.apache.pulsar.client.api.Message;
import org.apache.pulsar.client.api.Producer;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.SubscriptionType;
import org.apache.pulsar.client.impl.auth.AuthenticationTls;
-import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.naming.NamespaceBundle;
+import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.policies.data.BundlesData;
import org.apache.pulsar.common.policies.data.LocalPolicies;
import org.apache.pulsar.common.policies.data.PersistentTopicStats;
@@ -77,6 +75,8 @@
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
+import lombok.Cleanup;
+
/**
*/
public class BrokerServiceTest extends BrokerTestBase {
@@ -140,9 +140,7 @@ public void testBrokerServicePersistentTopicStats() throws Exception {
PersistentTopicStats stats;
SubscriptionStats subStats;
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
- Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();
Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
@@ -157,7 +155,7 @@ public void testBrokerServicePersistentTopicStats() throws Exception {
assertEquals(subStats.msgBacklog, 0);
assertEquals(subStats.consumers.size(), 1);
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
for (int i = 0; i < 10; i++) {
@@ -195,7 +193,7 @@ public void testBrokerServicePersistentTopicStats() throws Exception {
assertEquals(stats.msgThroughputOut, subStats.consumers.get(0).msgThroughputOut);
assertNotNull(subStats.consumers.get(0).clientVersion);
- Message msg;
+ Message<byte[]> msg;
for (int i = 0; i < 10; i++) {
msg = consumer.receive();
consumer.acknowledge(msg);
@@ -218,9 +216,8 @@ public void testBrokerServicePersistentRedeliverTopicStats() throws Exception {
PersistentTopicStats stats;
SubscriptionStats subStats;
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Shared);
- Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .subscriptionType(SubscriptionType.Shared).subscribe();
Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
@@ -235,7 +232,7 @@ public void testBrokerServicePersistentRedeliverTopicStats() throws Exception {
assertEquals(subStats.msgBacklog, 0);
assertEquals(subStats.consumers.size(), 1);
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
for (int i = 0; i < 10; i++) {
@@ -285,7 +282,7 @@ public void testBrokerServicePersistentRedeliverTopicStats() throws Exception {
assertTrue(subStats.msgRateRedeliver > 0.0);
assertEquals(subStats.msgRateRedeliver, subStats.consumers.get(0).msgRateRedeliver);
- Message msg;
+ Message<byte[]> msg;
for (int i = 0; i < 10; i++) {
msg = consumer.receive();
consumer.acknowledge(msg);
@@ -306,12 +303,10 @@ public void testBrokerStatsMetrics() throws Exception {
final String subName = "newSub";
BrokerStats brokerStatsClient = admin.brokerStats();
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
- Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();
Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
for (int i = 0; i < 10; i++) {
@@ -319,7 +314,7 @@ public void testBrokerStatsMetrics() throws Exception {
producer.send(message.getBytes());
}
Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
- Message msg = null;
+ Message<byte[]> msg = null;
for (int i = 0; i < 10; i++) {
msg = consumer.receive();
consumer.acknowledge(msg);
@@ -333,7 +328,7 @@ public void testBrokerStatsMetrics() throws Exception {
// is the order really relevant here?
boolean namespaceDimensionFound = false;
boolean topicLoadTimesDimensionFound = false;
- for ( int i=0; i<metrics.size(); i++ ) {
+ for (int i = 0; i < metrics.size(); i++) {
try {
String data = metrics.get(i).getAsJsonObject().get("dimensions").toString();
if (!namespaceDimensionFound && data.contains("prop/use/ns-abc")) {
@@ -342,7 +337,8 @@ public void testBrokerStatsMetrics() throws Exception {
if (!topicLoadTimesDimensionFound && data.contains("prop/use/ns-abc")) {
topicLoadTimesDimensionFound = true;
}
- } catch (Exception e) { /* it's possible there's no dimensions */ }
+ } catch (Exception e) {
+ /* it's possible there's no dimensions */ }
}
assertTrue(namespaceDimensionFound && topicLoadTimesDimensionFound);
@@ -350,7 +346,6 @@ public void testBrokerStatsMetrics() throws Exception {
Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
}
- @SuppressWarnings("unchecked")
@Test
public void testBrokerServiceNamespaceStats() throws Exception {
final int numBundles = 4;
@@ -358,16 +353,16 @@ public void testBrokerServiceNamespaceStats() throws Exception {
final String ns2 = "prop/use/stats2";
List<String> nsList = Lists.newArrayList(ns1, ns2);
- List<Producer> producerList = Lists.newArrayList();
+ List<Producer<byte[]>> producerList = Lists.newArrayList();
BrokerStats brokerStatsClient = admin.brokerStats();
for (String ns : nsList) {
admin.namespaces().createNamespace(ns, numBundles);
String topic1 = String.format("persistent://%s/topic1", ns);
- producerList.add(pulsarClient.createProducer(topic1));
+ producerList.add(pulsarClient.newProducer().topic(topic1).create());
String topic2 = String.format("persistent://%s/topic2", ns);
- producerList.add(pulsarClient.createProducer(topic2));
+ producerList.add(pulsarClient.newProducer().topic(topic2).create());
}
rolloverPerIntervalStats();
@@ -378,8 +373,7 @@ public void testBrokerServiceNamespaceStats() throws Exception {
JsonObject nsObject = topicStats.getAsJsonObject(ns);
List<String> topicList = admin.namespaces().getTopics(ns);
for (String topic : topicList) {
- NamespaceBundle bundle = (NamespaceBundle) pulsar.getNamespaceService()
- .getBundle(TopicName.get(topic));
+ NamespaceBundle bundle = (NamespaceBundle) pulsar.getNamespaceService().getBundle(TopicName.get(topic));
JsonObject bundleObject = nsObject.getAsJsonObject(bundle.getBundleRange());
JsonObject topicObject = bundleObject.getAsJsonObject("persistent");
AtomicBoolean topicPresent = new AtomicBoolean();
@@ -392,9 +386,10 @@ public void testBrokerServiceNamespaceStats() throws Exception {
}
}
- for (Producer producer : producerList) {
+ for (Producer<?> producer : producerList) {
producer.close();
}
+
for (String ns : nsList) {
List<String> topics = admin.namespaces().getTopics(ns);
for (String dest : topics) {
@@ -408,9 +403,6 @@ public void testBrokerServiceNamespaceStats() throws Exception {
public void testTlsDisabled() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/newTopic";
final String subName = "newSub";
- ClientConfiguration clientConfig;
- ConsumerConfiguration consumerConfig;
- Consumer consumer;
PulsarClient pulsarClient = null;
conf.setAuthenticationEnabled(false);
@@ -419,13 +411,11 @@ public void testTlsDisabled() throws Exception {
// Case 1: Access without TLS
try {
- clientConfig = new ClientConfiguration();
- clientConfig.setStatsInterval(0, TimeUnit.SECONDS);
- pulsarClient = PulsarClient.create(brokerUrl.toString(), clientConfig);
- consumerConfig = new ConsumerConfiguration();
- consumerConfig.setSubscriptionType(SubscriptionType.Exclusive);
- consumer = pulsarClient.subscribe(topicName, subName, consumerConfig);
- consumer.close();
+ pulsarClient = PulsarClient.builder().serviceUrl(brokerUrl.toString()).statsInterval(0, TimeUnit.SECONDS)
+ .build();
+ @Cleanup
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .subscribe();
} catch (Exception e) {
fail("should not fail");
} finally {
@@ -434,14 +424,13 @@ public void testTlsDisabled() throws Exception {
// Case 2: Access with TLS
try {
- clientConfig = new ClientConfiguration();
- clientConfig.setUseTls(true);
- clientConfig.setStatsInterval(0, TimeUnit.SECONDS);
- pulsarClient = PulsarClient.create(brokerUrlTls.toString(), clientConfig);
- consumerConfig = new ConsumerConfiguration();
- consumerConfig.setSubscriptionType(SubscriptionType.Exclusive);
- consumer = pulsarClient.subscribe(topicName, subName, consumerConfig);
- consumer.close();
+ pulsarClient = PulsarClient.builder().serviceUrl(brokerUrlTls.toString()).enableTls(true)
+ .statsInterval(0, TimeUnit.SECONDS).build();
+
+ @Cleanup
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .subscribe();
+
fail("TLS connection should fail");
} catch (Exception e) {
assertTrue(e.getMessage().contains("ConnectException"));
@@ -454,9 +443,6 @@ public void testTlsDisabled() throws Exception {
public void testTlsEnabled() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/newTopic";
final String subName = "newSub";
- ClientConfiguration clientConfig;
- ConsumerConfiguration consumerConfig;
- Consumer consumer;
conf.setAuthenticationEnabled(false);
conf.setTlsEnabled(true);
@@ -467,13 +453,11 @@ public void testTlsEnabled() throws Exception {
// Case 1: Access without TLS
PulsarClient pulsarClient = null;
try {
- clientConfig = new ClientConfiguration();
- clientConfig.setStatsInterval(0, TimeUnit.SECONDS);
- pulsarClient = PulsarClient.create(brokerUrl.toString(), clientConfig);
- consumerConfig = new ConsumerConfiguration();
- consumerConfig.setSubscriptionType(SubscriptionType.Exclusive);
- consumer = pulsarClient.subscribe(topicName, subName, consumerConfig);
- consumer.close();
+ pulsarClient = PulsarClient.builder().serviceUrl(brokerUrl.toString()).statsInterval(0, TimeUnit.SECONDS)
+ .build();
+ @Cleanup
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .subscribe();
} catch (Exception e) {
fail("should not fail");
} finally {
@@ -482,15 +466,13 @@ public void testTlsEnabled() throws Exception {
// Case 2: Access with TLS (Allow insecure TLS connection)
try {
- clientConfig = new ClientConfiguration();
- clientConfig.setUseTls(true);
- clientConfig.setTlsAllowInsecureConnection(true);
- clientConfig.setStatsInterval(0, TimeUnit.SECONDS);
- pulsarClient = PulsarClient.create(brokerUrlTls.toString(), clientConfig);
- consumerConfig = new ConsumerConfiguration();
- consumerConfig.setSubscriptionType(SubscriptionType.Exclusive);
- consumer = pulsarClient.subscribe(topicName, subName, consumerConfig);
- consumer.close();
+ pulsarClient = PulsarClient.builder().serviceUrl(brokerUrlTls.toString()).enableTls(true)
+ .allowTlsInsecureConnection(true).statsInterval(0, TimeUnit.SECONDS).build();
+
+ @Cleanup
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .subscribe();
+
} catch (Exception e) {
fail("should not fail");
} finally {
@@ -499,15 +481,13 @@ public void testTlsEnabled() throws Exception {
// Case 3: Access with TLS (Disallow insecure TLS connection)
try {
- clientConfig = new ClientConfiguration();
- clientConfig.setUseTls(true);
- clientConfig.setTlsAllowInsecureConnection(false);
- clientConfig.setStatsInterval(0, TimeUnit.SECONDS);
- pulsarClient = PulsarClient.create(brokerUrlTls.toString(), clientConfig);
- consumerConfig = new ConsumerConfiguration();
- consumerConfig.setSubscriptionType(SubscriptionType.Exclusive);
- consumer = pulsarClient.subscribe(topicName, subName, consumerConfig);
- consumer.close();
+ pulsarClient = PulsarClient.builder().serviceUrl(brokerUrlTls.toString()).enableTls(true)
+ .allowTlsInsecureConnection(false).statsInterval(0, TimeUnit.SECONDS).build();
+
+ @Cleanup
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .subscribe();
+
fail("should fail");
} catch (Exception e) {
assertTrue(e.getMessage().contains("General OpenSslEngine problem"));
@@ -517,16 +497,13 @@ public void testTlsEnabled() throws Exception {
// Case 4: Access with TLS (Use trusted certificates)
try {
- clientConfig = new ClientConfiguration();
- clientConfig.setUseTls(true);
- clientConfig.setTlsAllowInsecureConnection(false);
- clientConfig.setTlsTrustCertsFilePath(TLS_SERVER_CERT_FILE_PATH);
- clientConfig.setStatsInterval(0, TimeUnit.SECONDS);
- pulsarClient = PulsarClient.create(brokerUrlTls.toString(), clientConfig);
- consumerConfig = new ConsumerConfiguration();
- consumerConfig.setSubscriptionType(SubscriptionType.Exclusive);
- consumer = pulsarClient.subscribe(topicName, subName, consumerConfig);
- consumer.close();
+ pulsarClient = PulsarClient.builder().serviceUrl(brokerUrlTls.toString()).enableTls(true)
+ .allowTlsInsecureConnection(false).tlsTrustCertsFilePath(TLS_SERVER_CERT_FILE_PATH)
+ .statsInterval(0, TimeUnit.SECONDS).build();
+
+ @Cleanup
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .subscribe();
} catch (Exception e) {
fail("should not fail");
} finally {
@@ -534,13 +511,11 @@ public void testTlsEnabled() throws Exception {
}
}
+ @SuppressWarnings("deprecation")
@Test
public void testTlsAuthAllowInsecure() throws Exception {
final String topicName = "persistent://prop/usw/my-ns/newTopic";
final String subName = "newSub";
- ClientConfiguration clientConfig;
- ConsumerConfiguration consumerConfig;
- Consumer consumer;
Authentication auth;
Set<String> providers = new HashSet<>();
@@ -562,15 +537,13 @@ public void testTlsAuthAllowInsecure() throws Exception {
// Case 1: Access without client certificate
try {
- clientConfig = new ClientConfiguration();
- clientConfig.setUseTls(true);
- clientConfig.setTlsAllowInsecureConnection(true);
- clientConfig.setStatsInterval(0, TimeUnit.SECONDS);
- pulsarClient = PulsarClient.create(brokerUrlTls.toString(), clientConfig);
- consumerConfig = new ConsumerConfiguration();
- consumerConfig.setSubscriptionType(SubscriptionType.Exclusive);
- consumer = pulsarClient.subscribe(topicName, subName, consumerConfig);
- consumer.close();
+ pulsarClient = PulsarClient.builder().serviceUrl(brokerUrlTls.toString()).enableTls(true)
+ .allowTlsInsecureConnection(true).statsInterval(0, TimeUnit.SECONDS).build();
+
+ @Cleanup
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .subscribe();
+
fail("should fail");
} catch (Exception e) {
assertTrue(e.getMessage().contains("Authentication required"));
@@ -582,16 +555,14 @@ public void testTlsAuthAllowInsecure() throws Exception {
try {
auth = new AuthenticationTls();
auth.configure(authParams);
- clientConfig = new ClientConfiguration();
- clientConfig.setAuthentication(auth);
- clientConfig.setUseTls(true);
- clientConfig.setTlsAllowInsecureConnection(true);
- clientConfig.setStatsInterval(0, TimeUnit.SECONDS);
- pulsarClient = PulsarClient.create(brokerUrlTls.toString(), clientConfig);
- consumerConfig = new ConsumerConfiguration();
- consumerConfig.setSubscriptionType(SubscriptionType.Exclusive);
- consumer = pulsarClient.subscribe(topicName, subName, consumerConfig);
- consumer.close();
+
+ pulsarClient = PulsarClient.builder().authentication(auth).serviceUrl(brokerUrlTls.toString())
+ .enableTls(true).allowTlsInsecureConnection(true).statsInterval(0, TimeUnit.SECONDS).build();
+
+ @Cleanup
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .subscribe();
+
} catch (Exception e) {
fail("should not fail");
} finally {
@@ -599,13 +570,11 @@ public void testTlsAuthAllowInsecure() throws Exception {
}
}
+ @SuppressWarnings("deprecation")
@Test
public void testTlsAuthDisallowInsecure() throws Exception {
final String topicName = "persistent://prop/usw/my-ns/newTopic";
final String subName = "newSub";
- ClientConfiguration clientConfig;
- ConsumerConfiguration consumerConfig;
- Consumer consumer;
Authentication auth;
Set<String> providers = new HashSet<>();
@@ -627,15 +596,13 @@ public void testTlsAuthDisallowInsecure() throws Exception {
// Case 1: Access without client certificate
try {
- clientConfig = new ClientConfiguration();
- clientConfig.setUseTls(true);
- clientConfig.setTlsAllowInsecureConnection(true);
- clientConfig.setStatsInterval(0, TimeUnit.SECONDS);
- pulsarClient = PulsarClient.create(brokerUrlTls.toString(), clientConfig);
- consumerConfig = new ConsumerConfiguration();
- consumerConfig.setSubscriptionType(SubscriptionType.Exclusive);
- consumer = pulsarClient.subscribe(topicName, subName, consumerConfig);
- consumer.close();
+ pulsarClient = PulsarClient.builder().serviceUrl(brokerUrlTls.toString()).enableTls(true)
+ .allowTlsInsecureConnection(true).statsInterval(0, TimeUnit.SECONDS).build();
+
+ @Cleanup
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .subscribe();
+
fail("should fail");
} catch (Exception e) {
assertTrue(e.getMessage().contains("Authentication required"));
@@ -647,16 +614,12 @@ public void testTlsAuthDisallowInsecure() throws Exception {
try {
auth = new AuthenticationTls();
auth.configure(authParams);
- clientConfig = new ClientConfiguration();
- clientConfig.setAuthentication(auth);
- clientConfig.setUseTls(true);
- clientConfig.setTlsAllowInsecureConnection(true);
- clientConfig.setStatsInterval(0, TimeUnit.SECONDS);
- pulsarClient = PulsarClient.create(brokerUrlTls.toString(), clientConfig);
- consumerConfig = new ConsumerConfiguration();
- consumerConfig.setSubscriptionType(SubscriptionType.Exclusive);
- consumer = pulsarClient.subscribe(topicName, subName, consumerConfig);
- consumer.close();
+ pulsarClient = PulsarClient.builder().authentication(auth).serviceUrl(brokerUrlTls.toString())
+ .enableTls(true).allowTlsInsecureConnection(true).statsInterval(0, TimeUnit.SECONDS).build();
+
+ @Cleanup
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .subscribe();
fail("should fail");
} catch (Exception e) {
assertTrue(e.getMessage().contains("Authentication required"));
@@ -665,13 +628,11 @@ public void testTlsAuthDisallowInsecure() throws Exception {
}
}
+ @SuppressWarnings("deprecation")
@Test
public void testTlsAuthUseTrustCert() throws Exception {
final String topicName = "persistent://prop/usw/my-ns/newTopic";
final String subName = "newSub";
- ClientConfiguration clientConfig;
- ConsumerConfiguration consumerConfig;
- Consumer consumer;
Authentication auth;
Set<String> providers = new HashSet<>();
@@ -694,15 +655,12 @@ public void testTlsAuthUseTrustCert() throws Exception {
// Case 1: Access without client certificate
try {
- clientConfig = new ClientConfiguration();
- clientConfig.setUseTls(true);
- clientConfig.setTlsAllowInsecureConnection(true);
- clientConfig.setStatsInterval(0, TimeUnit.SECONDS);
- pulsarClient = PulsarClient.create(brokerUrlTls.toString(), clientConfig);
- consumerConfig = new ConsumerConfiguration();
- consumerConfig.setSubscriptionType(SubscriptionType.Exclusive);
- consumer = pulsarClient.subscribe(topicName, subName, consumerConfig);
- consumer.close();
+ pulsarClient = PulsarClient.builder().serviceUrl(brokerUrlTls.toString()).enableTls(true)
+ .allowTlsInsecureConnection(true).statsInterval(0, TimeUnit.SECONDS).build();
+
+ @Cleanup
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .subscribe();
fail("should fail");
} catch (Exception e) {
assertTrue(e.getMessage().contains("Authentication required"));
@@ -714,16 +672,12 @@ public void testTlsAuthUseTrustCert() throws Exception {
try {
auth = new AuthenticationTls();
auth.configure(authParams);
- clientConfig = new ClientConfiguration();
- clientConfig.setAuthentication(auth);
- clientConfig.setUseTls(true);
- clientConfig.setTlsAllowInsecureConnection(true);
- clientConfig.setStatsInterval(0, TimeUnit.SECONDS);
- pulsarClient = PulsarClient.create(brokerUrlTls.toString(), clientConfig);
- consumerConfig = new ConsumerConfiguration();
- consumerConfig.setSubscriptionType(SubscriptionType.Exclusive);
- consumer = pulsarClient.subscribe(topicName, subName, consumerConfig);
- consumer.close();
+ pulsarClient = PulsarClient.builder().authentication(auth).serviceUrl(brokerUrlTls.toString())
+ .enableTls(true).allowTlsInsecureConnection(true).statsInterval(0, TimeUnit.SECONDS).build();
+
+ @Cleanup
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .subscribe();
} catch (Exception e) {
fail("should not fail");
} finally {
@@ -740,14 +694,12 @@ public void testTlsAuthUseTrustCert() throws Exception {
public void testLookupThrottlingForClientByClient() throws Exception {
final String topicName = "persistent://prop/usw/my-ns/newTopic";
- org.apache.pulsar.client.api.ClientConfiguration clientConf = new org.apache.pulsar.client.api.ClientConfiguration();
- clientConf.setStatsInterval(0, TimeUnit.SECONDS);
- clientConf.setConcurrentLookupRequest(0);
String lookupUrl = new URI("pulsar://localhost:" + BROKER_PORT).toString();
- PulsarClient pulsarClient = PulsarClient.create(lookupUrl, clientConf);
+ PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(lookupUrl).statsInterval(0, TimeUnit.SECONDS)
+ .maxConcurrentLookupRequests(0).build();
try {
- Consumer consumer = pulsarClient.subscribe(topicName, "mysub", new ConsumerConfiguration());
+ pulsarClient.newConsumer().topic(topicName).subscriptionName("mysub").subscribe();
fail("It should fail as throttling should not receive any request");
} catch (org.apache.pulsar.client.api.PulsarClientException.TooManyRequestsException e) {
// ok as throttling set to 0
@@ -762,7 +714,7 @@ public void testTopicLoadingOnDisableNamespaceBundle() throws Exception {
// own namespace bundle
final String topicName = "persistent://" + namespace + "/my-topic";
TopicName topic = TopicName.get(topicName);
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
producer.close();
// disable namespace-bundle
@@ -796,7 +748,7 @@ public void testTopicFailureShouldNotHaveDeadLock() {
// let this broker own this namespace bundle by creating a topic
try {
final String successfulTopic = "persistent://" + namespace + "/ownBundleTopic";
- Producer producer = pulsarClient.createProducer(successfulTopic);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(successfulTopic).create();
producer.close();
} catch (Exception e) {
fail(e.getMessage());
@@ -839,7 +791,7 @@ public void testLedgerOpenFailureShouldNotHaveDeadLock() throws Exception {
// let this broker own this namespace bundle by creating a topic
try {
final String successfulTopic = "persistent://" + namespace + "/ownBundleTopic";
- Producer producer = pulsarClient.createProducer(successfulTopic);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(successfulTopic).create();
producer.close();
} catch (Exception e) {
fail(e.getMessage());
@@ -856,6 +808,7 @@ public void testLedgerOpenFailureShouldNotHaveDeadLock() throws Exception {
// fail managed-ledger future
Field ledgerField = ManagedLedgerFactoryImpl.class.getDeclaredField("ledgers");
ledgerField.setAccessible(true);
+ @SuppressWarnings("unchecked")
ConcurrentHashMap<String, CompletableFuture<ManagedLedgerImpl>> ledgers = (ConcurrentHashMap<String, CompletableFuture<ManagedLedgerImpl>>) ledgerField
.get(pulsar.getManagedLedgerFactory());
CompletableFuture<ManagedLedgerImpl> future = new CompletableFuture<>();
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceThrottlingTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceThrottlingTest.java
index 89f477144..a3171c7a5 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceThrottlingTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceThrottlingTest.java
@@ -34,7 +34,6 @@
import org.apache.bookkeeper.util.ZkUtils;
import org.apache.pulsar.client.api.Consumer;
-import org.apache.pulsar.client.api.ConsumerConfiguration;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.client.api.SubscriptionType;
@@ -48,8 +47,6 @@
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.retryStrategically;
-import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.retryStrategically;
/**
*/
@@ -92,13 +89,11 @@ public void testLookupThrottlingForClientByBroker0Permit() throws Exception {
final String topicName = "persistent://prop/usw/my-ns/newTopic";
- org.apache.pulsar.client.api.ClientConfiguration clientConf = new org.apache.pulsar.client.api.ClientConfiguration();
- clientConf.setStatsInterval(0, TimeUnit.SECONDS);
String lookupUrl = new URI("pulsar://localhost:" + BROKER_PORT).toString();
- PulsarClient pulsarClient = PulsarClient.create(lookupUrl, clientConf);
+ PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(lookupUrl).statsInterval(0, TimeUnit.SECONDS)
+ .build();
- ConsumerConfiguration consumerConfig = new ConsumerConfiguration();
- Consumer consumer = pulsarClient.subscribe(topicName, "mysub", consumerConfig);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("mysub").subscribe();
consumer.close();
int newPermits = 0;
@@ -113,7 +108,7 @@ public void testLookupThrottlingForClientByBroker0Permit() throws Exception {
}
try {
- consumer = pulsarClient.subscribe(topicName, "mysub", consumerConfig);
+ consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("mysub").subscribe();
consumer.close();
fail("It should fail as throttling should not receive any request");
} catch (org.apache.pulsar.client.api.PulsarClientException.TooManyRequestsException e) {
@@ -135,15 +130,9 @@ public void testLookupThrottlingForClientByBroker0Permit() throws Exception {
public void testLookupThrottlingForClientByBroker() throws Exception {
final String topicName = "persistent://prop/usw/my-ns/newTopic";
- org.apache.pulsar.client.api.ClientConfiguration clientConf = new org.apache.pulsar.client.api.ClientConfiguration();
- clientConf.setStatsInterval(0, TimeUnit.SECONDS);
- clientConf.setIoThreads(20);
- clientConf.setConnectionsPerBroker(20);
String lookupUrl = new URI("pulsar://localhost:" + BROKER_PORT).toString();
- PulsarClient pulsarClient = PulsarClient.create(lookupUrl, clientConf);
-
- ConsumerConfiguration consumerConfig = new ConsumerConfiguration();
- consumerConfig.setSubscriptionType(SubscriptionType.Shared);
+ PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(lookupUrl).statsInterval(0, TimeUnit.SECONDS)
+ .ioThreads(20).connectionsPerBroker(20).build();
int newPermits = 1;
admin.brokers().updateDynamicConfiguration("maxConcurrentLookupRequest", Integer.toString(newPermits));
@@ -156,14 +145,15 @@ public void testLookupThrottlingForClientByBroker() throws Exception {
}
}
- List<Consumer> successfulConsumers = Collections.synchronizedList(Lists.newArrayList());
+ List<Consumer<byte[]>> successfulConsumers = Collections.synchronizedList(Lists.newArrayList());
ExecutorService executor = Executors.newFixedThreadPool(10);
final int totalConsumers = 20;
CountDownLatch latch = new CountDownLatch(totalConsumers);
for (int i = 0; i < totalConsumers; i++) {
executor.execute(() -> {
try {
- successfulConsumers.add(pulsarClient.subscribe(topicName, "mysub", consumerConfig));
+ successfulConsumers.add(pulsarClient.newConsumer().topic(topicName).subscriptionName("mysub")
+ .subscriptionType(SubscriptionType.Shared).subscribe());
} catch (PulsarClientException.TooManyRequestsException e) {
// ok
} catch (Exception e) {
@@ -174,7 +164,7 @@ public void testLookupThrottlingForClientByBroker() throws Exception {
}
latch.await();
- for (Consumer c : successfulConsumers) {
+ for (Consumer<?> c : successfulConsumers) {
if (c != null) {
c.close();
}
@@ -202,23 +192,19 @@ public void testLookupThrottlingForClientByBrokerInternalRetry() throws Exceptio
final String topicName = "persistent://prop/usw/my-ns/newTopic";
- org.apache.pulsar.client.api.ClientConfiguration clientConf = new org.apache.pulsar.client.api.ClientConfiguration();
- clientConf.setStatsInterval(0, TimeUnit.SECONDS);
- clientConf.setIoThreads(20);
- clientConf.setConnectionsPerBroker(20);
String lookupUrl = new URI("pulsar://localhost:" + BROKER_PORT).toString();
- PulsarClient pulsarClient = PulsarClient.create(lookupUrl, clientConf);
+ PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(lookupUrl).statsInterval(0, TimeUnit.SECONDS)
+ .ioThreads(20).connectionsPerBroker(20).build();
upsertLookupPermits(100);
- ConsumerConfiguration consumerConfig = new ConsumerConfiguration();
- consumerConfig.setSubscriptionType(SubscriptionType.Shared);
- List<Consumer> consumers = Collections.synchronizedList(Lists.newArrayList());
+ List<Consumer<byte[]>> consumers = Collections.synchronizedList(Lists.newArrayList());
ExecutorService executor = Executors.newFixedThreadPool(10);
final int totalConsumers = 8;
CountDownLatch latch = new CountDownLatch(totalConsumers);
for (int i = 0; i < totalConsumers; i++) {
executor.execute(() -> {
try {
- consumers.add(pulsarClient.subscribe(topicName, "mysub", consumerConfig));
+ consumers.add(pulsarClient.newConsumer().topic(topicName).subscriptionName("mysub")
+ .subscriptionType(SubscriptionType.Shared).subscribe());
} catch (PulsarClientException.TooManyRequestsException e) {
// ok
} catch (Exception e) {
@@ -238,7 +224,7 @@ public void testLookupThrottlingForClientByBrokerInternalRetry() throws Exceptio
int totalConnectedConsumers = 0;
for (int i = 0; i < consumers.size(); i++) {
- if (((ConsumerImpl) consumers.get(i)).isConnected()) {
+ if (((ConsumerImpl<?>) consumers.get(i)).isConnected()) {
totalConnectedConsumers++;
}
consumers.get(i).close();
@@ -250,9 +236,9 @@ public void testLookupThrottlingForClientByBrokerInternalRetry() throws Exceptio
pulsarClient.close();
}
- private boolean areAllConsumersConnected(List<Consumer> consumers) {
+ private boolean areAllConsumersConnected(List<Consumer<byte[]>> consumers) {
for (int i = 0; i < consumers.size(); i++) {
- if (!((ConsumerImpl) consumers.get(i)).isConnected()) {
+ if (!((ConsumerImpl<?>) consumers.get(i)).isConnected()) {
return false;
}
}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PartitionKeyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PartitionKeyTest.java
index cedaf3a59..d467ed145 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PartitionKeyTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PartitionKeyTest.java
@@ -24,7 +24,6 @@
import org.apache.pulsar.client.api.Message;
import org.apache.pulsar.client.api.MessageBuilder;
import org.apache.pulsar.client.api.Producer;
-import org.apache.pulsar.client.api.ProducerConfiguration;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@@ -49,15 +48,16 @@ protected void cleanup() throws Exception {
public void testPartitionKey() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/testPartitionKey";
- org.apache.pulsar.client.api.Consumer consumer = pulsarClient.subscribe(topicName, "my-subscription");
+ org.apache.pulsar.client.api.Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName("my-subscription").subscribe();
// 1. producer with batch enabled
- ProducerConfiguration conf = new ProducerConfiguration();
- conf.setBatchingEnabled(true);
- Producer producerWithBatches = pulsarClient.createProducer(topicName, conf);
+ Producer<byte[]> producerWithBatches = pulsarClient.newProducer().topic(topicName).enableBatching(true)
+ .create();
+
// 2. Producer without batches
- Producer producerWithoutBatches = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producerWithoutBatches = pulsarClient.newProducer().topic(topicName).create();
producerWithBatches.sendAsync(MessageBuilder.create().setKey("key-1").setContent("msg-1".getBytes()).build());
producerWithBatches.sendAsync(MessageBuilder.create().setKey("key-2").setContent("msg-2".getBytes()).build())
@@ -67,7 +67,7 @@ public void testPartitionKey() throws Exception {
.sendAsync(MessageBuilder.create().setKey("key-3").setContent("msg-3".getBytes()).build());
for (int i = 1; i <= 3; i++) {
- Message msg = consumer.receive();
+ Message<byte[]> msg = consumer.receive();
assertTrue(msg.hasKey());
assertEquals(msg.getKey(), "key-" + i);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PeerReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PeerReplicatorTest.java
index d5456fcca..d23356e2b 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PeerReplicatorTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PeerReplicatorTest.java
@@ -28,13 +28,10 @@
import org.apache.pulsar.broker.service.persistent.PersistentTopic;
import org.apache.pulsar.client.admin.PulsarAdminException;
-import org.apache.pulsar.client.api.ClientConfiguration;
import org.apache.pulsar.client.api.Producer;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.common.policies.data.PersistentTopicStats;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.DataProvider;
@@ -73,9 +70,9 @@ void shutdown() throws Exception {
* 5. Try to create producer using broker in cluster r3
* 6. Success : "r3" finds "r1" in peer cluster which owns n1 and redirects to "r1"
* 7. call admin-api to "r3" which redirects request to "r1"
- *
+ *
* </pre>
- *
+ *
* @param protocol
* @throws Exception
*/
@@ -99,14 +96,11 @@ public void testPeerClusterTopicLookup(String protocol) throws Exception {
final String topic1 = "persistent://" + namespace1 + "/topic1";
final String topic2 = "persistent://" + namespace2 + "/topic2";
- ClientConfiguration conf = new ClientConfiguration();
- conf.setStatsInterval(0, TimeUnit.SECONDS);
- PulsarClient client3 = PulsarClient.create(serviceUrl, conf);
- Producer producer;
+ PulsarClient client3 = PulsarClient.builder().serviceUrl(serviceUrl).statsInterval(0, TimeUnit.SECONDS).build();
try {
// try to create producer for topic1 (part of cluster: r1) by calling cluster: r3
- producer = client3.createProducer(topic1);
+ client3.newProducer().topic(topic1).create();
fail("should have failed as cluster:r3 doesn't own namespace");
} catch (PulsarClientException e) {
// Ok
@@ -114,7 +108,7 @@ public void testPeerClusterTopicLookup(String protocol) throws Exception {
try {
// try to create producer for topic2 (part of cluster: r2) by calling cluster: r3
- producer = client3.createProducer(topic2);
+ client3.newProducer().topic(topic2).create();
fail("should have failed as cluster:r3 doesn't own namespace");
} catch (PulsarClientException e) {
// Ok
@@ -122,7 +116,7 @@ public void testPeerClusterTopicLookup(String protocol) throws Exception {
// set peer-clusters : r3->r1
admin1.clusters().updatePeerClusterNames("r3", Sets.newLinkedHashSet(Lists.newArrayList("r1")));
- producer = client3.createProducer(topic1);
+ Producer<byte[]> producer = client3.newProducer().topic(topic1).create();
PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopic(topic1).get();
assertNotNull(topic);
pulsar1.getBrokerService().updateRates();
@@ -137,7 +131,7 @@ public void testPeerClusterTopicLookup(String protocol) throws Exception {
// set peer-clusters : r3->r2
admin2.clusters().updatePeerClusterNames("r3", Sets.newLinkedHashSet(Lists.newArrayList("r2")));
- producer = client3.createProducer(topic2);
+ producer = client3.newProducer().topic(topic2).create();
topic = (PersistentTopic) pulsar2.getBrokerService().getTopic(topic2).get();
assertNotNull(topic);
pulsar2.getBrokerService().updateRates();
@@ -169,7 +163,4 @@ public void testGetPeerClusters() throws Exception {
}, 5, 100);
assertEquals(admin1.clusters().getPeerClusterNames(mainClusterName), peerClusters);
}
-
- private static final Logger log = LoggerFactory.getLogger(PeerReplicatorTest.class);
-
}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java
index e25581f77..b9dee2b33 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java
@@ -47,7 +47,7 @@
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
-
+import org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.InitialPosition;
import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback;
import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback;
import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteLedgerCallback;
@@ -225,7 +225,7 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
((OpenCursorCallback) invocationOnMock.getArguments()[1]).openCursorComplete(cursorMock, null);
return null;
}
- }).when(ledgerMock).asyncOpenCursor(matches(".*success.*"), any(OpenCursorCallback.class), anyObject());
+ }).when(ledgerMock).asyncOpenCursor(matches(".*success.*"), any(InitialPosition.class), any(OpenCursorCallback.class), anyObject());
// call deleteLedgerComplete on ledger asyncDelete
doAnswer(new Answer<Object>() {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentFailoverE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentFailoverE2ETest.java
index fa4da0b68..d609cd4e2 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentFailoverE2ETest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentFailoverE2ETest.java
@@ -24,7 +24,6 @@
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
-import com.google.common.collect.Sets;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -36,15 +35,14 @@
import org.apache.pulsar.broker.service.persistent.PersistentSubscription;
import org.apache.pulsar.broker.service.persistent.PersistentTopic;
import org.apache.pulsar.client.api.Consumer;
-import org.apache.pulsar.client.api.ConsumerConfiguration;
+import org.apache.pulsar.client.api.ConsumerBuilder;
import org.apache.pulsar.client.api.ConsumerEventListener;
import org.apache.pulsar.client.api.Message;
import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageRoutingMode;
import org.apache.pulsar.client.api.Producer;
-import org.apache.pulsar.client.api.ProducerConfiguration;
import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.client.api.SubscriptionType;
-import org.apache.pulsar.client.api.ProducerConfiguration.MessageRoutingMode;
import org.apache.pulsar.client.impl.MessageIdImpl;
import org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.SubType;
import org.apache.pulsar.common.naming.TopicName;
@@ -55,6 +53,7 @@
import org.testng.annotations.Test;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
public class PersistentFailoverE2ETest extends BrokerTestBase {
@@ -78,7 +77,7 @@ protected void cleanup() throws Exception {
final LinkedBlockingQueue<Integer> inActiveQueue = new LinkedBlockingQueue<>();
@Override
- public void becameActive(Consumer consumer, int partitionId) {
+ public void becameActive(Consumer<?> consumer, int partitionId) {
try {
activeQueue.put(partitionId);
} catch (InterruptedException e) {
@@ -86,7 +85,7 @@ public void becameActive(Consumer consumer, int partitionId) {
}
@Override
- public void becameInactive(Consumer consumer, int partitionId) {
+ public void becameInactive(Consumer<?> consumer, int partitionId) {
try {
inActiveQueue.put(partitionId);
} catch (InterruptedException e) {
@@ -115,13 +114,13 @@ private void verifyConsumerInactive(TestConsumerStateEventListener listener, int
private final Set<Integer> inactivePtns = Sets.newHashSet();
@Override
- public synchronized void becameActive(Consumer consumer, int partitionId) {
+ public synchronized void becameActive(Consumer<?> consumer, int partitionId) {
activePtns.add(partitionId);
inactivePtns.remove(partitionId);
}
@Override
- public synchronized void becameInactive(Consumer consumer, int partitionId) {
+ public synchronized void becameInactive(Consumer<?> consumer, int partitionId) {
activePtns.remove(partitionId);
inactivePtns.add(partitionId);
}
@@ -134,20 +133,17 @@ public void testSimpleConsumerEventsWithoutPartition() throws Exception {
final int numMsgs = 100;
TestConsumerStateEventListener listener1 = new TestConsumerStateEventListener();
- ConsumerConfiguration consumerConf1 = new ConsumerConfiguration();
- consumerConf1.setSubscriptionType(SubscriptionType.Failover);
- consumerConf1.setConsumerName("1");
- consumerConf1.setConsumerEventListener(listener1);
-
TestConsumerStateEventListener listener2 = new TestConsumerStateEventListener();
- ConsumerConfiguration consumerConf2 = new ConsumerConfiguration();
- consumerConf2.setSubscriptionType(SubscriptionType.Failover);
- consumerConf2.setConsumerName("2");
- consumerConf2.setConsumerEventListener(listener2);
+ ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .subscriptionType(SubscriptionType.Failover);
+
// 1. two consumers on the same subscription
- Consumer consumer1 = pulsarClient.subscribe(topicName, subName, consumerConf1);
- Consumer consumer2 = pulsarClient.subscribe(topicName, subName, consumerConf2);
+ ConsumerBuilder<byte[]> consumerBulder1 = consumerBuilder.clone().consumerName("1")
+ .consumerEventListener(listener1);
+ Consumer<byte[]> consumer1 = consumerBulder1.subscribe();
+ Consumer<byte[]> consumer2 = consumerBuilder.clone().consumerName("2").consumerEventListener(listener2)
+ .subscribe();
verifyConsumerActive(listener1, -1);
verifyConsumerInactive(listener2, -1);
@@ -162,7 +158,7 @@ public void testSimpleConsumerEventsWithoutPartition() throws Exception {
assertEquals(subRef.getDispatcher().getType(), SubType.Failover);
List<CompletableFuture<MessageId>> futures = Lists.newArrayListWithCapacity(numMsgs);
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
for (int i = 0; i < numMsgs; i++) {
String message = "my-message-" + i;
futures.add(producer.sendAsync(message.getBytes()));
@@ -176,7 +172,7 @@ public void testSimpleConsumerEventsWithoutPartition() throws Exception {
Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
// 3. consumer1 should have all the messages while consumer2 should have no messages
- Message msg = null;
+ Message<byte[]> msg = null;
Assert.assertNull(consumer2.receive(1, TimeUnit.SECONDS));
for (int i = 0; i < numMsgs; i++) {
msg = consumer1.receive(1, TimeUnit.SECONDS);
@@ -242,7 +238,7 @@ public void testSimpleConsumerEventsWithoutPartition() throws Exception {
Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
consumer2.acknowledge(msg);
}
- consumer1 = pulsarClient.subscribe(topicName, subName, consumerConf1);
+ consumer1 = consumerBulder1.subscribe();
Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
for (int i = 5; i < numMsgs; i++) {
msg = consumer1.receive(1, TimeUnit.SECONDS);
@@ -264,18 +260,15 @@ public void testSimpleConsumerEventsWithoutPartition() throws Exception {
futures.clear();
// 7. consumer subscription should not send messages to the new consumer if its name is not highest in the list
- TestConsumerStateEventListener listener3 = new TestConsumerStateEventListener();
- ConsumerConfiguration consumerConf3 = new ConsumerConfiguration();
- consumerConf3.setSubscriptionType(SubscriptionType.Failover);
- consumerConf3.setConsumerName("3");
- consumerConf3.setConsumerEventListener(listener3);
for (int i = 0; i < 5; i++) {
msg = consumer1.receive(1, TimeUnit.SECONDS);
Assert.assertNotNull(msg);
Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
consumer1.acknowledge(msg);
}
- Consumer consumer3 = pulsarClient.subscribe(topicName, subName, consumerConf3);
+ TestConsumerStateEventListener listener3 = new TestConsumerStateEventListener();
+ Consumer<byte[]> consumer3 = consumerBuilder.clone().consumerName("3").consumerEventListener(listener3)
+ .subscribe();
Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
verifyConsumerInactive(listener3, -1);
@@ -333,24 +326,18 @@ public void testSimpleConsumerEventsWithPartition() throws Exception {
admin.persistentTopics().createPartitionedTopic(topicName, numPartitions);
- ProducerConfiguration producerConf = new ProducerConfiguration();
- producerConf.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
+ ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .subscriptionType(SubscriptionType.Failover);
- ActiveInactiveListenerEvent listener1 = new ActiveInactiveListenerEvent();
- ConsumerConfiguration consumerConf1 = new ConsumerConfiguration();
- consumerConf1.setSubscriptionType(SubscriptionType.Failover);
- consumerConf1.setConsumerName("1");
- consumerConf1.setConsumerEventListener(listener1);
+ // 1. two consumers on the same subscription
+ ActiveInactiveListenerEvent listener1 = new ActiveInactiveListenerEvent();
ActiveInactiveListenerEvent listener2 = new ActiveInactiveListenerEvent();
- ConsumerConfiguration consumerConf2 = new ConsumerConfiguration();
- consumerConf2.setSubscriptionType(SubscriptionType.Failover);
- consumerConf2.setConsumerName("2");
- consumerConf2.setConsumerEventListener(listener2);
- // 1. two consumers on the same subscription
- Consumer consumer1 = pulsarClient.subscribe(topicName, subName, consumerConf1);
- Consumer consumer2 = pulsarClient.subscribe(topicName, subName, consumerConf2);
+ Consumer<byte[]> consumer1 = consumerBuilder.clone().consumerName("1").consumerEventListener(listener1)
+ .subscribe();
+ Consumer<byte[]> consumer2 = consumerBuilder.clone().consumerName("2").consumerEventListener(listener2)
+ .subscribe();
PersistentTopic topicRef;
topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(destName.getPartition(0).toString());
@@ -367,7 +354,8 @@ public void testSimpleConsumerEventsWithPartition() throws Exception {
.getSubscription(subName).getDispatcher();
List<CompletableFuture<MessageId>> futures = Lists.newArrayListWithCapacity(numMsgs);
- Producer producer = pulsarClient.createProducer(topicName, producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
+ .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();
for (int i = 0; i < numMsgs; i++) {
String message = "my-message-" + i;
futures.add(producer.sendAsync(message.getBytes()));
@@ -377,7 +365,7 @@ public void testSimpleConsumerEventsWithPartition() throws Exception {
// equal distribution between both consumers
int totalMessages = 0;
- Message msg = null;
+ Message<byte[]> msg = null;
Set<Integer> receivedPtns = Sets.newHashSet();
while (true) {
msg = consumer1.receive(1, TimeUnit.SECONDS);
@@ -410,10 +398,10 @@ public void testSimpleConsumerEventsWithPartition() throws Exception {
assertTrue(Sets.difference(listener2.activePtns, receivedPtns).isEmpty());
Assert.assertEquals(totalMessages, numMsgs);
- Assert.assertEquals(disp0.getActiveConsumer().consumerName(), consumerConf1.getConsumerName());
- Assert.assertEquals(disp1.getActiveConsumer().consumerName(), consumerConf2.getConsumerName());
- Assert.assertEquals(disp2.getActiveConsumer().consumerName(), consumerConf1.getConsumerName());
- Assert.assertEquals(disp3.getActiveConsumer().consumerName(), consumerConf2.getConsumerName());
+ Assert.assertEquals(disp0.getActiveConsumer().consumerName(), "1");
+ Assert.assertEquals(disp1.getActiveConsumer().consumerName(), "2");
+ Assert.assertEquals(disp2.getActiveConsumer().consumerName(), "1");
+ Assert.assertEquals(disp3.getActiveConsumer().consumerName(), "2");
totalMessages = 0;
for (int i = 0; i < numMsgs; i++) {
@@ -424,16 +412,13 @@ public void testSimpleConsumerEventsWithPartition() throws Exception {
futures.clear();
// add a consumer
- ConsumerConfiguration consumerConf3 = new ConsumerConfiguration();
- consumerConf3.setSubscriptionType(SubscriptionType.Failover);
- consumerConf3.setConsumerName("3");
for (int i = 0; i < 20; i++) {
msg = consumer1.receive(1, TimeUnit.SECONDS);
Assert.assertNotNull(msg);
uniqueMessages.add(new String(msg.getData()));
consumer1.acknowledge(msg);
}
- Consumer consumer3 = pulsarClient.subscribe(topicName, subName, consumerConf3);
+ Consumer<byte[]> consumer3 = consumerBuilder.clone().consumerName("3").subscribe();
Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
int consumer1Messages = 0;
while (true) {
@@ -470,10 +455,10 @@ public void testSimpleConsumerEventsWithPartition() throws Exception {
}
Assert.assertEquals(uniqueMessages.size(), numMsgs);
- Assert.assertEquals(disp0.getActiveConsumer().consumerName(), consumerConf1.getConsumerName());
- Assert.assertEquals(disp1.getActiveConsumer().consumerName(), consumerConf2.getConsumerName());
- Assert.assertEquals(disp2.getActiveConsumer().consumerName(), consumerConf3.getConsumerName());
- Assert.assertEquals(disp3.getActiveConsumer().consumerName(), consumerConf1.getConsumerName());
+ Assert.assertEquals(disp0.getActiveConsumer().consumerName(), "1");
+ Assert.assertEquals(disp1.getActiveConsumer().consumerName(), "2");
+ Assert.assertEquals(disp2.getActiveConsumer().consumerName(), "3");
+ Assert.assertEquals(disp3.getActiveConsumer().consumerName(), "1");
uniqueMessages.clear();
for (int i = 0; i < numMsgs; i++) {
@@ -516,10 +501,10 @@ public void testSimpleConsumerEventsWithPartition() throws Exception {
}
Assert.assertEquals(uniqueMessages.size(), numMsgs);
- Assert.assertEquals(disp0.getActiveConsumer().consumerName(), consumerConf2.getConsumerName());
- Assert.assertEquals(disp1.getActiveConsumer().consumerName(), consumerConf3.getConsumerName());
- Assert.assertEquals(disp2.getActiveConsumer().consumerName(), consumerConf2.getConsumerName());
- Assert.assertEquals(disp3.getActiveConsumer().consumerName(), consumerConf3.getConsumerName());
+ Assert.assertEquals(disp0.getActiveConsumer().consumerName(), "2");
+ Assert.assertEquals(disp1.getActiveConsumer().consumerName(), "3");
+ Assert.assertEquals(disp2.getActiveConsumer().consumerName(), "2");
+ Assert.assertEquals(disp3.getActiveConsumer().consumerName(), "3");
producer.close();
consumer2.close();
@@ -533,48 +518,35 @@ public void testActiveConsumerFailoverWithDelay() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/failover-topic3";
final String subName = "sub1";
final int numMsgs = 100;
- List<Message> receivedMessages = Lists.newArrayList();
-
- ConsumerConfiguration consumerConf1 = new ConsumerConfiguration();
- consumerConf1.setSubscriptionType(SubscriptionType.Failover);
- consumerConf1.setConsumerName("1");
- consumerConf1.setMessageListener((consumer, msg) -> {
- try {
- synchronized (receivedMessages) {
- receivedMessages.add(msg);
- }
- consumer.acknowledge(msg);
- } catch (Exception e) {
- fail("Should not fail");
- }
- });
-
- ConsumerConfiguration consumerConf2 = new ConsumerConfiguration();
- consumerConf2.setSubscriptionType(SubscriptionType.Failover);
- consumerConf2.setConsumerName("2");
- consumerConf2.setMessageListener((consumer, msg) -> {
- try {
- synchronized (receivedMessages) {
- receivedMessages.add(msg);
- }
- consumer.acknowledge(msg);
- } catch (Exception e) {
- fail("Should not fail");
- }
- });
+ List<Message<byte[]>> receivedMessages = Lists.newArrayList();
+
+ ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .subscriptionType(SubscriptionType.Failover).messageListener((consumer, msg) -> {
+ try {
+ synchronized (receivedMessages) {
+ receivedMessages.add(msg);
+ }
+ consumer.acknowledge(msg);
+ } catch (Exception e) {
+ fail("Should not fail");
+ }
+ });
+
+ ConsumerBuilder<byte[]> consumerBuilder1 = consumerBuilder.clone().consumerName("1");
+ ConsumerBuilder<byte[]> consumerBuilder2 = consumerBuilder.clone().consumerName("2");
conf.setActiveConsumerFailoverDelayTimeMillis(500);
restartBroker();
// create subscription
- Consumer consumer = pulsarClient.subscribe(topicName, subName, consumerConf1);
+ Consumer<byte[]> consumer = consumerBuilder1.subscribe();
consumer.close();
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
PersistentSubscription subRef = topicRef.getSubscription(subName);
// enqueue messages
List<CompletableFuture<MessageId>> futures = Lists.newArrayListWithCapacity(numMsgs);
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
for (int i = 0; i < numMsgs; i++) {
String message = "my-message-" + i;
futures.add(producer.sendAsync(message.getBytes()));
@@ -584,8 +556,8 @@ public void testActiveConsumerFailoverWithDelay() throws Exception {
producer.close();
// two consumers subscribe at almost the same time
- CompletableFuture<Consumer> subscribeFuture2 = pulsarClient.subscribeAsync(topicName, subName, consumerConf2);
- CompletableFuture<Consumer> subscribeFuture1 = pulsarClient.subscribeAsync(topicName, subName, consumerConf1);
+ CompletableFuture<Consumer<byte[]>> subscribeFuture2 = consumerBuilder2.subscribeAsync();
+ CompletableFuture<Consumer<byte[]>> subscribeFuture1 = consumerBuilder1.subscribeAsync();
// wait for all messages to be dequeued
int retry = 20;
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentQueueE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentQueueE2ETest.java
index f311f4ce8..8cc11d707 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentQueueE2ETest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentQueueE2ETest.java
@@ -36,15 +36,13 @@
import org.apache.pulsar.broker.service.persistent.PersistentSubscription;
import org.apache.pulsar.broker.service.persistent.PersistentTopic;
import org.apache.pulsar.client.api.Consumer;
-import org.apache.pulsar.client.api.ConsumerConfiguration;
+import org.apache.pulsar.client.api.ConsumerBuilder;
import org.apache.pulsar.client.api.Message;
import org.apache.pulsar.client.api.MessageId;
import org.apache.pulsar.client.api.Producer;
-import org.apache.pulsar.client.api.ProducerConfiguration;
import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.client.api.SubscriptionType;
import org.apache.pulsar.client.impl.ConsumerImpl;
-import org.apache.pulsar.client.impl.MessageIdImpl;
import org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.SubType;
import org.apache.pulsar.common.policies.data.ConsumerStats;
import org.apache.pulsar.common.policies.data.PersistentTopicStats;
@@ -83,12 +81,12 @@ public void testSimpleConsumerEvents() throws Exception {
final String subName = "sub1";
final int numMsgs = 100;
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Shared);
+ ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .subscriptionType(SubscriptionType.Shared);
// 1. two consumers on the same subscription
- Consumer consumer1 = pulsarClient.subscribe(topicName, subName, conf);
- Consumer consumer2 = pulsarClient.subscribe(topicName, subName, conf);
+ Consumer<byte[]> consumer1 = consumerBuilder.subscribe();
+ Consumer<byte[]> consumer2 = consumerBuilder.subscribe();
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
PersistentSubscription subRef = topicRef.getSubscription(subName);
@@ -101,7 +99,7 @@ public void testSimpleConsumerEvents() throws Exception {
assertEquals(subRef.getDispatcher().getType(), SubType.Shared);
List<CompletableFuture<MessageId>> futures = Lists.newArrayListWithCapacity(numMsgs * 2);
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
for (int i = 0; i < numMsgs * 2; i++) {
String message = "my-message-" + i;
futures.add(producer.sendAsync(message.getBytes()));
@@ -114,14 +112,15 @@ public void testSimpleConsumerEvents() throws Exception {
Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
// both consumers will together consumer all messages
- Message msg;
- Consumer c = consumer1;
+ Message<byte[]> msg;
+ Consumer<byte[]> c = consumer1;
while (true) {
try {
msg = c.receive(1, TimeUnit.SECONDS);
c.acknowledge(msg);
} catch (PulsarClientException e) {
if (c.equals(consumer1)) {
+ consumer1.close();
c = consumer2;
} else {
break;
@@ -181,34 +180,24 @@ public void testReplayOnConsumerDisconnect() throws Exception {
final List<String> messagesProduced = Lists.newArrayListWithCapacity(numMsgs);
final List<String> messagesConsumed = new BlockingArrayQueue<>(numMsgs);
- ConsumerConfiguration conf1 = new ConsumerConfiguration();
- conf1.setSubscriptionType(SubscriptionType.Shared);
- conf1.setMessageListener((consumer, msg) -> {
- try {
- consumer.acknowledge(msg);
- messagesConsumed.add(new String(msg.getData()));
- } catch (Exception e) {
- fail("Should not fail");
- }
- });
-
- ConsumerConfiguration conf2 = new ConsumerConfiguration();
- conf2.setSubscriptionType(SubscriptionType.Shared);
- conf2.setMessageListener((consumer, msg) -> {
- try {
- // do nothing
- } catch (Exception e) {
- fail("Should not fail");
- }
- });
-
- Consumer consumer1 = pulsarClient.subscribe(topicName, subName, conf1);
+ Consumer<byte[]> consumer1 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .subscriptionType(SubscriptionType.Shared).messageListener((consumer, msg) -> {
+ try {
+ consumer.acknowledge(msg);
+ messagesConsumed.add(new String(msg.getData()));
+ } catch (Exception e) {
+ fail("Should not fail");
+ }
+ }).subscribe();
// consumer2 does not ack messages
- Consumer consumer2 = pulsarClient.subscribe(topicName, subName, conf2);
+ Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .subscriptionType(SubscriptionType.Shared).messageListener((consumer, msg) -> {
+ // do notthing
+ }).subscribe();
List<CompletableFuture<MessageId>> futures = Lists.newArrayListWithCapacity(numMsgs * 2);
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
for (int i = 0; i < numMsgs; i++) {
String message = "msg-" + i;
futures.add(producer.sendAsync(message.getBytes()));
@@ -241,40 +230,34 @@ public void testConsumersWithDifferentPermits() throws Exception {
final CountDownLatch latch = new CountDownLatch(numMsgs);
int recvQ1 = 10;
- ConsumerConfiguration conf1 = new ConsumerConfiguration();
- conf1.setSubscriptionType(SubscriptionType.Shared);
- conf1.setReceiverQueueSize(recvQ1);
- conf1.setMessageListener((consumer, msg) -> {
- msgCountConsumer1.incrementAndGet();
- try {
- consumer.acknowledge(msg);
- latch.countDown();
- } catch (PulsarClientException e) {
- fail("Should not fail");
- }
- });
+ Consumer<byte[]> consumer1 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .subscriptionType(SubscriptionType.Shared).receiverQueueSize(recvQ1)
+ .messageListener((consumer, msg) -> {
+ msgCountConsumer1.incrementAndGet();
+ try {
+ consumer.acknowledge(msg);
+ latch.countDown();
+ } catch (PulsarClientException e) {
+ fail("Should not fail");
+ }
+ }).subscribe();
int recvQ2 = 1;
- ConsumerConfiguration conf2 = new ConsumerConfiguration();
- conf2.setSubscriptionType(SubscriptionType.Shared);
- conf2.setReceiverQueueSize(recvQ2);
- conf2.setMessageListener((consumer, msg) -> {
- msgCountConsumer2.incrementAndGet();
- try {
- consumer.acknowledge(msg);
- latch.countDown();
- } catch (PulsarClientException e) {
- fail("Should not fail");
- }
- });
-
- Consumer consumer1 = pulsarClient.subscribe(topicName, subName, conf1);
- Consumer consumer2 = pulsarClient.subscribe(topicName, subName, conf2);
+ Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .subscriptionType(SubscriptionType.Shared).receiverQueueSize(recvQ2)
+ .messageListener((consumer, msg) -> {
+ msgCountConsumer2.incrementAndGet();
+ try {
+ consumer.acknowledge(msg);
+ latch.countDown();
+ } catch (PulsarClientException e) {
+ fail("Should not fail");
+ }
+ }).subscribe();
List<CompletableFuture<MessageId>> futures = Lists.newArrayListWithCapacity(numMsgs);
- ProducerConfiguration conf = new ProducerConfiguration();
- conf.setMaxPendingMessages(numMsgs + 1);
- Producer producer = pulsarClient.createProducer(topicName, conf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).maxPendingMessages(numMsgs + 1)
+ .create();
for (int i = 0; i < numMsgs; i++) {
String message = "msg-" + i;
futures.add(producer.sendAsync(message.getBytes()));
@@ -307,10 +290,10 @@ public void testRoundRobinBatchDistribution() throws Exception {
final CountDownLatch latch = new CountDownLatch(numMsgs * 3);
- ConsumerConfiguration conf1 = new ConsumerConfiguration();
- conf1.setSubscriptionType(SubscriptionType.Shared);
- conf1.setReceiverQueueSize(10);
- conf1.setMessageListener((consumer, msg) -> {
+ ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .receiverQueueSize(10).subscriptionType(SubscriptionType.Shared);
+
+ Consumer<byte[]> consumer1 = consumerBuilder.clone().messageListener((consumer, msg) -> {
try {
counter1.incrementAndGet();
consumer.acknowledge(msg);
@@ -318,12 +301,9 @@ public void testRoundRobinBatchDistribution() throws Exception {
} catch (Exception e) {
fail("Should not fail");
}
- });
+ }).subscribe();
- ConsumerConfiguration conf2 = new ConsumerConfiguration();
- conf2.setSubscriptionType(SubscriptionType.Shared);
- conf2.setReceiverQueueSize(10);
- conf2.setMessageListener((consumer, msg) -> {
+ Consumer<byte[]> consumer2 = consumerBuilder.clone().messageListener((consumer, msg) -> {
try {
counter2.incrementAndGet();
consumer.acknowledge(msg);
@@ -331,29 +311,20 @@ public void testRoundRobinBatchDistribution() throws Exception {
} catch (Exception e) {
fail("Should not fail");
}
- });
+ }).subscribe();
- ConsumerConfiguration conf3 = new ConsumerConfiguration();
- conf3.setSubscriptionType(SubscriptionType.Shared);
- conf3.setReceiverQueueSize(10);
- conf3.setMessageListener((consumer, msg) -> {
+ Consumer<byte[]> consumer3 = consumerBuilder.clone().messageListener((consumer, msg) -> {
try {
- counter3.incrementAndGet();
+ counter1.incrementAndGet();
consumer.acknowledge(msg);
latch.countDown();
} catch (Exception e) {
fail("Should not fail");
}
- });
-
- // subscribe and close, so that distribution can be checked after
- // all messages are published
- Consumer consumer1 = pulsarClient.subscribe(topicName, subName, conf1);
- Consumer consumer2 = pulsarClient.subscribe(topicName, subName, conf2);
- Consumer consumer3 = pulsarClient.subscribe(topicName, subName, conf3);
+ }).subscribe();
List<CompletableFuture<MessageId>> futures = Lists.newArrayListWithCapacity(numMsgs);
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
for (int i = 0; i < numMsgs * 3; i++) {
String message = "msg-" + i;
futures.add(producer.sendAsync(message.getBytes()));
@@ -386,17 +357,16 @@ public void testSharedSingleAckedNormalTopic() throws Exception {
final int totalMessages = 50;
// 1. producer connect
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
assertNotNull(topicRef);
assertEquals(topicRef.getProducers().size(), 1);
// 2. Create consumer
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setReceiverQueueSize(10);
- conf.setSubscriptionType(SubscriptionType.Shared);
- Consumer consumer1 = pulsarClient.subscribe(topicName, subscriptionName, conf);
- Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, conf);
+ ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName(subscriptionName).receiverQueueSize(10).subscriptionType(SubscriptionType.Shared);
+ Consumer<byte[]> consumer1 = consumerBuilder.subscribe();
+ Consumer<byte[]> consumer2 = consumerBuilder.subscribe();
// 3. Producer publishes messages
for (int i = 0; i < totalMessages; i++) {
@@ -407,8 +377,8 @@ public void testSharedSingleAckedNormalTopic() throws Exception {
// 4. Receive messages
int receivedConsumer1 = 0, receivedConsumer2 = 0;
- Message message1 = consumer1.receive();
- Message message2 = consumer2.receive();
+ Message<byte[]> message1 = consumer1.receive();
+ Message<byte[]> message2 = consumer2.receive();
do {
if (message1 != null) {
log.info("Consumer 1 Received: " + new String(message1.getData()));
@@ -453,17 +423,16 @@ public void testCancelReadRequestOnLastDisconnect() throws Exception {
final int totalMessages = 10;
// 1. producer connect
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
assertNotNull(topicRef);
assertEquals(topicRef.getProducers().size(), 1);
// 2. Create consumer
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setReceiverQueueSize(1000);
- conf.setSubscriptionType(SubscriptionType.Shared);
- Consumer consumer1 = pulsarClient.subscribe(topicName, subscriptionName, conf);
- Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, conf);
+ ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName(subscriptionName).receiverQueueSize(1000).subscriptionType(SubscriptionType.Shared);
+ Consumer<byte[]> consumer1 = consumerBuilder.subscribe();
+ Consumer<byte[]> consumer2 = consumerBuilder.subscribe();
// 3. Producer publishes messages
for (int i = 0; i < totalMessages; i++) {
@@ -474,8 +443,8 @@ public void testCancelReadRequestOnLastDisconnect() throws Exception {
// 4. Receive messages
int receivedConsumer1 = 0, receivedConsumer2 = 0;
- Message message1 = consumer1.receive();
- Message message2 = consumer2.receive();
+ Message<byte[]> message1 = consumer1.receive();
+ Message<byte[]> message2 = consumer2.receive();
do {
if (message1 != null) {
log.info("Consumer 1 Received: " + new String(message1.getData()));
@@ -508,7 +477,7 @@ public void testCancelReadRequestOnLastDisconnect() throws Exception {
}
// 7. Consumer reconnects
- consumer1 = pulsarClient.subscribe(topicName, subscriptionName, conf);
+ consumer1 = consumerBuilder.subscribe();
// 8. Check number of messages received
receivedConsumer1 = 0;
@@ -528,12 +497,11 @@ public void testUnackedCountWithRedeliveries() throws Exception {
final String subName = "sub3";
final int numMsgs = 10;
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Shared);
- conf.setReceiverQueueSize(0);
- ConsumerImpl consumer1 = (ConsumerImpl) pulsarClient.subscribe(topicName, subName, conf);
+ ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .receiverQueueSize(10).subscriptionType(SubscriptionType.Shared);
+ ConsumerImpl<byte[]> consumer1 = (ConsumerImpl<byte[]>) consumerBuilder.subscribe();
for (int i = 0; i < numMsgs; i++) {
producer.send(("hello-" + i).getBytes());
@@ -547,7 +515,7 @@ public void testUnackedCountWithRedeliveries() throws Exception {
}
// C-2 will not get any message initially, since everything went to C-1 already
- Consumer consumer2 = pulsarClient.subscribe(topicName, subName, conf);
+ Consumer<byte[]> consumer2 = consumerBuilder.subscribe();
// Trigger C-1 to redeliver everything, half will go C-1 again and the other half to C-2
consumer1.redeliverUnacknowledgedMessages(c1_receivedMessages);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicConcurrentTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicConcurrentTest.java
index eda685a41..77660d39c 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicConcurrentTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicConcurrentTest.java
@@ -48,6 +48,7 @@
import org.apache.pulsar.broker.service.ServerCnx;
import org.apache.pulsar.broker.service.persistent.PersistentSubscription;
import org.apache.pulsar.broker.service.persistent.PersistentTopic;
+import org.apache.pulsar.broker.service.PersistentTopicTest;
import org.apache.pulsar.common.api.proto.PulsarApi;
import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.naming.NamespaceBundle;
@@ -120,7 +121,7 @@ public void testConcurrentTopicAndSubscriptionDelete() throws Exception {
.setSubType(PulsarApi.CommandSubscribe.SubType.Exclusive).build();
Future<Consumer> f1 = topic.subscribe(serverCnx, cmd.getSubscription(), cmd.getConsumerId(), cmd.getSubType(),
- 0, cmd.getConsumerName(), cmd.getDurable(), null, Collections.emptyMap(), cmd.getReadCompacted());
+ 0, cmd.getConsumerName(), cmd.getDurable(), null, Collections.emptyMap(), cmd.getReadCompacted(), cmd.getInitialPosition());
f1.get();
final CyclicBarrier barrier = new CyclicBarrier(2);
@@ -178,7 +179,7 @@ public void testConcurrentTopicGCAndSubscriptionDelete() throws Exception {
.setSubType(PulsarApi.CommandSubscribe.SubType.Exclusive).build();
Future<Consumer> f1 = topic.subscribe(serverCnx, cmd.getSubscription(), cmd.getConsumerId(), cmd.getSubType(),
- 0, cmd.getConsumerName(), cmd.getDurable(), null, Collections.emptyMap(), cmd.getReadCompacted());
+ 0, cmd.getConsumerName(), cmd.getDurable(), null, Collections.emptyMap(), cmd.getReadCompacted(), cmd.getInitialPosition());
f1.get();
final CyclicBarrier barrier = new CyclicBarrier(2);
@@ -240,7 +241,7 @@ public void testConcurrentTopicDeleteAndUnsubscribe() throws Exception {
.setSubType(PulsarApi.CommandSubscribe.SubType.Exclusive).build();
Future<Consumer> f1 = topic.subscribe(serverCnx, cmd.getSubscription(), cmd.getConsumerId(), cmd.getSubType(),
- 0, cmd.getConsumerName(), cmd.getDurable(), null, Collections.emptyMap(), cmd.getReadCompacted());
+ 0, cmd.getConsumerName(), cmd.getDurable(), null, Collections.emptyMap(), cmd.getReadCompacted(), cmd.getInitialPosition());
f1.get();
final CyclicBarrier barrier = new CyclicBarrier(2);
@@ -298,7 +299,7 @@ public void testConcurrentTopicDeleteAndSubsUnsubscribe() throws Exception {
.setSubType(PulsarApi.CommandSubscribe.SubType.Exclusive).build();
Future<Consumer> f1 = topic.subscribe(serverCnx, cmd.getSubscription(), cmd.getConsumerId(), cmd.getSubType(),
- 0, cmd.getConsumerName(), cmd.getDurable(), null, Collections.emptyMap(), cmd.getReadCompacted());
+ 0, cmd.getConsumerName(), cmd.getDurable(), null, Collections.emptyMap(), cmd.getReadCompacted(), cmd.getInitialPosition());
f1.get();
final CyclicBarrier barrier = new CyclicBarrier(2);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicE2ETest.java
index ad461792b..8b82c884a 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicE2ETest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicE2ETest.java
@@ -18,12 +18,6 @@
*/
package org.apache.pulsar.broker.service;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyObject;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNotNull;
@@ -32,10 +26,8 @@
import static org.testng.Assert.fail;
import java.lang.reflect.Field;
-import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import java.util.TreeSet;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CountDownLatch;
@@ -46,26 +38,20 @@
import java.util.concurrent.TimeUnit;
import org.apache.bookkeeper.mledger.ManagedCursor;
-import org.apache.bookkeeper.mledger.ManagedLedger;
-import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
-import org.apache.bookkeeper.mledger.AsyncCallbacks.OpenLedgerCallback;
import org.apache.bookkeeper.mledger.impl.EntryCacheImpl;
import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl;
import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl;
-import org.apache.bookkeeper.mledger.impl.PositionImpl;
-import org.apache.pulsar.broker.service.BrokerService;
import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers;
import org.apache.pulsar.broker.service.persistent.PersistentSubscription;
import org.apache.pulsar.broker.service.persistent.PersistentTopic;
import org.apache.pulsar.client.admin.PulsarAdminException;
import org.apache.pulsar.client.api.CompressionType;
import org.apache.pulsar.client.api.Consumer;
-import org.apache.pulsar.client.api.ConsumerConfiguration;
import org.apache.pulsar.client.api.Message;
import org.apache.pulsar.client.api.MessageBuilder;
import org.apache.pulsar.client.api.MessageId;
import org.apache.pulsar.client.api.Producer;
-import org.apache.pulsar.client.api.ProducerConfiguration;
+import org.apache.pulsar.client.api.ProducerBuilder;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.client.api.PulsarClientException.ProducerBusyException;
@@ -79,16 +65,12 @@
import org.apache.pulsar.common.stats.Metrics;
import org.apache.pulsar.common.util.collections.ConcurrentLongPairSet;
import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
-import com.google.common.collect.Sets;
-
/**
*/
@Test
@@ -110,7 +92,7 @@ public void testSimpleProducerEvents() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/topic0";
// 1. producer connect
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
assertNotNull(topicRef);
@@ -138,11 +120,8 @@ public void testSimpleConsumerEvents() throws Exception {
final String subName = "sub1";
final int numMsgs = 10;
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
-
// 1. client connect
- Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
PersistentSubscription subRef = topicRef.getSubscription(subName);
@@ -154,7 +133,7 @@ public void testSimpleConsumerEvents() throws Exception {
Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
assertEquals(getAvailablePermits(subRef), 1000 /* default */);
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
for (int i = 0; i < numMsgs * 2; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
@@ -168,7 +147,7 @@ public void testSimpleConsumerEvents() throws Exception {
Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
assertEquals(getAvailablePermits(subRef), 1000 - numMsgs * 2);
- Message msg = null;
+ Message<byte[]> msg = null;
for (int i = 0; i < numMsgs; i++) {
msg = consumer.receive();
// 3. in-order message delivery
@@ -223,15 +202,12 @@ public void testConsumerFlowControl() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/topic2";
final String subName = "sub2";
- Message msg;
+ Message<byte[]> msg;
int recvQueueSize = 4;
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
- conf.setReceiverQueueSize(recvQueueSize);
-
- Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
- Producer producer = pulsarClient.createProducer(topicName);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .receiverQueueSize(recvQueueSize).subscribe();
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
assertNotNull(topicRef);
@@ -269,16 +245,13 @@ public void testActiveSubscriptionWithCache() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/topic2";
final String subName = "sub2";
- Message msg;
+ Message<byte[]> msg;
int recvQueueSize = 4;
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
- conf.setReceiverQueueSize(recvQueueSize);
-
// (1) Create subscription
- Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
- Producer producer = pulsarClient.createProducer(topicName);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .receiverQueueSize(recvQueueSize).subscribe();
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
// (2) Produce Messages
for (int i = 0; i < recvQueueSize / 2; i++) {
@@ -329,10 +302,6 @@ public void testConcurrentConsumerThreads() throws Exception {
final int recvQueueSize = 100;
final int numConsumersThreads = 10;
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
- conf.setReceiverQueueSize(recvQueueSize);
-
ExecutorService executor = Executors.newCachedThreadPool();
final CyclicBarrier barrier = new CyclicBarrier(numConsumersThreads + 1);
@@ -342,9 +311,10 @@ public void testConcurrentConsumerThreads() throws Exception {
public Void call() throws Exception {
barrier.await();
- Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .receiverQueueSize(recvQueueSize).subscribe();
for (int i = 0; i < recvQueueSize / numConsumersThreads; i++) {
- Message msg = consumer.receive();
+ Message<byte[]> msg = consumer.receive();
consumer.acknowledge(msg);
}
return null;
@@ -352,7 +322,7 @@ public Void call() throws Exception {
});
}
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
for (int i = 0; i < recvQueueSize * numConsumersThreads; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
@@ -370,6 +340,7 @@ public Void call() throws Exception {
// 2. flow control works the same as single consumer single thread
Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
assertEquals(getAvailablePermits(subRef), recvQueueSize);
+ executor.shutdown();
}
@Test(enabled = false)
@@ -378,7 +349,7 @@ public void testGracefulClose() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/topic4";
final String subName = "sub4";
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
@@ -405,14 +376,12 @@ public void testGracefulClose() throws Exception {
// wait for the spawned thread to complete
latch.await();
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
- Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();
PersistentSubscription subRef = topicRef.getSubscription(subName);
assertNotNull(subRef);
- Message msg = null;
+ Message<byte[]> msg = null;
for (int i = 0; i < 10; i++) {
msg = consumer.receive();
}
@@ -433,6 +402,8 @@ public void testGracefulClose() throws Exception {
consumer.close();
Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
assertTrue(subRef.getDispatcher().isConsumerConnected());
+
+ executor.shutdown();
}
@Test
@@ -440,18 +411,15 @@ public void testSimpleCloseTopic() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/topic5";
final String subName = "sub5";
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
-
- Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
- Producer producer = pulsarClient.createProducer(topicName);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
assertNotNull(topicRef);
PersistentSubscription subRef = topicRef.getSubscription(subName);
assertNotNull(subRef);
- Message msg;
+ Message<byte[]> msg;
for (int i = 0; i < 10; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
@@ -471,13 +439,10 @@ public void testSingleClientMultipleSubscriptions() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/topic6";
final String subName = "sub6";
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
-
- pulsarClient.subscribe(topicName, subName, conf);
- pulsarClient.createProducer(topicName);
+ pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();
+ pulsarClient.newProducer().topic(topicName).create();
try {
- pulsarClient.subscribe(topicName, subName, conf);
+ pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();
fail("Should have thrown an exception since one consumer is already connected");
} catch (PulsarClientException cce) {
Assert.assertTrue(cce.getMessage().contains("Exclusive consumer is already connected"));
@@ -489,19 +454,16 @@ public void testMultipleClientsMultipleSubscriptions() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/topic7";
final String subName = "sub7";
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
-
- PulsarClient client1 = PulsarClient.create(brokerUrl.toString());
- PulsarClient client2 = PulsarClient.create(brokerUrl.toString());
+ PulsarClient client1 = PulsarClient.builder().serviceUrl(brokerUrl.toString()).build();
+ PulsarClient client2 = PulsarClient.builder().serviceUrl(brokerUrl.toString()).build();
try {
- client1.subscribe(topicName, subName, conf);
- client1.createProducer(topicName);
+ client1.newConsumer().topic(topicName).subscriptionName(subName).subscribe();
+ client1.newProducer().topic(topicName).create();
- client2.createProducer(topicName);
+ client2.newProducer().topic(topicName).create();
- client2.subscribe(topicName, subName, conf);
+ client2.newConsumer().topic(topicName).subscriptionName(subName).subscribe();
fail("Should have thrown an exception since one consumer is already connected");
} catch (PulsarClientException cce) {
Assert.assertTrue(cce.getMessage().contains("Exclusive consumer is already connected"));
@@ -516,11 +478,8 @@ public void testTopicDeleteWithDisconnectedSubscription() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/topic8";
final String subName = "sub1";
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
-
// 1. client connect
- Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
PersistentSubscription subRef = topicRef.getSubscription(subName);
@@ -550,7 +509,7 @@ int getAvailablePermits(PersistentSubscription sub) {
public void testUnloadNamespace() throws Exception {
String topic = "persistent://prop/use/ns-abc/topic-9";
TopicName topicName = TopicName.get(topic);
- pulsarClient.createProducer(topic);
+ pulsarClient.newProducer().topic(topic).create();
pulsarClient.close();
assertTrue(pulsar.getBrokerService().getTopicReference(topic) != null);
@@ -579,7 +538,7 @@ public void testUnloadNamespace() throws Exception {
public void testGC() throws Exception {
// 1. Simple successful GC
String topicName = "persistent://prop/use/ns-abc/topic-10";
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
producer.close();
assertNotNull(pulsar.getBrokerService().getTopicReference(topicName));
@@ -587,10 +546,8 @@ public void testGC() throws Exception {
assertNull(pulsar.getBrokerService().getTopicReference(topicName));
// 2. Topic is not GCed with live connection
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
String subName = "sub1";
- Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();
runGC();
assertNotNull(pulsar.getBrokerService().getTopicReference(topicName));
@@ -609,8 +566,8 @@ public void testGC() throws Exception {
}
/**
- * A topic that has retention policy set to non-0, should not be GCed
- * until it has been inactive for at least the retention time.
+ * A topic that has retention policy set to non-0, should not be GCed until it has been inactive for at least the
+ * retention time.
*/
@Test
public void testGcAndRetentionPolicy() throws Exception {
@@ -620,7 +577,7 @@ public void testGcAndRetentionPolicy() throws Exception {
// 1. Simple successful GC
String topicName = "persistent://prop/use/ns-abc/topic-10";
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
producer.close();
assertNotNull(pulsar.getBrokerService().getTopicReference(topicName));
@@ -628,16 +585,13 @@ public void testGcAndRetentionPolicy() throws Exception {
// Should not have been deleted, since we have retention
assertNotNull(pulsar.getBrokerService().getTopicReference(topicName));
-
// Remove retention
admin.namespaces().setRetention("prop/use/ns-abc", new RetentionPolicies(0, 10));
Thread.sleep(300);
// 2. Topic is not GCed with live connection
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
String subName = "sub1";
- Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();
runGC();
assertNotNull(pulsar.getBrokerService().getTopicReference(topicName));
@@ -656,9 +610,8 @@ public void testGcAndRetentionPolicy() throws Exception {
}
/**
- * A topic that has retention policy set to -1, should not be GCed
- * until it has been inactive for at least the retention time and the data
- * should never be deleted
+ * A topic that has retention policy set to -1, should not be GCed until it has been inactive for at least the
+ * retention time and the data should never be deleted
*/
@Test
public void testInfiniteRetentionPolicy() throws Exception {
@@ -667,7 +620,7 @@ public void testInfiniteRetentionPolicy() throws Exception {
// 1. Simple successful GC
String topicName = "persistent://prop/use/ns-abc/topic-10";
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
producer.close();
assertNotNull(pulsar.getBrokerService().getTopicReference(topicName));
@@ -675,16 +628,13 @@ public void testInfiniteRetentionPolicy() throws Exception {
// Should not have been deleted, since we have retention
assertNotNull(pulsar.getBrokerService().getTopicReference(topicName));
-
// Remove retention
admin.namespaces().setRetention("prop/use/ns-abc", new RetentionPolicies(0, 10));
Thread.sleep(300);
// 2. Topic is not GCed with live connection
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
String subName = "sub1";
- Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();
runGC();
assertNotNull(pulsar.getBrokerService().getTopicReference(topicName));
@@ -714,10 +664,7 @@ public void testMessageExpiry() throws Exception {
final String subName = "sub1";
final int numMsgs = 10;
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
-
- Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
PersistentSubscription subRef = topicRef.getSubscription(subName);
@@ -725,7 +672,7 @@ public void testMessageExpiry() throws Exception {
consumer.close();
assertFalse(subRef.getDispatcher().isConsumerConnected());
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
for (int i = 0; i < numMsgs; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
@@ -760,17 +707,14 @@ public void testMessageExpiryWithFewExpiredBacklog() throws Exception {
final String subName = "sub1";
final int numMsgs = 10;
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
-
- pulsarClient.subscribe(topicName, subName, conf);
+ pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
PersistentSubscription subRef = topicRef.getSubscription(subName);
assertTrue(subRef.getDispatcher().isConsumerConnected());
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
for (int i = 0; i < numMsgs; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
@@ -795,25 +739,18 @@ public void testSubscriptionTypeTransitions() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/shared-topic2";
final String subName = "sub2";
- ConsumerConfiguration conf1 = new ConsumerConfiguration();
- conf1.setSubscriptionType(SubscriptionType.Exclusive);
-
- ConsumerConfiguration conf2 = new ConsumerConfiguration();
- conf2.setSubscriptionType(SubscriptionType.Shared);
-
- ConsumerConfiguration conf3 = new ConsumerConfiguration();
- conf3.setSubscriptionType(SubscriptionType.Failover);
-
- Consumer consumer1 = pulsarClient.subscribe(topicName, subName, conf1);
- Consumer consumer2 = null;
- Consumer consumer3 = null;
+ Consumer<byte[]> consumer1 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .subscriptionType(SubscriptionType.Exclusive).subscribe();
+ Consumer<byte[]> consumer2 = null;
+ Consumer<byte[]> consumer3 = null;
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
PersistentSubscription subRef = topicRef.getSubscription(subName);
// 1. shared consumer on an exclusive sub fails
try {
- consumer2 = pulsarClient.subscribe(topicName, subName, conf2);
+ consumer2 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .subscriptionType(SubscriptionType.Shared).subscribe();
fail("should have failed");
} catch (PulsarClientException e) {
assertTrue(e.getMessage().contains("Subscription is of different type"));
@@ -821,7 +758,8 @@ public void testSubscriptionTypeTransitions() throws Exception {
// 2. failover consumer on an exclusive sub fails
try {
- consumer3 = pulsarClient.subscribe(topicName, subName, conf3);
+ consumer3 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .subscriptionType(SubscriptionType.Failover).subscribe();
fail("should have failed");
} catch (PulsarClientException e) {
assertTrue(e.getMessage().contains("Subscription is of different type"));
@@ -830,7 +768,8 @@ public void testSubscriptionTypeTransitions() throws Exception {
// 3. disconnected sub can be converted in shared
consumer1.close();
try {
- consumer2 = pulsarClient.subscribe(topicName, subName, conf2);
+ consumer2 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .subscriptionType(SubscriptionType.Shared).subscribe();
assertEquals(subRef.getDispatcher().getType(), SubType.Shared);
} catch (PulsarClientException e) {
fail("should not fail");
@@ -838,7 +777,8 @@ public void testSubscriptionTypeTransitions() throws Exception {
// 4. exclusive fails on shared sub
try {
- consumer1 = pulsarClient.subscribe(topicName, subName, conf1);
+ consumer1 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .subscriptionType(SubscriptionType.Exclusive).subscribe();
fail("should have failed");
} catch (PulsarClientException e) {
assertTrue(e.getMessage().contains("Subscription is of different type"));
@@ -847,7 +787,8 @@ public void testSubscriptionTypeTransitions() throws Exception {
// 5. disconnected sub can be converted in failover
consumer2.close();
try {
- consumer3 = pulsarClient.subscribe(topicName, subName, conf3);
+ consumer3 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .subscriptionType(SubscriptionType.Failover).subscribe();
assertEquals(subRef.getDispatcher().getType(), SubType.Failover);
} catch (PulsarClientException e) {
fail("should not fail");
@@ -856,7 +797,8 @@ public void testSubscriptionTypeTransitions() throws Exception {
// 5. exclusive consumer can connect after failover disconnects
consumer3.close();
try {
- consumer1 = pulsarClient.subscribe(topicName, subName, conf1);
+ consumer1 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .subscriptionType(SubscriptionType.Exclusive).subscribe();
assertEquals(subRef.getDispatcher().getType(), SubType.Exclusive);
} catch (PulsarClientException e) {
fail("should not fail");
@@ -871,16 +813,13 @@ public void testReceiveWithTimeout() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/topic-receive-timeout";
final String subName = "sub";
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
- conf.setReceiverQueueSize(1000);
-
- ConsumerImpl consumer = (ConsumerImpl) pulsarClient.subscribe(topicName, subName, conf);
- Producer producer = pulsarClient.createProducer(topicName);
+ ConsumerImpl<byte[]> consumer = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName(subName).receiverQueueSize(1000).subscribe();
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
assertEquals(consumer.getAvailablePermits(), 0);
- Message msg = consumer.receive(10, TimeUnit.MILLISECONDS);
+ Message<byte[]> msg = consumer.receive(10, TimeUnit.MILLISECONDS);
assertNull(msg);
assertEquals(consumer.getAvailablePermits(), 0);
@@ -903,7 +842,7 @@ public void testProducerReturnedMessageId() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/topic-xyz";
// 1. producer connect
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
assertNotNull(topicRef);
@@ -927,7 +866,7 @@ public void testProducerReturnedMessageId() throws Exception {
for (int i = SyncMessages; i < (SyncMessages + AsyncMessages); i++) {
String content = "my-message-" + i;
- Message msg = MessageBuilder.create().setContent(content.getBytes()).build();
+ Message<byte[]> msg = MessageBuilder.create().setContent(content.getBytes()).build();
final int index = i;
producer.sendAsync(msg).thenRun(() -> {
@@ -949,12 +888,11 @@ public void testProducerQueueFullBlocking() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/topic-xyzx";
final int messages = 10;
- PulsarClient client = PulsarClient.create(brokerUrl.toString());
+ PulsarClient client = PulsarClient.builder().serviceUrl(brokerUrl.toString()).build();
// 1. Producer connect
- ProducerConfiguration producerConfiguration = new ProducerConfiguration().setMaxPendingMessages(messages)
- .setBlockIfQueueFull(true).setSendTimeout(1, TimeUnit.SECONDS);
- ProducerImpl producer = (ProducerImpl) client.createProducer(topicName, producerConfiguration);
+ ProducerImpl<byte[]> producer = (ProducerImpl<byte[]>) client.newProducer().topic(topicName)
+ .maxPendingMessages(messages).blockIfQueueFull(true).sendTimeout(1, TimeUnit.SECONDS).create();
// 2. Stop broker
cleanup();
@@ -993,10 +931,9 @@ public void testProducerQueueFullNonBlocking() throws Exception {
final int messages = 10;
// 1. Producer connect
- PulsarClient client = PulsarClient.create(brokerUrl.toString());
- ProducerConfiguration producerConfiguration = new ProducerConfiguration().setMaxPendingMessages(messages)
- .setBlockIfQueueFull(false).setSendTimeout(1, TimeUnit.SECONDS);
- ProducerImpl producer = (ProducerImpl) client.createProducer(topicName, producerConfiguration);
+ PulsarClient client = PulsarClient.builder().serviceUrl(brokerUrl.toString()).build();
+ ProducerImpl<byte[]> producer = (ProducerImpl<byte[]>) client.newProducer().topic(topicName)
+ .maxPendingMessages(messages).blockIfQueueFull(false).sendTimeout(1, TimeUnit.SECONDS).create();
// 2. Stop broker
cleanup();
@@ -1038,8 +975,9 @@ public void testDeleteTopics() throws Exception {
BrokerService brokerService = pulsar.getBrokerService();
// 1. producers connect
- Producer producer1 = pulsarClient.createProducer("persistent://prop/use/ns-abc/topic-1");
- Producer producer2 = pulsarClient.createProducer("persistent://prop/use/ns-abc/topic-2");
+ Producer<byte[]> producer1 = pulsarClient.newProducer().topic("persistent://prop/use/ns-abc/topic-1").create();
+ /* Producer<byte[]> producer2 = */ pulsarClient.newProducer().topic("persistent://prop/use/ns-abc/topic-2")
+ .create();
brokerService.updateRates();
@@ -1078,11 +1016,9 @@ public void testCompression(CompressionType compressionType) throws Exception {
final String topicName = "persistent://prop/use/ns-abc/topic0" + compressionType;
// 1. producer connect
- ProducerConfiguration producerConf = new ProducerConfiguration();
- producerConf.setCompressionType(compressionType);
- Producer producer = pulsarClient.createProducer(topicName, producerConf);
-
- Consumer consumer = pulsarClient.subscribe(topicName, "my-sub");
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).compressionType(compressionType)
+ .create();
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").subscribe();
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
assertNotNull(topicRef);
@@ -1095,7 +1031,7 @@ public void testCompression(CompressionType compressionType) throws Exception {
}
for (int i = 0; i < 10; i++) {
- Message msg = consumer.receive(5, TimeUnit.SECONDS);
+ Message<byte[]> msg = consumer.receive(5, TimeUnit.SECONDS);
assertNotNull(msg);
assertEquals(msg.getData(), ("my-message-" + i).getBytes());
}
@@ -1116,8 +1052,7 @@ public void testBrokerTopicStats() throws Exception {
statsUpdater.shutdown();
final String namespace = "prop/use/ns-abc";
- ProducerConfiguration producerConf = new ProducerConfiguration();
- Producer producer = pulsarClient.createProducer("persistent://" + namespace + "/topic0", producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://" + namespace + "/topic0").create();
// 1. producer publish messages
for (int i = 0; i < 10; i++) {
String message = "my-message-" + i;
@@ -1146,11 +1081,10 @@ public void testPayloadCorruptionDetection() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/topic1";
// 1. producer connect
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").subscribe();
- Consumer consumer = pulsarClient.subscribe(topicName, "my-sub");
-
- Message msg1 = MessageBuilder.create().setContent("message-1".getBytes()).build();
+ Message<byte[]> msg1 = MessageBuilder.create().setContent("message-1".getBytes()).build();
CompletableFuture<MessageId> future1 = producer.sendAsync(msg1);
// Stop the broker, and publishes messages. Messages are accumulated in the producer queue and they're checksums
@@ -1158,8 +1092,7 @@ public void testPayloadCorruptionDetection() throws Exception {
// checksum validation error
stopBroker();
-
- Message msg2 = MessageBuilder.create().setContent("message-2".getBytes()).build();
+ Message<byte[]> msg2 = MessageBuilder.create().setContent("message-2".getBytes()).build();
CompletableFuture<MessageId> future2 = producer.sendAsync(msg2);
// Taint msg2
@@ -1178,7 +1111,7 @@ public void testPayloadCorruptionDetection() throws Exception {
}
// We should only receive msg1
- Message msg = consumer.receive(1, TimeUnit.SECONDS);
+ Message<byte[]> msg = consumer.receive(1, TimeUnit.SECONDS);
assertEquals(new String(msg.getData()), "message-1");
while ((msg = consumer.receive(1, TimeUnit.SECONDS)) != null) {
@@ -1187,25 +1120,23 @@ public void testPayloadCorruptionDetection() throws Exception {
}
/**
- * Verify: Broker should not replay already acknowledged messages again and should clear them from messageReplay bucket
+ * Verify: Broker should not replay already acknowledged messages again and should clear them from messageReplay
+ * bucket
*
- * 1. produce messages
- * 2. consume messages and ack all except 1 msg
- * 3. Verification: should replay only 1 unacked message
+ * 1. produce messages 2. consume messages and ack all except 1 msg 3. Verification: should replay only 1 unacked
+ * message
*/
@Test()
public void testMessageRedelivery() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/topic2";
final String subName = "sub2";
- Message msg;
+ Message<byte[]> msg;
int totalMessages = 10;
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Shared);
-
- Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
- Producer producer = pulsarClient.createProducer(topicName);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .subscriptionType(SubscriptionType.Shared).subscribe();
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
// (1) Produce messages
for (int i = 0; i < totalMessages; i++) {
@@ -1213,8 +1144,8 @@ public void testMessageRedelivery() throws Exception {
producer.send(message.getBytes());
}
- //(2) Consume and ack messages except first message
- Message unAckedMsg = null;
+ // (2) Consume and ack messages except first message
+ Message<byte[]> unAckedMsg = null;
for (int i = 0; i < totalMessages; i++) {
msg = consumer.receive();
if (i == 0) {
@@ -1243,10 +1174,8 @@ public void testMessageRedelivery() throws Exception {
}
/**
- * Verify:
- * 1. Broker should not replay already acknowledged messages
- * 2. Dispatcher should not stuck while dispatching new messages due to previous-replay
- * of invalid/already-acked messages
+ * Verify: 1. Broker should not replay already acknowledged messages 2. Dispatcher should not stuck while
+ * dispatching new messages due to previous-replay of invalid/already-acked messages
*
* @throws Exception
*/
@@ -1256,16 +1185,13 @@ public void testMessageReplay() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/topic2";
final String subName = "sub2";
- Message msg;
+ Message<byte[]> msg;
int totalMessages = 10;
int replayIndex = totalMessages / 2;
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Shared);
- conf.setReceiverQueueSize(1);
-
- Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
- Producer producer = pulsarClient.createProducer(topicName);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .subscriptionType(SubscriptionType.Shared).receiverQueueSize(1).subscribe();
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
assertNotNull(topicRef);
@@ -1327,13 +1253,12 @@ public void testMessageReplay() throws Exception {
public void testCreateProducerWithSameName() throws Exception {
String topic = "persistent://prop/use/ns-abc/testCreateProducerWithSameName";
- ProducerConfiguration conf = new ProducerConfiguration();
- conf.setProducerName("test-producer-a");
-
- Producer p1 = pulsarClient.createProducer(topic, conf);
+ ProducerBuilder<byte[]> producerBuilder = pulsarClient.newProducer().topic(topic)
+ .producerName("test-producer-a");
+ Producer<byte[]> p1 = producerBuilder.create();
try {
- pulsarClient.createProducer(topic, conf);
+ producerBuilder.create();
fail("Should have thrown ProducerBusyException");
} catch (ProducerBusyException e) {
// Expected
@@ -1342,7 +1267,7 @@ public void testCreateProducerWithSameName() throws Exception {
p1.close();
// Now p2 should succeed
- Producer p2 = pulsarClient.createProducer(topic, conf);
+ Producer<byte[]> p2 = producerBuilder.create();
p2.close();
}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java
index c1b44c0b6..6392bb619 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java
@@ -91,6 +91,7 @@
import org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType;
import org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe;
import org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.SubType;
+import org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.InitialPosition;
import org.apache.pulsar.common.api.proto.PulsarApi.MessageMetadata;
import org.apache.pulsar.common.naming.NamespaceBundle;
import org.apache.pulsar.common.naming.TopicName;
@@ -420,7 +421,7 @@ public void testSubscribeFail() throws Exception {
.setSubscription("").setRequestId(1).setSubType(SubType.Exclusive).build();
Future<Consumer> f1 = topic.subscribe(serverCnx, cmd.getSubscription(), cmd.getConsumerId(), cmd.getSubType(),
- 0, cmd.getConsumerName(), cmd.getDurable(), null, Collections.emptyMap(), cmd.getReadCompacted());
+ 0, cmd.getConsumerName(), cmd.getDurable(), null, Collections.emptyMap(), cmd.getReadCompacted(), cmd.getInitialPosition());
try {
f1.get();
fail("should fail with exception");
@@ -439,12 +440,12 @@ public void testSubscribeUnsubscribe() throws Exception {
// 1. simple subscribe
Future<Consumer> f1 = topic.subscribe(serverCnx, cmd.getSubscription(), cmd.getConsumerId(), cmd.getSubType(),
- 0, cmd.getConsumerName(), cmd.getDurable(), null, Collections.emptyMap(), cmd.getReadCompacted());
+ 0, cmd.getConsumerName(), cmd.getDurable(), null, Collections.emptyMap(), cmd.getReadCompacted(), cmd.getInitialPosition());
f1.get();
// 2. duplicate subscribe
Future<Consumer> f2 = topic.subscribe(serverCnx, cmd.getSubscription(), cmd.getConsumerId(), cmd.getSubType(),
- 0, cmd.getConsumerName(), cmd.getDurable(), null, Collections.emptyMap(), cmd.getReadCompacted());
+ 0, cmd.getConsumerName(), cmd.getDurable(), null, Collections.emptyMap(), cmd.getReadCompacted(), cmd.getInitialPosition());
try {
f2.get();
@@ -731,7 +732,7 @@ public void testDeleteTopic() throws Exception {
.setSubscription(successSubName).setRequestId(1).setSubType(SubType.Exclusive).build();
Future<Consumer> f1 = topic.subscribe(serverCnx, cmd.getSubscription(), cmd.getConsumerId(), cmd.getSubType(),
- 0, cmd.getConsumerName(), cmd.getDurable(), null, Collections.emptyMap(), false /* read compacted */);
+ 0, cmd.getConsumerName(), cmd.getDurable(), null, Collections.emptyMap(), false /* read compacted */, cmd.getInitialPosition());
f1.get();
assertTrue(topic.delete().isCompletedExceptionally());
@@ -746,7 +747,7 @@ public void testDeleteAndUnsubscribeTopic() throws Exception {
.setSubscription(successSubName).setRequestId(1).setSubType(SubType.Exclusive).build();
Future<Consumer> f1 = topic.subscribe(serverCnx, cmd.getSubscription(), cmd.getConsumerId(), cmd.getSubType(),
- 0, cmd.getConsumerName(), cmd.getDurable(), null, Collections.emptyMap(), cmd.getReadCompacted());
+ 0, cmd.getConsumerName(), cmd.getDurable(), null, Collections.emptyMap(), cmd.getReadCompacted(), cmd.getInitialPosition());
f1.get();
final CyclicBarrier barrier = new CyclicBarrier(2);
@@ -800,7 +801,7 @@ public void testConcurrentTopicAndSubscriptionDelete() throws Exception {
.setSubscription(successSubName).setRequestId(1).setSubType(SubType.Exclusive).build();
Future<Consumer> f1 = topic.subscribe(serverCnx, cmd.getSubscription(), cmd.getConsumerId(), cmd.getSubType(),
- 0, cmd.getConsumerName(), cmd.getDurable(), null, Collections.emptyMap(), cmd.getReadCompacted());
+ 0, cmd.getConsumerName(), cmd.getDurable(), null, Collections.emptyMap(), cmd.getReadCompacted(), cmd.getInitialPosition());
f1.get();
final CyclicBarrier barrier = new CyclicBarrier(2);
@@ -887,8 +888,8 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
.setSubscription(successSubName).setRequestId(1).setSubType(SubType.Exclusive).build();
Future<Consumer> f = topic.subscribe(serverCnx, cmd.getSubscription(), cmd.getConsumerId(), cmd.getSubType(),
- 0, cmd.getConsumerName(), cmd.getDurable(), null, Collections.emptyMap(), cmd.getReadCompacted());
-
+ 0, cmd.getConsumerName(), cmd.getDurable(), null, Collections.emptyMap(), cmd.getReadCompacted(), cmd.getInitialPosition());
+
try {
f.get();
fail("should have failed");
@@ -962,14 +963,16 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
}
}).when(ledgerMock).asyncAddEntry(any(ByteBuf.class), any(AddEntryCallback.class), anyObject());
+
// call openCursorComplete on cursor asyncOpen
doAnswer(new Answer<Object>() {
@Override
public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
- ((OpenCursorCallback) invocationOnMock.getArguments()[1]).openCursorComplete(cursorMock, null);
+ ((OpenCursorCallback) invocationOnMock.getArguments()[2]).openCursorComplete(cursorMock, null);
return null;
}
- }).when(ledgerMock).asyncOpenCursor(matches(".*success.*"), any(OpenCursorCallback.class), anyObject());
+ }).when(ledgerMock).asyncOpenCursor(matches(".*success.*"), any(InitialPosition.class), any(OpenCursorCallback.class), anyObject());
+
// call deleteLedgerComplete on ledger asyncDelete
doAnswer(new Answer<Object>() {
@@ -1005,7 +1008,7 @@ public void testFailoverSubscription() throws Exception {
// 1. Subscribe with non partition topic
Future<Consumer> f1 = topic1.subscribe(serverCnx, cmd1.getSubscription(), cmd1.getConsumerId(),
cmd1.getSubType(), 0, cmd1.getConsumerName(), cmd1.getDurable(), null, Collections.emptyMap(),
- cmd1.getReadCompacted());
+ cmd1.getReadCompacted(), cmd1.getInitialPosition());
f1.get();
// 2. Subscribe with partition topic
@@ -1017,7 +1020,7 @@ public void testFailoverSubscription() throws Exception {
Future<Consumer> f2 = topic2.subscribe(serverCnx, cmd2.getSubscription(), cmd2.getConsumerId(),
cmd2.getSubType(), 0, cmd2.getConsumerName(), cmd2.getDurable(), null, Collections.emptyMap(),
- cmd2.getReadCompacted());
+ cmd2.getReadCompacted(), cmd2.getInitialPosition());
f2.get();
// 3. Subscribe and create second consumer
@@ -1027,7 +1030,7 @@ public void testFailoverSubscription() throws Exception {
Future<Consumer> f3 = topic2.subscribe(serverCnx, cmd3.getSubscription(), cmd3.getConsumerId(),
cmd3.getSubType(), 0, cmd3.getConsumerName(), cmd3.getDurable(), null, Collections.emptyMap(),
- cmd3.getReadCompacted());
+ cmd3.getReadCompacted(), cmd3.getInitialPosition());
f3.get();
assertEquals(
@@ -1048,7 +1051,7 @@ public void testFailoverSubscription() throws Exception {
Future<Consumer> f4 = topic2.subscribe(serverCnx, cmd4.getSubscription(), cmd4.getConsumerId(),
cmd4.getSubType(), 0, cmd4.getConsumerName(), cmd4.getDurable(), null, Collections.emptyMap(),
- cmd4.getReadCompacted());
+ cmd4.getReadCompacted(), cmd4.getInitialPosition());
f4.get();
assertEquals(
@@ -1074,7 +1077,7 @@ public void testFailoverSubscription() throws Exception {
Future<Consumer> f5 = topic2.subscribe(serverCnx, cmd5.getSubscription(), cmd5.getConsumerId(),
cmd5.getSubType(), 0, cmd5.getConsumerName(), cmd5.getDurable(), null, Collections.emptyMap(),
- cmd5.getReadCompacted());
+ cmd5.getReadCompacted(), cmd5.getInitialPosition());
try {
f5.get();
@@ -1091,7 +1094,7 @@ public void testFailoverSubscription() throws Exception {
Future<Consumer> f6 = topic2.subscribe(serverCnx, cmd6.getSubscription(), cmd6.getConsumerId(),
cmd6.getSubType(), 0, cmd6.getConsumerName(), cmd6.getDurable(), null, Collections.emptyMap(),
- cmd6.getReadCompacted());
+ cmd6.getReadCompacted(), cmd6.getInitialPosition());
f6.get();
// 7. unsubscribe exclusive sub
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java
index 6f9a51c26..799a07d43 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java
@@ -34,7 +34,6 @@
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
-import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
@@ -52,12 +51,12 @@
import org.apache.pulsar.broker.service.persistent.PersistentReplicator;
import org.apache.pulsar.broker.service.persistent.PersistentTopic;
import org.apache.pulsar.client.admin.PulsarAdminException.PreconditionFailedException;
-import org.apache.pulsar.client.api.ClientConfiguration;
import org.apache.pulsar.client.api.MessageBuilder;
import org.apache.pulsar.client.api.Producer;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.RawMessage;
import org.apache.pulsar.client.api.RawReader;
+import org.apache.pulsar.client.api.Schema;
import org.apache.pulsar.client.impl.ProducerImpl;
import org.apache.pulsar.client.impl.PulsarClientImpl;
import org.apache.pulsar.client.impl.conf.ProducerConfigurationData;
@@ -96,7 +95,6 @@ public void beforeMethod(Method m) throws Exception {
methodName = m.getName();
}
-
@Override
@BeforeClass(timeOut = 30000)
void setup() throws Exception {
@@ -106,13 +104,7 @@ void setup() throws Exception {
@Override
@AfterClass(timeOut = 30000)
void shutdown() throws Exception {
- ForkJoinPool.commonPool().execute(() -> {
- try {
- super.shutdown();
- } catch (Exception e) {
- e.printStackTrace();
- }
- });
+ super.shutdown();
}
@DataProvider(name = "partitionedTopic")
@@ -128,8 +120,7 @@ public void testConfigChange() throws Exception {
// Run a set of producer tasks to create the topics
List<Future<Void>> results = Lists.newArrayList();
for (int i = 0; i < 10; i++) {
- final TopicName dest = TopicName
- .get(String.format("persistent://pulsar/global/ns/topic-%d", i));
+ final TopicName dest = TopicName.get(String.format("persistent://pulsar/global/ns/topic-%d", i));
results.add(executor.submit(new Callable<Void>() {
@Override
@@ -205,6 +196,7 @@ public Void call() throws Exception {
// Case 3: TODO: Once automatic cleanup is implemented, add tests case to verify auto removal of clusters
}
+ @SuppressWarnings("unchecked")
@Test(timeOut = 30000)
public void testConcurrentReplicator() throws Exception {
@@ -213,13 +205,13 @@ public void testConcurrentReplicator() throws Exception {
final String namespace = "pulsar/global/concurrent";
admin1.namespaces().createNamespace(namespace);
admin1.namespaces().setNamespaceReplicationClusters(namespace, Lists.newArrayList("r1", "r2"));
- final TopicName dest = TopicName.get(String.format("persistent://" + namespace + "/topic-%d", 0));
- ClientConfiguration conf = new ClientConfiguration();
- conf.setStatsInterval(0, TimeUnit.SECONDS);
- Producer producer = PulsarClient.create(url1.toString(), conf).createProducer(dest.toString());
+ final TopicName topicName = TopicName.get(String.format("persistent://" + namespace + "/topic-%d", 0));
+ PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).statsInterval(0, TimeUnit.SECONDS)
+ .build();
+ Producer<byte[]> producer = client1.newProducer().topic(topicName.toString()).create();
producer.close();
- PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopic(dest.toString()).get();
+ PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopic(topicName.toString()).get();
PulsarClientImpl pulsarClient = spy((PulsarClientImpl) pulsar1.getBrokerService().getReplicationClient("r3"));
final Method startRepl = PersistentTopic.class.getDeclaredMethod("startReplicator", String.class);
@@ -244,8 +236,10 @@ public void testConcurrentReplicator() throws Exception {
}
Thread.sleep(3000);
- Mockito.verify(pulsarClient, Mockito.times(1)).createProducerAsync(Mockito.any(ProducerConfigurationData.class));
+ Mockito.verify(pulsarClient, Mockito.times(1)).createProducerAsync(Mockito.any(ProducerConfigurationData.class),
+ Mockito.any(Schema.class));
+ client1.shutdown();
}
@Test(enabled = false, timeOut = 30000)
@@ -293,8 +287,7 @@ public void testReplication() throws Exception {
SortedSet<String> testDests = new TreeSet<String>();
List<Future<Void>> results = Lists.newArrayList();
for (int i = 0; i < 3; i++) {
- final TopicName dest = TopicName
- .get(String.format("persistent://pulsar/global/ns/repltopic-%d", i));
+ final TopicName dest = TopicName.get(String.format("persistent://pulsar/global/ns/repltopic-%d", i));
testDests.add(dest.toString());
results.add(executor.submit(new Callable<Void>() {
@@ -394,8 +387,7 @@ public void testReplicationOverrides() throws Exception {
SortedSet<String> testDests = new TreeSet<String>();
List<Future<Void>> results = Lists.newArrayList();
for (int i = 0; i < 10; i++) {
- final TopicName dest = TopicName
- .get(String.format("persistent://pulsar/global/ns/repltopic-%d", i));
+ final TopicName dest = TopicName.get(String.format("persistent://pulsar/global/ns/repltopic-%d", i));
testDests.add(dest.toString());
results.add(executor.submit(new Callable<Void>() {
@@ -420,11 +412,6 @@ public Void call() throws Exception {
MessageConsumer consumer3 = new MessageConsumer(url3, dest);
log.info("--- Starting Consumer --- " + url3);
- // Produce from cluster1 for this test
- int nr1 = 0;
- int nr2 = 0;
- int nR3 = 0;
-
// Produce a message that isn't replicated
producer1.produce(1, MessageBuilder.create().disableReplication());
@@ -474,8 +461,7 @@ public void testFailures() throws Exception {
try {
// 1. Create a consumer using the reserved consumer id prefix "pulsar.repl."
- final TopicName dest = TopicName
- .get(String.format("persistent://pulsar/global/ns/res-cons-id"));
+ final TopicName dest = TopicName.get(String.format("persistent://pulsar/global/ns/res-cons-id"));
// Create another consumer using replication prefix as sub id
MessageConsumer consumer = new MessageConsumer(url2, dest, "pulsar.repl.");
@@ -502,7 +488,8 @@ public void testReplicatePeekAndSkip() throws Exception {
producer1.produce(2);
producer1.close();
PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(dest.toString());
- PersistentReplicator replicator = (PersistentReplicator) topic.getReplicators().get(topic.getReplicators().keys().get(0));
+ PersistentReplicator replicator = (PersistentReplicator) topic.getReplicators()
+ .get(topic.getReplicators().keys().get(0));
replicator.skipMessages(2);
CompletableFuture<Entry> result = replicator.peekNthMessage(1);
Entry entry = result.get(50, TimeUnit.MILLISECONDS);
@@ -526,7 +513,8 @@ public void testReplicatorClearBacklog() throws Exception {
producer1.produce(2);
producer1.close();
PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(dest.toString());
- PersistentReplicator replicator = (PersistentReplicator) spy(topic.getReplicators().get(topic.getReplicators().keys().get(0)));
+ PersistentReplicator replicator = (PersistentReplicator) spy(
+ topic.getReplicators().get(topic.getReplicators().keys().get(0)));
replicator.readEntriesFailed(new ManagedLedgerException.InvalidCursorPositionException("failed"), null);
replicator.clearBacklog().get();
Thread.sleep(100);
@@ -546,8 +534,7 @@ public void testResetCursorNotFail() throws Exception {
SortedSet<String> testDests = new TreeSet<String>();
List<Future<Void>> results = Lists.newArrayList();
for (int i = 0; i < 1; i++) {
- final TopicName dest = TopicName
- .get(String.format("persistent://pulsar/global/ns/resetrepltopic-%d", i));
+ final TopicName dest = TopicName.get(String.format("persistent://pulsar/global/ns/resetrepltopic-%d", i));
testDests.add(dest.toString());
results.add(executor.submit(new Callable<Void>() {
@@ -592,8 +579,7 @@ public void testReplicationForBatchMessages() throws Exception {
SortedSet<String> testDests = new TreeSet<String>();
List<Future<Void>> results = Lists.newArrayList();
for (int i = 0; i < 3; i++) {
- final TopicName dest = TopicName
- .get(String.format("persistent://pulsar/global/ns/repltopicbatch-%d", i));
+ final TopicName dest = TopicName.get(String.format("persistent://pulsar/global/ns/repltopicbatch-%d", i));
testDests.add(dest.toString());
results.add(executor.submit(new Callable<Void>() {
@@ -636,6 +622,12 @@ public Void call() throws Exception {
consumer3.receive(10);
+ producer1.close();
+ producer2.close();
+ producer3.close();
+ consumer1.close();
+ consumer2.close();
+ consumer3.close();
return null;
}
}));
@@ -686,6 +678,7 @@ public void deleteCursorFailed(ManagedLedgerException exception, Object ctx) {
removeReplicator.setAccessible(true);
// invoke removeReplicator : it fails as cursor is not present: but still it should remove the replicator from
// list without restarting it
+ @SuppressWarnings("unchecked")
CompletableFuture<Void> result = (CompletableFuture<Void>) removeReplicator.invoke(topic,
replicatorClusterName);
result.thenApply((v) -> {
@@ -696,6 +689,7 @@ public void deleteCursorFailed(ManagedLedgerException exception, Object ctx) {
producer1.close();
}
+ @SuppressWarnings("unchecked")
@Test(priority = 5, timeOut = 30000)
public void testReplicatorProducerClosing() throws Exception {
log.info("--- Starting ReplicatorTest::testDeleteReplicatorFailure ---");
@@ -711,7 +705,7 @@ public void testReplicatorProducerClosing() throws Exception {
Thread.sleep(100);
Field field = AbstractReplicator.class.getDeclaredField("producer");
field.setAccessible(true);
- ProducerImpl producer = (ProducerImpl) field.get(replicator);
+ ProducerImpl<byte[]> producer = (ProducerImpl<byte[]>) field.get(replicator);
assertNull(producer);
producer1.close();
}
@@ -749,7 +743,8 @@ public void testResumptionAfterBacklogRelaxed() throws Exception {
PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(dest.toString());
Replicator replicator = topic.getPersistentReplicator("r2");
- // Produce 1 message in r1. This message will be replicated immediately into r2 and it will become part of local backlog
+ // Produce 1 message in r1. This message will be replicated immediately into r2 and it will become part of
+ // local backlog
producer1.produce(1);
Thread.sleep(500);
@@ -840,7 +835,8 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
// Replicator producer must be closed
Field producerField = AbstractReplicator.class.getDeclaredField("producer");
producerField.setAccessible(true);
- ProducerImpl replicatorProducer = (ProducerImpl) producerField.get(replicator);
+ @SuppressWarnings("unchecked")
+ ProducerImpl<byte[]> replicatorProducer = (ProducerImpl<byte[]>) producerField.get(replicator);
assertEquals(replicatorProducer, null);
producer1.close();
@@ -852,10 +848,10 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
public void verifyChecksumAfterReplication() throws Exception {
final String topicName = "persistent://pulsar/global/ns/checksumAfterReplication";
- PulsarClient c1 = PulsarClient.create(url1.toString());
- Producer p1 = c1.createProducer(topicName);
+ PulsarClient c1 = PulsarClient.builder().serviceUrl(url1.toString()).build();
+ Producer<byte[]> p1 = c1.newProducer().topic(topicName).create();
- PulsarClient c2 = PulsarClient.create(url2.toString());
+ PulsarClient c2 = PulsarClient.builder().serviceUrl(url2.toString()).build();
RawReader reader2 = RawReader.create(c2, topicName, "sub").get();
p1.send("Hello".getBytes());
@@ -899,8 +895,8 @@ public void testReplicatorOnPartitionedTopic(boolean isPartitionedTopic) throws
}
// load namespace with dummy topic on ns
- PulsarClient client = PulsarClient.create(url1.toString());
- client.createProducer("persistent://" + namespace + "/dummyTopic");
+ PulsarClient client = PulsarClient.builder().serviceUrl(url1.toString()).build();
+ client.newProducer().topic("persistent://" + namespace + "/dummyTopic").create();
// persistent topic test
try {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTestBase.java
index d71f48491..abde14255 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTestBase.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTestBase.java
@@ -32,16 +32,14 @@
import org.apache.bookkeeper.test.PortManager;
import org.apache.pulsar.broker.PulsarService;
import org.apache.pulsar.broker.ServiceConfiguration;
-import org.apache.pulsar.broker.service.BrokerService;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.api.Authentication;
-import org.apache.pulsar.client.api.ClientConfiguration;
import org.apache.pulsar.client.api.Consumer;
import org.apache.pulsar.client.api.Message;
import org.apache.pulsar.client.api.MessageBuilder;
import org.apache.pulsar.client.api.MessageId;
import org.apache.pulsar.client.api.Producer;
-import org.apache.pulsar.client.api.ProducerConfiguration;
+import org.apache.pulsar.client.api.ProducerBuilder;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.policies.data.ClusterData;
@@ -263,16 +261,14 @@ void shutdown() throws Exception {
String namespace;
String topicName;
PulsarClient client;
- Producer producer;
+ Producer<byte[]> producer;
MessageProducer(URL url, final TopicName dest) throws Exception {
this.url = url;
this.namespace = dest.getNamespace();
this.topicName = dest.toString();
- ClientConfiguration conf = new ClientConfiguration();
- conf.setStatsInterval(0, TimeUnit.SECONDS);
- client = PulsarClient.create(url.toString(), conf);
- producer = client.createProducer(topicName);
+ client = PulsarClient.builder().serviceUrl(url.toString()).statsInterval(0, TimeUnit.SECONDS).build();
+ producer = client.newProducer().topic(topicName).create();
}
@@ -280,16 +276,14 @@ void shutdown() throws Exception {
this.url = url;
this.namespace = dest.getNamespace();
this.topicName = dest.toString();
- ClientConfiguration conf = new ClientConfiguration();
- conf.setStatsInterval(0, TimeUnit.SECONDS);
- client = PulsarClient.create(url.toString(), conf);
- ProducerConfiguration producerConfiguration = new ProducerConfiguration();
+ client = PulsarClient.builder().serviceUrl(url.toString()).statsInterval(0, TimeUnit.SECONDS).build();
+ ProducerBuilder<byte[]> producerBuilder = client.newProducer().topic(topicName);
if (batch) {
- producerConfiguration.setBatchingEnabled(true);
- producerConfiguration.setBatchingMaxPublishDelay(1, TimeUnit.SECONDS);
- producerConfiguration.setBatchingMaxMessages(5);
+ producerBuilder.enableBatching(true);
+ producerBuilder.batchingMaxPublishDelay(1, TimeUnit.SECONDS);
+ producerBuilder.batchingMaxMessages(5);
}
- producer = client.createProducer(topicName, producerConfiguration);
+ producer = producerBuilder.create();
}
@@ -314,7 +308,7 @@ void produce(int messages) throws Exception {
}
- void produce(int messages, MessageBuilder messageBuilder) throws Exception {
+ void produce(int messages, MessageBuilder<byte[]> messageBuilder) throws Exception {
log.info("Start sending messages");
for (int i = 0; i < messages; i++) {
final String m = new String("test-builder-" + i);
@@ -335,7 +329,7 @@ void close() throws Exception {
final String namespace;
final String topicName;
final PulsarClient client;
- final Consumer consumer;
+ final Consumer<byte[]> consumer;
MessageConsumer(URL url, final TopicName dest) throws Exception {
this(url, dest, "sub-id");
@@ -345,12 +339,11 @@ void close() throws Exception {
this.url = url;
this.namespace = dest.getNamespace();
this.topicName = dest.toString();
- ClientConfiguration conf = new ClientConfiguration();
- conf.setStatsInterval(0, TimeUnit.SECONDS);
- client = PulsarClient.create(url.toString(), conf);
+
+ client = PulsarClient.builder().serviceUrl(url.toString()).statsInterval(0, TimeUnit.SECONDS).build();
try {
- consumer = client.subscribe(topicName, subId);
+ consumer = client.newConsumer().topic(topicName).subscriptionName(subId).subscribe();
} catch (Exception e) {
client.close();
throw e;
@@ -359,7 +352,7 @@ void close() throws Exception {
void receive(int messages) throws Exception {
log.info("Start receiving messages");
- Message msg = null;
+ Message<byte[]> msg = null;
for (int i = 0; i < messages; i++) {
msg = consumer.receive();
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ResendRequestTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ResendRequestTest.java
index a429180bd..433904ca7 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ResendRequestTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ResendRequestTest.java
@@ -30,13 +30,12 @@
import org.apache.pulsar.broker.service.persistent.PersistentTopic;
import org.apache.pulsar.client.api.Consumer;
-import org.apache.pulsar.client.api.ConsumerConfiguration;
+import org.apache.pulsar.client.api.ConsumerBuilder;
import org.apache.pulsar.client.api.Message;
import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageRoutingMode;
import org.apache.pulsar.client.api.Producer;
-import org.apache.pulsar.client.api.ProducerConfiguration;
import org.apache.pulsar.client.api.SubscriptionType;
-import org.apache.pulsar.client.api.ProducerConfiguration.MessageRoutingMode;
import org.apache.pulsar.client.impl.ConsumerBase;
import org.apache.pulsar.common.policies.data.PropertyAdmin;
import org.slf4j.Logger;
@@ -74,16 +73,15 @@ public void testExclusiveSingleAckedNormalTopic() throws Exception {
HashSet<String> messageDataHashSet = new HashSet<String>();
// 1. producer connect
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
assertNotNull(topicRef);
assertEquals(topicRef.getProducers().size(), 1);
// 2. Create consumer
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setReceiverQueueSize(7);
- Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName, conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
+ .receiverQueueSize(7).subscribe();
// 3. producer publish messages
for (int i = 0; i < totalMessages; i++) {
@@ -92,11 +90,11 @@ public void testExclusiveSingleAckedNormalTopic() throws Exception {
}
// 4. Receive messages
- Message message = consumer.receive();
+ Message<byte[]> message = consumer.receive();
log.info("Message received " + new String(message.getData()));
-
+
for (int i = 1; i < totalMessages; i++) {
- Message msg = consumer.receive();
+ Message<byte[]> msg = consumer.receive();
log.info("Message received " + new String(msg.getData()));
messageDataHashSet.add(new String(msg.getData()));
}
@@ -107,7 +105,7 @@ public void testExclusiveSingleAckedNormalTopic() throws Exception {
log.info("Message acked " + new String(message.getData()));
messageIdHashSet.add(message.getMessageId());
messageDataHashSet.add(new String(message.getData()));
-
+
consumer.redeliverUnacknowledgedMessages();
log.info("Resend Messages Request sent");
@@ -138,7 +136,7 @@ public void testExclusiveSingleAckedNormalTopic() throws Exception {
messageDataHashSet.add(new String(message.getData()));
message = consumer.receive(5000, TimeUnit.MILLISECONDS);
}
-
+
assertEquals(messageIdHashSet.size(), totalMessages);
assertEquals(messageDataHashSet.size(), totalMessages);
printIncomingMessageQueue(consumer);
@@ -146,7 +144,7 @@ public void testExclusiveSingleAckedNormalTopic() throws Exception {
// 9. Calling resend after acking all messages - expectin 0 messages
consumer.redeliverUnacknowledgedMessages();
assertEquals(consumer.receive(2000, TimeUnit.MILLISECONDS), null);
-
+
// 10. Checking message contents
for (int i = 0; i < totalMessages; i++) {
assertTrue(messageDataHashSet.contains(messagePredicate + i));
@@ -161,17 +159,17 @@ public void testSharedSingleAckedNormalTopic() throws Exception {
final String messagePredicate = "my-message-" + key + "-";
final int totalMessages = 10;
// 1. producer connect
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
assertNotNull(topicRef);
assertEquals(topicRef.getProducers().size(), 1);
// 2. Create consumer
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setReceiverQueueSize(totalMessages / 2);
- conf.setSubscriptionType(SubscriptionType.Shared);
- Consumer consumer1 = pulsarClient.subscribe(topicName, subscriptionName, conf);
- Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, conf);
+ ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName(subscriptionName).receiverQueueSize(totalMessages / 2)
+ .subscriptionType(SubscriptionType.Shared);
+ Consumer<byte[]> consumer1 = consumerBuilder.subscribe();
+ Consumer<byte[]> consumer2 = consumerBuilder.subscribe();
// 3. Producer publishes messages
for (int i = 0; i < totalMessages; i++) {
@@ -182,8 +180,8 @@ public void testSharedSingleAckedNormalTopic() throws Exception {
// 4. Receive messages
int receivedConsumer1 = 0, receivedConsumer2 = 0;
- Message message1 = consumer1.receive();
- Message message2 = consumer2.receive();
+ Message<byte[]> message1 = consumer1.receive();
+ Message<byte[]> message2 = consumer2.receive();
do {
if (message1 != null) {
log.info("Consumer 1 Received: " + new String(message1.getData()));
@@ -241,19 +239,16 @@ public void testFailoverSingleAckedNormalTopic() throws Exception {
final String messagePredicate = "my-message-" + key + "-";
final int totalMessages = 10;
// 1. producer connect
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
assertNotNull(topicRef);
assertEquals(topicRef.getProducers().size(), 1);
// 2. Create consumer
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setReceiverQueueSize(10);
- conf.setSubscriptionType(SubscriptionType.Failover);
- conf.setConsumerName("consumer-1");
- Consumer consumer1 = pulsarClient.subscribe(topicName, subscriptionName, conf);
- conf.setConsumerName("consumer-2");
- Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, conf);
+ ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName(subscriptionName).receiverQueueSize(10).subscriptionType(SubscriptionType.Failover);
+ Consumer<byte[]> consumer1 = consumerBuilder.clone().consumerName("consumer-1").subscribe();
+ Consumer<byte[]> consumer2 = consumerBuilder.clone().consumerName("consumer-2").subscribe();
// 3. Producer publishes messages
for (int i = 0; i < totalMessages; i++) {
@@ -264,8 +259,8 @@ public void testFailoverSingleAckedNormalTopic() throws Exception {
// 4. Receive messages
int receivedConsumer1 = 0, receivedConsumer2 = 0;
- Message message1;
- Message message2;
+ Message<byte[]> message1;
+ Message<byte[]> message2;
do {
message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
@@ -359,16 +354,15 @@ public void testExclusiveCumulativeAckedNormalTopic() throws Exception {
final int totalMessages = 10;
// 1. producer connect
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
assertNotNull(topicRef);
assertEquals(topicRef.getProducers().size(), 1);
// 2. Create consumer
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setReceiverQueueSize(7);
- Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName, conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
+ .receiverQueueSize(7).subscribe();
// 3. producer publish messages
for (int i = 0; i < totalMessages; i++) {
@@ -377,7 +371,7 @@ public void testExclusiveCumulativeAckedNormalTopic() throws Exception {
}
// 4. Receive messages
- Message message = consumer.receive();
+ Message<byte[]> message = consumer.receive();
log.info("Message received " + new String(message.getData()));
for (int i = 0; i < 7; i++) {
printIncomingMessageQueue(consumer);
@@ -418,14 +412,12 @@ public void testExclusiveSingleAckedPartitionedTopic() throws Exception {
// Special step to create partitioned topic
// 1. producer connect
- ProducerConfiguration prodConfig = new ProducerConfiguration();
- prodConfig.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
- Producer producer = pulsarClient.createProducer(topicName, prodConfig);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
+ .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();
// 2. Create consumer
- ConsumerConfiguration consumerConfig = new ConsumerConfiguration();
- consumerConfig.setReceiverQueueSize(7);
- Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName, consumerConfig);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
+ .receiverQueueSize(7).subscribe();
// 3. producer publish messages
for (int i = 0; i < totalMessages; i++) {
@@ -435,7 +427,7 @@ public void testExclusiveSingleAckedPartitionedTopic() throws Exception {
}
// 4. Receive messages
- Message message = consumer.receive();
+ Message<byte[]> message = consumer.receive();
int messageCount = 0;
log.info("Message received " + new String(message.getData()));
do {
@@ -474,16 +466,14 @@ public void testSharedSingleAckedPartitionedTopic() throws Exception {
// Special step to create partitioned topic
// 1. producer connect
- ProducerConfiguration prodConfig = new ProducerConfiguration();
- prodConfig.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
- Producer producer = pulsarClient.createProducer(topicName, prodConfig);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
+ .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();
// 2. Create consumer
- ConsumerConfiguration consumerConfig = new ConsumerConfiguration();
- consumerConfig.setReceiverQueueSize(7);
- consumerConfig.setSubscriptionType(SubscriptionType.Shared);
- Consumer consumer1 = pulsarClient.subscribe(topicName, subscriptionName, consumerConfig);
- Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, consumerConfig);
+ ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName(subscriptionName).receiverQueueSize(7).subscriptionType(SubscriptionType.Shared);
+ Consumer<byte[]> consumer1 = consumerBuilder.subscribe();
+ Consumer<byte[]> consumer2 = consumerBuilder.subscribe();
// 3. producer publish messages
for (int i = 0; i < totalMessages; i++) {
@@ -493,8 +483,8 @@ public void testSharedSingleAckedPartitionedTopic() throws Exception {
}
// 4. Receive messages
- Message message1 = consumer1.receive();
- Message message2 = consumer2.receive();
+ Message<byte[]> message1 = consumer1.receive();
+ Message<byte[]> message2 = consumer2.receive();
int messageCount1 = 0;
int messageCount2 = 0;
int ackCount1 = 0;
@@ -572,18 +562,15 @@ public void testFailoverSingleAckedPartitionedTopic() throws Exception {
// Special step to create partitioned topic
// 1. producer connect
- ProducerConfiguration prodConfig = new ProducerConfiguration();
- prodConfig.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
- Producer producer = pulsarClient.createProducer(topicName, prodConfig);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
+ .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();
// 2. Create consumer
- ConsumerConfiguration consumerConfig = new ConsumerConfiguration();
- consumerConfig.setReceiverQueueSize(7);
- consumerConfig.setSubscriptionType(SubscriptionType.Failover);
- consumerConfig.setConsumerName("Consumer-1");
- Consumer consumer1 = pulsarClient.subscribe(topicName, subscriptionName, consumerConfig);
- consumerConfig.setConsumerName("Consumer-2");
- Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, consumerConfig);
+ ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName(subscriptionName).receiverQueueSize(7).subscriptionType(SubscriptionType.Failover);
+ Consumer<byte[]> consumer1 = consumerBuilder.clone().consumerName("Consumer-1").subscribe();
+ Consumer<byte[]> consumer2 = consumerBuilder.clone().consumerName("Consumer-2").subscribe();
+
Thread.sleep(1000);
// 3. producer publish messages
for (int i = 0; i < totalMessages; i++) {
@@ -593,8 +580,8 @@ public void testFailoverSingleAckedPartitionedTopic() throws Exception {
}
// 4. Receive messages
- Message message1 = consumer1.receive();
- Message message2 = consumer2.receive();
+ Message<byte[]> message1 = consumer1.receive();
+ Message<byte[]> message2 = consumer2.receive();
int messageCount1 = 0;
int messageCount2 = 0;
int ackCount1 = 0;
@@ -657,19 +644,16 @@ public void testFailoverInactiveConsumer() throws Exception {
final String messagePredicate = "my-message-" + key + "-";
final int totalMessages = 10;
// 1. producer connect
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
assertNotNull(topicRef);
assertEquals(topicRef.getProducers().size(), 1);
// 2. Create consumer
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setReceiverQueueSize(10);
- conf.setSubscriptionType(SubscriptionType.Failover);
- conf.setConsumerName("consumer-1");
- Consumer consumer1 = pulsarClient.subscribe(topicName, subscriptionName, conf);
- conf.setConsumerName("consumer-2");
- Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, conf);
+ ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName(subscriptionName).receiverQueueSize(10).subscriptionType(SubscriptionType.Failover);
+ Consumer<byte[]> consumer1 = consumerBuilder.clone().consumerName("Consumer-1").subscribe();
+ Consumer<byte[]> consumer2 = consumerBuilder.clone().consumerName("Consumer-2").subscribe();
// 3. Producer publishes messages
for (int i = 0; i < totalMessages; i++) {
@@ -680,8 +664,8 @@ public void testFailoverInactiveConsumer() throws Exception {
// 4. Receive messages
int receivedConsumer1 = 0, receivedConsumer2 = 0;
- Message message1;
- Message message2;
+ Message<byte[]> message1;
+ Message<byte[]> message2;
do {
message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
if (message1 != null) {
@@ -707,12 +691,13 @@ public void testFailoverInactiveConsumer() throws Exception {
assertEquals(message2, null);
}
- private BlockingQueue<Message> printIncomingMessageQueue(Consumer consumer) throws Exception {
- BlockingQueue<Message> imq = null;
- ConsumerBase c = (ConsumerBase) consumer;
+ @SuppressWarnings("unchecked")
+ private BlockingQueue<Message<byte[]>> printIncomingMessageQueue(Consumer<byte[]> consumer) throws Exception {
+ BlockingQueue<Message<byte[]>> imq = null;
+ ConsumerBase<byte[]> c = (ConsumerBase<byte[]>) consumer;
Field field = ConsumerBase.class.getDeclaredField("incomingMessages");
field.setAccessible(true);
- imq = (BlockingQueue<Message>) field.get(c);
+ imq = (BlockingQueue<Message<byte[]>>) field.get(c);
log.info("Incoming MEssage Queue: {}", imq);
return imq;
}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java
index 486f1cdee..4a0441c7b 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java
@@ -83,6 +83,7 @@
import org.apache.pulsar.common.api.proto.PulsarApi.CommandProducerSuccess;
import org.apache.pulsar.common.api.proto.PulsarApi.CommandSendError;
import org.apache.pulsar.common.api.proto.PulsarApi.CommandSendReceipt;
+import org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.InitialPosition;
import org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.SubType;
import org.apache.pulsar.common.api.proto.PulsarApi.CommandSuccess;
import org.apache.pulsar.common.api.proto.PulsarApi.EncryptionKeys;
@@ -1198,7 +1199,8 @@ public void testAckCommand() throws Exception {
channel.writeInbound(clientCommand);
assertTrue(getResponse() instanceof CommandSuccess);
- PositionImpl pos = new PositionImpl(0, 0);
+ // PositionImpl pos = new PositionImpl(0, 0);
+ PositionImpl pos = PositionImpl.latest;
clientCommand = Commands.newAck(1 /* consumer id */, pos.getLedgerId(), pos.getEntryId(), AckType.Individual,
null, Collections.emptyMap());
@@ -1438,20 +1440,20 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
@Override
public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
Thread.sleep(300);
- ((OpenCursorCallback) invocationOnMock.getArguments()[1]).openCursorComplete(cursorMock, null);
+ ((OpenCursorCallback) invocationOnMock.getArguments()[2]).openCursorComplete(cursorMock, null);
return null;
}
- }).when(ledgerMock).asyncOpenCursor(matches(".*success.*"), any(OpenCursorCallback.class), anyObject());
+ }).when(ledgerMock).asyncOpenCursor(matches(".*success.*"), any(InitialPosition.class), any(OpenCursorCallback.class), anyObject());
doAnswer(new Answer<Object>() {
@Override
public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
Thread.sleep(300);
- ((OpenCursorCallback) invocationOnMock.getArguments()[1])
+ ((OpenCursorCallback) invocationOnMock.getArguments()[2])
.openCursorFailed(new ManagedLedgerException("Managed ledger failure"), null);
return null;
}
- }).when(ledgerMock).asyncOpenCursor(matches(".*fail.*"), any(OpenCursorCallback.class), anyObject());
+ }).when(ledgerMock).asyncOpenCursor(matches(".*fail.*"), any(InitialPosition.class), any(OpenCursorCallback.class), anyObject());
doAnswer(new Answer<Object>() {
@Override
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java
index cd383849a..284e6c6ee 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java
@@ -27,7 +27,6 @@
import org.apache.pulsar.broker.service.persistent.PersistentSubscription;
import org.apache.pulsar.broker.service.persistent.PersistentTopic;
-import org.apache.pulsar.client.api.ConsumerConfiguration;
import org.apache.pulsar.client.api.MessageId;
import org.apache.pulsar.client.api.Producer;
import org.apache.pulsar.client.api.PulsarClientException;
@@ -55,14 +54,11 @@ protected void cleanup() throws Exception {
public void testSeek() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/testSeek";
- Producer producer = pulsarClient.createProducer(topicName);
-
- ConsumerConfiguration consumerConf = new ConsumerConfiguration();
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
// Disable pre-fetch in consumer to track the messages received
- consumerConf.setReceiverQueueSize(0);
- org.apache.pulsar.client.api.Consumer consumer = pulsarClient.subscribe(topicName, "my-subscription",
- consumerConf);
+ org.apache.pulsar.client.api.Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName("my-subscription").receiverQueueSize(0).subscribe();
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
assertNotNull(topicRef);
@@ -97,7 +93,8 @@ public void testSeekOnPartitionedTopic() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/testSeekPartitions";
admin.persistentTopics().createPartitionedTopic(topicName, 2);
- org.apache.pulsar.client.api.Consumer consumer = pulsarClient.subscribe(topicName, "my-subscription");
+ org.apache.pulsar.client.api.Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName("my-subscription").subscribe();
try {
consumer.seek(MessageId.latest);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicTerminationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicTerminationTest.java
index bcc5470e6..696a33181 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicTerminationTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicTerminationTest.java
@@ -35,14 +35,12 @@
import org.apache.pulsar.client.admin.PulsarAdminException.NotAllowedException;
import org.apache.pulsar.client.api.Consumer;
-import org.apache.pulsar.client.api.ConsumerConfiguration;
import org.apache.pulsar.client.api.Message;
import org.apache.pulsar.client.api.MessageId;
import org.apache.pulsar.client.api.MessageListener;
import org.apache.pulsar.client.api.Producer;
import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.client.api.Reader;
-import org.apache.pulsar.client.api.ReaderConfiguration;
import org.apache.pulsar.client.api.ReaderListener;
import org.apache.pulsar.common.util.FutureUtil;
import org.testng.annotations.AfterMethod;
@@ -67,7 +65,7 @@ protected void cleanup() throws Exception {
@Test
public void testSimpleTermination() throws Exception {
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
/* MessageId msgId1 = */producer.send("test-msg-1".getBytes());
/* MessageId msgId2 = */producer.send("test-msg-2".getBytes());
@@ -86,7 +84,7 @@ public void testSimpleTermination() throws Exception {
@Test
public void testCreateProducerOnTerminatedTopic() throws Exception {
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
/* MessageId msgId1 = */producer.send("test-msg-1".getBytes());
/* MessageId msgId2 = */producer.send("test-msg-2".getBytes());
@@ -96,7 +94,7 @@ public void testCreateProducerOnTerminatedTopic() throws Exception {
assertEquals(lastMessageId, msgId3);
try {
- pulsarClient.createProducer(topicName);
+ pulsarClient.newProducer().topic(topicName).create();
fail("Should have thrown exception");
} catch (PulsarClientException.TopicTerminatedException e) {
// Expected
@@ -105,7 +103,7 @@ public void testCreateProducerOnTerminatedTopic() throws Exception {
@Test(timeOut = 20000)
public void testTerminateWhilePublishing() throws Exception {
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
CyclicBarrier barrier = new CyclicBarrier(2);
List<CompletableFuture<MessageId>> futures = new ArrayList<>();
@@ -150,7 +148,7 @@ public void testTerminateWhilePublishing() throws Exception {
@Test
public void testDoubleTerminate() throws Exception {
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
/* MessageId msgId1 = */producer.send("test-msg-1".getBytes());
/* MessageId msgId2 = */producer.send("test-msg-2".getBytes());
@@ -178,13 +176,14 @@ public void testTerminatePartitionedTopic() throws Exception {
@Test(timeOut = 20000)
public void testSimpleTerminationConsumer() throws Exception {
- Producer producer = pulsarClient.createProducer(topicName);
- org.apache.pulsar.client.api.Consumer consumer = pulsarClient.subscribe(topicName, "my-sub");
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
+ org.apache.pulsar.client.api.Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName("my-sub").subscribe();
MessageId msgId1 = producer.send("test-msg-1".getBytes());
MessageId msgId2 = producer.send("test-msg-2".getBytes());
- Message msg1 = consumer.receive();
+ Message<byte[]> msg1 = consumer.receive();
assertEquals(msg1.getMessageId(), msgId1);
consumer.acknowledge(msg1);
@@ -195,43 +194,41 @@ public void testSimpleTerminationConsumer() throws Exception {
MessageId lastMessageId = admin.persistentTopics().terminateTopicAsync(topicName).get();
assertEquals(lastMessageId, msgId3);
- Message msg2 = consumer.receive();
+ Message<byte[]> msg2 = consumer.receive();
assertEquals(msg2.getMessageId(), msgId2);
consumer.acknowledge(msg2);
- Message msg3 = consumer.receive();
+ Message<byte[]> msg3 = consumer.receive();
assertEquals(msg3.getMessageId(), msgId3);
consumer.acknowledge(msg3);
- Message msg4 = consumer.receive(100, TimeUnit.MILLISECONDS);
+ Message<byte[]> msg4 = consumer.receive(100, TimeUnit.MILLISECONDS);
assertNull(msg4);
Thread.sleep(100);
assertTrue(consumer.hasReachedEndOfTopic());
}
- @SuppressWarnings("serial")
@Test(timeOut = 20000)
public void testSimpleTerminationMessageListener() throws Exception {
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
CountDownLatch latch = new CountDownLatch(1);
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setMessageListener(new MessageListener() {
+ org.apache.pulsar.client.api.Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName("my-sub").messageListener(new MessageListener<byte[]>() {
- @Override
- public void received(Consumer consumer, Message msg) {
- // do nothing
- }
+ @Override
+ public void received(Consumer<byte[]> consumer, Message<byte[]> msg) {
+ // do nothing
+ }
- @Override
- public void reachedEndOfTopic(Consumer consumer) {
- latch.countDown();
- assertTrue(consumer.hasReachedEndOfTopic());
- }
- });
- org.apache.pulsar.client.api.Consumer consumer = pulsarClient.subscribe(topicName, "my-sub", conf);
+ @Override
+ public void reachedEndOfTopic(Consumer<byte[]> consumer) {
+ latch.countDown();
+ assertTrue(consumer.hasReachedEndOfTopic());
+ }
+ }).subscribe();
/* MessageId msgId1 = */ producer.send("test-msg-1".getBytes());
/* MessageId msgId2 = */ producer.send("test-msg-2".getBytes());
@@ -251,7 +248,7 @@ public void reachedEndOfTopic(Consumer consumer) {
@Test(timeOut = 20000)
public void testSimpleTerminationReader() throws Exception {
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
MessageId msgId1 = producer.send("test-msg-1".getBytes());
MessageId msgId2 = producer.send("test-msg-2".getBytes());
@@ -260,46 +257,43 @@ public void testSimpleTerminationReader() throws Exception {
MessageId lastMessageId = admin.persistentTopics().terminateTopicAsync(topicName).get();
assertEquals(lastMessageId, msgId3);
- Reader reader = pulsarClient.createReader(topicName, MessageId.earliest, new ReaderConfiguration());
+ Reader<byte[]> reader = pulsarClient.newReader().topic(topicName).startMessageId(MessageId.earliest).create();
- Message msg1 = reader.readNext();
+ Message<byte[]> msg1 = reader.readNext();
assertEquals(msg1.getMessageId(), msgId1);
- Message msg2 = reader.readNext();
+ Message<byte[]> msg2 = reader.readNext();
assertEquals(msg2.getMessageId(), msgId2);
- Message msg3 = reader.readNext();
+ Message<byte[]> msg3 = reader.readNext();
assertEquals(msg3.getMessageId(), msgId3);
- Message msg4 = reader.readNext(100, TimeUnit.MILLISECONDS);
+ Message<byte[]> msg4 = reader.readNext(100, TimeUnit.MILLISECONDS);
assertNull(msg4);
Thread.sleep(100);
assertTrue(reader.hasReachedEndOfTopic());
}
- @SuppressWarnings("serial")
@Test(timeOut = 20000)
public void testSimpleTerminationReaderListener() throws Exception {
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
CountDownLatch latch = new CountDownLatch(1);
- ReaderConfiguration conf = new ReaderConfiguration();
- conf.setReaderListener(new ReaderListener() {
+ Reader<byte[]> reader = pulsarClient.newReader().topic(topicName).startMessageId(MessageId.latest)
+ .readerListener(new ReaderListener<byte[]>() {
+ @Override
+ public void received(Reader<byte[]> reader, Message<byte[]> msg) {
+ // do nothing
+ }
- @Override
- public void received(Reader r, Message msg) {
- // do nothing
- }
-
- @Override
- public void reachedEndOfTopic(Reader reader) {
- latch.countDown();
- assertTrue(reader.hasReachedEndOfTopic());
- }
- });
- Reader reader = pulsarClient.createReader(topicName, MessageId.latest, conf);
+ @Override
+ public void reachedEndOfTopic(Reader<byte[]> reader) {
+ latch.countDown();
+ assertTrue(reader.hasReachedEndOfTopic());
+ }
+ }).create();
/* MessageId msgId1 = */ producer.send("test-msg-1".getBytes());
/* MessageId msgId2 = */ producer.send("test-msg-2".getBytes());
@@ -317,14 +311,15 @@ public void reachedEndOfTopic(Reader reader) {
@Test(timeOut = 20000)
public void testSubscribeOnTerminatedTopic() throws Exception {
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
/* MessageId msgId1 = */ producer.send("test-msg-1".getBytes());
MessageId msgId2 = producer.send("test-msg-2".getBytes());
MessageId lastMessageId = admin.persistentTopics().terminateTopicAsync(topicName).get();
assertEquals(lastMessageId, msgId2);
- org.apache.pulsar.client.api.Consumer consumer = pulsarClient.subscribe(topicName, "my-sub");
+ org.apache.pulsar.client.api.Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName("my-sub").subscribe();
Thread.sleep(200);
assertTrue(consumer.hasReachedEndOfTopic());
@@ -332,10 +327,11 @@ public void testSubscribeOnTerminatedTopic() throws Exception {
@Test(timeOut = 20000)
public void testSubscribeOnTerminatedTopicWithNoMessages() throws Exception {
- pulsarClient.createProducer(topicName);
+ pulsarClient.newProducer().topic(topicName).create();
admin.persistentTopics().terminateTopicAsync(topicName).get();
- org.apache.pulsar.client.api.Consumer consumer = pulsarClient.subscribe(topicName, "my-sub");
+ org.apache.pulsar.client.api.Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName("my-sub").subscribe();
Thread.sleep(200);
assertTrue(consumer.hasReachedEndOfTopic());
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ChecksumTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ChecksumTest.java
index 6d7a858eb..3e6b3bd43 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ChecksumTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ChecksumTest.java
@@ -59,7 +59,7 @@ protected void cleanup() throws Exception {
public void verifyChecksumStoredInManagedLedger() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/topic0";
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
@@ -86,7 +86,7 @@ public void verifyChecksumStoredInManagedLedger() throws Exception {
public void verifyChecksumSentToConsumer() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/topic-1";
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
RawReader reader = RawReader.create(pulsarClient, topicName, "sub").get();
producer.send("Hello".getBytes());
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ManagedLedgerMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ManagedLedgerMetricsTest.java
index 45d541fe4..f74f19238 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ManagedLedgerMetricsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ManagedLedgerMetricsTest.java
@@ -58,7 +58,8 @@ public void testManagedLedgerMetrics() throws Exception {
List<Metrics> list1 = metrics.generate();
Assert.assertTrue(list1.isEmpty());
- Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1");
+ Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1")
+ .create();
for (int i = 0; i < 10; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
index 8aa6fdc2c..62d490e13 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
@@ -55,8 +55,8 @@ protected void cleanup() throws Exception {
@Test
public void testPerTopicStats() throws Exception {
- Producer p1 = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1");
- Producer p2 = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic2");
+ Producer<byte[]> p1 = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1").create();
+ Producer<byte[]> p2 = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic2").create();
for (int i = 0; i < 10; i++) {
String message = "my-message-" + i;
p1.send(message.getBytes());
@@ -101,8 +101,8 @@ public void testPerTopicStats() throws Exception {
@Test
public void testPerNamespaceStats() throws Exception {
- Producer p1 = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1");
- Producer p2 = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic2");
+ Producer<byte[]> p1 = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1").create();
+ Producer<byte[]> p2 = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic2").create();
for (int i = 0; i < 10; i++) {
String message = "my-message-" + i;
p1.send(message.getBytes());
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/zookeeper/ZooKeeperClientAspectJTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/zookeeper/ZooKeeperClientAspectJTest.java
index 5454c8bc1..99ac8057d 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/zookeeper/ZooKeeperClientAspectJTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/zookeeper/ZooKeeperClientAspectJTest.java
@@ -189,7 +189,7 @@ public void testZkOpStatsMetrics() throws Exception {
PulsarClient pulsarClient = mockPulsar.getClient();
PulsarService pulsar = mockPulsar.getPulsar();
- pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1");
+ pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1").create();
Metrics zkOpMetric = getMetric(pulsar, "zk_write_latency");
Assert.assertNotNull(zkOpMetric);
Assert.assertTrue(zkOpMetric.getMetrics().containsKey("brk_zk_write_rate_s"));
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java
index 7c6f6a9f0..eafe9c8fc 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java
@@ -109,7 +109,9 @@ protected final void internalSetup(Authentication auth) throws Exception {
} else {
lookupUrl = new URI("pulsar+ssl://localhost:" + BROKER_PORT_TLS).toString();
}
- pulsarClient = PulsarClient.create(lookupUrl, clientConf);
+ pulsarClient = PulsarClient.builder().serviceUrl(lookupUrl).statsInterval(0, TimeUnit.SECONDS)
+ .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).allowTlsInsecureConnection(true).authentication(auth)
+ .enableTls(true).build();
}
@AfterMethod
@@ -124,26 +126,24 @@ protected void cleanup() throws Exception {
}
public void testSyncProducerAndConsumer(int batchMessageDelayMs) throws Exception {
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
- Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic", "my-subscriber-name",
- conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://my-property/use/my-ns/my-topic")
+ .subscriptionName("my-subscriber-name").subscribe();
- ProducerConfiguration producerConf = new ProducerConfiguration();
+ ProducerBuilder<byte[]> producerBuilder = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic");
if (batchMessageDelayMs != 0) {
- producerConf.setBatchingEnabled(true);
- producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
- producerConf.setBatchingMaxMessages(5);
+ producerBuilder.enableBatching(true);
+ producerBuilder.batchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
+ producerBuilder.batchingMaxMessages(5);
}
- Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic", producerConf);
+ Producer<byte[]> producer = producerBuilder.create();
for (int i = 0; i < 10; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
- Message msg = null;
+ Message<byte[]> msg = null;
Set<String> messageSet = Sets.newHashSet();
for (int i = 0; i < 10; i++) {
msg = consumer.receive(5, TimeUnit.SECONDS);
@@ -184,8 +184,8 @@ public void testBasicCryptSyncProducerAndConsumer(int batchMessageDelayMs) throw
authPassword.configure("{\"userId\":\"superUser\",\"password\":\"supepass\"}");
internalSetup(authPassword);
- admin.properties()
- .createProperty("my-property", new PropertyAdmin(Lists.newArrayList(), Sets.newHashSet("use")));
+ admin.properties().createProperty("my-property",
+ new PropertyAdmin(Lists.newArrayList(), Sets.newHashSet("use")));
admin.namespaces().createNamespace("my-property/use/my-ns");
testSyncProducerAndConsumer(batchMessageDelayMs);
@@ -200,8 +200,8 @@ public void testBasicArp1SyncProducerAndConsumer(int batchMessageDelayMs) throws
authPassword.configure("{\"userId\":\"superUser2\",\"password\":\"superpassword\"}");
internalSetup(authPassword);
- admin.properties()
- .createProperty("my-property", new PropertyAdmin(Lists.newArrayList(), Sets.newHashSet("use")));
+ admin.properties().createProperty("my-property",
+ new PropertyAdmin(Lists.newArrayList(), Sets.newHashSet("use")));
admin.namespaces().createNamespace("my-property/use/my-ns");
testSyncProducerAndConsumer(batchMessageDelayMs);
@@ -209,7 +209,6 @@ public void testBasicArp1SyncProducerAndConsumer(int batchMessageDelayMs) throws
log.info("-- Exiting {} test --", methodName);
}
-
@Test(dataProvider = "batch")
public void testAnonymousSyncProducerAndConsumer(int batchMessageDelayMs) throws Exception {
log.info("-- Starting {} test --", methodName);
@@ -232,17 +231,19 @@ public void testAnonymousSyncProducerAndConsumer(int batchMessageDelayMs) throws
clientConf.setOperationTimeout(1, TimeUnit.SECONDS);
admin = spy(new PulsarAdmin(brokerUrl, clientConf));
admin.namespaces().createNamespace("my-property/use/my-ns");
- admin.persistentTopics().grantPermission("persistent://my-property/use/my-ns/my-topic", "anonymousUser", EnumSet
- .allOf(AuthAction.class));
+ admin.persistentTopics().grantPermission("persistent://my-property/use/my-ns/my-topic", "anonymousUser",
+ EnumSet.allOf(AuthAction.class));
// setup the client
pulsarClient.close();
- pulsarClient = PulsarClient.create("pulsar://localhost:" + BROKER_PORT, clientConf);
+ pulsarClient = PulsarClient.builder().serviceUrl("pulsar://localhost:" + BROKER_PORT)
+ .operationTimeout(1, TimeUnit.SECONDS).build();
// unauthorized topic test
Exception pulsarClientException = null;
try {
- pulsarClient.subscribe("persistent://my-property/use/my-ns/other-topic", "my-subscriber-name");
+ pulsarClient.newConsumer().topic("persistent://my-property/use/my-ns/other-topic")
+ .subscriptionName("my-subscriber-name").subscribe();
} catch (Exception e) {
pulsarClientException = e;
}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java
index 5ccfc142b..4338ae32d 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java
@@ -118,7 +118,10 @@ protected void setupClient() throws Exception {
admin = spy(new PulsarAdmin(brokerUrlTls, clientConf));
String lookupUrl;
lookupUrl = new URI("pulsar+ssl://" + brokerHostName + ":" + BROKER_PORT_TLS).toString();
- pulsarClient = PulsarClient.create(lookupUrl, clientConf);
+ pulsarClient = PulsarClient.builder().serviceUrl(lookupUrl).statsInterval(0, TimeUnit.SECONDS)
+ .tlsTrustCertsFilePath(TLS_MIM_TRUST_CERT_FILE_PATH).allowTlsInsecureConnection(true)
+ .authentication(authTls).enableTls(true).enableTlsHostnameVerification(hostnameVerificationEnabled)
+ .build();
admin.properties().createProperty("my-property",
new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet("use")));
@@ -140,13 +143,13 @@ protected void cleanup() throws Exception {
/**
* It verifies that client performs host-verification in order to create producer/consumer.
- *
+ *
* <pre>
* 1. Client tries to connect to broker with hostname="localhost"
* 2. Broker sends x509 certificates with CN = "pulsar"
* 3. Client verifies the host-name and closes the connection and fails consumer creation
* </pre>
- *
+ *
* @throws Exception
*/
@Test(dataProvider = "hostnameVerification")
@@ -164,11 +167,9 @@ public void testTlsSyncProducerAndConsumerWithInvalidBrokerHost(boolean hostname
setup();
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
try {
- Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic",
- "my-subscriber-name", conf);
+ pulsarClient.newConsumer().topic("persistent://my-property/use/my-ns/my-topic")
+ .subscriptionName("my-subscriber-name").subscribe();
if (hostnameVerificationEnabled) {
Assert.fail("Connection should be failed due to hostnameVerification enabled");
}
@@ -183,13 +184,13 @@ public void testTlsSyncProducerAndConsumerWithInvalidBrokerHost(boolean hostname
/**
* It verifies that client performs host-verification in order to create producer/consumer.
- *
+ *
* <pre>
* 1. Client tries to connect to broker with hostname="localhost"
* 2. Broker sends x509 certificates with CN = "localhost"
* 3. Client verifies the host-name and continues
* </pre>
- *
+ *
* @throws Exception
*/
@Test
@@ -203,20 +204,17 @@ public void testTlsSyncProducerAndConsumerCorrectBrokerHost() throws Exception {
setup();
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
- Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic", "my-subscriber-name",
- conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://my-property/use/my-ns/my-topic")
+ .subscriptionName("my-subscriber-name").subscribe();
- ProducerConfiguration producerConf = new ProducerConfiguration();
-
- Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic", producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic")
+ .create();
for (int i = 0; i < 10; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
- Message msg = null;
+ Message<byte[]> msg = null;
Set<String> messageSet = Sets.newHashSet();
for (int i = 0; i < 10; i++) {
msg = consumer.receive(5, TimeUnit.SECONDS);
@@ -234,7 +232,7 @@ public void testTlsSyncProducerAndConsumerCorrectBrokerHost() throws Exception {
/**
* This test verifies {@link DefaultHostnameVerifier} behavior and gives fair idea about host matching result
- *
+ *
* @throws Exception
*/
@Test
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java
index 1eb27db13..ec2ddd1a4 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java
@@ -110,37 +110,36 @@ public void testProducerAndConsumerAuthorization() throws Exception {
String lookupUrl;
lookupUrl = new URI("pulsar://localhost:" + BROKER_PORT).toString();
- ClientConfiguration clientConfValid = new ClientConfiguration();
Authentication authentication = new ClientAuthentication(clientRole);
- clientConfValid.setAuthentication(authentication);
-
- ClientConfiguration clientConfInvalidRole = new ClientConfiguration();
Authentication authenticationInvalidRole = new ClientAuthentication("test-role");
- clientConfInvalidRole.setAuthentication(authenticationInvalidRole);
- pulsarClient = PulsarClient.create(lookupUrl, clientConfValid);
- PulsarClient pulsarClientInvalidRole = PulsarClient.create(lookupUrl, clientConfInvalidRole);
+ pulsarClient = PulsarClient.builder().serviceUrl(lookupUrl).authentication(authentication).build();
+ PulsarClient pulsarClientInvalidRole = PulsarClient.builder().serviceUrl(lookupUrl)
+ .authentication(authenticationInvalidRole).build();
admin.properties().createProperty("my-property",
new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet("use")));
admin.namespaces().createNamespace("my-property/use/my-ns");
// (1) Valid Producer and consumer creation
- Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic", "my-subscriber-name");
- Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic");
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://my-property/use/my-ns/my-topic")
+ .subscriptionName("my-subscriber-name").subscribe();
+ Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic")
+ .create();
consumer.close();
producer.close();
// (2) InValid user auth-role will be rejected by authorization service
try {
- consumer = pulsarClientInvalidRole.subscribe("persistent://my-property/use/my-ns/my-topic",
- "my-subscriber-name");
+ consumer = pulsarClientInvalidRole.newConsumer().topic("persistent://my-property/use/my-ns/my-topic")
+ .subscriptionName("my-subscriber-name").subscribe();
Assert.fail("should have failed with authorization error");
} catch (PulsarClientException.AuthorizationException pa) {
// Ok
}
try {
- producer = pulsarClientInvalidRole.createProducer("persistent://my-property/use/my-ns/my-topic");
+ producer = pulsarClientInvalidRole.newProducer().topic("persistent://my-property/use/my-ns/my-topic")
+ .create();
Assert.fail("should have failed with authorization error");
} catch (PulsarClientException.AuthorizationException pa) {
// Ok
@@ -164,23 +163,23 @@ public void testSubscriptionPrefixAuthorization() throws Exception {
String lookupUrl;
lookupUrl = new URI("pulsar://localhost:" + BROKER_PORT).toString();
- ClientConfiguration clientConfValid = new ClientConfiguration();
Authentication authentication = new ClientAuthentication(clientRole);
- clientConfValid.setAuthentication(authentication);
- pulsarClient = PulsarClient.create(lookupUrl, clientConfValid);
+ pulsarClient = PulsarClient.builder().serviceUrl(lookupUrl).authentication(authentication).build();
admin.properties().createProperty("prop-prefix",
new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet("use")));
admin.namespaces().createNamespace("prop-prefix/use/ns");
// (1) Valid subscription name will be approved by authorization service
- Consumer consumer = pulsarClient.subscribe("persistent://prop-prefix/use/ns/t1", clientRole + "-sub1");
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://prop-prefix/use/ns/t1")
+ .subscriptionName(clientRole + "-sub1").subscribe();
consumer.close();
// (2) InValid subscription name will be rejected by authorization service
try {
- consumer = pulsarClient.subscribe("persistent://prop-prefix/use/ns/t1", "sub1");
+ consumer = pulsarClient.newConsumer().topic("persistent://prop-prefix/use/ns/t1").subscriptionName("sub1")
+ .subscribe();
Assert.fail("should have failed with authorization error");
} catch (PulsarClientException.AuthorizationException pa) {
// Ok
@@ -201,8 +200,7 @@ public void testGrantPermission() throws Exception {
String role = "test-role";
Assert.assertFalse(authorizationService.canProduce(topicName, role, null));
Assert.assertFalse(authorizationService.canConsume(topicName, role, null, "sub1"));
- authorizationService
- .grantPermissionAsync(topicName, null, role, "auth-json").get();
+ authorizationService.grantPermissionAsync(topicName, null, role, "auth-json").get();
Assert.assertTrue(authorizationService.canProduce(topicName, role, null));
Assert.assertTrue(authorizationService.canConsume(topicName, role, null, "sub1"));
@@ -219,16 +217,13 @@ public void testAuthData() throws Exception {
AuthorizationService authorizationService = new AuthorizationService(conf, null);
TopicName topicName = TopicName.get("persistent://prop/cluster/ns/t1");
String role = "test-role";
- authorizationService
- .grantPermissionAsync(topicName, null, role, "auth-json")
- .get();
+ authorizationService.grantPermissionAsync(topicName, null, role, "auth-json").get();
Assert.assertEquals(TestAuthorizationProviderWithGrantPermission.authDataJson, "auth-json");
- Assert.assertTrue(
- authorizationService.canProduce(topicName, role, new AuthenticationDataCommand("prod-auth")));
+ Assert.assertTrue(authorizationService.canProduce(topicName, role, new AuthenticationDataCommand("prod-auth")));
Assert.assertEquals(TestAuthorizationProviderWithGrantPermission.authenticationData.getCommandData(),
"prod-auth");
- Assert.assertTrue(authorizationService.canConsume(topicName, role, new AuthenticationDataCommand("cons-auth"),
- "sub1"));
+ Assert.assertTrue(
+ authorizationService.canConsume(topicName, role, new AuthenticationDataCommand("cons-auth"), "sub1"));
Assert.assertEquals(TestAuthorizationProviderWithGrantPermission.authenticationData.getCommandData(),
"cons-auth");
@@ -348,8 +343,8 @@ public void initialize(ServiceConfiguration conf, ConfigurationCacheService conf
}
@Override
- public CompletableFuture<Void> grantPermissionAsync(TopicName topicname, Set<AuthAction> actions,
- String role, String authenticationData) {
+ public CompletableFuture<Void> grantPermissionAsync(TopicName topicname, Set<AuthAction> actions, String role,
+ String authenticationData) {
return CompletableFuture.completedFuture(null);
}
}
@@ -433,8 +428,8 @@ public void initialize(ServiceConfiguration conf, ConfigurationCacheService conf
}
@Override
- public CompletableFuture<Void> grantPermissionAsync(TopicName topicname, Set<AuthAction> actions,
- String role, String authData) {
+ public CompletableFuture<Void> grantPermissionAsync(TopicName topicname, Set<AuthAction> actions, String role,
+ String authData) {
this.authDataJson = authData;
grantRoles.add(role);
return CompletableFuture.completedFuture(null);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java
index c72aa9e43..57d8b478b 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java
@@ -70,11 +70,10 @@
import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerWrapper;
import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceUnit;
import org.apache.pulsar.broker.namespace.NamespaceService;
-import org.apache.pulsar.client.api.ProducerConfiguration.MessageRoutingMode;
import org.apache.pulsar.client.impl.auth.AuthenticationTls;
-import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.naming.NamespaceBundle;
import org.apache.pulsar.common.naming.ServiceUnitId;
+import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.partition.PartitionedTopicMetadata;
import org.apache.pulsar.common.policies.data.ClusterData;
import org.apache.pulsar.common.policies.data.PropertyAdmin;
@@ -105,6 +104,7 @@
import io.netty.handler.codec.http.HttpRequest;
import io.netty.handler.codec.http.HttpResponse;
import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
+import lombok.Cleanup;
public class BrokerServiceLookupTest extends ProducerConsumerBase {
private static final Logger log = LoggerFactory.getLogger(BrokerServiceLookupTest.class);
@@ -114,10 +114,9 @@
protected void setup() throws Exception {
conf.setDefaultNumberOfNamespaceBundles(1);
super.init();
- org.apache.pulsar.client.api.ClientConfiguration clientConf = new org.apache.pulsar.client.api.ClientConfiguration();
- clientConf.setStatsInterval(0, TimeUnit.SECONDS);
URI brokerServiceUrl = new URI("pulsar://localhost:" + BROKER_PORT);
- pulsarClient = PulsarClient.create(brokerServiceUrl.toString(), clientConf);
+ pulsarClient = PulsarClient.builder().serviceUrl(brokerServiceUrl.toString()).statsInterval(0, TimeUnit.SECONDS)
+ .build();
super.producerBaseSetup();
}
@@ -127,14 +126,13 @@ protected void cleanup() throws Exception {
super.internalCleanup();
}
-
/**
* UsecaseL Multiple Broker => Lookup Redirection test
*
- * 1. Broker1 is a leader
- * 2. Lookup request reaches to Broker2 which redirects to leader (Broker1) with authoritative = false
- * 3. Leader (Broker1) finds out least loaded broker as Broker2 and redirects request to Broker2 with authoritative = true
- * 4. Broker2 receives final request to own a bundle with authoritative = true and client connects to Broker2
+ * 1. Broker1 is a leader 2. Lookup request reaches to Broker2 which redirects to leader (Broker1) with
+ * authoritative = false 3. Leader (Broker1) finds out least loaded broker as Broker2 and redirects request to
+ * Broker2 with authoritative = true 4. Broker2 receives final request to own a bundle with authoritative = true and
+ * client connects to Broker2
*
* @throws Exception
*/
@@ -156,7 +154,6 @@ public void testMultipleBrokerLookup() throws Exception {
pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
pulsar2.getLoadManager().get().writeLoadReportOnZookeeper();
-
LoadManager loadManager1 = spy(pulsar.getLoadManager().get());
LoadManager loadManager2 = spy(pulsar2.getLoadManager().get());
Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager");
@@ -175,19 +172,20 @@ public void testMultipleBrokerLookup() throws Exception {
/**** started broker-2 ****/
URI brokerServiceUrl = new URI("pulsar://localhost:" + conf2.getBrokerServicePort());
- PulsarClient pulsarClient2 = PulsarClient.create(brokerServiceUrl.toString(), new ClientConfiguration());
+ PulsarClient pulsarClient2 = PulsarClient.builder().serviceUrl(brokerServiceUrl.toString()).build();
// load namespace-bundle by calling Broker2
- Consumer consumer = pulsarClient2.subscribe("persistent://my-property/use/my-ns/my-topic1", "my-subscriber-name",
- new ConsumerConfiguration());
- Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1", new ProducerConfiguration());
+ Consumer<byte[]> consumer = pulsarClient2.newConsumer().topic("persistent://my-property/use/my-ns/my-topic1")
+ .subscriptionName("my-subscriber-name").subscribe();
+ Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1")
+ .create();
for (int i = 0; i < 10; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
- Message msg = null;
+ Message<byte[]> msg = null;
Set<String> messageSet = Sets.newHashSet();
for (int i = 0; i < 10; i++) {
msg = consumer.receive(5, TimeUnit.SECONDS);
@@ -209,11 +207,9 @@ public void testMultipleBrokerLookup() throws Exception {
}
/**
- * Usecase: Redirection due to different cluster
- * 1. Broker1 runs on cluster: "use" and Broker2 runs on cluster: "use2"
- * 2. Broker1 receives "use2" cluster request => Broker1 reads "/clusters" from global-zookkeeper and
- * redirects request to Broker2 whch serves "use2"
- * 3. Broker2 receives redirect request and own namespace bundle
+ * Usecase: Redirection due to different cluster 1. Broker1 runs on cluster: "use" and Broker2 runs on cluster:
+ * "use2" 2. Broker1 receives "use2" cluster request => Broker1 reads "/clusters" from global-zookkeeper and
+ * redirects request to Broker2 whch serves "use2" 3. Broker2 receives redirect request and own namespace bundle
*
* @throws Exception
*/
@@ -235,18 +231,18 @@ public void testMultipleBrokerDifferentClusterLookup() throws Exception {
conf2.setZookeeperServers("localhost:2181");
String broker2ServiceUrl = "pulsar://localhost:" + conf2.getBrokerServicePort();
- admin.clusters().createCluster(newCluster, new ClusterData("http://127.0.0.1:" + BROKER_WEBSERVICE_PORT, null, broker2ServiceUrl, null));
+ admin.clusters().createCluster(newCluster,
+ new ClusterData("http://127.0.0.1:" + BROKER_WEBSERVICE_PORT, null, broker2ServiceUrl, null));
admin.properties().createProperty(property,
new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet(newCluster)));
admin.namespaces().createNamespace(property + "/" + newCluster + "/my-ns");
-
PulsarService pulsar2 = startBroker(conf2);
pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
pulsar2.getLoadManager().get().writeLoadReportOnZookeeper();
URI brokerServiceUrl = new URI(broker2ServiceUrl);
- PulsarClient pulsarClient2 = PulsarClient.create(brokerServiceUrl.toString(), new ClientConfiguration());
+ PulsarClient pulsarClient2 = PulsarClient.builder().serviceUrl(brokerServiceUrl.toString()).build();
// enable authorization: so, broker can validate cluster and redirect if finds different cluster
pulsar.getConfiguration().setAuthorizationEnabled(true);
@@ -266,16 +262,17 @@ public void testMultipleBrokerDifferentClusterLookup() throws Exception {
/**** started broker-2 ****/
// load namespace-bundle by calling Broker2
- Consumer consumer = pulsarClient.subscribe("persistent://my-property2/use2/my-ns/my-topic1", "my-subscriber-name",
- new ConsumerConfiguration());
- Producer producer = pulsarClient2.createProducer("persistent://my-property2/use2/my-ns/my-topic1", new ProducerConfiguration());
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://my-property2/use2/my-ns/my-topic1")
+ .subscriptionName("my-subscriber-name").subscribe();
+ Producer<byte[]> producer = pulsarClient2.newProducer().topic("persistent://my-property2/use2/my-ns/my-topic1")
+ .create();
for (int i = 0; i < 10; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
- Message msg = null;
+ Message<byte[]> msg = null;
Set<String> messageSet = Sets.newHashSet();
for (int i = 0; i < 10; i++) {
msg = consumer.receive(5, TimeUnit.SECONDS);
@@ -298,10 +295,8 @@ public void testMultipleBrokerDifferentClusterLookup() throws Exception {
}
/**
- * Create #PartitionedTopic and let it served by multiple brokers which requries
- * a. tcp partitioned-metadata-lookup
- * b. multiple topic-lookup
- * c. partitioned producer-consumer
+ * Create #PartitionedTopic and let it served by multiple brokers which requries a. tcp partitioned-metadata-lookup
+ * b. multiple topic-lookup c. partitioned producer-consumer
*
* @throws Exception
*/
@@ -312,9 +307,6 @@ public void testPartitionTopicLookup() throws Exception {
int numPartitions = 8;
TopicName topicName = TopicName.get("persistent://my-property/use/my-ns/my-partitionedtopic1");
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
-
admin.persistentTopics().createPartitionedTopic(topicName.toString(), numPartitions);
/**** start broker-2 ****/
@@ -331,7 +323,6 @@ public void testPartitionTopicLookup() throws Exception {
pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
pulsar2.getLoadManager().get().writeLoadReportOnZookeeper();
-
LoadManager loadManager1 = spy(pulsar.getLoadManager().get());
LoadManager loadManager2 = spy(pulsar2.getLoadManager().get());
Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager");
@@ -344,20 +335,20 @@ public void testPartitionTopicLookup() throws Exception {
// mock: redirect request to leader
doReturn(true).when(loadManager2).isCentralized();
loadManagerField.set(pulsar2.getNamespaceService(), new AtomicReference<>(loadManager2));
- /**** broker-2 started ****/
+ /**** broker-2 started ****/
- ProducerConfiguration producerConf = new ProducerConfiguration();
- producerConf.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
- Producer producer = pulsarClient.createProducer(topicName.toString(), producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName.toString())
+ .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();
- Consumer consumer = pulsarClient.subscribe(topicName.toString(), "my-partitioned-subscriber", conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName.toString())
+ .subscriptionName("my-partitioned-subscriber").subscribe();
for (int i = 0; i < 20; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
- Message msg = null;
+ Message<byte[]> msg = null;
Set<String> messageSet = Sets.newHashSet();
for (int i = 0; i < 20; i++) {
msg = consumer.receive(5, TimeUnit.SECONDS);
@@ -380,109 +371,105 @@ public void testPartitionTopicLookup() throws Exception {
}
/**
- * 1. Start broker1 and broker2 with tls enable
- * 2. Hit HTTPS lookup url at broker2 which redirects to HTTPS broker1
+ * 1. Start broker1 and broker2 with tls enable 2. Hit HTTPS lookup url at broker2 which redirects to HTTPS broker1
*
* @throws Exception
*/
@Test
- public void testWebserviceServiceTls() throws Exception {
- log.info("-- Starting {} test --", methodName);
- final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/certificate/server.crt";
- final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/certificate/server.key";
- final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/certificate/client.crt";
- final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/certificate/client.key";
-
- /**** start broker-2 ****/
- ServiceConfiguration conf2 = new ServiceConfiguration();
- conf2.setAdvertisedAddress("localhost");
- conf2.setBrokerServicePort(PortManager.nextFreePort());
- conf2.setBrokerServicePortTls(PortManager.nextFreePort());
- conf2.setWebServicePort(PortManager.nextFreePort());
- conf2.setWebServicePortTls(PortManager.nextFreePort());
- conf2.setAdvertisedAddress("localhost");
- conf2.setTlsAllowInsecureConnection(true);
- conf2.setTlsEnabled(true);
- conf2.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
- conf2.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
- conf2.setClusterName(conf.getClusterName());
- conf2.setZookeeperServers("localhost:2181");
- PulsarService pulsar2 = startBroker(conf2);
-
- // restart broker1 with tls enabled
- conf.setTlsAllowInsecureConnection(true);
- conf.setTlsEnabled(true);
- conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
- conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
- stopBroker();
- startBroker();
- pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
- pulsar2.getLoadManager().get().writeLoadReportOnZookeeper();
-
- LoadManager loadManager1 = spy(pulsar.getLoadManager().get());
- LoadManager loadManager2 = spy(pulsar2.getLoadManager().get());
- Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager");
- loadManagerField.setAccessible(true);
-
- // mock: redirect request to leader [2]
- doReturn(true).when(loadManager2).isCentralized();
- loadManagerField.set(pulsar2.getNamespaceService(), new AtomicReference<>(loadManager2));
-
- // mock: return Broker2 as a Least-loaded broker when leader receies
- // request [3]
- doReturn(true).when(loadManager1).isCentralized();
- SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar2.getWebServiceAddress(), null);
- doReturn(Optional.of(resourceUnit)).when(loadManager1).getLeastLoaded(any(ServiceUnitId.class));
- loadManagerField.set(pulsar.getNamespaceService(), new AtomicReference<>(loadManager1));
-
- /**** started broker-2 ****/
-
- URI brokerServiceUrl = new URI("pulsar://localhost:" + conf2.getBrokerServicePort());
- PulsarClient pulsarClient2 = PulsarClient.create(brokerServiceUrl.toString(), new ClientConfiguration());
-
- final String lookupResourceUrl = "/lookup/v2/destination/persistent/my-property/use/my-ns/my-topic1";
-
- // set client cert_key file
- KeyManager[] keyManagers = null;
- Certificate[] tlsCert = SecurityUtility.loadCertificatesFromPemFile(TLS_CLIENT_CERT_FILE_PATH);
- PrivateKey tlsKey = SecurityUtility.loadPrivateKeyFromPemFile(TLS_CLIENT_KEY_FILE_PATH);
- KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType());
- ks.load(null, null);
- ks.setKeyEntry("private", tlsKey, "".toCharArray(), tlsCert);
- KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
- kmf.init(ks, "".toCharArray());
- keyManagers = kmf.getKeyManagers();
- TrustManager[] trustManagers = InsecureTrustManagerFactory.INSTANCE.getTrustManagers();
- SSLContext sslCtx = SSLContext.getInstance("TLS");
- sslCtx.init(keyManagers, trustManagers, new SecureRandom());
- HttpsURLConnection.setDefaultSSLSocketFactory(sslCtx.getSocketFactory());
-
-
- // hit broker2 url
- URLConnection con = new URL(pulsar2.getWebServiceAddressTls() + lookupResourceUrl).openConnection();
- log.info("orignal url: {}", con.getURL());
- con.connect();
- log.info("connected url: {} ", con.getURL());
- // assert connect-url: broker2-https
- assertEquals(con.getURL().getPort(), conf2.getWebServicePortTls());
- InputStream is = con.getInputStream();
- // assert redirect-url: broker1-https only
- log.info("redirected url: {}", con.getURL());
- assertEquals(con.getURL().getPort(), conf.getWebServicePortTls());
- is.close();
-
- pulsarClient2.close();
- pulsar2.close();
- loadManager1 = null;
- loadManager2 = null;
-
- }
+ public void testWebserviceServiceTls() throws Exception {
+ log.info("-- Starting {} test --", methodName);
+ final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/certificate/server.crt";
+ final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/certificate/server.key";
+ final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/certificate/client.crt";
+ final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/certificate/client.key";
+
+ /**** start broker-2 ****/
+ ServiceConfiguration conf2 = new ServiceConfiguration();
+ conf2.setAdvertisedAddress("localhost");
+ conf2.setBrokerServicePort(PortManager.nextFreePort());
+ conf2.setBrokerServicePortTls(PortManager.nextFreePort());
+ conf2.setWebServicePort(PortManager.nextFreePort());
+ conf2.setWebServicePortTls(PortManager.nextFreePort());
+ conf2.setAdvertisedAddress("localhost");
+ conf2.setTlsAllowInsecureConnection(true);
+ conf2.setTlsEnabled(true);
+ conf2.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
+ conf2.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
+ conf2.setClusterName(conf.getClusterName());
+ conf2.setZookeeperServers("localhost:2181");
+ PulsarService pulsar2 = startBroker(conf2);
+
+ // restart broker1 with tls enabled
+ conf.setTlsAllowInsecureConnection(true);
+ conf.setTlsEnabled(true);
+ conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
+ conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
+ stopBroker();
+ startBroker();
+ pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
+ pulsar2.getLoadManager().get().writeLoadReportOnZookeeper();
+
+ LoadManager loadManager1 = spy(pulsar.getLoadManager().get());
+ LoadManager loadManager2 = spy(pulsar2.getLoadManager().get());
+ Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager");
+ loadManagerField.setAccessible(true);
+
+ // mock: redirect request to leader [2]
+ doReturn(true).when(loadManager2).isCentralized();
+ loadManagerField.set(pulsar2.getNamespaceService(), new AtomicReference<>(loadManager2));
+
+ // mock: return Broker2 as a Least-loaded broker when leader receies
+ // request [3]
+ doReturn(true).when(loadManager1).isCentralized();
+ SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar2.getWebServiceAddress(), null);
+ doReturn(Optional.of(resourceUnit)).when(loadManager1).getLeastLoaded(any(ServiceUnitId.class));
+ loadManagerField.set(pulsar.getNamespaceService(), new AtomicReference<>(loadManager1));
+
+ /**** started broker-2 ****/
+
+ URI brokerServiceUrl = new URI("pulsar://localhost:" + conf2.getBrokerServicePort());
+ PulsarClient pulsarClient2 = PulsarClient.builder().serviceUrl(brokerServiceUrl.toString()).build();
+
+ final String lookupResourceUrl = "/lookup/v2/destination/persistent/my-property/use/my-ns/my-topic1";
+
+ // set client cert_key file
+ KeyManager[] keyManagers = null;
+ Certificate[] tlsCert = SecurityUtility.loadCertificatesFromPemFile(TLS_CLIENT_CERT_FILE_PATH);
+ PrivateKey tlsKey = SecurityUtility.loadPrivateKeyFromPemFile(TLS_CLIENT_KEY_FILE_PATH);
+ KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType());
+ ks.load(null, null);
+ ks.setKeyEntry("private", tlsKey, "".toCharArray(), tlsCert);
+ KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
+ kmf.init(ks, "".toCharArray());
+ keyManagers = kmf.getKeyManagers();
+ TrustManager[] trustManagers = InsecureTrustManagerFactory.INSTANCE.getTrustManagers();
+ SSLContext sslCtx = SSLContext.getInstance("TLS");
+ sslCtx.init(keyManagers, trustManagers, new SecureRandom());
+ HttpsURLConnection.setDefaultSSLSocketFactory(sslCtx.getSocketFactory());
+
+ // hit broker2 url
+ URLConnection con = new URL(pulsar2.getWebServiceAddressTls() + lookupResourceUrl).openConnection();
+ log.info("orignal url: {}", con.getURL());
+ con.connect();
+ log.info("connected url: {} ", con.getURL());
+ // assert connect-url: broker2-https
+ assertEquals(con.getURL().getPort(), conf2.getWebServicePortTls());
+ InputStream is = con.getInputStream();
+ // assert redirect-url: broker1-https only
+ log.info("redirected url: {}", con.getURL());
+ assertEquals(con.getURL().getPort(), conf.getWebServicePortTls());
+ is.close();
+
+ pulsarClient2.close();
+ pulsar2.close();
+ loadManager1 = null;
+ loadManager2 = null;
+
+ }
/**
- * Discovery-Service lookup over binary-protocol
- * 1. Start discovery service
- * 2. start broker
- * 3. Create Producer/Consumer: by calling Discovery service for partitionedMetadata and topic lookup
+ * Discovery-Service lookup over binary-protocol 1. Start discovery service 2. start broker 3. Create
+ * Producer/Consumer: by calling Discovery service for partitionedMetadata and topic lookup
*
* @throws Exception
*/
@@ -499,18 +486,18 @@ public void testDiscoveryLookup() throws Exception {
// (2) lookup using discovery service
final String discoverySvcUrl = discoveryService.getServiceUrl();
- ClientConfiguration clientConfig = new ClientConfiguration();
- PulsarClient pulsarClient2 = PulsarClient.create(discoverySvcUrl, clientConfig);
- Consumer consumer = pulsarClient2.subscribe("persistent://my-property2/use2/my-ns/my-topic1", "my-subscriber-name",
- new ConsumerConfiguration());
- Producer producer = pulsarClient2.createProducer("persistent://my-property2/use2/my-ns/my-topic1", new ProducerConfiguration());
+ PulsarClient pulsarClient2 = PulsarClient.builder().serviceUrl(discoverySvcUrl).build();
+ Consumer<byte[]> consumer = pulsarClient2.newConsumer().topic("persistent://my-property2/use2/my-ns/my-topic1")
+ .subscriptionName("my-subscriber-name").subscribe();
+ Producer<byte[]> producer = pulsarClient2.newProducer().topic("persistent://my-property2/use2/my-ns/my-topic1")
+ .create();
for (int i = 0; i < 10; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
- Message msg = null;
+ Message<byte[]> msg = null;
Set<String> messageSet = Sets.newHashSet();
for (int i = 0; i < 10; i++) {
msg = consumer.receive(5, TimeUnit.SECONDS);
@@ -526,12 +513,12 @@ public void testDiscoveryLookup() throws Exception {
}
-
/**
* Verify discovery-service binary-proto lookup using tls
*
* @throws Exception
*/
+ @SuppressWarnings("deprecation")
@Test
public void testDiscoveryLookupTls() throws Exception {
@@ -562,29 +549,26 @@ public void testDiscoveryLookupTls() throws Exception {
// (3) lookup using discovery service
final String discoverySvcUrl = discoveryService.getServiceUrlTls();
- ClientConfiguration clientConfig = new ClientConfiguration();
Map<String, String> authParams = new HashMap<>();
authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
Authentication auth = new AuthenticationTls();
auth.configure(authParams);
- clientConfig.setAuthentication(auth);
- clientConfig.setUseTls(true);
- clientConfig.setTlsAllowInsecureConnection(true);
-
- PulsarClient pulsarClient2 = PulsarClient.create(discoverySvcUrl, clientConfig);
- Consumer consumer = pulsarClient2.subscribe("persistent://my-property2/use2/my-ns/my-topic1", "my-subscriber-name",
- new ConsumerConfiguration());
- Producer producer = pulsarClient2.createProducer("persistent://my-property2/use2/my-ns/my-topic1", new ProducerConfiguration());
+ PulsarClient pulsarClient2 = PulsarClient.builder().serviceUrl(discoverySvcUrl).authentication(auth)
+ .enableTls(true).allowTlsInsecureConnection(true).build();
+ Consumer<byte[]> consumer = pulsarClient2.newConsumer().topic("persistent://my-property2/use2/my-ns/my-topic1")
+ .subscriptionName("my-subscriber-name").subscribe();
+ Producer<byte[]> producer = pulsarClient2.newProducer().topic("persistent://my-property2/use2/my-ns/my-topic1")
+ .create();
for (int i = 0; i < 10; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
- Message msg = null;
+ Message<byte[]> msg = null;
Set<String> messageSet = Sets.newHashSet();
for (int i = 0; i < 10; i++) {
msg = consumer.receive(5, TimeUnit.SECONDS);
@@ -619,42 +603,45 @@ public void testDiscoveryLookupAuthAndAuthSuccess() throws Exception {
// (2) lookup using discovery service
final String discoverySvcUrl = discoveryService.getServiceUrl();
- ClientConfiguration clientConfig = new ClientConfiguration();
// set authentication data
- clientConfig.setAuthentication(new Authentication() {
+ Authentication auth = new Authentication() {
private static final long serialVersionUID = 1L;
@Override
public void close() throws IOException {
}
+
@Override
public String getAuthMethodName() {
return "auth";
}
+
@Override
public AuthenticationDataProvider getAuthData() throws PulsarClientException {
return new AuthenticationDataProvider() {
private static final long serialVersionUID = 1L;
};
}
+
@Override
public void configure(Map<String, String> authParams) {
}
+
@Override
public void start() throws PulsarClientException {
}
- });
+ };
- PulsarClient pulsarClient = PulsarClient.create(discoverySvcUrl, clientConfig);
- Consumer consumer = pulsarClient.subscribe("persistent://my-property/use2/my-ns/my-topic1",
- "my-subscriber-name", new ConsumerConfiguration());
- Producer producer = pulsarClient.createProducer("persistent://my-property/use2/my-ns/my-topic1",
- new ProducerConfiguration());
+ PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(discoverySvcUrl).authentication(auth).build();
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://my-property/use2/my-ns/my-topic1")
+ .subscriptionName("my-subscriber-name").subscribe();
+ Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://my-property/use2/my-ns/my-topic1")
+ .create();
for (int i = 0; i < 10; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
- Message msg = null;
+ Message<byte[]> msg = null;
Set<String> messageSet = Sets.newHashSet();
for (int i = 0; i < 10; i++) {
msg = consumer.receive(5, TimeUnit.SECONDS);
@@ -687,35 +674,41 @@ public void testDiscoveryLookupAuthenticationFailure() throws Exception {
discoveryService.start();
// (2) lookup using discovery service
final String discoverySvcUrl = discoveryService.getServiceUrl();
- ClientConfiguration clientConfig = new ClientConfiguration();
+
// set authentication data
- clientConfig.setAuthentication(new Authentication() {
+ Authentication auth = new Authentication() {
private static final long serialVersionUID = 1L;
@Override
public void close() throws IOException {
}
+
@Override
public String getAuthMethodName() {
return "auth";
}
+
@Override
public AuthenticationDataProvider getAuthData() throws PulsarClientException {
return new AuthenticationDataProvider() {
private static final long serialVersionUID = 1L;
};
}
+
@Override
public void configure(Map<String, String> authParams) {
}
+
@Override
public void start() throws PulsarClientException {
}
- });
- PulsarClient pulsarClient = PulsarClient.create(discoverySvcUrl, clientConfig);
+ };
+
+ @Cleanup
+ PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(discoverySvcUrl).authentication(auth).build();
try {
- pulsarClient.subscribe("persistent://my-property/use2/my-ns/my-topic1", "my-subscriber-name",
- new ConsumerConfiguration());
+ pulsarClient.newConsumer().topic("persistent://my-property/use2/my-ns/my-topic1")
+ .subscriptionName("my-subscriber-name").subscribe();
fail("should have failed due to authentication");
} catch (PulsarClientException e) {
// Ok: expected
@@ -740,35 +733,41 @@ public void testDiscoveryLookupAuthorizationFailure() throws Exception {
discoveryService.start();
// (2) lookup using discovery service
final String discoverySvcUrl = discoveryService.getServiceUrl();
- ClientConfiguration clientConfig = new ClientConfiguration();
+
// set authentication data
- clientConfig.setAuthentication(new Authentication() {
+ Authentication auth = new Authentication() {
private static final long serialVersionUID = 1L;
@Override
public void close() throws IOException {
}
+
@Override
public String getAuthMethodName() {
return "auth";
}
+
@Override
public AuthenticationDataProvider getAuthData() throws PulsarClientException {
return new AuthenticationDataProvider() {
private static final long serialVersionUID = 1L;
};
}
+
@Override
public void configure(Map<String, String> authParams) {
}
+
@Override
public void start() throws PulsarClientException {
}
- });
- PulsarClient pulsarClient = PulsarClient.create(discoverySvcUrl, clientConfig);
+ };
+
+ @Cleanup
+ PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(discoverySvcUrl).authentication(auth).build();
try {
- pulsarClient.subscribe("persistent://my-property/use2/my-ns/my-topic1", "my-subscriber-name",
- new ConsumerConfiguration());
+ pulsarClient.newConsumer().topic("persistent://my-property/use2/my-ns/my-topic1")
+ .subscriptionName("my-subscriber-name").subscribe();
fail("should have failed due to authentication");
} catch (PulsarClientException e) {
// Ok: expected
@@ -833,11 +832,12 @@ public void testSplitUnloadLookupTest() throws Exception {
loadManagerField.set(pulsar.getNamespaceService(), new AtomicReference<>(loadManager1));
URI broker2ServiceUrl = new URI("pulsar://localhost:" + conf2.getBrokerServicePort());
- PulsarClient pulsarClient2 = PulsarClient.create(broker2ServiceUrl.toString(), new ClientConfiguration());
+ PulsarClient pulsarClient2 = PulsarClient.builder().serviceUrl(broker2ServiceUrl.toString()).build();
// (3) Broker-2 receives topic-1 request, creates local-policies and sets the watch
final String topic1 = "persistent://" + namespace + "/topic1";
- Consumer consumer1 = pulsarClient2.subscribe(topic1, "my-subscriber-name", new ConsumerConfiguration());
+ Consumer<byte[]> consumer1 = pulsarClient2.newConsumer().topic(topic1).subscriptionName("my-subscriber-name")
+ .subscribe();
Set<String> serviceUnits1 = pulsar.getNamespaceService().getOwnedServiceUnits().stream()
.map(nb -> nb.toString()).collect(Collectors.toSet());
@@ -865,10 +865,10 @@ public void testSplitUnloadLookupTest() throws Exception {
// (7) Make lookup request again to Broker-2 which should succeed.
final String topic2 = "persistent://" + namespace + "/topic2";
- Consumer consumer2 = pulsarClient.subscribe(topic2, "my-subscriber-name", new ConsumerConfiguration());
+ Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic(topic2).subscriptionName("my-subscriber-name")
+ .subscribe();
- NamespaceBundle bundleInBroker1AfterSplit = pulsar2.getNamespaceService()
- .getBundle(TopicName.get(topic2));
+ NamespaceBundle bundleInBroker1AfterSplit = pulsar2.getNamespaceService().getBundle(TopicName.get(topic2));
assertFalse(bundleInBroker1AfterSplit.equals(unsplitBundle));
consumer1.close();
@@ -942,11 +942,12 @@ public void testModularLoadManagerSplitBundle() throws Exception {
loadManagerField.set(pulsar.getNamespaceService(), new AtomicReference<>(loadManager1));
URI broker2ServiceUrl = new URI("pulsar://localhost:" + conf2.getBrokerServicePort());
- PulsarClient pulsarClient2 = PulsarClient.create(broker2ServiceUrl.toString(), new ClientConfiguration());
+ PulsarClient pulsarClient2 = PulsarClient.builder().serviceUrl(broker2ServiceUrl.toString()).build();
// (3) Broker-2 receives topic-1 request, creates local-policies and sets the watch
final String topic1 = "persistent://" + namespace + "/topic1";
- Consumer consumer1 = pulsarClient2.subscribe(topic1, "my-subscriber-name", new ConsumerConfiguration());
+ Consumer<byte[]> consumer1 = pulsarClient2.newConsumer().topic(topic1)
+ .subscriptionName("my-subscriber-name").subscribe();
Set<String> serviceUnits1 = pulsar.getNamespaceService().getOwnedServiceUnits().stream()
.map(nb -> nb.toString()).collect(Collectors.toSet());
@@ -994,10 +995,10 @@ public void testModularLoadManagerSplitBundle() throws Exception {
// (7) Make lookup request again to Broker-2 which should succeed.
final String topic2 = "persistent://" + namespace + "/topic2";
- Consumer consumer2 = pulsarClient.subscribe(topic2, "my-subscriber-name", new ConsumerConfiguration());
+ Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic(topic2).subscriptionName("my-subscriber-name")
+ .subscribe();
- NamespaceBundle bundleInBroker1AfterSplit = pulsar2.getNamespaceService()
- .getBundle(TopicName.get(topic2));
+ NamespaceBundle bundleInBroker1AfterSplit = pulsar2.getNamespaceService().getBundle(TopicName.get(topic2));
assertFalse(bundleInBroker1AfterSplit.equals(unsplitBundle));
consumer1.close();
@@ -1152,13 +1153,16 @@ public boolean keepAlive(Request ahcRequest, HttpRequest request, HttpResponse r
@Override
public void close() throws IOException {
}
+
@Override
public void initialize(ServiceConfiguration config) throws IOException {
}
+
@Override
public String getAuthMethodName() {
return "auth";
}
+
@Override
public String authenticate(AuthenticationDataSource authData) throws AuthenticationException {
return "appid1";
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientDeduplicationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientDeduplicationTest.java
index fb5cc0fa8..de8c24c7d 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientDeduplicationTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientDeduplicationTest.java
@@ -46,9 +46,9 @@ public void testProducerSequenceAfterReconnect() throws Exception {
String topic = "persistent://my-property/use/my-ns/testProducerSequenceAfterReconnect";
admin.namespaces().setDeduplicationStatus("my-property/use/my-ns", true);
- ProducerConfiguration conf = new ProducerConfiguration();
- conf.setProducerName("my-producer-name");
- Producer producer = pulsarClient.createProducer(topic, conf);
+ ProducerBuilder<byte[]> producerBuilder = pulsarClient.newProducer().topic(topic)
+ .producerName("my-producer-name");
+ Producer<byte[]> producer = producerBuilder.create();
assertEquals(producer.getLastSequenceId(), -1L);
@@ -60,7 +60,7 @@ public void testProducerSequenceAfterReconnect() throws Exception {
producer.close();
- producer = pulsarClient.createProducer(topic, conf);
+ producer = producerBuilder.create();
assertEquals(producer.getLastSequenceId(), 9L);
for (int i = 10; i < 20; i++) {
@@ -77,9 +77,9 @@ public void testProducerSequenceAfterRestart() throws Exception {
String topic = "persistent://my-property/use/my-ns/testProducerSequenceAfterRestart";
admin.namespaces().setDeduplicationStatus("my-property/use/my-ns", true);
- ProducerConfiguration conf = new ProducerConfiguration();
- conf.setProducerName("my-producer-name");
- Producer producer = pulsarClient.createProducer(topic, conf);
+ ProducerBuilder<byte[]> producerBuilder = pulsarClient.newProducer().topic(topic)
+ .producerName("my-producer-name");
+ Producer<byte[]> producer = producerBuilder.create();
assertEquals(producer.getLastSequenceId(), -1L);
@@ -94,7 +94,7 @@ public void testProducerSequenceAfterRestart() throws Exception {
// Kill and restart broker
restartBroker();
- producer = pulsarClient.createProducer(topic, conf);
+ producer = producerBuilder.create();
assertEquals(producer.getLastSequenceId(), 9L);
for (int i = 10; i < 20; i++) {
@@ -111,16 +111,15 @@ public void testProducerDeduplication() throws Exception {
String topic = "persistent://my-property/use/my-ns/testProducerDeduplication";
admin.namespaces().setDeduplicationStatus("my-property/use/my-ns", true);
- ProducerConfiguration conf = new ProducerConfiguration();
- conf.setProducerName("my-producer-name");
-
// Set infinite timeout
- conf.setSendTimeout(0, TimeUnit.SECONDS);
- Producer producer = pulsarClient.createProducer(topic, conf);
+ ProducerBuilder<byte[]> producerBuilder = pulsarClient.newProducer().topic(topic)
+ .producerName("my-producer-name").sendTimeout(0, TimeUnit.SECONDS);
+ Producer<byte[]> producer = producerBuilder.create();
assertEquals(producer.getLastSequenceId(), -1L);
- Consumer consumer = pulsarClient.subscribe(topic, "my-subscription");
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("my-subscription")
+ .subscribe();
producer.send(MessageBuilder.create().setContent("my-message-0".getBytes()).setSequenceId(0).build());
producer.send(MessageBuilder.create().setContent("my-message-1".getBytes()).setSequenceId(1).build());
@@ -133,19 +132,19 @@ public void testProducerDeduplication() throws Exception {
producer.close();
for (int i = 0; i < 3; i++) {
- Message msg = consumer.receive();
+ Message<byte[]> msg = consumer.receive();
assertEquals(new String(msg.getData()), "my-message-" + i);
consumer.acknowledge(msg);
}
// No other messages should be received
- Message msg = consumer.receive(1, TimeUnit.SECONDS);
+ Message<byte[]> msg = consumer.receive(1, TimeUnit.SECONDS);
assertNull(msg);
// Kill and restart broker
restartBroker();
- producer = pulsarClient.createProducer(topic, conf);
+ producer = producerBuilder.create();
assertEquals(producer.getLastSequenceId(), 2L);
// Repeat the messages and verify they're not received by consumer
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientErrorsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientErrorsTest.java
index 2ef60c1b2..a0f007a83 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientErrorsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientErrorsTest.java
@@ -23,32 +23,24 @@
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
+import io.netty.channel.ChannelHandlerContext;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
-
import org.apache.bookkeeper.test.PortManager;
-import org.apache.pulsar.client.api.ClientConfiguration;
-import org.apache.pulsar.client.api.Consumer;
-import org.apache.pulsar.client.api.ConsumerConfiguration;
-import org.apache.pulsar.client.api.Producer;
-import org.apache.pulsar.client.api.PulsarClient;
-import org.apache.pulsar.client.api.PulsarClientException;
-import org.apache.pulsar.client.api.SubscriptionType;
import org.apache.pulsar.client.api.PulsarClientException.LookupException;
import org.apache.pulsar.client.impl.ConsumerBase;
import org.apache.pulsar.client.impl.ProducerBase;
import org.apache.pulsar.common.api.Commands;
-import org.apache.pulsar.common.api.proto.PulsarApi.ServerError;
import org.apache.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.LookupType;
+import org.apache.pulsar.common.api.proto.PulsarApi.ServerError;
+import org.apache.pulsar.common.schema.SchemaVersion;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
-import io.netty.channel.ChannelHandlerContext;
-
/**
*/
public class ClientErrorsTest {
@@ -78,13 +70,12 @@ public void teardown() {
public void testMockBrokerService() throws Exception {
// test default actions of mock broker service
try {
- PulsarClient client = PulsarClient.create("http://127.0.0.1:" + WEB_SERVICE_PORT);
+ PulsarClient client = PulsarClient.builder().serviceUrl("http://127.0.0.1:" + WEB_SERVICE_PORT).build();
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
- Consumer consumer = client.subscribe("persistent://prop/use/ns/t1", "sub1", conf);
+ Consumer<byte[]> consumer = client.newConsumer().topic("persistent://prop/use/ns/t1")
+ .subscriptionName("sub1").subscribe();
- Producer producer = client.createProducer("persistent://prop/use/ns/t1");
+ Producer<byte[]> producer = client.newProducer().topic("persistent://prop/use/ns/t1").create();
Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
producer.send("message".getBytes());
Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
@@ -110,7 +101,7 @@ public void testPartitionedProducerCreateFailWithoutRetry() throws Exception {
}
private void producerCreateFailWithoutRetry(String topic) throws Exception {
- PulsarClient client = PulsarClient.create("http://127.0.0.1:" + WEB_SERVICE_PORT);
+ PulsarClient client = PulsarClient.builder().serviceUrl("http://127.0.0.1:" + WEB_SERVICE_PORT).build();
final AtomicInteger counter = new AtomicInteger(0);
mockBrokerService.setHandleProducer((ctx, producer) -> {
@@ -124,7 +115,7 @@ private void producerCreateFailWithoutRetry(String topic) throws Exception {
});
try {
- Producer producer = client.createProducer(topic);
+ client.newProducer().topic(topic).create();
} catch (Exception e) {
if (e.getMessage().equals(ASSERTION_ERROR)) {
fail("Producer create should not retry on auth error");
@@ -146,19 +137,19 @@ public void testPartitionedProducerCreateSuccessAfterRetry() throws Exception {
}
private void producerCreateSuccessAfterRetry(String topic) throws Exception {
- PulsarClient client = PulsarClient.create("http://127.0.0.1:" + WEB_SERVICE_PORT);
+ PulsarClient client = PulsarClient.builder().serviceUrl("http://127.0.0.1:" + WEB_SERVICE_PORT).build();
final AtomicInteger counter = new AtomicInteger(0);
mockBrokerService.setHandleProducer((ctx, producer) -> {
if (counter.incrementAndGet() == 2) {
- ctx.writeAndFlush(Commands.newProducerSuccess(producer.getRequestId(), "default-producer"));
+ ctx.writeAndFlush(Commands.newProducerSuccess(producer.getRequestId(), "default-producer", SchemaVersion.Empty));
return;
}
ctx.writeAndFlush(Commands.newError(producer.getRequestId(), ServerError.ServiceNotReady, "msg"));
});
try {
- Producer producer = client.createProducer(topic);
+ client.newProducer().topic(topic).create();
} catch (Exception e) {
fail("Should not fail");
}
@@ -178,10 +169,8 @@ public void testPartitionedProducerCreateFailAfterRetryTimeout() throws Exceptio
}
private void producerCreateFailAfterRetryTimeout(String topic) throws Exception {
- ClientConfiguration conf = new ClientConfiguration();
- conf.setOperationTimeout(1, TimeUnit.SECONDS);
-
- PulsarClient client = PulsarClient.create("http://127.0.0.1:" + WEB_SERVICE_PORT, conf);
+ PulsarClient client = PulsarClient.builder().serviceUrl("http://127.0.0.1:" + WEB_SERVICE_PORT)
+ .operationTimeout(1, TimeUnit.SECONDS).build();
final AtomicInteger counter = new AtomicInteger(0);
mockBrokerService.setHandleProducer((ctx, producer) -> {
@@ -196,7 +185,7 @@ private void producerCreateFailAfterRetryTimeout(String topic) throws Exception
});
try {
- Producer producer = client.createProducer(topic);
+ client.newProducer().topic(topic).create();
fail("Should have failed");
} catch (Exception e) {
// we fail even on the retriable error
@@ -218,7 +207,7 @@ public void testPartitionedProducerFailDoesNotFailOtherProducer() throws Excepti
}
private void producerFailDoesNotFailOtherProducer(String topic1, String topic2) throws Exception {
- PulsarClient client = PulsarClient.create("http://127.0.0.1:" + WEB_SERVICE_PORT);
+ PulsarClient client = PulsarClient.builder().serviceUrl("http://127.0.0.1:" + WEB_SERVICE_PORT).build();
final AtomicInteger counter = new AtomicInteger(0);
mockBrokerService.setHandleProducer((ctx, producer) -> {
@@ -227,15 +216,15 @@ private void producerFailDoesNotFailOtherProducer(String topic1, String topic2)
ctx.writeAndFlush(Commands.newError(producer.getRequestId(), ServerError.AuthenticationError, "msg"));
return;
}
- ctx.writeAndFlush(Commands.newProducerSuccess(producer.getRequestId(), "default-producer"));
+ ctx.writeAndFlush(Commands.newProducerSuccess(producer.getRequestId(), "default-producer", SchemaVersion.Empty));
});
- ProducerBase producer1 = (ProducerBase) client.createProducer(topic1);
+ ProducerBase<byte[]> producer1 = (ProducerBase<byte[]>) client.newProducer().topic(topic1).create();
- ProducerBase producer2 = null;
+ ProducerBase<byte[]> producer2 = null;
try {
- producer2 = (ProducerBase) client.createProducer(topic2);
+ producer2 = (ProducerBase<byte[]>) client.newProducer().topic(topic2).create();
fail("Should have failed");
} catch (Exception e) {
// ok
@@ -259,14 +248,14 @@ public void testPartitionedProducerContinuousRetryAfterSendFail() throws Excepti
}
private void producerContinuousRetryAfterSendFail(String topic) throws Exception {
- PulsarClient client = PulsarClient.create("http://127.0.0.1:" + WEB_SERVICE_PORT);
+ PulsarClient client = PulsarClient.builder().serviceUrl("http://127.0.0.1:" + WEB_SERVICE_PORT).build();
final AtomicInteger counter = new AtomicInteger(0);
mockBrokerService.setHandleProducer((ctx, producer) -> {
int i = counter.incrementAndGet();
if (i == 1 || i == 5) {
// succeed on 1st and 5th attempts
- ctx.writeAndFlush(Commands.newProducerSuccess(producer.getRequestId(), "default-producer"));
+ ctx.writeAndFlush(Commands.newProducerSuccess(producer.getRequestId(), "default-producer", SchemaVersion.Empty));
return;
}
ctx.writeAndFlush(Commands.newError(producer.getRequestId(), ServerError.PersistenceError, "msg"));
@@ -283,7 +272,7 @@ private void producerContinuousRetryAfterSendFail(String topic) throws Exception
});
try {
- Producer producer = client.createProducer(topic);
+ Producer<byte[]> producer = client.newProducer().topic(topic).create();
producer.send("message".getBytes());
} catch (Exception e) {
fail("Should not fail");
@@ -307,7 +296,7 @@ public void testPartitionedSubscribeFailWithoutRetry() throws Exception {
@Test
public void testLookupWithDisconnection() throws Exception {
final String brokerUrl = "pulsar://127.0.0.1:" + BROKER_SERVICE_PORT;
- PulsarClient client = PulsarClient.create(brokerUrl);
+ PulsarClient client = PulsarClient.builder().serviceUrl(brokerUrl).build();
final AtomicInteger counter = new AtomicInteger(0);
String topic = "persistent://prop/use/ns/t1";
@@ -326,9 +315,7 @@ public void testLookupWithDisconnection() throws Exception {
});
try {
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
- Consumer consumer = client.subscribe(topic, "sub1", conf);
+ client.newConsumer().topic(topic).subscriptionName("sub1").subscribe();
} catch (Exception e) {
if (e.getMessage().equals(ASSERTION_ERROR)) {
fail("Subscribe should not retry on persistence error");
@@ -342,7 +329,7 @@ public void testLookupWithDisconnection() throws Exception {
}
private void subscribeFailWithoutRetry(String topic) throws Exception {
- PulsarClient client = PulsarClient.create("http://127.0.0.1:" + WEB_SERVICE_PORT);
+ PulsarClient client = PulsarClient.builder().serviceUrl("http://127.0.0.1:" + WEB_SERVICE_PORT).build();
final AtomicInteger counter = new AtomicInteger(0);
mockBrokerService.setHandleSubscribe((ctx, subscribe) -> {
@@ -356,9 +343,7 @@ private void subscribeFailWithoutRetry(String topic) throws Exception {
});
try {
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
- Consumer consumer = client.subscribe(topic, "sub1", conf);
+ client.newConsumer().topic(topic).subscriptionName("sub1").subscribe();
} catch (Exception e) {
if (e.getMessage().equals(ASSERTION_ERROR)) {
fail("Subscribe should not retry on persistence error");
@@ -380,7 +365,7 @@ public void testPartitionedSubscribeSuccessAfterRetry() throws Exception {
}
private void subscribeSuccessAfterRetry(String topic) throws Exception {
- PulsarClient client = PulsarClient.create("http://127.0.0.1:" + WEB_SERVICE_PORT);
+ PulsarClient client = PulsarClient.builder().serviceUrl("http://127.0.0.1:" + WEB_SERVICE_PORT).build();
final AtomicInteger counter = new AtomicInteger(0);
mockBrokerService.setHandleSubscribe((ctx, subscribe) -> {
@@ -392,9 +377,7 @@ private void subscribeSuccessAfterRetry(String topic) throws Exception {
});
try {
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
- Consumer consumer = client.subscribe(topic, "sub1", conf);
+ client.newConsumer().topic(topic).subscriptionName("sub1").subscribe();
} catch (Exception e) {
fail("Should not fail");
}
@@ -414,10 +397,8 @@ public void testPartitionedSubscribeFailAfterRetryTimeout() throws Exception {
}
private void subscribeFailAfterRetryTimeout(String topic) throws Exception {
- ClientConfiguration conf = new ClientConfiguration();
- conf.setOperationTimeout(200, TimeUnit.MILLISECONDS);
-
- PulsarClient client = PulsarClient.create("http://127.0.0.1:" + WEB_SERVICE_PORT, conf);
+ PulsarClient client = PulsarClient.builder().serviceUrl("http://127.0.0.1:" + WEB_SERVICE_PORT)
+ .operationTimeout(200, TimeUnit.MILLISECONDS).build();
final AtomicInteger counter = new AtomicInteger(0);
mockBrokerService.setHandleSubscribe((ctx, subscribe) -> {
@@ -432,9 +413,7 @@ private void subscribeFailAfterRetryTimeout(String topic) throws Exception {
});
try {
- ConsumerConfiguration cConf = new ConsumerConfiguration();
- cConf.setSubscriptionType(SubscriptionType.Exclusive);
- client.subscribe(topic, "sub1", cConf);
+ client.newConsumer().topic(topic).subscriptionName("sub1").subscribe();
fail("Should have failed");
} catch (Exception e) {
// we fail even on the retriable error
@@ -456,7 +435,7 @@ public void testPartitionedSubscribeFailDoesNotFailOtherConsumer() throws Except
}
private void subscribeFailDoesNotFailOtherConsumer(String topic1, String topic2) throws Exception {
- PulsarClient client = PulsarClient.create("http://127.0.0.1:" + WEB_SERVICE_PORT);
+ PulsarClient client = PulsarClient.builder().serviceUrl("http://127.0.0.1:" + WEB_SERVICE_PORT).build();
final AtomicInteger counter = new AtomicInteger(0);
mockBrokerService.setHandleSubscribe((ctx, subscribe) -> {
@@ -469,13 +448,12 @@ private void subscribeFailDoesNotFailOtherConsumer(String topic1, String topic2)
});
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
- ConsumerBase consumer1 = (ConsumerBase) client.subscribe(topic1, "sub1", conf);
+ ConsumerBase<byte[]> consumer1 = (ConsumerBase<byte[]>) client.newConsumer().topic(topic1)
+ .subscriptionName("sub1").subscribe();
- ConsumerBase consumer2 = null;
+ ConsumerBase<byte[]> consumer2 = null;
try {
- consumer2 = (ConsumerBase) client.subscribe(topic2, "sub1", conf);
+ consumer2 = (ConsumerBase<byte[]>) client.newConsumer().topic(topic2).subscriptionName("sub1").subscribe();
fail("Should have failed");
} catch (Exception e) {
// ok
@@ -492,7 +470,7 @@ private void subscribeFailDoesNotFailOtherConsumer(String topic1, String topic2)
// other producers and fail
@Test
public void testOneProducerFailShouldCloseAllProducersInPartitionedProducer() throws Exception {
- PulsarClient client = PulsarClient.create("http://127.0.0.1:" + WEB_SERVICE_PORT);
+ PulsarClient client = PulsarClient.builder().serviceUrl("http://127.0.0.1:" + WEB_SERVICE_PORT).build();
final AtomicInteger producerCounter = new AtomicInteger(0);
final AtomicInteger closeCounter = new AtomicInteger(0);
@@ -501,7 +479,7 @@ public void testOneProducerFailShouldCloseAllProducersInPartitionedProducer() th
ctx.writeAndFlush(Commands.newError(producer.getRequestId(), ServerError.AuthorizationError, "msg"));
return;
}
- ctx.writeAndFlush(Commands.newProducerSuccess(producer.getRequestId(), "default-producer"));
+ ctx.writeAndFlush(Commands.newProducerSuccess(producer.getRequestId(), "default-producer", SchemaVersion.Empty));
});
mockBrokerService.setHandleCloseProducer((ctx, closeProducer) -> {
@@ -510,7 +488,7 @@ public void testOneProducerFailShouldCloseAllProducersInPartitionedProducer() th
});
try {
- Producer producer = client.createProducer("persistent://prop/use/ns/multi-part-t1");
+ client.newProducer().topic("persistent://prop/use/ns/multi-part-t1").create();
fail("Should have failed with an authorization error");
} catch (Exception e) {
assertTrue(e instanceof PulsarClientException.AuthorizationException);
@@ -527,7 +505,7 @@ public void testOneProducerFailShouldCloseAllProducersInPartitionedProducer() th
// of other consumers and fail
@Test
public void testOneConsumerFailShouldCloseAllConsumersInPartitionedConsumer() throws Exception {
- PulsarClient client = PulsarClient.create("http://127.0.0.1:" + WEB_SERVICE_PORT);
+ PulsarClient client = PulsarClient.builder().serviceUrl("http://127.0.0.1:" + WEB_SERVICE_PORT).build();
final AtomicInteger subscribeCounter = new AtomicInteger(0);
final AtomicInteger closeCounter = new AtomicInteger(0);
@@ -546,7 +524,7 @@ public void testOneConsumerFailShouldCloseAllConsumersInPartitionedConsumer() th
});
try {
- Consumer consumer = client.subscribe("persistent://prop/use/ns/multi-part-t1", "my-sub");
+ client.newConsumer().topic("persistent://prop/use/ns/multi-part-t1").subscriptionName("sub1").subscribe();
fail("Should have failed with an authentication error");
} catch (Exception e) {
assertTrue(e instanceof PulsarClientException.AuthenticationException);
@@ -561,7 +539,7 @@ public void testOneConsumerFailShouldCloseAllConsumersInPartitionedConsumer() th
@Test
public void testFlowSendWhenPartitionedSubscribeCompletes() throws Exception {
- PulsarClient client = PulsarClient.create("http://127.0.0.1:" + WEB_SERVICE_PORT);
+ PulsarClient client = PulsarClient.builder().serviceUrl("http://127.0.0.1:" + WEB_SERVICE_PORT).build();
AtomicInteger subscribed = new AtomicInteger();
AtomicBoolean fail = new AtomicBoolean(false);
@@ -578,7 +556,7 @@ public void testFlowSendWhenPartitionedSubscribeCompletes() throws Exception {
}
});
- Consumer consumer = client.subscribe("persistent://prop/use/ns/multi-part-t1", "my-sub");
+ client.newConsumer().topic("persistent://prop/use/ns/multi-part-t1").subscriptionName("sub1").subscribe();
if (fail.get()) {
fail("Flow command should have been sent after all 4 partitions subscribe successfully");
@@ -605,7 +583,7 @@ public void testProducerReconnect() throws Exception {
});
mockBrokerService.setHandleProducer((ctx, produce) -> {
- ctx.writeAndFlush(Commands.newProducerSuccess(produce.getRequestId(), "default-producer"));
+ ctx.writeAndFlush(Commands.newProducerSuccess(produce.getRequestId(), "default-producer", SchemaVersion.Empty));
});
mockBrokerService.setHandleSend((ctx, sendCmd, headersAndPayload) -> {
@@ -613,8 +591,8 @@ public void testProducerReconnect() throws Exception {
ctx.writeAndFlush(Commands.newSendReceipt(0, 0, 1, 1));
});
- PulsarClient client = PulsarClient.create("http://127.0.0.1:" + WEB_SERVICE_PORT);
- Producer producer = client.createProducer("persistent://prop/use/ns/t1");
+ PulsarClient client = PulsarClient.builder().serviceUrl("http://127.0.0.1:" + WEB_SERVICE_PORT).build();
+ Producer<byte[]> producer = client.newProducer().topic("persistent://prop/use/ns/t1").create();
// close the cnx after creating the producer
channelCtx.get().channel().close().get();
@@ -651,8 +629,8 @@ public void testConsumerReconnect() throws Exception {
ctx.writeAndFlush(Commands.newSuccess(subscribe.getRequestId()));
});
- PulsarClient client = PulsarClient.create("http://127.0.0.1:" + WEB_SERVICE_PORT);
- client.subscribe("persistent://prop/use/ns/t1", "sub");
+ PulsarClient client = PulsarClient.builder().serviceUrl("http://127.0.0.1:" + WEB_SERVICE_PORT).build();
+ client.newConsumer().topic("persistent://prop/use/ns/t1").subscriptionName("sub1").subscribe();
// close the cnx after creating the producer
channelCtx.get().channel().close();
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DispatcherBlockConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DispatcherBlockConsumerTest.java
index 3167006ee..28e0a4a44 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DispatcherBlockConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DispatcherBlockConsumerTest.java
@@ -27,7 +27,6 @@
import static org.testng.Assert.fail;
import java.lang.reflect.Field;
-import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -91,7 +90,7 @@ protected void cleanup() throws Exception {
*
* @throws Exception
*/
- @Test(timeOut=10000)
+ @Test(timeOut = 10000)
public void testConsumerBlockingWithUnAckedMessagesAtDispatcher() throws Exception {
log.info("-- Starting {} test --", methodName);
@@ -106,18 +105,16 @@ public void testConsumerBlockingWithUnAckedMessagesAtDispatcher() throws Excepti
final String subscriberName = "subscriber-1";
pulsar.getConfiguration().setMaxUnackedMessagesPerSubscription(unackMsgAllowed);
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setReceiverQueueSize(receiverQueueSize);
- conf.setSubscriptionType(SubscriptionType.Shared);
- Consumer consumer1 = pulsarClient.subscribe(topicName, subscriberName, conf);
- Consumer consumer2 = pulsarClient.subscribe(topicName, subscriberName, conf);
- Consumer consumer3 = pulsarClient.subscribe(topicName, subscriberName, conf);
- Consumer[] consumers = { consumer1, consumer2, consumer3 };
-
- ProducerConfiguration producerConf = new ProducerConfiguration();
+ ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName(subscriberName).receiverQueueSize(receiverQueueSize)
+ .subscriptionType(SubscriptionType.Shared);
+ Consumer<byte[]> consumer1 = consumerBuilder.subscribe();
+ Consumer<byte[]> consumer2 = consumerBuilder.subscribe();
+ Consumer<byte[]> consumer3 = consumerBuilder.subscribe();
+ List<Consumer<?>> consumers = Lists.newArrayList(consumer1, consumer2, consumer3);
- Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic",
- producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer()
+ .topic("persistent://my-property/use/my-ns/unacked-topic").create();
// (1) Produced Messages
for (int i = 0; i < totalProducedMsgs; i++) {
@@ -126,13 +123,13 @@ public void testConsumerBlockingWithUnAckedMessagesAtDispatcher() throws Excepti
}
// (2) try to consume messages: but will be able to consume number of messages = unackMsgAllowed
- Message msg = null;
- Map<Message, Consumer> messages = Maps.newHashMap();
+ Message<?> msg = null;
+ Map<Message<?>, Consumer<?>> messages = Maps.newHashMap();
for (int i = 0; i < 3; i++) {
for (int j = 0; j < totalProducedMsgs; j++) {
- msg = consumers[i].receive(500, TimeUnit.MILLISECONDS);
+ msg = consumers.get(i).receive(500, TimeUnit.MILLISECONDS);
if (msg != null) {
- messages.put(msg, consumers[i]);
+ messages.put(msg, consumers.get(i));
} else {
break;
}
@@ -158,13 +155,13 @@ public void testConsumerBlockingWithUnAckedMessagesAtDispatcher() throws Excepti
// expecting messages which are not received
int expectedRemainingMessages = totalProducedMsgs - messages.size();
CountDownLatch latch = new CountDownLatch(expectedRemainingMessages);
- for (int i = 0; i < consumers.length; i++) {
+ for (int i = 0; i < consumers.size(); i++) {
final int consumerCount = i;
for (int j = 0; j < totalProducedMsgs; j++) {
- consumers[i].receiveAsync().thenAccept(m -> {
+ consumers.get(i).receiveAsync().thenAccept(m -> {
result.add(m.getMessageId());
try {
- consumers[consumerCount].acknowledge(m);
+ consumers.get(consumerCount).acknowledge(m);
} catch (PulsarClientException e) {
fail("failed to ack msg", e);
}
@@ -179,7 +176,7 @@ public void testConsumerBlockingWithUnAckedMessagesAtDispatcher() throws Excepti
assertTrue(result.size() >= expectedRemainingMessages);
producer.close();
- Arrays.asList(consumers).forEach(c -> {
+ consumers.forEach(c -> {
try {
c.close();
} catch (PulsarClientException e) {
@@ -200,7 +197,8 @@ public void testConsumerBlockingWithUnAckedMessagesAtDispatcher() throws Excepti
*
* @throws Exception
*/
- @Test(timeOut=10000)
+ @SuppressWarnings("unchecked")
+ @Test(timeOut = 10000)
public void testConsumerBlockingWithUnAckedMessagesAndRedelivery() throws Exception {
log.info("-- Starting {} test --", methodName);
@@ -213,18 +211,16 @@ public void testConsumerBlockingWithUnAckedMessagesAndRedelivery() throws Except
final String subscriberName = "subscriber-1";
pulsar.getConfiguration().setMaxUnackedMessagesPerSubscription(unackMsgAllowed);
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setReceiverQueueSize(receiverQueueSize);
- conf.setSubscriptionType(SubscriptionType.Shared);
- ConsumerImpl consumer1 = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriberName, conf);
- ConsumerImpl consumer2 = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriberName, conf);
- ConsumerImpl consumer3 = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriberName, conf);
- ConsumerImpl[] consumers = { consumer1, consumer2, consumer3 };
-
- ProducerConfiguration producerConf = new ProducerConfiguration();
+ ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName(subscriberName).receiverQueueSize(receiverQueueSize)
+ .subscriptionType(SubscriptionType.Shared);
+ ConsumerImpl<byte[]> consumer1 = (ConsumerImpl<byte[]>) consumerBuilder.subscribe();
+ ConsumerImpl<byte[]> consumer2 = (ConsumerImpl<byte[]>) consumerBuilder.subscribe();
+ ConsumerImpl<byte[]> consumer3 = (ConsumerImpl<byte[]>) consumerBuilder.subscribe();
+ List<ConsumerImpl<byte[]>> consumers = Lists.newArrayList(consumer1, consumer2, consumer3);
- Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic",
- producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer()
+ .topic("persistent://my-property/use/my-ns/unacked-topic").create();
// (1) Produced Messages
for (int i = 0; i < totalProducedMsgs; i++) {
@@ -233,13 +229,13 @@ public void testConsumerBlockingWithUnAckedMessagesAndRedelivery() throws Except
}
// (2) try to consume messages: but will be able to consume number of messages = unackMsgAllowed
- Message msg = null;
- Multimap<ConsumerImpl, MessageId> messages = ArrayListMultimap.create();
+ Message<?> msg = null;
+ Multimap<ConsumerImpl<?>, MessageId> messages = ArrayListMultimap.create();
for (int i = 0; i < 3; i++) {
for (int j = 0; j < totalProducedMsgs; j++) {
- msg = consumers[i].receive(500, TimeUnit.MILLISECONDS);
+ msg = consumers.get(i).receive(500, TimeUnit.MILLISECONDS);
if (msg != null) {
- messages.put(consumers[i], msg.getMessageId());
+ messages.put(consumers.get(i), msg.getMessageId());
log.info("Received message: " + new String(msg.getData()));
} else {
break;
@@ -260,13 +256,13 @@ public void testConsumerBlockingWithUnAckedMessagesAndRedelivery() throws Except
// all messages
Queue<MessageId> result = Queues.newConcurrentLinkedQueue();
CountDownLatch latch = new CountDownLatch(totalProducedMsgs);
- for (int i = 0; i < consumers.length; i++) {
+ for (int i = 0; i < consumers.size(); i++) {
final int consumerCount = i;
for (int j = 0; j < totalProducedMsgs; j++) {
- consumers[i].receiveAsync().thenAccept(m -> {
+ consumers.get(i).receiveAsync().thenAccept(m -> {
result.add(m.getMessageId());
try {
- consumers[consumerCount].acknowledge(m);
+ consumers.get(consumerCount).acknowledge(m);
} catch (PulsarClientException e) {
fail("failed to ack msg", e);
}
@@ -280,7 +276,7 @@ public void testConsumerBlockingWithUnAckedMessagesAndRedelivery() throws Except
// total received-messages should match to produced messages (it may have duplicate messages)
assertTrue(result.size() >= totalProducedMsgs);
producer.close();
- Arrays.asList(consumers).forEach(c -> {
+ consumers.forEach(c -> {
try {
c.close();
} catch (PulsarClientException e) {
@@ -315,15 +311,11 @@ public void testCloseConsumerBlockedDispatcher() throws Exception {
final String subscriberName = "subscriber-1";
pulsar.getConfiguration().setMaxUnackedMessagesPerSubscription(unackMsgAllowed);
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setReceiverQueueSize(receiverQueueSize);
- conf.setSubscriptionType(SubscriptionType.Shared);
- Consumer consumer1 = pulsarClient.subscribe(topicName, subscriberName, conf);
+ Consumer<byte[]> consumer1 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriberName)
+ .receiverQueueSize(receiverQueueSize).subscriptionType(SubscriptionType.Shared).subscribe();
- ProducerConfiguration producerConf = new ProducerConfiguration();
-
- Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic",
- producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer()
+ .topic("persistent://my-property/use/my-ns/unacked-topic").create();
// (1) Produced Messages
for (int i = 0; i < totalProducedMsgs; i++) {
@@ -332,8 +324,8 @@ public void testCloseConsumerBlockedDispatcher() throws Exception {
}
// (2) try to consume messages: but will be able to consume number of messages = unackMsgAllowed
- Message msg = null;
- Map<Message, Consumer> messages = Maps.newHashMap();
+ Message<?> msg = null;
+ Map<Message<?>, Consumer<?>> messages = Maps.newHashMap();
for (int i = 0; i < totalProducedMsgs; i++) {
msg = consumer1.receive(500, TimeUnit.MILLISECONDS);
if (msg != null) {
@@ -350,8 +342,9 @@ public void testCloseConsumerBlockedDispatcher() throws Exception {
// close consumer1: all messages of consumer1 must be replayed and received by consumer2
consumer1.close();
// create consumer2
- Consumer consumer2 = pulsarClient.subscribe(topicName, subscriberName, conf);
- Map<Message, Consumer> messages2 = Maps.newHashMap();
+ Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriberName)
+ .receiverQueueSize(receiverQueueSize).subscriptionType(SubscriptionType.Shared).subscribe();
+ Map<Message<?>, Consumer<?>> messages2 = Maps.newHashMap();
// try to consume remaining messages: broker may take time to deliver so, retry multiple time to consume
// all messages
for (int retry = 0; retry < 5; retry++) {
@@ -389,7 +382,7 @@ public void testCloseConsumerBlockedDispatcher() throws Exception {
*
* @throws Exception
*/
- @Test(timeOut=10000)
+ @Test(timeOut = 10000)
public void testRedeliveryOnBlockedDistpatcher() throws Exception {
log.info("-- Starting {} test --", methodName);
@@ -404,18 +397,16 @@ public void testRedeliveryOnBlockedDistpatcher() throws Exception {
final String subscriberName = "subscriber-1";
pulsar.getConfiguration().setMaxUnackedMessagesPerSubscription(unackMsgAllowed);
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Shared);
- conf.setReceiverQueueSize(receiverQueueSize);
- ConsumerImpl consumer1 = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriberName, conf);
- ConsumerImpl consumer2 = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriberName, conf);
- ConsumerImpl consumer3 = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriberName, conf);
- ConsumerImpl[] consumers = { consumer1, consumer2, consumer3 };
+ ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName(subscriberName).receiverQueueSize(receiverQueueSize)
+ .subscriptionType(SubscriptionType.Shared);
+ ConsumerImpl<byte[]> consumer1 = (ConsumerImpl<byte[]>) consumerBuilder.subscribe();
+ ConsumerImpl<byte[]> consumer2 = (ConsumerImpl<byte[]>) consumerBuilder.subscribe();
+ ConsumerImpl<byte[]> consumer3 = (ConsumerImpl<byte[]>) consumerBuilder.subscribe();
+ List<ConsumerImpl<?>> consumers = Lists.newArrayList(consumer1, consumer2, consumer3);
- ProducerConfiguration producerConf = new ProducerConfiguration();
-
- Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic",
- producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer()
+ .topic("persistent://my-property/use/my-ns/unacked-topic").create();
// (1) Produced Messages
for (int i = 0; i < totalProducedMsgs; i++) {
@@ -424,11 +415,11 @@ public void testRedeliveryOnBlockedDistpatcher() throws Exception {
}
// (2) try to consume messages: but will be able to consume number of messages = unackMsgAllowed
- Message msg = null;
+ Message<?> msg = null;
Set<MessageId> messages = Sets.newHashSet();
for (int i = 0; i < 3; i++) {
for (int j = 0; j < totalProducedMsgs; j++) {
- msg = consumers[i].receive(500, TimeUnit.MILLISECONDS);
+ msg = consumers.get(i).receive(500, TimeUnit.MILLISECONDS);
if (msg != null) {
messages.add(msg.getMessageId());
log.info("Received message: " + new String(msg.getData()));
@@ -443,7 +434,7 @@ public void testRedeliveryOnBlockedDistpatcher() throws Exception {
assertEquals(totalConsumedMsgs, unackMsgAllowed, 3 * receiverQueueSize);
// trigger redelivery
- Arrays.asList(consumers).forEach(c -> {
+ consumers.forEach(c -> {
c.redeliverUnacknowledgedMessages();
});
@@ -451,13 +442,13 @@ public void testRedeliveryOnBlockedDistpatcher() throws Exception {
Thread.sleep(1000);
// now, broker must have redelivered all unacked messages
- Map<ConsumerImpl, Set<MessageId>> messages1 = Maps.newHashMap();
+ Map<ConsumerImpl<?>, Set<MessageId>> messages1 = Maps.newHashMap();
for (int i = 0; i < 3; i++) {
for (int j = 0; j < totalProducedMsgs; j++) {
- msg = consumers[i].receive(500, TimeUnit.MILLISECONDS);
+ msg = consumers.get(i).receive(500, TimeUnit.MILLISECONDS);
if (msg != null) {
- messages1.putIfAbsent(consumers[i], Sets.newHashSet());
- messages1.get(consumers[i]).add(msg.getMessageId());
+ messages1.putIfAbsent(consumers.get(i), Sets.newHashSet());
+ messages1.get(consumers.get(i)).add(msg.getMessageId());
log.info("Received message: " + new String(msg.getData()));
} else {
break;
@@ -466,7 +457,7 @@ public void testRedeliveryOnBlockedDistpatcher() throws Exception {
}
Set<MessageId> result = Sets.newHashSet();
- messages1.values().forEach(s -> result.addAll(s));
+ messages1.values().forEach(result::addAll);
// check all unacked messages have been redelivered
@@ -483,20 +474,20 @@ public void testRedeliveryOnBlockedDistpatcher() throws Exception {
});
});
- messages1.values().forEach(s -> result.addAll(s));
+ messages1.values().forEach(result::addAll);
// try to consume remaining messages
int remainingMessages = totalProducedMsgs - result.size();
// try to consume remaining messages: broker may take time to deliver so, retry multiple time to consume
// all messages
CountDownLatch latch = new CountDownLatch(remainingMessages);
Queue<MessageId> consumedMessages = Queues.newConcurrentLinkedQueue();
- for (int i = 0; i < consumers.length; i++) {
+ for (int i = 0; i < consumers.size(); i++) {
final int counsumerIndex = i;
for (int j = 0; j < remainingMessages; j++) {
- consumers[i].receiveAsync().thenAccept(m -> {
+ consumers.get(i).receiveAsync().thenAccept(m -> {
consumedMessages.add(m.getMessageId());
try {
- consumers[counsumerIndex].acknowledge(m);
+ consumers.get(counsumerIndex).acknowledge(m);
} catch (PulsarClientException e) {
fail("failed to ack", e);
}
@@ -509,7 +500,7 @@ public void testRedeliveryOnBlockedDistpatcher() throws Exception {
// total received-messages should match remaining messages excluding duplicate
assertTrue(consumedMessages.size() >= remainingMessages);
producer.close();
- Arrays.asList(consumers).forEach(c -> {
+ consumers.forEach(c -> {
try {
c.close();
} catch (PulsarClientException e) {
@@ -540,9 +531,8 @@ public void testBlockDispatcherStats() throws Exception {
stopBroker();
startBroker();
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Shared);
- Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
+ .subscriptionType(SubscriptionType.Shared).subscribe();
Thread.sleep(timeWaitToSync);
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
@@ -557,7 +547,7 @@ public void testBlockDispatcherStats() throws Exception {
assertEquals(subStats.msgBacklog, 0);
assertEquals(subStats.consumers.size(), 1);
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
Thread.sleep(timeWaitToSync);
for (int i = 0; i < 100; i++) {
@@ -606,14 +596,11 @@ public void testBrokerSubscriptionRecovery(boolean unloadBundleGracefully) throw
final String subscriberName = "subscriber-1";
final int totalProducedMsgs = 500;
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Shared);
- Consumer consumer = pulsarClient.subscribe(topicName, subscriberName, conf);
-
- ProducerConfiguration producerConf = new ProducerConfiguration();
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriberName)
+ .subscriptionType(SubscriptionType.Shared).subscribe();
- Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic",
- producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/unacked-topic")
+ .create();
CountDownLatch latch = new CountDownLatch(totalProducedMsgs);
// (1) Produced Messages
@@ -626,7 +613,7 @@ public void testBrokerSubscriptionRecovery(boolean unloadBundleGracefully) throw
Set<Integer> unackMessages = Sets.newHashSet(5, 10, 20, 21, 22, 23, 25, 26, 30, 32, 40, 80, 160, 320);
int receivedMsgCount = 0;
for (int i = 0; i < totalProducedMsgs; i++) {
- Message msg = consumer.receive(500, TimeUnit.MILLISECONDS);
+ Message<?> msg = consumer.receive(500, TimeUnit.MILLISECONDS);
if (!unackMessages.contains(i)) {
consumer.acknowledge(msg);
}
@@ -645,13 +632,14 @@ public void testBrokerSubscriptionRecovery(boolean unloadBundleGracefully) throw
// start broker which will recover topic-cursor from the ledger
startBroker();
- consumer = pulsarClient.subscribe(topicName, subscriberName, conf);
+ consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriberName)
+ .subscriptionType(SubscriptionType.Shared).subscribe();
// consumer should only receive unakced messages
Set<String> unackMsgs = unackMessages.stream().map(i -> "my-message-" + i).collect(Collectors.toSet());
Set<String> receivedMsgs = Sets.newHashSet();
for (int i = 0; i < totalProducedMsgs; i++) {
- Message msg = consumer.receive(500, TimeUnit.MILLISECONDS);
+ Message<?> msg = consumer.receive(500, TimeUnit.MILLISECONDS);
if (msg == null) {
break;
}
@@ -666,18 +654,14 @@ public void testBrokerSubscriptionRecovery(boolean unloadBundleGracefully) throw
* verifies perBroker dispatching blocking. A. maxUnAckPerBroker = 200, maxUnAckPerDispatcher = 20 Now, it tests
* with 3 subscriptions.
*
- * 1. Subscription-1: try to consume without acking
- * a. consumer will be blocked after 200 (maxUnAckPerBroker) msgs
- * b. even second consumer will not receive any new messages
- * c. broker will have 1 blocked dispatcher
- * 2. Subscription-2: try to consume without acking
- * a. as broker is already blocked it will block subscription after 20 msgs (maxUnAckPerDispatcher)
- * b. broker will have 2 blocked dispatchers
- * 3. Subscription-3: try to consume with acking
- * a. as consumer is acking not reached maxUnAckPerDispatcher=20 unack msg => consumes all produced msgs
- * 4.Subscription-1 : acks all pending msgs and consume by acking
- * a. broker unblocks all dispatcher and sub-1 consumes all messages
- * 5. Subscription-2 : it triggers redelivery and acks all messages so, it consumes all produced messages
+ * 1. Subscription-1: try to consume without acking a. consumer will be blocked after 200 (maxUnAckPerBroker) msgs
+ * b. even second consumer will not receive any new messages c. broker will have 1 blocked dispatcher 2.
+ * Subscription-2: try to consume without acking a. as broker is already blocked it will block subscription after 20
+ * msgs (maxUnAckPerDispatcher) b. broker will have 2 blocked dispatchers 3. Subscription-3: try to consume with
+ * acking a. as consumer is acking not reached maxUnAckPerDispatcher=20 unack msg => consumes all produced msgs
+ * 4.Subscription-1 : acks all pending msgs and consume by acking a. broker unblocks all dispatcher and sub-1
+ * consumes all messages 5. Subscription-2 : it triggers redelivery and acks all messages so, it consumes all
+ * produced messages
* </pre>
*
* @throws Exception
@@ -704,6 +688,7 @@ public void testBlockBrokerDispatching() throws Exception {
Field field = BrokerService.class.getDeclaredField("blockedDispatchers");
field.setAccessible(true);
+ @SuppressWarnings("unchecked")
ConcurrentOpenHashSet<PersistentDispatcherMultipleConsumers> blockedDispatchers = (ConcurrentOpenHashSet<PersistentDispatcherMultipleConsumers>) field
.get(pulsar.getBrokerService());
@@ -714,18 +699,21 @@ public void testBlockBrokerDispatching() throws Exception {
final String subscriberName2 = "subscriber-2";
final String subscriberName3 = "subscriber-3";
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Shared);
- conf.setReceiverQueueSize(receiverQueueSize);
- ConsumerImpl consumer1Sub1 = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriberName1, conf);
+ ConsumerImpl<byte[]> consumer1Sub1 = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName(subscriberName1).receiverQueueSize(receiverQueueSize)
+ .subscriptionType(SubscriptionType.Shared).subscribe();
// create subscription-2 and 3
- ConsumerImpl consumer1Sub2 = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriberName2, conf);
+ ConsumerImpl<byte[]> consumer1Sub2 = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName(subscriberName2).receiverQueueSize(receiverQueueSize)
+ .subscriptionType(SubscriptionType.Shared).subscribe();
consumer1Sub2.close();
- ConsumerImpl consumer1Sub3 = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriberName3, conf);
+ ConsumerImpl<byte[]> consumer1Sub3 = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName(subscriberName3).receiverQueueSize(receiverQueueSize)
+ .subscriptionType(SubscriptionType.Shared).subscribe();
consumer1Sub3.close();
- Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic",
- new ProducerConfiguration());
+ Producer<byte[]> producer = pulsarClient.newProducer()
+ .topic("persistent://my-property/use/my-ns/unacked-topic").create();
// continuously checks unack-message dispatching
ScheduledExecutorService executor = Executors.newScheduledThreadPool(1);
@@ -741,7 +729,7 @@ public void testBlockBrokerDispatching() throws Exception {
* (1) try to consume messages: without acking messages and dispatcher will be blocked once it reaches
* maxUnAckPerBroker limit
***/
- Message msg = null;
+ Message<byte[]> msg = null;
Set<MessageId> messages1 = Sets.newHashSet();
for (int j = 0; j < totalProducedMsgs; j++) {
msg = consumer1Sub1.receive(100, TimeUnit.MILLISECONDS);
@@ -759,7 +747,9 @@ public void testBlockBrokerDispatching() throws Exception {
// client must receive number of messages = maxUnAckPerbroker rather all produced messages
assertNotEquals(messages1.size(), totalProducedMsgs);
// (1.b) consumer2 with same sub should not receive any more messages as subscription is blocked
- ConsumerImpl consumer2Sub1 = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriberName1, conf);
+ ConsumerImpl<byte[]> consumer2Sub1 = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName(subscriberName1).receiverQueueSize(receiverQueueSize)
+ .subscriptionType(SubscriptionType.Shared).subscribe();
int consumer2Msgs = 0;
for (int j = 0; j < totalProducedMsgs; j++) {
msg = consumer2Sub1.receive(100, TimeUnit.MILLISECONDS);
@@ -782,7 +772,9 @@ public void testBlockBrokerDispatching() throws Exception {
* (2) However, other subscription2 should still be able to consume messages until it reaches to
* maxUnAckPerDispatcher limit
**/
- ConsumerImpl consumerSub2 = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriberName2, conf);
+ ConsumerImpl<byte[]> consumerSub2 = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName(subscriberName2).receiverQueueSize(receiverQueueSize)
+ .subscriptionType(SubscriptionType.Shared).subscribe();
Set<MessageId> messages2 = Sets.newHashSet();
for (int j = 0; j < totalProducedMsgs; j++) {
msg = consumerSub2.receive(100, TimeUnit.MILLISECONDS);
@@ -797,7 +789,9 @@ public void testBlockBrokerDispatching() throws Exception {
assertEquals(blockedDispatchers.size(), 2);
/** (3) if Subscription3 is acking then it shouldn't be blocked **/
- consumer1Sub3 = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriberName3, conf);
+ consumer1Sub3 = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName(subscriberName3).receiverQueueSize(receiverQueueSize)
+ .subscriptionType(SubscriptionType.Shared).subscribe();
int consumedMsgsSub3 = 0;
for (int j = 0; j < totalProducedMsgs; j++) {
msg = consumer1Sub3.receive(100, TimeUnit.MILLISECONDS);
@@ -871,6 +865,7 @@ public void testBlockBrokerDispatching() throws Exception {
* </pre>
*
*/
+ @SuppressWarnings("unchecked")
@Test
public void testBrokerDispatchBlockAndSubAckBackRequiredMsgs() {
@@ -903,12 +898,13 @@ public void testBrokerDispatchBlockAndSubAckBackRequiredMsgs() {
final String subscriberName1 = "subscriber-1";
final String subscriberName2 = "subscriber-2";
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Shared);
- conf.setReceiverQueueSize(receiverQueueSize);
- ConsumerImpl consumer1Sub1 = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriberName1, conf);
+ ConsumerImpl<byte[]> consumer1Sub1 = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName(subscriberName1).receiverQueueSize(receiverQueueSize)
+ .subscriptionType(SubscriptionType.Shared).subscribe();
// create subscription-2 and 3
- ConsumerImpl consumer1Sub2 = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriberName2, conf);
+ ConsumerImpl<byte[]> consumer1Sub2 = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName(subscriberName2).receiverQueueSize(receiverQueueSize)
+ .subscriptionType(SubscriptionType.Shared).subscribe();
consumer1Sub2.close();
// continuously checks unack-message dispatching
@@ -916,8 +912,8 @@ public void testBrokerDispatchBlockAndSubAckBackRequiredMsgs() {
executor.scheduleAtFixedRate(() -> pulsar.getBrokerService().checkUnAckMessageDispatching(), 10, 10,
TimeUnit.MILLISECONDS);
- Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic",
- new ProducerConfiguration());
+ Producer<byte[]> producer = pulsarClient.newProducer()
+ .topic("persistent://my-property/use/my-ns/unacked-topic").create();
// Produced Messages
for (int i = 0; i < totalProducedMsgs; i++) {
@@ -929,7 +925,7 @@ public void testBrokerDispatchBlockAndSubAckBackRequiredMsgs() {
* (1) try to consume messages: without acking messages and dispatcher will be blocked once it reaches
* maxUnAckPerBroker limit
***/
- Message msg = null;
+ Message<?> msg = null;
Set<MessageId> messages1 = Sets.newHashSet();
for (int j = 0; j < totalProducedMsgs; j++) {
msg = consumer1Sub1.receive(100, TimeUnit.MILLISECONDS);
@@ -947,7 +943,9 @@ public void testBrokerDispatchBlockAndSubAckBackRequiredMsgs() {
// client must receive number of messages = maxUnAckPerbroker rather all produced messages
assertNotEquals(messages1.size(), totalProducedMsgs);
// (1.b) consumer2 with same sub should not receive any more messages as subscription is blocked
- ConsumerImpl consumer2Sub1 = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriberName1, conf);
+ ConsumerImpl<byte[]> consumer2Sub1 = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName(subscriberName1).receiverQueueSize(receiverQueueSize)
+ .subscriptionType(SubscriptionType.Shared).subscribe();
int consumer2Msgs = 0;
for (int j = 0; j < totalProducedMsgs; j++) {
msg = consumer2Sub1.receive(100, TimeUnit.MILLISECONDS);
@@ -970,7 +968,9 @@ public void testBrokerDispatchBlockAndSubAckBackRequiredMsgs() {
* (2) However, other subscription2 should still be able to consume messages until it reaches to
* maxUnAckPerDispatcher limit
**/
- consumer1Sub2 = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriberName2, conf);
+ consumer1Sub2 = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName(subscriberName2).receiverQueueSize(receiverQueueSize)
+ .subscriptionType(SubscriptionType.Shared).subscribe();
Set<MessageId> messages2 = Sets.newHashSet();
for (int j = 0; j < totalProducedMsgs; j++) {
msg = consumer1Sub2.receive(100, TimeUnit.MILLISECONDS);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MessageDispatchThrottlingTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MessageDispatchThrottlingTest.java
index 9163e0dfd..44257660c 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MessageDispatchThrottlingTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MessageDispatchThrottlingTest.java
@@ -107,7 +107,7 @@ public void testMessageRateDynamicallyChange() throws Exception {
admin.namespaces().createNamespace(namespace);
// create producer and topic
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
// (1) verify message-rate is -1 initially
Assert.assertEquals(topic.getDispatchRateLimiter().getDispatchRateOnMsg(), -1);
@@ -177,7 +177,7 @@ public void testMessageRateLimitingNotReceiveAllMessages(SubscriptionType subscr
admin.namespaces().createNamespace(namespace);
admin.namespaces().setDispatchRate(namespace, dispatchRate);
// create producer and topic
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
boolean isMessageRateUpdate = false;
int retry = 5;
@@ -199,15 +199,13 @@ public void testMessageRateLimitingNotReceiveAllMessages(SubscriptionType subscr
final AtomicInteger totalReceived = new AtomicInteger(0);
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(subscription);
- conf.setMessageListener((consumer, msg) -> {
- Assert.assertNotNull(msg, "Message cannot be null");
- String receivedMessage = new String(msg.getData());
- log.debug("Received message [{}] in the listener", receivedMessage);
- totalReceived.incrementAndGet();
- });
- Consumer consumer = pulsarClient.subscribe(topicName, "my-subscriber-name", conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name")
+ .subscriptionType(subscription).messageListener((c1, msg) -> {
+ Assert.assertNotNull(msg, "Message cannot be null");
+ String receivedMessage = new String(msg.getData());
+ log.debug("Received message [{}] in the listener", receivedMessage);
+ totalReceived.incrementAndGet();
+ }).subscribe();
// deactive cursors
deactiveCursors((ManagedLedgerImpl) topic.getManagedLedger());
@@ -242,7 +240,8 @@ public void testClusterMsgByteRateLimitingClusterConfig() throws Exception {
int initValue = pulsar.getConfiguration().getDispatchThrottlingRatePerTopicInMsg();
// (1) Update message-dispatch-rate limit
- admin.brokers().updateDynamicConfiguration("dispatchThrottlingRatePerTopicInMsg", Integer.toString(messageRate));
+ admin.brokers().updateDynamicConfiguration("dispatchThrottlingRatePerTopicInMsg",
+ Integer.toString(messageRate));
admin.brokers().updateDynamicConfiguration("dispatchThrottlingRatePerTopicInByte", Long.toString(byteRate));
// sleep incrementally as zk-watch notification is async and may take some time
for (int i = 0; i < 5; i++) {
@@ -254,21 +253,20 @@ public void testClusterMsgByteRateLimitingClusterConfig() throws Exception {
admin.namespaces().createNamespace(namespace);
// create producer and topic
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
int numMessages = 500;
final AtomicInteger totalReceived = new AtomicInteger(0);
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Shared);
- conf.setMessageListener((consumer, msg) -> {
- Assert.assertNotNull(msg, "Message cannot be null");
- String receivedMessage = new String(msg.getData());
- log.debug("Received message [{}] in the listener", receivedMessage);
- totalReceived.incrementAndGet();
- });
- Consumer consumer = pulsarClient.subscribe(topicName, "my-subscriber-name", conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name")
+ .subscriptionType(SubscriptionType.Shared).messageListener((c1, msg) -> {
+ Assert.assertNotNull(msg, "Message cannot be null");
+ String receivedMessage = new String(msg.getData());
+ log.debug("Received message [{}] in the listener", receivedMessage);
+ totalReceived.incrementAndGet();
+ }).subscribe();
+
// deactive cursors
deactiveCursors((ManagedLedgerImpl) topic.getManagedLedger());
@@ -315,7 +313,7 @@ public void testMessageRateLimitingReceiveAllMessagesAfterThrottling(Subscriptio
admin.namespaces().createNamespace(namespace);
admin.namespaces().setDispatchRate(namespace, dispatchRate);
// create producer and topic
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
boolean isMessageRateUpdate = false;
int retry = 5;
@@ -337,16 +335,14 @@ public void testMessageRateLimitingReceiveAllMessagesAfterThrottling(Subscriptio
final AtomicInteger totalReceived = new AtomicInteger(0);
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(subscription);
- conf.setMessageListener((consumer, msg) -> {
- Assert.assertNotNull(msg, "Message cannot be null");
- String receivedMessage = new String(msg.getData());
- log.debug("Received message [{}] in the listener", receivedMessage);
- totalReceived.incrementAndGet();
- latch.countDown();
- });
- Consumer consumer = pulsarClient.subscribe(topicName, "my-subscriber-name", conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name")
+ .subscriptionType(subscription).messageListener((c1, msg) -> {
+ Assert.assertNotNull(msg, "Message cannot be null");
+ String receivedMessage = new String(msg.getData());
+ log.debug("Received message [{}] in the listener", receivedMessage);
+ totalReceived.incrementAndGet();
+ latch.countDown();
+ }).subscribe();
// deactive cursors
deactiveCursors((ManagedLedgerImpl) topic.getManagedLedger());
@@ -388,7 +384,7 @@ public void testBytesRateLimitingReceiveAllMessagesAfterThrottling(SubscriptionT
admin.namespaces().createNamespace(namespace);
admin.namespaces().setDispatchRate(namespace, dispatchRate);
// create producer and topic
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
boolean isMessageRateUpdate = false;
int retry = 5;
@@ -410,16 +406,14 @@ public void testBytesRateLimitingReceiveAllMessagesAfterThrottling(SubscriptionT
final AtomicInteger totalReceived = new AtomicInteger(0);
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(subscription);
- conf.setMessageListener((consumer, msg) -> {
- Assert.assertNotNull(msg, "Message cannot be null");
- String receivedMessage = new String(msg.getData());
- log.debug("Received message [{}] in the listener", receivedMessage);
- totalReceived.incrementAndGet();
- latch.countDown();
- });
- Consumer consumer = pulsarClient.subscribe(topicName, "my-subscriber-name-" + subscription, conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name")
+ .subscriptionType(subscription).messageListener((c1, msg) -> {
+ Assert.assertNotNull(msg, "Message cannot be null");
+ String receivedMessage = new String(msg.getData());
+ log.debug("Received message [{}] in the listener", receivedMessage);
+ totalReceived.incrementAndGet();
+ latch.countDown();
+ }).subscribe();
// deactive cursors
deactiveCursors((ManagedLedgerImpl) topic.getManagedLedger());
@@ -453,7 +447,7 @@ public void testRateLimitingMultipleConsumers() throws Exception {
admin.namespaces().createNamespace(namespace);
admin.namespaces().setDispatchRate(namespace, dispatchRate);
// create producer and topic
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
boolean isMessageRateUpdate = false;
int retry = 5;
@@ -474,19 +468,19 @@ public void testRateLimitingMultipleConsumers() throws Exception {
final AtomicInteger totalReceived = new AtomicInteger(0);
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Shared);
- conf.setMessageListener((consumer, msg) -> {
- Assert.assertNotNull(msg, "Message cannot be null");
- String receivedMessage = new String(msg.getData());
- log.debug("Received message [{}] in the listener", receivedMessage);
- totalReceived.incrementAndGet();
- });
- Consumer consumer1 = pulsarClient.subscribe(topicName, "my-subscriber-name", conf);
- Consumer consumer2 = pulsarClient.subscribe(topicName, "my-subscriber-name", conf);
- Consumer consumer3 = pulsarClient.subscribe(topicName, "my-subscriber-name", conf);
- Consumer consumer4 = pulsarClient.subscribe(topicName, "my-subscriber-name", conf);
- Consumer consumer5 = pulsarClient.subscribe(topicName, "my-subscriber-name", conf);
+ ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName("my-subscriber-name").subscriptionType(SubscriptionType.Shared).messageListener((c1, msg) -> {
+ Assert.assertNotNull(msg, "Message cannot be null");
+ String receivedMessage = new String(msg.getData());
+ log.debug("Received message [{}] in the listener", receivedMessage);
+ totalReceived.incrementAndGet();
+ });
+ Consumer<byte[]> consumer1 = consumerBuilder.subscribe();
+ Consumer<byte[]> consumer2 = consumerBuilder.subscribe();
+ Consumer<byte[]> consumer3 = consumerBuilder.subscribe();
+ Consumer<byte[]> consumer4 = consumerBuilder.subscribe();
+ Consumer<byte[]> consumer5 = consumerBuilder.subscribe();
+
// deactive cursors
deactiveCursors((ManagedLedgerImpl) topic.getManagedLedger());
@@ -521,7 +515,8 @@ public void testClusterRateLimitingConfiguration(SubscriptionType subscription)
int initValue = pulsar.getConfiguration().getDispatchThrottlingRatePerTopicInMsg();
// (1) Update message-dispatch-rate limit
- admin.brokers().updateDynamicConfiguration("dispatchThrottlingRatePerTopicInMsg", Integer.toString(messageRate));
+ admin.brokers().updateDynamicConfiguration("dispatchThrottlingRatePerTopicInMsg",
+ Integer.toString(messageRate));
// sleep incrementally as zk-watch notification is async and may take some time
for (int i = 0; i < 5; i++) {
if (pulsar.getConfiguration().getDispatchThrottlingRatePerTopicInMsg() == initValue) {
@@ -532,21 +527,19 @@ public void testClusterRateLimitingConfiguration(SubscriptionType subscription)
admin.namespaces().createNamespace(namespace);
// create producer and topic
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
int numMessages = 500;
final AtomicInteger totalReceived = new AtomicInteger(0);
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(subscription);
- conf.setMessageListener((consumer, msg) -> {
- Assert.assertNotNull(msg, "Message cannot be null");
- String receivedMessage = new String(msg.getData());
- log.debug("Received message [{}] in the listener", receivedMessage);
- totalReceived.incrementAndGet();
- });
- Consumer consumer = pulsarClient.subscribe(topicName, "my-subscriber-name", conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name")
+ .subscriptionType(subscription).messageListener((c1, msg) -> {
+ Assert.assertNotNull(msg, "Message cannot be null");
+ String receivedMessage = new String(msg.getData());
+ log.debug("Received message [{}] in the listener", receivedMessage);
+ totalReceived.incrementAndGet();
+ }).subscribe();
// deactive cursors
deactiveCursors((ManagedLedgerImpl) topic.getManagedLedger());
@@ -587,7 +580,7 @@ public void testMessageByteRateThrottlingCombined(SubscriptionType subscription)
admin.namespaces().createNamespace(namespace);
admin.namespaces().setDispatchRate(namespace, dispatchRate);
// create producer and topic
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
boolean isMessageRateUpdate = false;
int retry = 5;
@@ -609,15 +602,14 @@ public void testMessageByteRateThrottlingCombined(SubscriptionType subscription)
final AtomicInteger totalReceived = new AtomicInteger(0);
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(subscription);
- conf.setMessageListener((consumer, msg) -> {
- Assert.assertNotNull(msg, "Message cannot be null");
- String receivedMessage = new String(msg.getData());
- log.debug("Received message [{}] in the listener", receivedMessage);
- totalReceived.incrementAndGet();
- });
- Consumer consumer = pulsarClient.subscribe(topicName, "my-subscriber-name", conf);
+ ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName("my-subscriber-name").subscriptionType(subscription).messageListener((c1, msg) -> {
+ Assert.assertNotNull(msg, "Message cannot be null");
+ String receivedMessage = new String(msg.getData());
+ log.debug("Received message [{}] in the listener", receivedMessage);
+ totalReceived.incrementAndGet();
+ });
+ Consumer<byte[]> consumer = consumerBuilder.subscribe();
// deactive cursors
deactiveCursors((ManagedLedgerImpl) topic.getManagedLedger());
consumer.close();
@@ -629,7 +621,7 @@ public void testMessageByteRateThrottlingCombined(SubscriptionType subscription)
producer.send(data);
}
- consumer = pulsarClient.subscribe(topicName, "my-subscriber-name", conf);
+ consumer = consumerBuilder.subscribe();
final int totalReceivedBytes = dataSize * totalReceived.get();
Assert.assertNotEquals(totalReceivedBytes, byteRate * 2);
@@ -646,6 +638,7 @@ public void testMessageByteRateThrottlingCombined(SubscriptionType subscription)
* 3. applies dispatch rate
*
* </pre>
+ *
* @throws Exception
*/
@Test
@@ -655,7 +648,6 @@ public void testGlobalNamespaceThrottling() throws Exception {
final String namespace = "my-property/global/throttling_ns";
final String topicName = "persistent://" + namespace + "/throttlingBlock";
-
final int messageRate = 5;
DispatchRate dispatchRate = new DispatchRate(messageRate, -1, 360);
@@ -665,7 +657,7 @@ public void testGlobalNamespaceThrottling() throws Exception {
admin.namespaces().setDispatchRate(namespace, dispatchRate);
// create producer and topic
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
boolean isMessageRateUpdate = false;
int retry = 5;
@@ -687,15 +679,13 @@ public void testGlobalNamespaceThrottling() throws Exception {
final AtomicInteger totalReceived = new AtomicInteger(0);
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Shared);
- conf.setMessageListener((consumer, msg) -> {
- Assert.assertNotNull(msg, "Message cannot be null");
- String receivedMessage = new String(msg.getData());
- log.debug("Received message [{}] in the listener", receivedMessage);
- totalReceived.incrementAndGet();
- });
- Consumer consumer = pulsarClient.subscribe(topicName, "my-subscriber-name", conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name")
+ .subscriptionType(SubscriptionType.Shared).messageListener((c1, msg) -> {
+ Assert.assertNotNull(msg, "Message cannot be null");
+ String receivedMessage = new String(msg.getData());
+ log.debug("Received message [{}] in the listener", receivedMessage);
+ totalReceived.incrementAndGet();
+ }).subscribe();
// deactive cursors
deactiveCursors((ManagedLedgerImpl) topic.getManagedLedger());
@@ -736,7 +726,7 @@ public void testNonBacklogConsumerWithThrottlingEnabled(SubscriptionType subscri
admin.brokers().updateDynamicConfiguration("dispatchThrottlingOnNonBacklogConsumerEnabled",
Boolean.TRUE.toString());
// create producer and topic
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
boolean isUpdated = false;
int retry = 5;
@@ -760,15 +750,13 @@ public void testNonBacklogConsumerWithThrottlingEnabled(SubscriptionType subscri
final AtomicInteger totalReceived = new AtomicInteger(0);
- ConsumerConfiguration consumerConf = new ConsumerConfiguration();
- consumerConf.setSubscriptionType(subscription);
- consumerConf.setMessageListener((consumer, msg) -> {
- Assert.assertNotNull(msg, "Message cannot be null");
- String receivedMessage = new String(msg.getData());
- log.debug("Received message [{}] in the listener", receivedMessage);
- totalReceived.incrementAndGet();
- });
- Consumer consumer = pulsarClient.subscribe(topicName, "my-subscriber-name", consumerConf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name")
+ .subscriptionType(subscription).messageListener((c1, msg) -> {
+ Assert.assertNotNull(msg, "Message cannot be null");
+ String receivedMessage = new String(msg.getData());
+ log.debug("Received message [{}] in the listener", receivedMessage);
+ totalReceived.incrementAndGet();
+ }).subscribe();
// Asynchronously produce messages
for (int i = 0; i < numMessages; i++) {
@@ -785,7 +773,7 @@ public void testNonBacklogConsumerWithThrottlingEnabled(SubscriptionType subscri
log.info("-- Exiting {} test --", methodName);
}
- /**
+ /**
* <pre>
* It verifies that cluster-throttling value gets considered when namespace-policy throttling is disabled.
*
@@ -821,7 +809,7 @@ public void testClusterPolicyOverrideConfiguration() throws Exception {
admin.namespaces().createNamespace(namespace);
// create producer and topic
- Producer producer = pulsarClient.createProducer(topicName1);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName1).create();
PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName1).get();
// (1) Update dispatch rate on cluster-config update
@@ -849,7 +837,7 @@ public void testClusterPolicyOverrideConfiguration() throws Exception {
Assert.assertEquals(clusterMessageRate, topic.getDispatchRateLimiter().getDispatchRateOnMsg());
// (5) Namespace throttling is disabled so, new topic should take cluster throttling limit
- Producer producer2 = pulsarClient.createProducer(topicName2);
+ Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topicName2).create();
PersistentTopic topic2 = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName2).get();
Assert.assertEquals(clusterMessageRate, topic2.getDispatchRateLimiter().getDispatchRateOnMsg());
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MockBrokerService.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MockBrokerService.java
index 13c114647..91e5bbe6d 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MockBrokerService.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MockBrokerService.java
@@ -47,6 +47,7 @@
import org.apache.pulsar.common.api.proto.PulsarApi.CommandSend;
import org.apache.pulsar.common.lookup.data.LookupData;
import org.apache.pulsar.common.partition.PartitionedTopicMetadata;
+import org.apache.pulsar.common.schema.SchemaVersion;
import org.apache.pulsar.common.util.netty.EventLoopUtil;
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.Server;
@@ -174,7 +175,7 @@ protected void handleProducer(PulsarApi.CommandProducer producer) {
return;
}
// default
- ctx.writeAndFlush(Commands.newProducerSuccess(producer.getRequestId(), "default-producer"));
+ ctx.writeAndFlush(Commands.newProducerSuccess(producer.getRequestId(), "default-producer", SchemaVersion.Empty));
}
@Override
@@ -244,7 +245,6 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws E
EventLoopGroup workerGroup;
private final int webServicePort;
- private final int webServicePortTls;
private final int brokerServicePort;
private final int brokerServicePortTls;
@@ -268,7 +268,6 @@ public MockBrokerService() {
public MockBrokerService(int webServicePort, int webServicePortTls, int brokerServicePort,
int brokerServicePortTls) {
this.webServicePort = webServicePort;
- this.webServicePortTls = webServicePortTls;
this.brokerServicePort = brokerServicePort;
this.brokerServicePortTls = brokerServicePortTls;
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java
index fefe8e72e..41dfb7cbf 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java
@@ -104,14 +104,10 @@ public void testNonPersistentTopic(SubscriptionType type) throws Exception {
log.info("-- Starting {} test --", methodName);
final String topic = "non-persistent://my-property/use/my-ns/unacked-topic";
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(type);
+ ConsumerImpl<byte[]> consumer = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topic)
+ .subscriptionName("subscriber-1").subscriptionType(type).subscribe();
- ProducerConfiguration producerConf = new ProducerConfiguration();
-
- ConsumerImpl consumer = (ConsumerImpl) pulsarClient.subscribe(topic, "subscriber-1", conf);
-
- Producer producer = pulsarClient.createProducer(topic, producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topic).create();
int totalProduceMsg = 500;
for (int i = 0; i < totalProduceMsg; i++) {
@@ -120,7 +116,7 @@ public void testNonPersistentTopic(SubscriptionType type) throws Exception {
Thread.sleep(10);
}
- Message msg = null;
+ Message<?> msg = null;
Set<String> messageSet = Sets.newHashSet();
for (int i = 0; i < totalProduceMsg; i++) {
msg = consumer.receive(1, TimeUnit.SECONDS);
@@ -148,14 +144,10 @@ public void testPartitionedNonPersistentTopic(SubscriptionType type) throws Exce
final String topic = "non-persistent://my-property/use/my-ns/partitioned-topic";
admin.nonPersistentTopics().createPartitionedTopic(topic, 5);
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(type);
-
- ProducerConfiguration producerConf = new ProducerConfiguration();
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("subscriber-1")
+ .subscriptionType(type).subscribe();
- Consumer consumer = pulsarClient.subscribe(topic, "subscriber-1", conf);
-
- Producer producer = pulsarClient.createProducer(topic, producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topic).create();
int totalProduceMsg = 500;
for (int i = 0; i < totalProduceMsg; i++) {
@@ -164,7 +156,7 @@ public void testPartitionedNonPersistentTopic(SubscriptionType type) throws Exce
Thread.sleep(10);
}
- Message msg = null;
+ Message<?> msg = null;
Set<String> messageSet = Sets.newHashSet();
for (int i = 0; i < totalProduceMsg; i++) {
msg = consumer.receive(1, TimeUnit.SECONDS);
@@ -196,10 +188,10 @@ public void testPartitionedNonPersistentTopicWithTcpLookup(SubscriptionType type
PulsarClient client = PulsarClient.builder().serviceUrl("pulsar://localhost:" + BROKER_PORT)
.statsInterval(0, TimeUnit.SECONDS).build();
- Consumer consumer = client.newConsumer().topic(topic).subscriptionName("subscriber-1").subscriptionType(type)
- .subscribe();
+ Consumer<byte[]> consumer = client.newConsumer().topic(topic).subscriptionName("subscriber-1")
+ .subscriptionType(type).subscribe();
- Producer producer = client.newProducer().topic(topic).create();
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topic).create();
// Ensure all partitions exist
for (int i = 0; i < numPartitions; i++) {
@@ -214,7 +206,7 @@ public void testPartitionedNonPersistentTopicWithTcpLookup(SubscriptionType type
Thread.sleep(10);
}
- Message msg = null;
+ Message<?> msg = null;
Set<String> messageSet = Sets.newHashSet();
for (int i = 0; i < totalProduceMsg; i++) {
msg = consumer.receive(1, TimeUnit.SECONDS);
@@ -237,8 +229,7 @@ public void testPartitionedNonPersistentTopicWithTcpLookup(SubscriptionType type
}
/**
- * It verifies that broker doesn't dispatch messages if consumer runs out of permits
- * filled out with messages
+ * It verifies that broker doesn't dispatch messages if consumer runs out of permits filled out with messages
*/
@Test(dataProvider = "subscriptionType")
public void testConsumerInternalQueueMaxOut(SubscriptionType type) throws Exception {
@@ -246,15 +237,10 @@ public void testConsumerInternalQueueMaxOut(SubscriptionType type) throws Except
final String topic = "non-persistent://my-property/use/my-ns/unacked-topic";
final int queueSize = 10;
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(type);
- conf.setReceiverQueueSize(queueSize);
-
- ProducerConfiguration producerConf = new ProducerConfiguration();
-
- ConsumerImpl consumer = (ConsumerImpl) pulsarClient.subscribe(topic, "subscriber-1", conf);
+ ConsumerImpl<byte[]> consumer = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topic)
+ .receiverQueueSize(queueSize).subscriptionName("subscriber-1").subscriptionType(type).subscribe();
- Producer producer = pulsarClient.createProducer(topic, producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topic).create();
int totalProduceMsg = 50;
for (int i = 0; i < totalProduceMsg; i++) {
@@ -263,7 +249,7 @@ public void testConsumerInternalQueueMaxOut(SubscriptionType type) throws Except
Thread.sleep(10);
}
- Message msg = null;
+ Message<?> msg = null;
Set<String> messageSet = Sets.newHashSet();
for (int i = 0; i < totalProduceMsg; i++) {
msg = consumer.receive(1, TimeUnit.SECONDS);
@@ -300,9 +286,9 @@ public void testProducerRateLimit() throws Exception {
// produce message concurrently
ExecutorService executor = Executors.newFixedThreadPool(5);
AtomicBoolean failed = new AtomicBoolean(false);
- ProducerConfiguration producerConf = new ProducerConfiguration();
- Consumer consumer = pulsarClient.subscribe(topic, "subscriber-1");
- Producer producer = pulsarClient.createProducer(topic, producerConf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("subscriber-1")
+ .subscribe();
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topic).create();
byte[] msgData = "testData".getBytes();
final int totalProduceMessages = 10;
CountDownLatch latch = new CountDownLatch(totalProduceMessages);
@@ -319,7 +305,7 @@ public void testProducerRateLimit() throws Exception {
}
latch.await();
- Message msg = null;
+ Message<?> msg = null;
Set<String> messageSet = Sets.newHashSet();
for (int i = 0; i < totalProduceMessages; i++) {
msg = consumer.receive(500, TimeUnit.MILLISECONDS);
@@ -352,19 +338,19 @@ public void testMultipleSubscription() throws Exception {
log.info("-- Starting {} test --", methodName);
final String topic = "non-persistent://my-property/use/my-ns/unacked-topic";
- ConsumerConfiguration sharedConf = new ConsumerConfiguration();
- sharedConf.setSubscriptionType(SubscriptionType.Shared);
- ConsumerConfiguration excConf = new ConsumerConfiguration();
- excConf.setSubscriptionType(SubscriptionType.Failover);
+ ConsumerImpl<byte[]> consumer1Shared = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topic)
+ .subscriptionName("subscriber-shared").subscriptionType(SubscriptionType.Shared).subscribe();
- ProducerConfiguration producerConf = new ProducerConfiguration();
+ ConsumerImpl<byte[]> consumer2Shared = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topic)
+ .subscriptionName("subscriber-shared").subscriptionType(SubscriptionType.Shared).subscribe();
- ConsumerImpl consumer1Shared = (ConsumerImpl) pulsarClient.subscribe(topic, "subscriber-shared", sharedConf);
- ConsumerImpl consumer2Shared = (ConsumerImpl) pulsarClient.subscribe(topic, "subscriber-shared", sharedConf);
- ConsumerImpl consumer1FailOver = (ConsumerImpl) pulsarClient.subscribe(topic, "subscriber-fo", excConf);
- ConsumerImpl consumer2FailOver = (ConsumerImpl) pulsarClient.subscribe(topic, "subscriber-fo", excConf);
+ ConsumerImpl<byte[]> consumer1FailOver = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topic)
+ .subscriptionName("subscriber-fo").subscriptionType(SubscriptionType.Failover).subscribe();
- Producer producer = pulsarClient.createProducer(topic, producerConf);
+ ConsumerImpl<byte[]> consumer2FailOver = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topic)
+ .subscriptionName("subscriber-fo").subscriptionType(SubscriptionType.Failover).subscribe();
+
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topic).create();
int totalProduceMsg = 500;
for (int i = 0; i < totalProduceMsg; i++) {
@@ -374,7 +360,7 @@ public void testMultipleSubscription() throws Exception {
}
// consume from shared-subscriptions
- Message msg = null;
+ Message<?> msg = null;
Set<String> messageSet = Sets.newHashSet();
for (int i = 0; i < totalProduceMsg; i++) {
msg = consumer1Shared.receive(500, TimeUnit.MILLISECONDS);
@@ -436,9 +422,8 @@ public void testTopicStats() throws Exception {
NonPersistentTopicStats stats;
SubscriptionStats subStats;
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Shared);
- Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName)
+ .subscriptionType(SubscriptionType.Shared).subscriptionName(subName).subscribe();
Thread.sleep(timeWaitToSync);
NonPersistentTopic topicRef = (NonPersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
@@ -452,7 +437,7 @@ public void testTopicStats() throws Exception {
assertEquals(stats.getSubscriptions().keySet().size(), 1);
assertEquals(subStats.consumers.size(), 1);
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
Thread.sleep(timeWaitToSync);
int totalProducedMessages = 100;
@@ -494,21 +479,21 @@ public void testReplicator() throws Exception {
NonPersistentTopicStats stats;
SubscriptionStats subStats;
- TopicName dest = TopicName.get(globalTopicName);
+ PulsarClient client1 = PulsarClient.builder().serviceUrl(replication.url1.toString()).build();
+ PulsarClient client2 = PulsarClient.builder().serviceUrl(replication.url2.toString()).build();
+ PulsarClient client3 = PulsarClient.builder().serviceUrl(replication.url3.toString()).build();
+
+ ConsumerImpl<byte[]> consumer1 = (ConsumerImpl<byte[]>) client1.newConsumer().topic(globalTopicName)
+ .subscriptionName("subscriber-1").subscribe();
+ ConsumerImpl<byte[]> consumer2 = (ConsumerImpl<byte[]>) client1.newConsumer().topic(globalTopicName)
+ .subscriptionName("subscriber-2").subscribe();
- PulsarClient client1 = PulsarClient.create(replication.url1.toString(), new ClientConfiguration());
- PulsarClient client2 = PulsarClient.create(replication.url2.toString(), new ClientConfiguration());
- PulsarClient client3 = PulsarClient.create(replication.url3.toString(), new ClientConfiguration());
+ ConsumerImpl<byte[]> repl2Consumer = (ConsumerImpl<byte[]>) client2.newConsumer().topic(globalTopicName)
+ .subscriptionName("subscriber-1").subscribe();
+ ConsumerImpl<byte[]> repl3Consumer = (ConsumerImpl<byte[]>) client3.newConsumer().topic(globalTopicName)
+ .subscriptionName("subscriber-1").subscribe();
- ProducerConfiguration producerConf = new ProducerConfiguration();
- ConsumerConfiguration consumerConf = new ConsumerConfiguration();
- ConsumerImpl consumer1 = (ConsumerImpl) client1.subscribe(globalTopicName, "subscriber-1", consumerConf);
- ConsumerImpl consumer2 = (ConsumerImpl) client1.subscribe(globalTopicName, "subscriber-2", consumerConf);
- ConsumerImpl repl2Consumer = (ConsumerImpl) client2.subscribe(globalTopicName, "subscriber-1",
- consumerConf);
- ConsumerImpl repl3Consumer = (ConsumerImpl) client3.subscribe(globalTopicName, "subscriber-1",
- consumerConf);
- Producer producer = client1.createProducer(globalTopicName, producerConf);
+ Producer<byte[]> producer = client1.newProducer().topic(globalTopicName).create();
Thread.sleep(timeWaitToSync);
@@ -516,7 +501,7 @@ public void testReplicator() throws Exception {
// Replicator for r1 -> r2,r3
NonPersistentTopic topicRef = (NonPersistentTopic) replication.pulsar1.getBrokerService()
- .getTopicReference(dest.toString());
+ .getTopicReference(globalTopicName);
NonPersistentReplicator replicatorR2 = (NonPersistentReplicator) topicRef.getPersistentReplicator("r2");
NonPersistentReplicator replicatorR3 = (NonPersistentReplicator) topicRef.getPersistentReplicator("r3");
assertNotNull(topicRef);
@@ -540,7 +525,7 @@ public void testReplicator() throws Exception {
}
// (1) consume by consumer1
- Message msg = null;
+ Message<?> msg = null;
Set<String> messageSet = Sets.newHashSet();
for (int i = 0; i < totalProducedMessages; i++) {
msg = consumer1.receive(300, TimeUnit.MILLISECONDS);
@@ -623,6 +608,7 @@ public void testReplicator() throws Exception {
/**
* verifies load manager assigns topic only if broker started in non-persistent mode
+ *
* <pre>
* 1. Start broker with disable non-persistent topic mode
* 2. Create namespace with non-persistency set
@@ -647,6 +633,7 @@ public void testLoadManagerAssignmentForNonPersistentTestAssignment(String loadM
Field field = PulsarService.class.getDeclaredField("loadManager");
field.setAccessible(true);
+ @SuppressWarnings("unchecked")
AtomicReference<LoadManager> loadManagerRef = (AtomicReference<LoadManager>) field.get(pulsar);
LoadManager manager = LoadManager.create(pulsar);
manager.start();
@@ -662,9 +649,9 @@ public void testLoadManagerAssignmentForNonPersistentTestAssignment(String loadM
}
assertNull(broker);
- ProducerConfiguration producerConf = new ProducerConfiguration();
try {
- Producer producer = pulsarClient.createProducerAsync(topicName, producerConf).get(1, TimeUnit.SECONDS);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).createAsync().get(1,
+ TimeUnit.SECONDS);
producer.close();
fail("topic loading should have failed");
} catch (Exception e) {
@@ -697,9 +684,9 @@ public void testNonPersistentTopicUnderPersistentNamespace() throws Exception {
conf.setEnableNonPersistentTopics(false);
stopBroker();
startBroker();
- ProducerConfiguration producerConf = new ProducerConfiguration();
try {
- Producer producer = pulsarClient.createProducerAsync(topicName, producerConf).get(1, TimeUnit.SECONDS);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).createAsync().get(1,
+ TimeUnit.SECONDS);
producer.close();
fail("topic loading should have failed");
} catch (Exception e) {
@@ -736,6 +723,7 @@ public void testNonPersistentBrokerModeRejectPersistentTopic(String loadManagerN
Field field = PulsarService.class.getDeclaredField("loadManager");
field.setAccessible(true);
+ @SuppressWarnings("unchecked")
AtomicReference<LoadManager> loadManagerRef = (AtomicReference<LoadManager>) field.get(pulsar);
LoadManager manager = LoadManager.create(pulsar);
manager.start();
@@ -751,9 +739,9 @@ public void testNonPersistentBrokerModeRejectPersistentTopic(String loadManagerN
}
assertNull(broker);
- ProducerConfiguration producerConf = new ProducerConfiguration();
try {
- Producer producer = pulsarClient.createProducerAsync(topicName, producerConf).get(1, TimeUnit.SECONDS);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).createAsync().get(1,
+ TimeUnit.SECONDS);
producer.close();
fail("topic loading should have failed");
} catch (Exception e) {
@@ -785,15 +773,13 @@ public void testMsgDropStat() throws Exception {
conf.setMaxConcurrentNonPersistentMessagePerConnection(1);
stopBroker();
startBroker();
- ProducerConfiguration producerConf = new ProducerConfiguration();
- ConsumerConfiguration consumerConfig1 = new ConsumerConfiguration();
- consumerConfig1.setReceiverQueueSize(1);
- Consumer consumer = pulsarClient.subscribe(topicName, "subscriber-1", consumerConfig1);
- ConsumerConfiguration consumerConfig2 = new ConsumerConfiguration();
- consumerConfig2.setReceiverQueueSize(1);
- consumerConfig2.setSubscriptionType(SubscriptionType.Shared);
- Consumer consumer2 = pulsarClient.subscribe(topicName, "subscriber-2", consumerConfig2);
- ProducerImpl producer = (ProducerImpl) pulsarClient.createProducer(topicName, producerConf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("subscriber-1")
+ .receiverQueueSize(1).subscribe();
+
+ Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic(topicName).subscriptionName("subscriber-2")
+ .receiverQueueSize(1).subscriptionType(SubscriptionType.Shared).subscribe();
+
+ ProducerImpl<byte[]> producer = (ProducerImpl<byte[]>) pulsarClient.newProducer().topic(topicName).create();
String firstTimeConnected = producer.getConnectedSince();
ExecutorService executor = Executors.newFixedThreadPool(5);
byte[] msgData = "testData".getBytes();
@@ -801,7 +787,7 @@ public void testMsgDropStat() throws Exception {
CountDownLatch latch = new CountDownLatch(totalProduceMessages);
for (int i = 0; i < totalProduceMessages; i++) {
executor.submit(() -> {
- producer.sendAsync(msgData).handle((msg,e)->{
+ producer.sendAsync(msgData).handle((msg, e) -> {
latch.countDown();
return null;
});
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PartitionedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PartitionedProducerConsumerTest.java
index 3be9edcc3..0708229e2 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PartitionedProducerConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PartitionedProducerConsumerTest.java
@@ -25,20 +25,10 @@
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
-import org.apache.pulsar.client.api.Consumer;
-import org.apache.pulsar.client.api.ConsumerConfiguration;
-import org.apache.pulsar.client.api.Message;
-import org.apache.pulsar.client.api.MessageBuilder;
-import org.apache.pulsar.client.api.Producer;
-import org.apache.pulsar.client.api.ProducerConfiguration;
-import org.apache.pulsar.client.api.PulsarClientException;
-import org.apache.pulsar.client.api.SubscriptionType;
-import org.apache.pulsar.client.api.ProducerConfiguration.MessageRoutingMode;
import org.apache.pulsar.client.impl.PartitionedProducerImpl;
import org.apache.pulsar.common.naming.TopicName;
import org.slf4j.Logger;
@@ -80,23 +70,20 @@ public void testRoundRobinProducer() throws Exception {
int numPartitions = 4;
TopicName topicName = TopicName.get("persistent://my-property/use/my-ns/my-partitionedtopic1");
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
-
admin.persistentTopics().createPartitionedTopic(topicName.toString(), numPartitions);
- ProducerConfiguration producerConf = new ProducerConfiguration();
- producerConf.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
- Producer producer = pulsarClient.createProducer(topicName.toString(), producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName.toString())
+ .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();
- Consumer consumer = pulsarClient.subscribe(topicName.toString(), "my-partitioned-subscriber", conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName.toString())
+ .subscriptionName("my-partitioned-subscriber").subscribe();
for (int i = 0; i < 10; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
- Message msg = null;
+ Message<byte[]> msg = null;
Set<String> messageSet = Sets.newHashSet();
for (int i = 0; i < 10; i++) {
msg = consumer.receive(5, TimeUnit.SECONDS);
@@ -121,13 +108,13 @@ public void testPartitionedTopicNameWithSpecialCharacter() throws Exception {
int numPartitions = 4;
final String specialCharacter = "! * ' ( ) ; : @ & = + $ , /\\ ? % # [ ]";
- TopicName topicName = TopicName.get("persistent://my-property/use/my-ns/my-partitionedtopic1" + specialCharacter);
+ TopicName topicName = TopicName
+ .get("persistent://my-property/use/my-ns/my-partitionedtopic1" + specialCharacter);
admin.persistentTopics().createPartitionedTopic(topicName.toString(), numPartitions);
- ProducerConfiguration producerConf = new ProducerConfiguration();
- producerConf.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
// Try to create producer which does lookup and create connection with broker
- Producer producer = pulsarClient.createProducer(topicName.toString(), producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName.toString())
+ .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();
producer.close();
admin.persistentTopics().deletePartitionedTopic(topicName.toString());
log.info("-- Exiting {} test --", methodName);
@@ -140,23 +127,20 @@ public void testSinglePartitionProducer() throws Exception {
int numPartitions = 4;
TopicName topicName = TopicName.get("persistent://my-property/use/my-ns/my-partitionedtopic2");
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
-
admin.persistentTopics().createPartitionedTopic(topicName.toString(), numPartitions);
- ProducerConfiguration producerConf = new ProducerConfiguration();
- producerConf.setMessageRoutingMode(MessageRoutingMode.SinglePartition);
- Producer producer = pulsarClient.createProducer(topicName.toString(), producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName.toString())
+ .messageRoutingMode(MessageRoutingMode.SinglePartition).create();
- Consumer consumer = pulsarClient.subscribe(topicName.toString(), "my-partitioned-subscriber", conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName.toString())
+ .subscriptionName("my-partitioned-subscriber").subscribe();
for (int i = 0; i < 10; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
- Message msg = null;
+ Message<byte[]> msg = null;
Set<String> messageSet = Sets.newHashSet();
for (int i = 0; i < 10; i++) {
@@ -186,15 +170,13 @@ public void testKeyBasedProducer() throws Exception {
String dummyKey1 = "dummykey1";
String dummyKey2 = "dummykey2";
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
-
admin.persistentTopics().createPartitionedTopic(topicName.toString(), numPartitions);
- Producer producer = pulsarClient.createProducer(topicName.toString());
- Consumer consumer = pulsarClient.subscribe(topicName.toString(), "my-partitioned-subscriber", conf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName.toString()).create();
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName.toString())
+ .subscriptionName("my-partitioned-subscriber").subscribe();
- Message msg = null;
+ Message<byte[]> msg = null;
for (int i = 0; i < 5; i++) {
String message = "my-message-" + i;
msg = MessageBuilder.create().setContent(message.getBytes()).setKey(dummyKey1).build();
@@ -242,13 +224,11 @@ public void testInvalidSequence() throws Exception {
TopicName topicName = TopicName.get("persistent://my-property/use/my-ns/my-partitionedtopic4");
admin.persistentTopics().createPartitionedTopic(topicName.toString(), numPartitions);
- ConsumerConfiguration consumerConf = new ConsumerConfiguration();
- consumerConf.setSubscriptionType(SubscriptionType.Exclusive);
-
- Consumer consumer = pulsarClient.subscribe(topicName.toString(), "my-subscriber-name", consumerConf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName.toString())
+ .subscriptionName("my-subscriber-name").subscribe();
try {
- Message msg = MessageBuilder.create().setContent("InvalidMessage".getBytes()).build();
+ Message<byte[]> msg = MessageBuilder.create().setContent("InvalidMessage".getBytes()).build();
consumer.acknowledge(msg);
} catch (PulsarClientException.InvalidMessageException e) {
// ok
@@ -270,7 +250,7 @@ public void testInvalidSequence() throws Exception {
// ok
}
- Producer producer = pulsarClient.createProducer(topicName.toString());
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName.toString()).create();
producer.close();
try {
@@ -291,47 +271,30 @@ public void testSillyUser() throws Exception {
TopicName topicName = TopicName.get("persistent://my-property/use/my-ns/my-partitionedtopic5");
admin.persistentTopics().createPartitionedTopic(topicName.toString(), numPartitions);
- ProducerConfiguration producerConf = new ProducerConfiguration();
- Producer producer = null;
- Consumer consumer = null;
+ Producer<byte[]> producer = null;
+ Consumer<byte[]> consumer = null;
try {
- producerConf.setMessageRouter(null);
+ pulsarClient.newProducer().messageRouter(null);
Assert.fail("should fail");
} catch (NullPointerException e) {
// ok
}
try {
- producerConf.setMessageRoutingMode(null);
+ pulsarClient.newProducer().messageRoutingMode(null);
Assert.fail("should fail");
} catch (NullPointerException e) {
// ok
}
try {
- producer = pulsarClient.createProducer(topicName.toString(), null);
- Assert.fail("should fail");
- } catch (PulsarClientException e) {
- Assert.assertTrue(e instanceof PulsarClientException.InvalidConfigurationException);
- }
-
- ConsumerConfiguration consumerConf = new ConsumerConfiguration();
-
- try {
- consumer = pulsarClient.subscribe(topicName.toString(), "my-subscriber-name", null);
- Assert.fail("Should fail");
- } catch (PulsarClientException e) {
- Assert.assertTrue(e instanceof PulsarClientException.InvalidConfigurationException);
- }
-
- try {
- producer = pulsarClient.createProducer(topicName.toString());
- consumer = pulsarClient.subscribe(topicName.toString(), "my-sub");
+ producer = pulsarClient.newProducer().topic(topicName.toString()).create();
+ consumer = pulsarClient.newConsumer().topic(topicName.toString()).subscriptionName("my-sub").subscribe();
producer.send("message1".getBytes());
producer.send("message2".getBytes());
- Message msg1 = consumer.receive();
- Message msg2 = consumer.receive();
+ /* Message<byte[]> msg1 = */ consumer.receive();
+ Message<byte[]> msg2 = consumer.receive();
consumer.acknowledgeCumulative(msg2);
Assert.fail("should fail since ack cumulative is not supported for partitioned topic");
} catch (PulsarClientException e) {
@@ -352,15 +315,16 @@ public void testDeletePartitionedTopic() throws Exception {
TopicName topicName = TopicName.get("persistent://my-property/use/my-ns/my-partitionedtopic6");
admin.persistentTopics().createPartitionedTopic(topicName.toString(), numPartitions);
- Producer producer = pulsarClient.createProducer(topicName.toString());
- Consumer consumer = pulsarClient.subscribe(topicName.toString(), "my-sub");
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName.toString()).create();
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName.toString()).subscriptionName("my-sub")
+ .subscribe();
consumer.unsubscribe();
consumer.close();
producer.close();
admin.persistentTopics().deletePartitionedTopic(topicName.toString());
- Producer producer1 = pulsarClient.createProducer(topicName.toString());
+ Producer<byte[]> producer1 = pulsarClient.newProducer().topic(topicName.toString()).create();
if (producer1 instanceof PartitionedProducerImpl) {
Assert.fail("should fail since partitioned topic was deleted");
}
@@ -377,16 +341,12 @@ public void testAsyncPartitionedProducerConsumer() throws Exception {
int numPartitions = 4;
TopicName topicName = TopicName.get("persistent://my-property/use/my-ns/my-partitionedtopic1");
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Shared);
-
admin.persistentTopics().createPartitionedTopic(topicName.toString(), numPartitions);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName.toString())
+ .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();
- ProducerConfiguration producerConf = new ProducerConfiguration();
- producerConf.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
- Producer producer = pulsarClient.createProducer(topicName.toString(), producerConf);
-
- Consumer consumer = pulsarClient.subscribe(topicName.toString(), "my-partitioned-subscriber", conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName.toString())
+ .subscriptionName("my-partitioned-subscriber").subscriptionType(SubscriptionType.Shared).subscribe();
// produce messages
for (int i = 0; i < totalMsg; i++) {
@@ -428,16 +388,13 @@ public void testAsyncPartitionedProducerConsumerQueueSizeOne() throws Exception
int numPartitions = 4;
TopicName topicName = TopicName.get("persistent://my-property/use/my-ns/my-partitionedtopic1");
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setReceiverQueueSize(1);
-
admin.persistentTopics().createPartitionedTopic(topicName.toString(), numPartitions);
- ProducerConfiguration producerConf = new ProducerConfiguration();
- producerConf.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
- Producer producer = pulsarClient.createProducer(topicName.toString(), producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName.toString())
+ .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();
- Consumer consumer = pulsarClient.subscribe(topicName.toString(), "my-partitioned-subscriber", conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName.toString())
+ .subscriptionName("my-partitioned-subscriber").receiverQueueSize(1).subscribe();
// produce messages
for (int i = 0; i < totalMsg; i++) {
@@ -483,17 +440,15 @@ public void testFairDistributionForPartitionConsumers() throws Exception {
final String producer2Msg = "producer2";
final int queueSize = 10;
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setReceiverQueueSize(queueSize);
-
admin.persistentTopics().createPartitionedTopic(topicName, numPartitions);
- ProducerConfiguration producerConf = new ProducerConfiguration();
- producerConf.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
- Producer producer1 = pulsarClient.createProducer(topicName + "-partition-0", producerConf);
- Producer producer2 = pulsarClient.createProducer(topicName + "-partition-1", producerConf);
+ Producer<byte[]> producer1 = pulsarClient.newProducer().topic(topicName + "-partition-0")
+ .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();
+ Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topicName + "-partition-1")
+ .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();
- Consumer consumer = pulsarClient.subscribe(topicName, "my-partitioned-subscriber", conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName("my-partitioned-subscriber").receiverQueueSize(queueSize).subscribe();
int partition2Msgs = 0;
@@ -513,7 +468,7 @@ public void testFairDistributionForPartitionConsumers() throws Exception {
// also: we will keep producing messages to partition-1
int produceMsgInPartition1AfterNumberOfConsumeMessages = 2;
for (int i = 0; i < 3 * queueSize; i++) {
- Message msg = consumer.receive();
+ Message<byte[]> msg = consumer.receive();
partition2Msgs += (new String(msg.getData())).startsWith(producer2Msg) ? 1 : 0;
if (i >= produceMsgInPartition1AfterNumberOfConsumeMessages) {
producer1.send(producer1Msg.getBytes());
@@ -532,10 +487,10 @@ public void testFairDistributionForPartitionConsumers() throws Exception {
log.info("-- Exiting {} test --", methodName);
}
- private void receiveAsync(Consumer consumer, int totalMessage, int currentMessage, CountDownLatch latch,
+ private void receiveAsync(Consumer<byte[]> consumer, int totalMessage, int currentMessage, CountDownLatch latch,
final Set<String> consumeMsg, ExecutorService executor) throws PulsarClientException {
if (currentMessage < totalMessage) {
- CompletableFuture<Message> future = consumer.receiveAsync();
+ CompletableFuture<Message<byte[]>> future = consumer.receiveAsync();
future.handle((msg, exception) -> {
if (exception == null) {
// add message to consumer-queue to verify with produced messages
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerStatTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerStatTest.java
index a13ac3f19..8a536f2a3 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerStatTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerStatTest.java
@@ -34,16 +34,6 @@
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.pulsar.client.admin.PulsarAdminException;
-import org.apache.pulsar.client.api.Consumer;
-import org.apache.pulsar.client.api.ConsumerConfiguration;
-import org.apache.pulsar.client.api.Message;
-import org.apache.pulsar.client.api.MessageId;
-import org.apache.pulsar.client.api.Producer;
-import org.apache.pulsar.client.api.ProducerConfiguration;
-import org.apache.pulsar.client.api.PulsarClientException;
-import org.apache.pulsar.client.api.SubscriptionType;
-import org.apache.pulsar.client.impl.ConsumerStats;
-import org.apache.pulsar.client.impl.ProducerStats;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.annotations.AfterMethod;
@@ -84,25 +74,25 @@ protected void cleanup() throws Exception {
@Test(dataProvider = "batch_with_timeout")
public void testSyncProducerAndConsumer(int batchMessageDelayMs, int ackTimeoutSec) throws Exception {
log.info("-- Starting {} test --", methodName);
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
+ ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer()
+ .topic("persistent://my-property/tp1/my-ns/my-topic1").subscriptionName("my-subscriber-name");
+
// Cumulative Ack-counter works if ackTimeOutTimer-task is enabled
boolean isAckTimeoutTaskEnabledForCumulativeAck = ackTimeoutSec > 0;
if (ackTimeoutSec > 0) {
- conf.setAckTimeout(ackTimeoutSec, TimeUnit.SECONDS);
+ consumerBuilder.ackTimeout(ackTimeoutSec, TimeUnit.SECONDS);
}
- Consumer consumer = pulsarClient.subscribe("persistent://my-property/tp1/my-ns/my-topic1", "my-subscriber-name",
- conf);
+ Consumer<byte[]> consumer = consumerBuilder.subscribe();
- ProducerConfiguration producerConf = new ProducerConfiguration();
+ ProducerBuilder<byte[]> producerBuilder = pulsarClient.newProducer()
+ .topic("persistent://my-property/tp1/my-ns/my-topic1");
if (batchMessageDelayMs != 0) {
- producerConf.setBatchingEnabled(true);
- producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
- producerConf.setBatchingMaxMessages(5);
+ producerBuilder.enableBatching(true).batchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS)
+ .batchingMaxMessages(5);
}
- Producer producer = pulsarClient.createProducer("persistent://my-property/tp1/my-ns/my-topic1", producerConf);
+ Producer<byte[]> producer = producerBuilder.create();
int numMessages = 11;
for (int i = 0; i < numMessages; i++) {
@@ -110,7 +100,7 @@ public void testSyncProducerAndConsumer(int batchMessageDelayMs, int ackTimeoutS
producer.send(message.getBytes());
}
- Message msg = null;
+ Message<byte[]> msg = null;
Set<String> messageSet = Sets.newHashSet();
for (int i = 0; i < numMessages; i++) {
msg = consumer.receive(5, TimeUnit.SECONDS);
@@ -131,23 +121,22 @@ public void testSyncProducerAndConsumer(int batchMessageDelayMs, int ackTimeoutS
@Test(dataProvider = "batch_with_timeout")
public void testAsyncProducerAndAsyncAck(int batchMessageDelayMs, int ackTimeoutSec) throws Exception {
log.info("-- Starting {} test --", methodName);
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
+ ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer()
+ .topic("persistent://my-property/tp1/my-ns/my-topic2").subscriptionName("my-subscriber-name");
if (ackTimeoutSec > 0) {
- conf.setAckTimeout(ackTimeoutSec, TimeUnit.SECONDS);
+ consumerBuilder.ackTimeout(ackTimeoutSec, TimeUnit.SECONDS);
}
- Consumer consumer = pulsarClient.subscribe("persistent://my-property/tp1/my-ns/my-topic2", "my-subscriber-name",
- conf);
+ Consumer<byte[]> consumer = consumerBuilder.subscribe();
- ProducerConfiguration producerConf = new ProducerConfiguration();
+ ProducerBuilder<byte[]> producerBuilder = pulsarClient.newProducer()
+ .topic("persistent://my-property/tp1/my-ns/my-topic2");
if (batchMessageDelayMs != 0) {
- producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
- producerConf.setBatchingMaxMessages(5);
- producerConf.setBatchingEnabled(true);
+ producerBuilder.enableBatching(true).batchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS)
+ .batchingMaxMessages(5);
}
- Producer producer = pulsarClient.createProducer("persistent://my-property/tp1/my-ns/my-topic2", producerConf);
+ Producer<byte[]> producer = producerBuilder.create();
List<Future<MessageId>> futures = Lists.newArrayList();
int numMessages = 50;
@@ -163,7 +152,7 @@ public void testAsyncProducerAndAsyncAck(int batchMessageDelayMs, int ackTimeout
future.get();
}
- Message msg = null;
+ Message<byte[]> msg = null;
Set<String> messageSet = Sets.newHashSet();
for (int i = 0; i < numMessages; i++) {
msg = consumer.receive(5, TimeUnit.SECONDS);
@@ -187,23 +176,22 @@ public void testAsyncProducerAndAsyncAck(int batchMessageDelayMs, int ackTimeout
public void testAsyncProducerAndReceiveAsyncAndAsyncAck(int batchMessageDelayMs, int ackTimeoutSec)
throws Exception {
log.info("-- Starting {} test --", methodName);
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
+ ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer()
+ .topic("persistent://my-property/tp1/my-ns/my-topic2").subscriptionName("my-subscriber-name");
if (ackTimeoutSec > 0) {
- conf.setAckTimeout(ackTimeoutSec, TimeUnit.SECONDS);
+ consumerBuilder.ackTimeout(ackTimeoutSec, TimeUnit.SECONDS);
}
- Consumer consumer = pulsarClient.subscribe("persistent://my-property/tp1/my-ns/my-topic2", "my-subscriber-name",
- conf);
+ Consumer<byte[]> consumer = consumerBuilder.subscribe();
- ProducerConfiguration producerConf = new ProducerConfiguration();
+ ProducerBuilder<byte[]> producerBuilder = pulsarClient.newProducer()
+ .topic("persistent://my-property/tp1/my-ns/my-topic2");
if (batchMessageDelayMs != 0) {
- producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
- producerConf.setBatchingMaxMessages(5);
- producerConf.setBatchingEnabled(true);
+ producerBuilder.enableBatching(true).batchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS)
+ .batchingMaxMessages(5);
}
- Producer producer = pulsarClient.createProducer("persistent://my-property/tp1/my-ns/my-topic2", producerConf);
+ Producer<byte[]> producer = producerBuilder.create();
List<Future<MessageId>> futures = Lists.newArrayList();
int numMessages = 101;
@@ -218,8 +206,8 @@ public void testAsyncProducerAndReceiveAsyncAndAsyncAck(int batchMessageDelayMs,
for (Future<MessageId> future : futures) {
future.get();
}
- Message msg = null;
- CompletableFuture<Message> future_msg = null;
+ Message<byte[]> msg = null;
+ CompletableFuture<Message<byte[]>> future_msg = null;
Set<String> messageSet = Sets.newHashSet();
for (int i = 0; i < numMessages; i++) {
future_msg = consumer.receiveAsync();
@@ -245,30 +233,28 @@ public void testAsyncProducerAndReceiveAsyncAndAsyncAck(int batchMessageDelayMs,
@Test(dataProvider = "batch", timeOut = 100000)
public void testMessageListener(int batchMessageDelayMs) throws Exception {
log.info("-- Starting {} test --", methodName);
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setAckTimeout(100, TimeUnit.SECONDS);
- conf.setSubscriptionType(SubscriptionType.Exclusive);
int numMessages = 100;
final CountDownLatch latch = new CountDownLatch(numMessages);
- conf.setMessageListener((consumer, msg) -> {
- assertNotNull(msg, "Message cannot be null");
- String receivedMessage = new String(msg.getData());
- log.debug("Received message [{}] in the listener", receivedMessage);
- consumer.acknowledgeAsync(msg);
- latch.countDown();
- });
-
- Consumer consumer = pulsarClient.subscribe("persistent://my-property/tp1/my-ns/my-topic3", "my-subscriber-name",
- conf);
- ProducerConfiguration producerConf = new ProducerConfiguration();
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://my-property/tp1/my-ns/my-topic3")
+ .subscriptionName("my-subscriber-name").ackTimeout(100, TimeUnit.SECONDS)
+ .messageListener((consumer1, msg) -> {
+ assertNotNull(msg, "Message cannot be null");
+ String receivedMessage = new String(msg.getData());
+ log.debug("Received message [{}] in the listener", receivedMessage);
+ consumer1.acknowledgeAsync(msg);
+ latch.countDown();
+ }).subscribe();
+
+ ProducerBuilder<byte[]> producerBuilder = pulsarClient.newProducer()
+ .topic("persistent://my-property/tp1/my-ns/my-topic3");
if (batchMessageDelayMs != 0) {
- producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
- producerConf.setBatchingMaxMessages(5);
- producerConf.setBatchingEnabled(true);
+ producerBuilder.enableBatching(true).batchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS)
+ .batchingMaxMessages(5);
}
- Producer producer = pulsarClient.createProducer("persistent://my-property/tp1/my-ns/my-topic3", producerConf);
+
+ Producer<byte[]> producer = producerBuilder.create();
List<Future<MessageId>> futures = Lists.newArrayList();
// Asynchronously produce messages
@@ -295,19 +281,18 @@ public void testMessageListener(int batchMessageDelayMs) throws Exception {
public void testSendTimeout(int batchMessageDelayMs) throws Exception {
log.info("-- Starting {} test --", methodName);
- ConsumerConfiguration consumerConf = new ConsumerConfiguration();
- consumerConf.setSubscriptionType(SubscriptionType.Exclusive);
- Consumer consumer = pulsarClient.subscribe("persistent://my-property/tp1/my-ns/my-topic5", "my-subscriber-name",
- consumerConf);
- ProducerConfiguration producerConf = new ProducerConfiguration();
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://my-property/tp1/my-ns/my-topic5")
+ .subscriptionName("my-subscriber-name").subscribe();
+
+ ProducerBuilder<byte[]> producerBuilder = pulsarClient.newProducer()
+ .topic("persistent://my-property/tp1/my-ns/my-topic5").sendTimeout(1, TimeUnit.SECONDS);
if (batchMessageDelayMs != 0) {
- producerConf.setBatchingMaxPublishDelay(2 * batchMessageDelayMs, TimeUnit.MILLISECONDS);
- producerConf.setBatchingMaxMessages(5);
- producerConf.setBatchingEnabled(true);
+ producerBuilder.enableBatching(true).batchingMaxPublishDelay(2 * batchMessageDelayMs, TimeUnit.MILLISECONDS)
+ .batchingMaxMessages(5);
}
- producerConf.setSendTimeout(1, TimeUnit.SECONDS);
- Producer producer = pulsarClient.createProducer("persistent://my-property/tp1/my-ns/my-topic5", producerConf);
+ Producer<byte[]> producer = producerBuilder.create();
+
final String message = "my-message";
// Trigger the send timeout
@@ -325,7 +310,7 @@ public void testSendTimeout(int batchMessageDelayMs) throws Exception {
startBroker();
// We should not have received any message
- Message msg = consumer.receive(3, TimeUnit.SECONDS);
+ Message<byte[]> msg = consumer.receive(3, TimeUnit.SECONDS);
assertNull(msg);
consumer.close();
producer.close();
@@ -338,28 +323,23 @@ public void testSendTimeout(int batchMessageDelayMs) throws Exception {
assertEquals(cStat.getTotalMsgsReceived(), cStat.getTotalAcksSent());
log.info("-- Exiting {} test --", methodName);
}
-
+
public void testBatchMessagesRateOut() throws PulsarClientException, InterruptedException, PulsarAdminException {
log.info("-- Starting {} test --", methodName);
String topicName = "persistent://my-property/cluster/my-ns/testBatchMessagesRateOut";
double produceRate = 17;
int batchSize = 5;
- ConsumerConfiguration consumerConf = new ConsumerConfiguration();
- consumerConf.setSubscriptionType(SubscriptionType.Exclusive);
- Consumer consumer = pulsarClient.subscribe(topicName, "my-subscriber-name", consumerConf);
- ProducerConfiguration producerConf = new ProducerConfiguration();
- producerConf.setBatchingMaxMessages(batchSize);
- producerConf.setBatchingEnabled(true);
- producerConf.setBatchingMaxPublishDelay(2, TimeUnit.SECONDS);
-
- Producer producer = pulsarClient.createProducer(topicName, producerConf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name")
+ .subscribe();
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).batchingMaxMessages(batchSize)
+ .enableBatching(true).batchingMaxPublishDelay(2, TimeUnit.SECONDS).create();
AtomicBoolean runTest = new AtomicBoolean(true);
Thread t1 = new Thread(() -> {
RateLimiter r = RateLimiter.create(produceRate);
while (runTest.get()) {
r.acquire();
producer.sendAsync("Hello World".getBytes());
- consumer.receiveAsync().thenAccept(message -> consumer.acknowledgeAsync(message));
+ consumer.receiveAsync().thenAccept(consumer::acknowledgeAsync);
}
});
t1.start();
@@ -372,7 +352,7 @@ public void testBatchMessagesRateOut() throws PulsarClientException, Interrupted
log.info("-- Exiting {} test --", methodName);
}
- public void validatingLogInfo(Consumer consumer, Producer producer, boolean verifyAckCount)
+ public void validatingLogInfo(Consumer<?> consumer, Producer<?> producer, boolean verifyAckCount)
throws InterruptedException {
// Waiting for recording last stat info
Thread.sleep(1000);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
index 33d10634b..a624ef0c7 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
@@ -54,6 +54,7 @@
import org.apache.bookkeeper.mledger.impl.EntryCacheImpl;
import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl;
import org.apache.pulsar.broker.service.persistent.PersistentTopic;
+import org.apache.pulsar.client.api.SubscriptionInitialPosition;
import org.apache.pulsar.client.impl.ConsumerImpl;
import org.apache.pulsar.client.impl.MessageIdImpl;
import org.apache.pulsar.common.api.PulsarDecoder;
@@ -2447,4 +2448,40 @@ public EncryptionKeyInfo getPrivateKey(String keyName, Map<String, String> keyMe
log.info("-- Exiting {} test --", methodName);
}
+ @Test
+ public void testConsumerSubscriptionInitialize() throws Exception {
+ log.info("-- Starting {} test --", methodName);
+ String topic = "persistent://my-property/use/my-ns/my-topic-test-subscription-initialize";
+
+ Producer producer = pulsarClient.createProducer(topic);
+
+ // first produce 5 messages
+ for (int i = 0; i < 5; i++) {
+ final String message = "my-message-" + i;
+ producer.send(message.getBytes());
+ }
+
+ // second create consumers
+ ConsumerConfiguration conf = new ConsumerConfiguration();
+ Consumer defaultConsumer = pulsarClient.subscribe(topic, "test-subscription-default");
+ conf.setSubscriptionInitialPosition(SubscriptionInitialPosition.Latest);
+ Consumer latestConsumer = pulsarClient.subscribe(topic, "test-subscription-latest", conf);
+ conf.setSubscriptionInitialPosition(SubscriptionInitialPosition.Earliest);
+ Consumer earlistConsumer = pulsarClient.subscribe(topic, "test-subscription-earliest", conf);
+
+ // third produce 5 messages
+ for (int i = 5; i < 10; i++) {
+ final String message = "my-message-" + i;
+ producer.send(message.getBytes());
+ }
+
+ assertEquals(defaultConsumer.receive().getData(), "my-message-5".getBytes());
+ assertEquals(latestConsumer.receive().getData(), "my-message-5".getBytes());
+ assertEquals(earlistConsumer.receive().getData(), "my-message-0".getBytes());
+
+ defaultConsumer.close();
+ latestConsumer.close();
+ earlistConsumer.close();
+ log.info("-- Exiting {} test --", methodName);
+ }
}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerBase.java
index 7b8e083f2..66b2265f2 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerBase.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerBase.java
@@ -35,8 +35,6 @@
import com.google.common.collect.Sets;
public class TlsProducerConsumerBase extends ProducerConsumerBase {
- private static final Logger log = LoggerFactory.getLogger(TlsProducerConsumerBase.class);
-
protected final String TLS_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem";
protected final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/broker-cert.pem";
protected final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/broker-key.pem";
@@ -67,11 +65,9 @@ protected void internalSetUpForBroker() throws Exception {
}
protected void internalSetUpForClient() throws Exception {
- ClientConfiguration clientConf = new ClientConfiguration();
- clientConf.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH);
- clientConf.setUseTls(true);
String lookupUrl = new URI("pulsar+ssl://localhost:" + BROKER_PORT_TLS).toString();
- pulsarClient = PulsarClient.create(lookupUrl, clientConf);
+ pulsarClient = PulsarClient.builder().serviceUrl(lookupUrl).tlsTrustCertsFilePath(TLS_SERVER_CERT_FILE_PATH)
+ .enableTls(true).build();
}
protected void internalSetUpForNamespace() throws Exception {
@@ -79,9 +75,8 @@ protected void internalSetUpForNamespace() throws Exception {
clientConf.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH);
clientConf.setUseTls(true);
admin = spy(new PulsarAdmin(brokerUrlTls, clientConf));
- admin.clusters().updateCluster(clusterName,
- new ClusterData(brokerUrl.toString(), brokerUrlTls.toString(), "pulsar://localhost:" + BROKER_PORT,
- "pulsar+ssl://localhost:" + BROKER_PORT_TLS));
+ admin.clusters().updateCluster(clusterName, new ClusterData(brokerUrl.toString(), brokerUrlTls.toString(),
+ "pulsar://localhost:" + BROKER_PORT, "pulsar+ssl://localhost:" + BROKER_PORT_TLS));
admin.properties().createProperty("my-property",
new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet("use")));
admin.namespaces().createNamespace("my-property/use/my-ns");
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerTest.java
index 109525f1d..a0d4bc275 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerTest.java
@@ -45,21 +45,18 @@ public void testTlsLargeSizeMessage() throws Exception {
internalSetUpForClient();
internalSetUpForNamespace();
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
- Consumer consumer = pulsarClient
- .subscribe("persistent://my-property/use/my-ns/my-topic1", "my-subscriber-name", conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://my-property/use/my-ns/my-topic1")
+ .subscriptionName("my-subscriber-name").subscribe();
- ProducerConfiguration producerConf = new ProducerConfiguration();
-
- Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1", producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1")
+ .create();
for (int i = 0; i < 10; i++) {
byte[] message = new byte[MESSAGE_SIZE];
Arrays.fill(message, (byte) i);
producer.send(message);
}
- Message msg = null;
+ Message<byte[]> msg = null;
for (int i = 0; i < 10; i++) {
msg = consumer.receive(5, TimeUnit.SECONDS);
byte[] expected = new byte[MESSAGE_SIZE];
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicReaderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicReaderTest.java
index e8f635fb7..12415dbb1 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicReaderTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicReaderTest.java
@@ -60,19 +60,17 @@ protected void cleanup() throws Exception {
@Test
public void testSimpleReader() throws Exception {
- ReaderConfiguration conf = new ReaderConfiguration();
- Reader reader = pulsarClient.createReader("persistent://my-property/use/my-ns/my-topic1", MessageId.earliest,
- conf);
+ Reader<byte[]> reader = pulsarClient.newReader().topic("persistent://my-property/use/my-ns/my-topic1")
+ .startMessageId(MessageId.earliest).create();
- ProducerConfiguration producerConf = new ProducerConfiguration();
-
- Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1", producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1")
+ .create();
for (int i = 0; i < 10; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
- Message msg = null;
+ Message<byte[]> msg = null;
Set<String> messageSet = Sets.newHashSet();
for (int i = 0; i < 10; i++) {
msg = reader.readNext(1, TimeUnit.SECONDS);
@@ -90,18 +88,17 @@ public void testSimpleReader() throws Exception {
@Test
public void testReaderAfterMessagesWerePublished() throws Exception {
- ProducerConfiguration producerConf = new ProducerConfiguration();
-
- Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1", producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1")
+ .create();
for (int i = 0; i < 10; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
- Reader reader = pulsarClient.createReader("persistent://my-property/use/my-ns/my-topic1", MessageId.earliest,
- new ReaderConfiguration());
+ Reader<byte[]> reader = pulsarClient.newReader().topic("persistent://my-property/use/my-ns/my-topic1")
+ .startMessageId(MessageId.earliest).create();
- Message msg = null;
+ Message<byte[]> msg = null;
Set<String> messageSet = Sets.newHashSet();
for (int i = 0; i < 10; i++) {
msg = reader.readNext(1, TimeUnit.SECONDS);
@@ -119,21 +116,20 @@ public void testReaderAfterMessagesWerePublished() throws Exception {
@Test
public void testMultipleReaders() throws Exception {
- ProducerConfiguration producerConf = new ProducerConfiguration();
-
- Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1", producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1")
+ .create();
for (int i = 0; i < 10; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
- Reader reader1 = pulsarClient.createReader("persistent://my-property/use/my-ns/my-topic1", MessageId.earliest,
- new ReaderConfiguration());
+ Reader<byte[]> reader1 = pulsarClient.newReader().topic("persistent://my-property/use/my-ns/my-topic1")
+ .startMessageId(MessageId.earliest).create();
- Reader reader2 = pulsarClient.createReader("persistent://my-property/use/my-ns/my-topic1", MessageId.earliest,
- new ReaderConfiguration());
+ Reader<byte[]> reader2 = pulsarClient.newReader().topic("persistent://my-property/use/my-ns/my-topic1")
+ .startMessageId(MessageId.earliest).create();
- Message msg = null;
+ Message<byte[]> msg = null;
Set<String> messageSet1 = Sets.newHashSet();
for (int i = 0; i < 10; i++) {
msg = reader1.readNext(1, TimeUnit.SECONDS);
@@ -163,9 +159,9 @@ public void testMultipleReaders() throws Exception {
public void testTopicStats() throws Exception {
String topicName = "persistent://my-property/use/my-ns/my-topic1";
- Reader reader1 = pulsarClient.createReader(topicName, MessageId.earliest, new ReaderConfiguration());
+ Reader<byte[]> reader1 = pulsarClient.newReader().topic(topicName).startMessageId(MessageId.earliest).create();
- Reader reader2 = pulsarClient.createReader(topicName, MessageId.earliest, new ReaderConfiguration());
+ Reader<byte[]> reader2 = pulsarClient.newReader().topic(topicName).startMessageId(MessageId.earliest).create();
PersistentTopicStats stats = admin.persistentTopics().getStats(topicName);
assertEquals(stats.subscriptions.size(), 2);
@@ -182,16 +178,15 @@ public void testTopicStats() throws Exception {
@Test
public void testReaderOnLastMessage() throws Exception {
- ProducerConfiguration producerConf = new ProducerConfiguration();
-
- Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1", producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1")
+ .create();
for (int i = 0; i < 10; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
- Reader reader = pulsarClient.createReader("persistent://my-property/use/my-ns/my-topic1", MessageId.latest,
- new ReaderConfiguration());
+ Reader<byte[]> reader = pulsarClient.newReader().topic("persistent://my-property/use/my-ns/my-topic1")
+ .startMessageId(MessageId.latest).create();
for (int i = 10; i < 20; i++) {
String message = "my-message-" + i;
@@ -200,7 +195,7 @@ public void testReaderOnLastMessage() throws Exception {
// Publish more messages and verify the readers only sees new messages
- Message msg = null;
+ Message<byte[]> msg = null;
Set<String> messageSet = Sets.newHashSet();
for (int i = 10; i < 20; i++) {
msg = reader.readNext(1, TimeUnit.SECONDS);
@@ -218,20 +213,19 @@ public void testReaderOnLastMessage() throws Exception {
@Test
public void testReaderOnSpecificMessage() throws Exception {
- ProducerConfiguration producerConf = new ProducerConfiguration();
-
- Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1", producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1")
+ .create();
List<MessageId> messageIds = new ArrayList<>();
for (int i = 0; i < 10; i++) {
String message = "my-message-" + i;
messageIds.add(producer.send(message.getBytes()));
}
- Reader reader = pulsarClient.createReader("persistent://my-property/use/my-ns/my-topic1", messageIds.get(4),
- new ReaderConfiguration());
+ Reader<byte[]> reader = pulsarClient.newReader().topic("persistent://my-property/use/my-ns/my-topic1")
+ .startMessageId(messageIds.get(4)).create();
// Publish more messages and verify the readers only sees messages starting from the intended message
- Message msg = null;
+ Message<byte[]> msg = null;
Set<String> messageSet = Sets.newHashSet();
for (int i = 5; i < 10; i++) {
msg = reader.readNext(1, TimeUnit.SECONDS);
@@ -249,10 +243,9 @@ public void testReaderOnSpecificMessage() throws Exception {
@Test
public void testReaderOnSpecificMessageWithBatches() throws Exception {
- ProducerConfiguration producerConf = new ProducerConfiguration();
- producerConf.setBatchingEnabled(true);
- producerConf.setBatchingMaxPublishDelay(100, TimeUnit.MILLISECONDS);
- Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/testReaderOnSpecificMessageWithBatches", producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer()
+ .topic("persistent://my-property/use/my-ns/testReaderOnSpecificMessageWithBatches").enableBatching(true)
+ .batchingMaxPublishDelay(100, TimeUnit.MILLISECONDS).create();
for (int i = 0; i < 10; i++) {
String message = "my-message-" + i;
producer.sendAsync(message.getBytes());
@@ -260,13 +253,13 @@ public void testReaderOnSpecificMessageWithBatches() throws Exception {
// Write one sync message to ensure everything before got persistend
producer.send("my-message-10".getBytes());
- Reader reader1 = pulsarClient.createReader(
- "persistent://my-property/use/my-ns/testReaderOnSpecificMessageWithBatches", MessageId.earliest,
- new ReaderConfiguration());
+ Reader<byte[]> reader1 = pulsarClient.newReader()
+ .topic("persistent://my-property/use/my-ns/testReaderOnSpecificMessageWithBatches")
+ .startMessageId(MessageId.earliest).create();
MessageId lastMessageId = null;
for (int i = 0; i < 5; i++) {
- Message msg = reader1.readNext();
+ Message<byte[]> msg = reader1.readNext();
lastMessageId = msg.getMessageId();
}
@@ -274,12 +267,12 @@ public void testReaderOnSpecificMessageWithBatches() throws Exception {
System.out.println("CREATING READER ON MSG ID: " + lastMessageId);
- Reader reader2 = pulsarClient.createReader(
- "persistent://my-property/use/my-ns/testReaderOnSpecificMessageWithBatches", lastMessageId,
- new ReaderConfiguration());
+ Reader<byte[]> reader2 = pulsarClient.newReader()
+ .topic("persistent://my-property/use/my-ns/testReaderOnSpecificMessageWithBatches")
+ .startMessageId(lastMessageId).create();
for (int i = 5; i < 11; i++) {
- Message msg = reader2.readNext(1, TimeUnit.SECONDS);
+ Message<byte[]> msg = reader2.readNext(1, TimeUnit.SECONDS);
String receivedMessage = new String(msg.getData());
log.debug("Received message: [{}]", receivedMessage);
@@ -297,6 +290,7 @@ public void testECDSAEncryption() throws Exception {
class EncKeyReader implements CryptoKeyReader {
EncryptionKeyInfo keyInfo = new EncryptionKeyInfo();
+
@Override
public EncryptionKeyInfo getPublicKey(String keyName, Map<String, String> keyMeta) {
String CERT_FILE_PATH = "./src/test/resources/certificate/public-key." + keyName;
@@ -333,22 +327,19 @@ public EncryptionKeyInfo getPrivateKey(String keyName, Map<String, String> keyMe
final int totalMsg = 10;
Set<String> messageSet = Sets.newHashSet();
- ReaderConfiguration conf = new ReaderConfiguration();
- conf.setCryptoKeyReader(new EncKeyReader());
- Reader reader = pulsarClient.createReader("persistent://my-property/use/my-ns/test-reader-myecdsa-topic1", MessageId.latest,
- conf);
+ Reader<byte[]> reader = pulsarClient.newReader()
+ .topic("persistent://my-property/use/my-ns/test-reader-myecdsa-topic1").startMessageId(MessageId.latest)
+ .cryptoKeyReader(new EncKeyReader()).create();
- ProducerConfiguration producerConf = new ProducerConfiguration();
- producerConf.addEncryptionKey("client-ecdsa.pem");
- producerConf.setCryptoKeyReader(new EncKeyReader());
-
- Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/test-reader-myecdsa-topic1", producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer()
+ .topic("persistent://my-property/use/my-ns/test-reader-myecdsa-topic1")
+ .addEncryptionKey("client-ecdsa.pem").cryptoKeyReader(new EncKeyReader()).create();
for (int i = 0; i < totalMsg; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
- Message msg = null;
+ Message<byte[]> msg = null;
for (int i = 0; i < totalMsg; i++) {
msg = reader.readNext(5, TimeUnit.SECONDS);
@@ -362,14 +353,12 @@ public EncryptionKeyInfo getPrivateKey(String keyName, Map<String, String> keyMe
log.info("-- Exiting {} test --", methodName);
}
-
@Test
public void testSimpleReaderReachEndofTopic() throws Exception {
- ReaderConfiguration conf = new ReaderConfiguration();
- Reader reader = pulsarClient.createReader("persistent://my-property/use/my-ns/my-topic1", MessageId.earliest,
- conf);
- ProducerConfiguration producerConf = new ProducerConfiguration();
- Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1", producerConf);
+ Reader<byte[]> reader = pulsarClient.newReader().topic("persistent://my-property/use/my-ns/my-topic1")
+ .startMessageId(MessageId.earliest).create();
+ Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1")
+ .create();
// no data write, should return false
assertFalse(reader.hasMessageAvailable());
@@ -380,16 +369,16 @@ public void testSimpleReaderReachEndofTopic() throws Exception {
producer.send(message.getBytes());
}
- MessageImpl msg = null;
+ MessageImpl<byte[]> msg = null;
Set<String> messageSet = Sets.newHashSet();
int index = 0;
// read message till end.
while (reader.hasMessageAvailable()) {
- msg = (MessageImpl) reader.readNext(1, TimeUnit.SECONDS);
+ msg = (MessageImpl<byte[]>) reader.readNext(1, TimeUnit.SECONDS);
String receivedMessage = new String(msg.getData());
log.debug("Received message: [{}]", receivedMessage);
- String expectedMessage = "my-message-" + (index ++);
+ String expectedMessage = "my-message-" + (index++);
testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
}
@@ -405,10 +394,10 @@ public void testSimpleReaderReachEndofTopic() throws Exception {
// read message till end again.
while (reader.hasMessageAvailable()) {
- msg = (MessageImpl) reader.readNext(1, TimeUnit.SECONDS);
+ msg = (MessageImpl<byte[]>) reader.readNext(1, TimeUnit.SECONDS);
String receivedMessage = new String(msg.getData());
log.debug("Received message: [{}]", receivedMessage);
- String expectedMessage = "my-message-" + (index ++);
+ String expectedMessage = "my-message-" + (index++);
testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
}
@@ -421,14 +410,13 @@ public void testSimpleReaderReachEndofTopic() throws Exception {
@Test
public void testReaderReachEndofTopicOnMessageWithBatches() throws Exception {
- Reader reader = pulsarClient.createReader(
- "persistent://my-property/use/my-ns/testReaderReachEndofTopicOnMessageWithBatches", MessageId.earliest,
- new ReaderConfiguration());
+ Reader<byte[]> reader = pulsarClient.newReader()
+ .topic("persistent://my-property/use/my-ns/testReaderReachEndofTopicOnMessageWithBatches")
+ .startMessageId(MessageId.earliest).create();
- ProducerConfiguration producerConf = new ProducerConfiguration();
- producerConf.setBatchingEnabled(true);
- producerConf.setBatchingMaxPublishDelay(100, TimeUnit.MILLISECONDS);
- Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/testReaderReachEndofTopicOnMessageWithBatches", producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer()
+ .topic("persistent://my-property/use/my-ns/testReaderReachEndofTopicOnMessageWithBatches")
+ .enableBatching(true).batchingMaxPublishDelay(100, TimeUnit.MILLISECONDS).create();
// no data write, should return false
assertFalse(reader.hasMessageAvailable());
@@ -446,7 +434,7 @@ public void testReaderReachEndofTopicOnMessageWithBatches() throws Exception {
assertTrue(reader.hasMessageAvailable());
if (reader.hasMessageAvailable()) {
- Message msg = reader.readNext();
+ Message<byte[]> msg = reader.readNext();
lastMessageId = msg.getMessageId();
assertEquals(lastMessageId.getClass(), BatchMessageIdImpl.class);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v1/V1_ProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v1/V1_ProducerConsumerTest.java
new file mode 100644
index 000000000..64df6a41b
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v1/V1_ProducerConsumerTest.java
@@ -0,0 +1,2459 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.client.api.v1;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotEquals;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+
+import org.apache.bookkeeper.mledger.impl.EntryCacheImpl;
+import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl;
+import org.apache.pulsar.broker.service.persistent.PersistentTopic;
+import org.apache.pulsar.client.api.CompressionType;
+import org.apache.pulsar.client.api.Consumer;
+import org.apache.pulsar.client.api.ConsumerConfiguration;
+import org.apache.pulsar.client.api.ConsumerCryptoFailureAction;
+import org.apache.pulsar.client.api.CryptoKeyReader;
+import org.apache.pulsar.client.api.EncryptionKeyInfo;
+import org.apache.pulsar.client.api.Message;
+import org.apache.pulsar.client.api.MessageBuilder;
+import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.client.api.ProducerConfiguration;
+import org.apache.pulsar.client.api.ProducerConsumerBase;
+import org.apache.pulsar.client.api.PulsarClient;
+import org.apache.pulsar.client.api.PulsarClientException;
+import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.client.impl.ConsumerImpl;
+import org.apache.pulsar.client.impl.MessageIdImpl;
+import org.apache.pulsar.common.api.PulsarDecoder;
+import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.util.FutureUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.Assert;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+/**
+ * Basic tests using the deprecated client APIs from Pulsar-1.x
+ */
+@SuppressWarnings({ "deprecation", "rawtypes", "unchecked" })
+public class V1_ProducerConsumerTest extends ProducerConsumerBase {
+ private static final Logger log = LoggerFactory.getLogger(V1_ProducerConsumerTest.class);
+
+ @BeforeMethod
+ @Override
+ protected void setup() throws Exception {
+ super.internalSetup();
+ super.producerBaseSetup();
+ }
+
+ @AfterMethod
+ @Override
+ protected void cleanup() throws Exception {
+ super.internalCleanup();
+ }
+
+ @DataProvider(name = "batch")
+ public Object[][] codecProvider() {
+ return new Object[][] { { 0 }, { 1000 } };
+ }
+
+ @Test(dataProvider = "batch")
+ public void testSyncProducerAndConsumer(int batchMessageDelayMs) throws Exception {
+ log.info("-- Starting {} test --", methodName);
+
+ ConsumerConfiguration conf = new ConsumerConfiguration();
+ conf.setSubscriptionType(SubscriptionType.Exclusive);
+ Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic1", "my-subscriber-name",
+ conf);
+
+ ProducerConfiguration producerConf = new ProducerConfiguration();
+
+ if (batchMessageDelayMs != 0) {
+ producerConf.setBatchingEnabled(true);
+ producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
+ producerConf.setBatchingMaxMessages(5);
+ }
+
+ Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1", producerConf);
+ for (int i = 0; i < 10; i++) {
+ String message = "my-message-" + i;
+ producer.send(message.getBytes());
+ }
+
+ Message msg = null;
+ Set<String> messageSet = Sets.newHashSet();
+ for (int i = 0; i < 10; i++) {
+ msg = consumer.receive(5, TimeUnit.SECONDS);
+ String receivedMessage = new String(msg.getData());
+ log.debug("Received message: [{}]", receivedMessage);
+ String expectedMessage = "my-message-" + i;
+ testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
+ }
+ // Acknowledge the consumption of all messages at once
+ consumer.acknowledgeCumulative(msg);
+ consumer.close();
+ log.info("-- Exiting {} test --", methodName);
+ }
+
+ @Test(dataProvider = "batch")
+ public void testAsyncProducerAndAsyncAck(int batchMessageDelayMs) throws Exception {
+ log.info("-- Starting {} test --", methodName);
+ ConsumerConfiguration conf = new ConsumerConfiguration();
+ conf.setSubscriptionType(SubscriptionType.Exclusive);
+ Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic2", "my-subscriber-name",
+ conf);
+ ProducerConfiguration producerConf = new ProducerConfiguration();
+ if (batchMessageDelayMs != 0) {
+ producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
+ producerConf.setBatchingMaxMessages(5);
+ producerConf.setBatchingEnabled(true);
+ }
+ Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic2", producerConf);
+ List<Future<MessageId>> futures = Lists.newArrayList();
+
+ // Asynchronously produce messages
+ for (int i = 0; i < 10; i++) {
+ final String message = "my-message-" + i;
+ Future<MessageId> future = producer.sendAsync(message.getBytes());
+ futures.add(future);
+ }
+
+ log.info("Waiting for async publish to complete");
+ for (Future<MessageId> future : futures) {
+ future.get();
+ }
+
+ Message msg = null;
+ Set<String> messageSet = Sets.newHashSet();
+ for (int i = 0; i < 10; i++) {
+ msg = consumer.receive(5, TimeUnit.SECONDS);
+ String receivedMessage = new String(msg.getData());
+ log.info("Received message: [{}]", receivedMessage);
+ String expectedMessage = "my-message-" + i;
+ testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
+ }
+
+ // Asynchronously acknowledge upto and including the last message
+ Future<Void> ackFuture = consumer.acknowledgeCumulativeAsync(msg);
+ log.info("Waiting for async ack to complete");
+ ackFuture.get();
+ consumer.close();
+ log.info("-- Exiting {} test --", methodName);
+ }
+
+ @Test(dataProvider = "batch", timeOut = 100000)
+ public void testMessageListener(int batchMessageDelayMs) throws Exception {
+ log.info("-- Starting {} test --", methodName);
+ ConsumerConfiguration conf = new ConsumerConfiguration();
+ conf.setSubscriptionType(SubscriptionType.Exclusive);
+
+ int numMessages = 100;
+ final CountDownLatch latch = new CountDownLatch(numMessages);
+
+ conf.setMessageListener((consumer, msg) -> {
+ Assert.assertNotNull(msg, "Message cannot be null");
+ String receivedMessage = new String(msg.getData());
+ log.debug("Received message [{}] in the listener", receivedMessage);
+ consumer.acknowledgeAsync(msg);
+ latch.countDown();
+ });
+ Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic3", "my-subscriber-name",
+ conf);
+ ProducerConfiguration producerConf = new ProducerConfiguration();
+ if (batchMessageDelayMs != 0) {
+ producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
+ producerConf.setBatchingMaxMessages(5);
+ producerConf.setBatchingEnabled(true);
+ }
+ Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic3", producerConf);
+ List<Future<MessageId>> futures = Lists.newArrayList();
+
+ // Asynchronously produce messages
+ for (int i = 0; i < numMessages; i++) {
+ final String message = "my-message-" + i;
+ Future<MessageId> future = producer.sendAsync(message.getBytes());
+ futures.add(future);
+ }
+
+ log.info("Waiting for async publish to complete");
+ for (Future<MessageId> future : futures) {
+ future.get();
+ }
+
+ log.info("Waiting for message listener to ack all messages");
+ assertEquals(latch.await(numMessages, TimeUnit.SECONDS), true, "Timed out waiting for message listener acks");
+ consumer.close();
+ log.info("-- Exiting {} test --", methodName);
+ }
+
+ @Test(dataProvider = "batch")
+ public void testBackoffAndReconnect(int batchMessageDelayMs) throws Exception {
+ log.info("-- Starting {} test --", methodName);
+ // Create consumer and producer
+ ConsumerConfiguration conf = new ConsumerConfiguration();
+ conf.setSubscriptionType(SubscriptionType.Exclusive);
+ Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic4", "my-subscriber-name",
+ conf);
+ ProducerConfiguration producerConf = new ProducerConfiguration();
+ if (batchMessageDelayMs != 0) {
+ producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
+ producerConf.setBatchingMaxMessages(5);
+ producerConf.setBatchingEnabled(true);
+ }
+ Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic4", producerConf);
+
+ // Produce messages
+ CompletableFuture<MessageId> lastFuture = null;
+ for (int i = 0; i < 10; i++) {
+ lastFuture = producer.sendAsync(("my-message-" + i).getBytes()).thenApply(msgId -> {
+ log.info("Published message id: {}", msgId);
+ return msgId;
+ });
+ }
+
+ lastFuture.get();
+
+ Message msg = null;
+ for (int i = 0; i < 10; i++) {
+ msg = consumer.receive(5, TimeUnit.SECONDS);
+ log.info("Received: [{}]", new String(msg.getData()));
+ }
+
+ // Restart the broker and wait for the backoff to kick in. The client library will try to reconnect, and once
+ // the broker is up, the consumer should receive the duplicate messages.
+ log.info("-- Restarting broker --");
+ restartBroker();
+
+ msg = null;
+ log.info("Receiving duplicate messages..");
+ for (int i = 0; i < 10; i++) {
+ msg = consumer.receive(5, TimeUnit.SECONDS);
+ log.info("Received: [{}]", new String(msg.getData()));
+ Assert.assertNotNull(msg, "Message cannot be null");
+ }
+ consumer.acknowledgeCumulative(msg);
+ consumer.close();
+ log.info("-- Exiting {} test --", methodName);
+ }
+
+ @Test(dataProvider = "batch")
+ public void testSendTimeout(int batchMessageDelayMs) throws Exception {
+ log.info("-- Starting {} test --", methodName);
+
+ ConsumerConfiguration consumerConf = new ConsumerConfiguration();
+ consumerConf.setSubscriptionType(SubscriptionType.Exclusive);
+ Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic5", "my-subscriber-name",
+ consumerConf);
+ ProducerConfiguration producerConf = new ProducerConfiguration();
+ if (batchMessageDelayMs != 0) {
+ producerConf.setBatchingMaxPublishDelay(2 * batchMessageDelayMs, TimeUnit.MILLISECONDS);
+ producerConf.setBatchingMaxMessages(5);
+ producerConf.setBatchingEnabled(true);
+ }
+ producerConf.setSendTimeout(1, TimeUnit.SECONDS);
+
+ Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic5", producerConf);
+ final String message = "my-message";
+
+ // Trigger the send timeout
+ stopBroker();
+
+ Future<MessageId> future = producer.sendAsync(message.getBytes());
+
+ try {
+ future.get();
+ Assert.fail("Send operation should have failed");
+ } catch (ExecutionException e) {
+ // Expected
+ }
+
+ startBroker();
+
+ // We should not have received any message
+ Message msg = consumer.receive(3, TimeUnit.SECONDS);
+ Assert.assertNull(msg);
+ consumer.close();
+ log.info("-- Exiting {} test --", methodName);
+ }
+
+ @Test
+ public void testInvalidSequence() throws Exception {
+ log.info("-- Starting {} test --", methodName);
+
+ PulsarClient client1 = PulsarClient.create("http://127.0.0.1:" + BROKER_WEBSERVICE_PORT);
+ client1.close();
+
+ ConsumerConfiguration consumerConf = new ConsumerConfiguration();
+ consumerConf.setSubscriptionType(SubscriptionType.Exclusive);
+
+ try {
+ client1.subscribe("persistent://my-property/use/my-ns/my-topic6", "my-subscriber-name", consumerConf);
+ Assert.fail("Should fail");
+ } catch (PulsarClientException e) {
+ Assert.assertTrue(e instanceof PulsarClientException.AlreadyClosedException);
+ }
+
+ try {
+ client1.createProducer("persistent://my-property/use/my-ns/my-topic6");
+ Assert.fail("Should fail");
+ } catch (PulsarClientException e) {
+ Assert.assertTrue(e instanceof PulsarClientException.AlreadyClosedException);
+ }
+
+ Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic6", "my-subscriber-name",
+ consumerConf);
+
+ try {
+ Message msg = MessageBuilder.create().setContent("InvalidMessage".getBytes()).build();
+ consumer.acknowledge(msg);
+ } catch (PulsarClientException.InvalidMessageException e) {
+ // ok
+ }
+
+ consumer.close();
+
+ try {
+ consumer.receive();
+ Assert.fail("Should fail");
+ } catch (PulsarClientException.AlreadyClosedException e) {
+ // ok
+ }
+
+ try {
+ consumer.unsubscribe();
+ Assert.fail("Should fail");
+ } catch (PulsarClientException.AlreadyClosedException e) {
+ // ok
+ }
+
+ Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic6");
+ producer.close();
+
+ try {
+ producer.send("message".getBytes());
+ Assert.fail("Should fail");
+ } catch (PulsarClientException.AlreadyClosedException e) {
+ // ok
+ }
+
+ }
+
+ @Test
+ public void testSillyUser() {
+ try {
+ PulsarClient.create("invalid://url");
+ Assert.fail("should fail");
+ } catch (PulsarClientException e) {
+ Assert.assertTrue(e instanceof PulsarClientException.InvalidServiceURL);
+ }
+
+ ProducerConfiguration producerConf = new ProducerConfiguration();
+
+ try {
+ producerConf.setSendTimeout(-1, TimeUnit.SECONDS);
+ Assert.fail("should fail");
+ } catch (IllegalArgumentException e) {
+ // ok
+ }
+
+ try {
+ producerConf.setMaxPendingMessages(0);
+ Assert.fail("should fail");
+ } catch (IllegalArgumentException e) {
+ // ok
+ }
+
+ try {
+ pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic7", (ProducerConfiguration) null);
+ Assert.fail("should fail");
+ } catch (PulsarClientException e) {
+ Assert.assertTrue(e instanceof PulsarClientException.InvalidConfigurationException);
+ }
+
+ try {
+ pulsarClient.createProducer("invalid://topic", producerConf);
+ Assert.fail("should fail");
+ } catch (PulsarClientException e) {
+ Assert.assertTrue(e instanceof PulsarClientException.InvalidTopicNameException);
+ }
+
+ ConsumerConfiguration consumerConf = new ConsumerConfiguration();
+
+ try {
+ consumerConf.setMessageListener(null);
+ Assert.fail("should fail");
+ } catch (NullPointerException e) {
+ // ok
+ }
+
+ try {
+ consumerConf.setSubscriptionType(null);
+ Assert.fail("should fail");
+ } catch (NullPointerException e) {
+ // ok
+ }
+
+ try {
+ consumerConf.setReceiverQueueSize(-1);
+ Assert.fail("should fail");
+ } catch (IllegalArgumentException e) {
+ // ok
+ }
+
+ try {
+ pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic7", "my-subscriber-name",
+ (ConsumerConfiguration) null);
+ Assert.fail("Should fail");
+ } catch (PulsarClientException e) {
+ Assert.assertTrue(e instanceof PulsarClientException.InvalidConfigurationException);
+ }
+
+ try {
+ pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic7", null, consumerConf);
+ Assert.fail("Should fail");
+ } catch (PulsarClientException e) {
+ Assert.assertTrue(e instanceof PulsarClientException.InvalidConfigurationException);
+ }
+
+ try {
+ pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic7", "", consumerConf);
+ Assert.fail("Should fail");
+ } catch (PulsarClientException e) {
+ Assert.assertTrue(e instanceof PulsarClientException.InvalidConfigurationException);
+ }
+
+ try {
+ pulsarClient.subscribe("invalid://topic7", "my-subscriber-name", consumerConf);
+ Assert.fail("Should fail");
+ } catch (PulsarClientException e) {
+ Assert.assertTrue(e instanceof PulsarClientException.InvalidTopicNameException);
+ }
+
+ }
+
+ // This is to test that the flow control counter doesn't get corrupted while concurrent receives during
+ // reconnections
+ @Test(dataProvider = "batch")
+ public void testConcurrentConsumerReceiveWhileReconnect(int batchMessageDelayMs) throws Exception {
+ final int recvQueueSize = 100;
+ final int numConsumersThreads = 10;
+
+ final ConsumerConfiguration conf = new ConsumerConfiguration();
+ conf.setReceiverQueueSize(recvQueueSize);
+ String subName = UUID.randomUUID().toString();
+ final Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic7", subName, conf);
+ ExecutorService executor = Executors.newCachedThreadPool();
+
+ final CyclicBarrier barrier = new CyclicBarrier(numConsumersThreads + 1);
+ for (int i = 0; i < numConsumersThreads; i++) {
+ executor.submit(new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ barrier.await();
+ consumer.receive();
+ return null;
+ }
+ });
+ }
+
+ barrier.await();
+ // there will be 10 threads calling receive() from the same consumer and will block
+ Thread.sleep(100);
+
+ // we restart the broker to reconnect
+ restartBroker();
+ Thread.sleep(2000);
+
+ // publish 100 messages so that the consumers blocked on receive() will now get the messages
+ ProducerConfiguration producerConf = new ProducerConfiguration();
+ if (batchMessageDelayMs != 0) {
+ producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
+ producerConf.setBatchingMaxMessages(5);
+ producerConf.setBatchingEnabled(true);
+ }
+ Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic7", producerConf);
+ for (int i = 0; i < recvQueueSize; i++) {
+ String message = "my-message-" + i;
+ producer.send(message.getBytes());
+ }
+ Thread.sleep(500);
+
+ ConsumerImpl consumerImpl = (ConsumerImpl) consumer;
+ // The available permits should be 10 and num messages in the queue should be 90
+ Assert.assertEquals(consumerImpl.getAvailablePermits(), numConsumersThreads);
+ Assert.assertEquals(consumerImpl.numMessagesInQueue(), recvQueueSize - numConsumersThreads);
+
+ barrier.reset();
+ for (int i = 0; i < numConsumersThreads; i++) {
+ executor.submit(new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ barrier.await();
+ consumer.receive();
+ return null;
+ }
+ });
+ }
+ barrier.await();
+ Thread.sleep(100);
+
+ // The available permits should be 20 and num messages in the queue should be 80
+ Assert.assertEquals(consumerImpl.getAvailablePermits(), numConsumersThreads * 2);
+ Assert.assertEquals(consumerImpl.numMessagesInQueue(), recvQueueSize - (numConsumersThreads * 2));
+
+ // clear the queue
+ while (true) {
+ Message msg = consumer.receive(1, TimeUnit.SECONDS);
+ if (msg == null) {
+ break;
+ }
+ }
+
+ // The available permits should be 0 and num messages in the queue should be 0
+ Assert.assertEquals(consumerImpl.getAvailablePermits(), 0);
+ Assert.assertEquals(consumerImpl.numMessagesInQueue(), 0);
+
+ barrier.reset();
+ for (int i = 0; i < numConsumersThreads; i++) {
+ executor.submit(new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ barrier.await();
+ consumer.receive();
+ return null;
+ }
+ });
+ }
+ barrier.await();
+ // we again make 10 threads call receive() and get blocked
+ Thread.sleep(100);
+
+ restartBroker();
+ Thread.sleep(2000);
+
+ // The available permits should be 10 and num messages in the queue should be 90
+ Assert.assertEquals(consumerImpl.getAvailablePermits(), numConsumersThreads);
+ Assert.assertEquals(consumerImpl.numMessagesInQueue(), recvQueueSize - numConsumersThreads);
+ consumer.close();
+ }
+
+ @Test
+ public void testSendBigMessageSize() throws Exception {
+ log.info("-- Starting {} test --", methodName);
+
+ // Messages are allowed up to MaxMessageSize
+ MessageBuilder.create().setContent(new byte[PulsarDecoder.MaxMessageSize]).build();
+
+ try {
+ final String topic = "persistent://my-property/use/my-ns/bigMsg";
+ Producer producer = pulsarClient.createProducer(topic);
+ Message message = MessageBuilder.create().setContent(new byte[PulsarDecoder.MaxMessageSize + 1]).build();
+ producer.send(message);
+ fail("Should have thrown exception");
+ } catch (PulsarClientException.InvalidMessageException e) {
+ // OK
+ }
+ }
+
+ /**
+ * Verifies non-batch message size being validated after performing compression while batch-messaging validates
+ * before compression of message
+ *
+ * <pre>
+ * send msg with size > MAX_SIZE (5 MB)
+ * a. non-batch with compression: pass
+ * b. batch-msg with compression: fail
+ * c. non-batch w/o compression: fail
+ * d. non-batch with compression, consumer consume: pass
+ * </pre>
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testSendBigMessageSizeButCompressed() throws Exception {
+ log.info("-- Starting {} test --", methodName);
+
+ final String topic = "persistent://my-property/use/my-ns/bigMsg";
+
+ // (a) non-batch msg with compression
+ ProducerConfiguration producerConf = new ProducerConfiguration();
+ producerConf.setCompressionType(CompressionType.LZ4);
+ Producer producer = pulsarClient.createProducer(topic, producerConf);
+ Message message = MessageBuilder.create().setContent(new byte[PulsarDecoder.MaxMessageSize + 1]).build();
+ producer.send(message);
+ producer.close();
+
+ // (b) batch-msg
+ producerConf = new ProducerConfiguration();
+ producerConf.setBatchingEnabled(true);
+ producerConf.setCompressionType(CompressionType.LZ4);
+ producer = pulsarClient.createProducer(topic, producerConf);
+ message = MessageBuilder.create().setContent(new byte[PulsarDecoder.MaxMessageSize + 1]).build();
+ try {
+ producer.send(message);
+ fail("Should have thrown exception");
+ } catch (PulsarClientException.InvalidMessageException e) {
+ // OK
+ }
+ producer.close();
+
+ // (c) non-batch msg without compression
+ producerConf = new ProducerConfiguration();
+ producerConf.setCompressionType(CompressionType.NONE);
+ producer = pulsarClient.createProducer(topic, producerConf);
+ message = MessageBuilder.create().setContent(new byte[PulsarDecoder.MaxMessageSize + 1]).build();
+ try {
+ producer.send(message);
+ fail("Should have thrown exception");
+ } catch (PulsarClientException.InvalidMessageException e) {
+ // OK
+ }
+ producer.close();
+
+ // (d) non-batch msg with compression and try to consume message
+ producerConf = new ProducerConfiguration();
+ producerConf.setCompressionType(CompressionType.LZ4);
+ producer = pulsarClient.createProducer(topic, producerConf);
+ Consumer consumer = pulsarClient.subscribe(topic, "sub1");
+ byte[] content = new byte[PulsarDecoder.MaxMessageSize + 10];
+ message = MessageBuilder.create().setContent(content).build();
+ producer.send(message);
+ assertEquals(consumer.receive().getData(), content);
+ producer.close();
+ consumer.close();
+
+ }
+
+ /**
+ * Usecase 1: Only 1 Active Subscription - 1 subscriber - Produce Messages - EntryCache should cache messages -
+ * EntryCache should be cleaned : Once active subscription consumes messages
+ *
+ * Usecase 2: 2 Active Subscriptions (faster and slower) and slower gets closed - 2 subscribers - Produce Messages -
+ * 1 faster-subscriber consumes all messages and another slower-subscriber none - EntryCache should have cached
+ * messages as slower-subscriber has not consumed messages yet - close slower-subscriber - EntryCache should be
+ * cleared
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testActiveAndInActiveConsumerEntryCacheBehavior() throws Exception {
+ log.info("-- Starting {} test --", methodName);
+
+ final long batchMessageDelayMs = 100;
+ final int receiverSize = 10;
+ final String topicName = "cache-topic";
+ final String sub1 = "faster-sub1";
+ final String sub2 = "slower-sub2";
+ ConsumerConfiguration conf = new ConsumerConfiguration();
+ conf.setSubscriptionType(SubscriptionType.Shared);
+ conf.setReceiverQueueSize(receiverSize);
+
+ ProducerConfiguration producerConf = new ProducerConfiguration();
+
+ if (batchMessageDelayMs != 0) {
+ producerConf.setBatchingEnabled(true);
+ producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
+ producerConf.setBatchingMaxMessages(5);
+ }
+
+ /************ usecase-1: *************/
+ // 1. Subscriber Faster subscriber
+ Consumer subscriber1 = pulsarClient.subscribe("persistent://my-property/use/my-ns/" + topicName, sub1, conf);
+ final String topic = "persistent://my-property/use/my-ns/" + topicName;
+ Producer producer = pulsarClient.createProducer(topic, producerConf);
+
+ PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic);
+ ManagedLedgerImpl ledger = (ManagedLedgerImpl) topicRef.getManagedLedger();
+ Field cacheField = ManagedLedgerImpl.class.getDeclaredField("entryCache");
+ cacheField.setAccessible(true);
+ Field modifiersField = Field.class.getDeclaredField("modifiers");
+ modifiersField.setAccessible(true);
+ modifiersField.setInt(cacheField, cacheField.getModifiers() & ~Modifier.FINAL);
+ EntryCacheImpl entryCache = spy((EntryCacheImpl) cacheField.get(ledger));
+ cacheField.set(ledger, entryCache);
+
+ Message msg = null;
+ // 2. Produce messages
+ for (int i = 0; i < 30; i++) {
+ String message = "my-message-" + i;
+ producer.send(message.getBytes());
+ }
+ // 3. Consume messages
+ for (int i = 0; i < 30; i++) {
+ msg = subscriber1.receive(5, TimeUnit.SECONDS);
+ subscriber1.acknowledge(msg);
+ }
+
+ // Verify: EntryCache has been invalidated
+ verify(entryCache, atLeastOnce()).invalidateEntries(any());
+
+ // sleep for a second: as ledger.updateCursorRateLimit RateLimiter will allow to invoke cursor-update after a
+ // second
+ Thread.sleep(1000);//
+ // produce-consume one more message to trigger : ledger.internalReadFromLedger(..) which updates cursor and
+ // EntryCache
+ producer.send("message".getBytes());
+ msg = subscriber1.receive(5, TimeUnit.SECONDS);
+
+ // Verify: cache has to be cleared as there is no message needs to be consumed by active subscriber
+ assertEquals(entryCache.getSize(), 0, 1);
+
+ /************ usecase-2: *************/
+ // 1.b Subscriber slower-subscriber
+ Consumer subscriber2 = pulsarClient.subscribe("persistent://my-property/use/my-ns/" + topicName, sub2, conf);
+ // Produce messages
+ final int moreMessages = 10;
+ for (int i = 0; i < receiverSize + moreMessages; i++) {
+ String message = "my-message-" + i;
+ producer.send(message.getBytes());
+ }
+ // Consume messages
+ for (int i = 0; i < receiverSize + moreMessages; i++) {
+ msg = subscriber1.receive(5, TimeUnit.SECONDS);
+ subscriber1.acknowledge(msg);
+ }
+
+ // sleep for a second: as ledger.updateCursorRateLimit RateLimiter will allow to invoke cursor-update after a
+ // second
+ Thread.sleep(1000);//
+ // produce-consume one more message to trigger : ledger.internalReadFromLedger(..) which updates cursor and
+ // EntryCache
+ producer.send("message".getBytes());
+ msg = subscriber1.receive(5, TimeUnit.SECONDS);
+
+ // Verify: as active-subscriber2 has not consumed messages: EntryCache must have those entries in cache
+ assertTrue(entryCache.getSize() != 0);
+
+ // 3.b Close subscriber2: which will trigger cache to clear the cache
+ subscriber2.close();
+
+ // retry strategically until broker clean up closed subscribers and invalidate all cache entries
+ retryStrategically((test) -> entryCache.getSize() == 0, 5, 100);
+
+ // Verify: EntryCache should be cleared
+ assertTrue(entryCache.getSize() == 0);
+ subscriber1.close();
+ log.info("-- Exiting {} test --", methodName);
+ }
+
+ @Test
+ public void testDeactivatingBacklogConsumer() throws Exception {
+ log.info("-- Starting {} test --", methodName);
+
+ final long batchMessageDelayMs = 100;
+ final int receiverSize = 10;
+ final String topicName = "cache-topic";
+ final String topic = "persistent://my-property/use/my-ns/" + topicName;
+ final String sub1 = "faster-sub1";
+ final String sub2 = "slower-sub2";
+ ConsumerConfiguration conf = new ConsumerConfiguration();
+ conf.setSubscriptionType(SubscriptionType.Shared);
+ conf.setReceiverQueueSize(receiverSize);
+
+ ProducerConfiguration producerConf = new ProducerConfiguration();
+
+ if (batchMessageDelayMs != 0) {
+ producerConf.setBatchingEnabled(true);
+ producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
+ producerConf.setBatchingMaxMessages(5);
+ }
+
+ // 1. Subscriber Faster subscriber: let it consume all messages immediately
+ Consumer subscriber1 = pulsarClient.subscribe("persistent://my-property/use/my-ns/" + topicName, sub1, conf);
+ // 1.b. Subscriber Slow subscriber:
+ conf.setReceiverQueueSize(receiverSize);
+ Consumer subscriber2 = pulsarClient.subscribe("persistent://my-property/use/my-ns/" + topicName, sub2, conf);
+ Producer producer = pulsarClient.createProducer(topic, producerConf);
+
+ PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic);
+ ManagedLedgerImpl ledger = (ManagedLedgerImpl) topicRef.getManagedLedger();
+
+ // reflection to set/get cache-backlog fields value:
+ final long maxMessageCacheRetentionTimeMillis = 100;
+ Field backlogThresholdField = ManagedLedgerImpl.class.getDeclaredField("maxActiveCursorBacklogEntries");
+ backlogThresholdField.setAccessible(true);
+ Field field = ManagedLedgerImpl.class.getDeclaredField("maxMessageCacheRetentionTimeMillis");
+ field.setAccessible(true);
+ Field modifiersField = Field.class.getDeclaredField("modifiers");
+ modifiersField.setAccessible(true);
+ modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
+ field.set(ledger, maxMessageCacheRetentionTimeMillis);
+ final long maxActiveCursorBacklogEntries = (long) backlogThresholdField.get(ledger);
+
+ Message msg = null;
+ final int totalMsgs = (int) maxActiveCursorBacklogEntries + receiverSize + 1;
+ // 2. Produce messages
+ for (int i = 0; i < totalMsgs; i++) {
+ String message = "my-message-" + i;
+ producer.send(message.getBytes());
+ }
+ // 3. Consume messages: at Faster subscriber
+ for (int i = 0; i < totalMsgs; i++) {
+ msg = subscriber1.receive(100, TimeUnit.MILLISECONDS);
+ subscriber1.acknowledge(msg);
+ }
+
+ // wait : so message can be eligible to to be evict from cache
+ Thread.sleep(maxMessageCacheRetentionTimeMillis);
+
+ // 4. deactivate subscriber which has built the backlog
+ ledger.checkBackloggedCursors();
+ Thread.sleep(100);
+
+ // 5. verify: active subscribers
+ Set<String> activeSubscriber = Sets.newHashSet();
+ ledger.getActiveCursors().forEach(c -> activeSubscriber.add(c.getName()));
+ assertTrue(activeSubscriber.contains(sub1));
+ assertFalse(activeSubscriber.contains(sub2));
+
+ // 6. consume messages : at slower subscriber
+ for (int i = 0; i < totalMsgs; i++) {
+ msg = subscriber2.receive(100, TimeUnit.MILLISECONDS);
+ subscriber2.acknowledge(msg);
+ }
+
+ ledger.checkBackloggedCursors();
+
+ activeSubscriber.clear();
+ ledger.getActiveCursors().forEach(c -> activeSubscriber.add(c.getName()));
+
+ assertTrue(activeSubscriber.contains(sub1));
+ assertTrue(activeSubscriber.contains(sub2));
+ }
+
+ @Test(timeOut = 2000)
+ public void testAsyncProducerAndConsumer() throws Exception {
+ log.info("-- Starting {} test --", methodName);
+
+ final int totalMsg = 100;
+ final Set<String> produceMsgs = Sets.newHashSet();
+ final Set<String> consumeMsgs = Sets.newHashSet();
+ final ProducerConfiguration producerConf = new ProducerConfiguration();
+ final ConsumerConfiguration conf = new ConsumerConfiguration();
+
+ conf.setSubscriptionType(SubscriptionType.Exclusive);
+ Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic1", "my-subscriber-name",
+ conf);
+
+ // produce message
+ Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1", producerConf);
+ for (int i = 0; i < totalMsg; i++) {
+ String message = "my-message-" + i;
+ producer.send(message.getBytes());
+ produceMsgs.add(message);
+ }
+
+ log.info(" start receiving messages :");
+ CountDownLatch latch = new CountDownLatch(totalMsg);
+ // receive messages
+ ExecutorService executor = Executors.newFixedThreadPool(1);
+ receiveAsync(consumer, totalMsg, 0, latch, consumeMsgs, executor);
+
+ latch.await();
+
+ // verify message produced correctly
+ assertEquals(produceMsgs.size(), totalMsg);
+ // verify produced and consumed messages must be exactly same
+ produceMsgs.removeAll(consumeMsgs);
+ assertTrue(produceMsgs.isEmpty());
+
+ producer.close();
+ consumer.close();
+ log.info("-- Exiting {} test --", methodName);
+ }
+
+ @Test(timeOut = 2000)
+ public void testAsyncProducerAndConsumerWithZeroQueueSize() throws Exception {
+ log.info("-- Starting {} test --", methodName);
+
+ final int totalMsg = 100;
+ final Set<String> produceMsgs = Sets.newHashSet();
+ final Set<String> consumeMsgs = Sets.newHashSet();
+ final ProducerConfiguration producerConf = new ProducerConfiguration();
+ final ConsumerConfiguration conf = new ConsumerConfiguration();
+
+ conf.setSubscriptionType(SubscriptionType.Exclusive);
+ Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic1", "my-subscriber-name",
+ conf);
+
+ // produce message
+ Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1", producerConf);
+ for (int i = 0; i < totalMsg; i++) {
+ String message = "my-message-" + i;
+ producer.send(message.getBytes());
+ produceMsgs.add(message);
+ }
+
+ log.info(" start receiving messages :");
+ CountDownLatch latch = new CountDownLatch(totalMsg);
+ // receive messages
+ ExecutorService executor = Executors.newFixedThreadPool(1);
+ receiveAsync(consumer, totalMsg, 0, latch, consumeMsgs, executor);
+
+ latch.await();
+
+ // verify message produced correctly
+ assertEquals(produceMsgs.size(), totalMsg);
+ // verify produced and consumed messages must be exactly same
+ produceMsgs.removeAll(consumeMsgs);
+ assertTrue(produceMsgs.isEmpty());
+
+ producer.close();
+ consumer.close();
+ log.info("-- Exiting {} test --", methodName);
+ }
+
+ @Test
+ public void testSendCallBack() throws Exception {
+ log.info("-- Starting {} test --", methodName);
+
+ final int totalMsg = 100;
+ final ProducerConfiguration producerConf = new ProducerConfiguration();
+ Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1", producerConf);
+ for (int i = 0; i < totalMsg; i++) {
+ final String message = "my-message-" + i;
+ Message msg = MessageBuilder.create().setContent(message.getBytes()).build();
+ final AtomicInteger msgLength = new AtomicInteger();
+ CompletableFuture<MessageId> future = producer.sendAsync(msg).handle((r, ex) -> {
+ if (ex != null) {
+ log.error("Message send failed:", ex);
+ } else {
+ msgLength.set(msg.getData().length);
+ }
+ return null;
+ });
+ future.get();
+ assertEquals(message.getBytes().length, msgLength.get());
+ }
+ }
+
+ /**
+ * consume message from consumer1 and send acknowledgement from different consumer subscribed under same
+ * subscription-name
+ *
+ * @throws Exception
+ */
+ @Test(timeOut = 30000)
+ public void testSharedConsumerAckDifferentConsumer() throws Exception {
+ log.info("-- Starting {} test --", methodName);
+
+ ConsumerConfiguration conf = new ConsumerConfiguration();
+ conf.setReceiverQueueSize(1);
+ conf.setSubscriptionType(SubscriptionType.Shared);
+ Consumer consumer1 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic1",
+ "my-subscriber-name", conf);
+ Consumer consumer2 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic1",
+ "my-subscriber-name", conf);
+
+ ProducerConfiguration producerConf = new ProducerConfiguration();
+
+ Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1", producerConf);
+ for (int i = 0; i < 10; i++) {
+ String message = "my-message-" + i;
+ producer.send(message.getBytes());
+ }
+
+ Message msg = null;
+ Set<Message> consumerMsgSet1 = Sets.newHashSet();
+ Set<Message> consumerMsgSet2 = Sets.newHashSet();
+ for (int i = 0; i < 5; i++) {
+ msg = consumer1.receive();
+ consumerMsgSet1.add(msg);
+
+ msg = consumer2.receive();
+ consumerMsgSet2.add(msg);
+ }
+
+ consumerMsgSet1.stream().forEach(m -> {
+ try {
+ consumer2.acknowledge(m);
+ } catch (PulsarClientException e) {
+ fail();
+ }
+ });
+ consumerMsgSet2.stream().forEach(m -> {
+ try {
+ consumer1.acknowledge(m);
+ } catch (PulsarClientException e) {
+ fail();
+ }
+ });
+
+ consumer1.redeliverUnacknowledgedMessages();
+ consumer2.redeliverUnacknowledgedMessages();
+
+ try {
+ if (consumer1.receive(100, TimeUnit.MILLISECONDS) != null
+ || consumer2.receive(100, TimeUnit.MILLISECONDS) != null) {
+ fail();
+ }
+ } finally {
+ consumer1.close();
+ consumer2.close();
+ }
+
+ log.info("-- Exiting {} test --", methodName);
+ }
+
+ private void receiveAsync(Consumer consumer, int totalMessage, int currentMessage, CountDownLatch latch,
+ final Set<String> consumeMsg, ExecutorService executor) throws PulsarClientException {
+ if (currentMessage < totalMessage) {
+ CompletableFuture<Message> future = consumer.receiveAsync();
+ future.handle((msg, exception) -> {
+ if (exception == null) {
+ // add message to consumer-queue to verify with produced messages
+ consumeMsg.add(new String(msg.getData()));
+ try {
+ consumer.acknowledge(msg);
+ } catch (PulsarClientException e1) {
+ fail("message acknowledge failed", e1);
+ }
+ // consume next message
+ executor.execute(() -> {
+ try {
+ receiveAsync(consumer, totalMessage, currentMessage + 1, latch, consumeMsg, executor);
+ } catch (PulsarClientException e) {
+ fail("message receive failed", e);
+ }
+ });
+ latch.countDown();
+ }
+ return null;
+ });
+ }
+ }
+
+ /**
+ * Verify: Consumer stops receiving msg when reach unack-msg limit and starts receiving once acks messages 1.
+ * Produce X (600) messages 2. Consumer has receive size (10) and receive message without acknowledging 3. Consumer
+ * will stop receiving message after unAckThreshold = 500 4. Consumer acks messages and starts consuming remanining
+ * messages This testcase enables checksum sending while producing message and broker verifies the checksum for the
+ * message.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testConsumerBlockingWithUnAckedMessages() throws Exception {
+ log.info("-- Starting {} test --", methodName);
+
+ int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
+ try {
+ final int unAckedMessagesBufferSize = 500;
+ final int receiverQueueSize = 10;
+ final int totalProducedMsgs = 600;
+
+ pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessagesBufferSize);
+ ConsumerConfiguration conf = new ConsumerConfiguration();
+ conf.setReceiverQueueSize(receiverQueueSize);
+ conf.setSubscriptionType(SubscriptionType.Shared);
+ Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic",
+ "subscriber-1", conf);
+
+ ProducerConfiguration producerConf = new ProducerConfiguration();
+
+ Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic",
+ producerConf);
+
+ // (1) Produced Messages
+ for (int i = 0; i < totalProducedMsgs; i++) {
+ String message = "my-message-" + i;
+ producer.send(message.getBytes());
+ }
+
+ // (2) try to consume messages: but will be able to consume number of messages = unAckedMessagesBufferSize
+ Message msg = null;
+ List<Message> messages = Lists.newArrayList();
+ for (int i = 0; i < totalProducedMsgs; i++) {
+ msg = consumer.receive(1, TimeUnit.SECONDS);
+ if (msg != null) {
+ messages.add(msg);
+ log.info("Received message: " + new String(msg.getData()));
+ } else {
+ break;
+ }
+ }
+ // client must receive number of messages = unAckedMessagesBufferSize rather all produced messages
+ assertEquals(messages.size(), unAckedMessagesBufferSize);
+
+ // start acknowledging messages
+ messages.forEach(m -> {
+ try {
+ consumer.acknowledge(m);
+ } catch (PulsarClientException e) {
+ fail("ack failed", e);
+ }
+ });
+
+ // try to consume remaining messages
+ int remainingMessages = totalProducedMsgs - messages.size();
+ for (int i = 0; i < remainingMessages; i++) {
+ msg = consumer.receive(1, TimeUnit.SECONDS);
+ if (msg != null) {
+ messages.add(msg);
+ log.info("Received message: " + new String(msg.getData()));
+ }
+ }
+
+ // total received-messages should match to produced messages
+ assertEquals(totalProducedMsgs, messages.size());
+ producer.close();
+ consumer.close();
+ log.info("-- Exiting {} test --", methodName);
+ } catch (Exception e) {
+ fail();
+ } finally {
+ pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
+ }
+ }
+
+ /**
+ * Verify: iteration of a. message receive w/o acking b. stop receiving msg c. ack msgs d. started receiving msgs
+ *
+ * 1. Produce total X (1500) messages 2. Consumer consumes messages without acking until stop receiving from broker
+ * due to reaching ack-threshold (500) 3. Consumer acks messages after stop getting messages 4. Consumer again tries
+ * to consume messages 5. Consumer should be able to complete consuming all 1500 messages in 3 iteration (1500/500)
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testConsumerBlockingWithUnAckedMessagesMultipleIteration() throws Exception {
+ log.info("-- Starting {} test --", methodName);
+
+ int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
+ try {
+ final int unAckedMessagesBufferSize = 500;
+ final int receiverQueueSize = 10;
+ final int totalProducedMsgs = 1500;
+
+ // receiver consumes messages in iteration after acknowledging broker
+ final int totalReceiveIteration = totalProducedMsgs / unAckedMessagesBufferSize;
+ pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessagesBufferSize);
+ ConsumerConfiguration conf = new ConsumerConfiguration();
+ conf.setReceiverQueueSize(receiverQueueSize);
+ conf.setSubscriptionType(SubscriptionType.Shared);
+ Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic",
+ "subscriber-1", conf);
+
+ ProducerConfiguration producerConf = new ProducerConfiguration();
+
+ Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic",
+ producerConf);
+
+ // (1) Produced Messages
+ for (int i = 0; i < totalProducedMsgs; i++) {
+ String message = "my-message-" + i;
+ producer.send(message.getBytes());
+ }
+
+ int totalReceivedMessages = 0;
+ // (2) Receive Messages
+ for (int j = 0; j < totalReceiveIteration; j++) {
+
+ Message msg = null;
+ List<Message> messages = Lists.newArrayList();
+ for (int i = 0; i < totalProducedMsgs; i++) {
+ msg = consumer.receive(1, TimeUnit.SECONDS);
+ if (msg != null) {
+ messages.add(msg);
+ log.info("Received message: " + new String(msg.getData()));
+ } else {
+ break;
+ }
+ }
+ // client must receive number of messages = unAckedMessagesBufferSize rather all produced messages
+ assertEquals(messages.size(), unAckedMessagesBufferSize);
+
+ // start acknowledging messages
+ messages.forEach(m -> {
+ try {
+ consumer.acknowledge(m);
+ } catch (PulsarClientException e) {
+ fail("ack failed", e);
+ }
+ });
+ totalReceivedMessages += messages.size();
+ }
+
+ // total received-messages should match to produced messages
+ assertEquals(totalReceivedMessages, totalProducedMsgs);
+ producer.close();
+ consumer.close();
+ log.info("-- Exiting {} test --", methodName);
+ } catch (Exception e) {
+ fail();
+ } finally {
+ pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
+ }
+ }
+
+ /**
+ * Verify: Consumer1 which doesn't send ack will not impact Consumer2 which sends ack for consumed message.
+ *
+ *
+ * @param batchMessageDelayMs
+ * @throws Exception
+ */
+ @Test
+ public void testMutlipleSharedConsumerBlockingWithUnAckedMessages() throws Exception {
+ log.info("-- Starting {} test --", methodName);
+
+ int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
+ try {
+ final int maxUnackedMessages = 20;
+ final int receiverQueueSize = 10;
+ final int totalProducedMsgs = 100;
+ int totalReceiveMessages = 0;
+
+ pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(maxUnackedMessages);
+ ConsumerConfiguration conf = new ConsumerConfiguration();
+ conf.setReceiverQueueSize(receiverQueueSize);
+ conf.setSubscriptionType(SubscriptionType.Shared);
+ Consumer consumer1 = pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic",
+ "subscriber-1", conf);
+ Consumer consumer2 = pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic",
+ "subscriber-1", conf);
+
+ ProducerConfiguration producerConf = new ProducerConfiguration();
+
+ Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic",
+ producerConf);
+
+ // (1) Produced Messages
+ for (int i = 0; i < totalProducedMsgs; i++) {
+ String message = "my-message-" + i;
+ producer.send(message.getBytes());
+ }
+
+ // (2) Consumer1: consume without ack:
+ // try to consume messages: but will be able to consume number of messages = maxUnackedMessages
+ Message msg = null;
+ List<Message> messages = Lists.newArrayList();
+ for (int i = 0; i < totalProducedMsgs; i++) {
+ msg = consumer1.receive(1, TimeUnit.SECONDS);
+ if (msg != null) {
+ messages.add(msg);
+ totalReceiveMessages++;
+ log.info("Received message: " + new String(msg.getData()));
+ } else {
+ break;
+ }
+ }
+ // client must receive number of messages = unAckedMessagesBufferSize rather all produced messages
+ assertEquals(messages.size(), maxUnackedMessages);
+
+ // (3.1) Consumer2 will start consuming messages without ack: it should stop after maxUnackedMessages
+ messages.clear();
+ for (int i = 0; i < totalProducedMsgs - maxUnackedMessages; i++) {
+ msg = consumer2.receive(1, TimeUnit.SECONDS);
+ if (msg != null) {
+ messages.add(msg);
+ totalReceiveMessages++;
+ log.info("Received message: " + new String(msg.getData()));
+ } else {
+ break;
+ }
+ }
+ assertEquals(messages.size(), maxUnackedMessages);
+ // (3.2) ack for all maxUnackedMessages
+ messages.forEach(m -> {
+ try {
+ consumer2.acknowledge(m);
+ } catch (PulsarClientException e) {
+ fail("shouldn't have failed ", e);
+ }
+ });
+
+ // (4) Consumer2 consumer and ack: so it should consume all remaining messages
+ messages.clear();
+ for (int i = 0; i < totalProducedMsgs - (2 * maxUnackedMessages); i++) {
+ msg = consumer2.receive(1, TimeUnit.SECONDS);
+ if (msg != null) {
+ messages.add(msg);
+ totalReceiveMessages++;
+ consumer2.acknowledge(msg);
+ log.info("Received message: " + new String(msg.getData()));
+ } else {
+ break;
+ }
+ }
+
+ // verify total-consumer messages = total-produce messages
+ assertEquals(totalProducedMsgs, totalReceiveMessages);
+ producer.close();
+ consumer1.close();
+ consumer2.close();
+ log.info("-- Exiting {} test --", methodName);
+ } catch (Exception e) {
+ fail();
+ } finally {
+ pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
+ }
+ }
+
+ @Test
+ public void testShouldNotBlockConsumerIfRedeliverBeforeReceive() throws Exception {
+ log.info("-- Starting {} test --", methodName);
+
+ int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
+ int totalReceiveMsg = 0;
+ try {
+ final int receiverQueueSize = 20;
+ final int totalProducedMsgs = 100;
+
+ ConsumerConfiguration conf = new ConsumerConfiguration();
+ conf.setReceiverQueueSize(receiverQueueSize);
+ conf.setAckTimeout(1, TimeUnit.SECONDS);
+ conf.setSubscriptionType(SubscriptionType.Shared);
+ ConsumerImpl consumer = (ConsumerImpl) pulsarClient
+ .subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
+
+ ProducerConfiguration producerConf = new ProducerConfiguration();
+
+ Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic",
+ producerConf);
+
+ // (1) Produced Messages
+ for (int i = 0; i < totalProducedMsgs; i++) {
+ String message = "my-message-" + i;
+ producer.send(message.getBytes());
+ }
+
+ // (2) wait for consumer to receive messages
+ Thread.sleep(1000);
+ assertEquals(consumer.numMessagesInQueue(), receiverQueueSize);
+
+ // (3) wait for messages to expire, we should've received more
+ Thread.sleep(2000);
+ assertEquals(consumer.numMessagesInQueue(), receiverQueueSize);
+
+ for (int i = 0; i < totalProducedMsgs; i++) {
+ Message msg = consumer.receive(1, TimeUnit.SECONDS);
+ if (msg != null) {
+ consumer.acknowledge(msg);
+ totalReceiveMsg++;
+ log.info("Received message: " + new String(msg.getData()));
+ } else {
+ break;
+ }
+ }
+
+ // total received-messages should match to produced messages
+ assertEquals(totalProducedMsgs, totalReceiveMsg);
+ producer.close();
+ consumer.close();
+ log.info("-- Exiting {} test --", methodName);
+ } catch (Exception e) {
+ fail();
+ } finally {
+ pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
+ }
+ }
+
+ @Test
+ public void testUnackBlockRedeliverMessages() throws Exception {
+ log.info("-- Starting {} test --", methodName);
+
+ int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
+ int totalReceiveMsg = 0;
+ try {
+ final int unAckedMessagesBufferSize = 20;
+ final int receiverQueueSize = 10;
+ final int totalProducedMsgs = 100;
+
+ pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessagesBufferSize);
+ ConsumerConfiguration conf = new ConsumerConfiguration();
+ conf.setReceiverQueueSize(receiverQueueSize);
+ conf.setSubscriptionType(SubscriptionType.Shared);
+ ConsumerImpl consumer = (ConsumerImpl) pulsarClient
+ .subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
+
+ ProducerConfiguration producerConf = new ProducerConfiguration();
+
+ Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic",
+ producerConf);
+
+ // (1) Produced Messages
+ for (int i = 0; i < totalProducedMsgs; i++) {
+ String message = "my-message-" + i;
+ producer.send(message.getBytes());
+ }
+
+ // (2) try to consume messages: but will be able to consume number of messages = unAckedMessagesBufferSize
+ Message msg = null;
+ List<Message> messages = Lists.newArrayList();
+ for (int i = 0; i < totalProducedMsgs; i++) {
+ msg = consumer.receive(1, TimeUnit.SECONDS);
+ if (msg != null) {
+ messages.add(msg);
+ totalReceiveMsg++;
+ log.info("Received message: " + new String(msg.getData()));
+ } else {
+ break;
+ }
+ }
+
+ consumer.redeliverUnacknowledgedMessages();
+
+ Thread.sleep(1000);
+ int alreadyConsumedMessages = messages.size();
+ messages.clear();
+ for (int i = 0; i < totalProducedMsgs; i++) {
+ msg = consumer.receive(1, TimeUnit.SECONDS);
+ if (msg != null) {
+ consumer.acknowledge(msg);
+ totalReceiveMsg++;
+ log.info("Received message: " + new String(msg.getData()));
+ } else {
+ break;
+ }
+ }
+
+ // total received-messages should match to produced messages
+ assertEquals(totalProducedMsgs + alreadyConsumedMessages, totalReceiveMsg);
+ producer.close();
+ consumer.close();
+ log.info("-- Exiting {} test --", methodName);
+ } catch (Exception e) {
+ fail();
+ } finally {
+ pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
+ }
+ }
+
+ @Test(dataProvider = "batch")
+ public void testUnackedBlockAtBatch(int batchMessageDelayMs) throws Exception {
+ log.info("-- Starting {} test --", methodName);
+
+ int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
+ try {
+ final int maxUnackedMessages = 20;
+ final int receiverQueueSize = 10;
+ final int totalProducedMsgs = 100;
+ int totalReceiveMessages = 0;
+
+ pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(maxUnackedMessages);
+ ConsumerConfiguration conf = new ConsumerConfiguration();
+ conf.setReceiverQueueSize(receiverQueueSize);
+ conf.setSubscriptionType(SubscriptionType.Shared);
+ Consumer consumer1 = pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic",
+ "subscriber-1", conf);
+
+ ProducerConfiguration producerConf = new ProducerConfiguration();
+
+ if (batchMessageDelayMs != 0) {
+ producerConf.setBatchingEnabled(true);
+ producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
+ producerConf.setBatchingMaxMessages(5);
+ }
+
+ Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic",
+ producerConf);
+
+ List<CompletableFuture<MessageId>> futures = Lists.newArrayList();
+ // (1) Produced Messages
+ for (int i = 0; i < totalProducedMsgs; i++) {
+ String message = "my-message-" + i;
+ futures.add(producer.sendAsync(message.getBytes()));
+ }
+
+ FutureUtil.waitForAll(futures).get();
+
+ // (2) Consumer1: consume without ack:
+ // try to consume messages: but will be able to consume number of messages = maxUnackedMessages
+ Message msg = null;
+ List<Message> messages = Lists.newArrayList();
+ for (int i = 0; i < totalProducedMsgs; i++) {
+ msg = consumer1.receive(1, TimeUnit.SECONDS);
+ if (msg != null) {
+ messages.add(msg);
+ totalReceiveMessages++;
+ log.info("Received message: " + new String(msg.getData()));
+ } else {
+ break;
+ }
+ }
+ // should be blocked due to unack-msgs and should not consume all msgs
+ assertNotEquals(messages.size(), totalProducedMsgs);
+ // ack for all maxUnackedMessages
+ messages.forEach(m -> {
+ try {
+ consumer1.acknowledge(m);
+ } catch (PulsarClientException e) {
+ fail("shouldn't have failed ", e);
+ }
+ });
+
+ // (3) Consumer consumes and ack: so it should consume all remaining messages
+ messages.clear();
+ for (int i = 0; i < totalProducedMsgs; i++) {
+ msg = consumer1.receive(1, TimeUnit.SECONDS);
+ if (msg != null) {
+ messages.add(msg);
+ totalReceiveMessages++;
+ consumer1.acknowledge(msg);
+ log.info("Received message: " + new String(msg.getData()));
+ } else {
+ break;
+ }
+ }
+ // verify total-consumer messages = total-produce messages
+ assertEquals(totalProducedMsgs, totalReceiveMessages);
+ producer.close();
+ consumer1.close();
+ log.info("-- Exiting {} test --", methodName);
+ } catch (Exception e) {
+ fail();
+ } finally {
+ pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
+ }
+ }
+
+ /**
+ * Verify: Consumer2 sends ack of Consumer1 and consumer1 should be unblock if it is blocked due to unack-messages
+ *
+ *
+ * @param batchMessageDelayMs
+ * @throws Exception
+ */
+ @Test
+ public void testBlockUnackConsumerAckByDifferentConsumer() throws Exception {
+ log.info("-- Starting {} test --", methodName);
+
+ int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
+ try {
+ final int maxUnackedMessages = 20;
+ final int receiverQueueSize = 10;
+ final int totalProducedMsgs = 100;
+ int totalReceiveMessages = 0;
+
+ pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(maxUnackedMessages);
+ ConsumerConfiguration conf = new ConsumerConfiguration();
+ conf.setReceiverQueueSize(receiverQueueSize);
+ conf.setSubscriptionType(SubscriptionType.Shared);
+ Consumer consumer1 = pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic",
+ "subscriber-1", conf);
+ Consumer consumer2 = pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic",
+ "subscriber-1", conf);
+
+ ProducerConfiguration producerConf = new ProducerConfiguration();
+
+ Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic",
+ producerConf);
+
+ // (1) Produced Messages
+ for (int i = 0; i < totalProducedMsgs; i++) {
+ String message = "my-message-" + i;
+ producer.send(message.getBytes());
+ }
+
+ // (2) Consumer1: consume without ack:
+ // try to consume messages: but will be able to consume number of messages = maxUnackedMessages
+ Message msg = null;
+ List<Message> messages = Lists.newArrayList();
+ for (int i = 0; i < totalProducedMsgs; i++) {
+ msg = consumer1.receive(1, TimeUnit.SECONDS);
+ if (msg != null) {
+ messages.add(msg);
+ totalReceiveMessages++;
+ log.info("Received message: " + new String(msg.getData()));
+ } else {
+ break;
+ }
+ }
+
+ assertEquals(messages.size(), maxUnackedMessages); // consumer1
+
+ // (3) ack for all UnackedMessages from consumer2
+ messages.forEach(m -> {
+ try {
+ consumer2.acknowledge(m);
+ } catch (PulsarClientException e) {
+ fail("shouldn't have failed ", e);
+ }
+ });
+
+ // (4) consumer1 will consumer remaining msgs and consumer2 will ack those messages
+ for (int i = 0; i < totalProducedMsgs; i++) {
+ msg = consumer1.receive(1, TimeUnit.SECONDS);
+ if (msg != null) {
+ totalReceiveMessages++;
+ consumer2.acknowledge(msg);
+ log.info("Received message: " + new String(msg.getData()));
+ } else {
+ break;
+ }
+ }
+
+ for (int i = 0; i < totalProducedMsgs; i++) {
+ msg = consumer2.receive(1, TimeUnit.SECONDS);
+ if (msg != null) {
+ totalReceiveMessages++;
+ log.info("Received message: " + new String(msg.getData()));
+ } else {
+ break;
+ }
+ }
+
+ // verify total-consumer messages = total-produce messages
+ assertEquals(totalProducedMsgs, totalReceiveMessages);
+ producer.close();
+ consumer1.close();
+ consumer2.close();
+ log.info("-- Exiting {} test --", methodName);
+ } catch (Exception e) {
+ fail();
+ } finally {
+ pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
+ }
+ }
+
+ @Test
+ public void testEnabledChecksumClient() throws Exception {
+ log.info("-- Starting {} test --", methodName);
+
+ final int totalMsg = 10;
+ ConsumerConfiguration conf = new ConsumerConfiguration();
+ conf.setSubscriptionType(SubscriptionType.Exclusive);
+ Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic1", "my-subscriber-name",
+ conf);
+
+ ProducerConfiguration producerConf = new ProducerConfiguration();
+ final int batchMessageDelayMs = 300;
+ if (batchMessageDelayMs != 0) {
+ producerConf.setBatchingEnabled(true);
+ producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
+ producerConf.setBatchingMaxMessages(5);
+ }
+
+ Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1", producerConf);
+ for (int i = 0; i < totalMsg; i++) {
+ String message = "my-message-" + i;
+ producer.send(message.getBytes());
+ }
+
+ Message msg = null;
+ Set<String> messageSet = Sets.newHashSet();
+ for (int i = 0; i < totalMsg; i++) {
+ msg = consumer.receive(5, TimeUnit.SECONDS);
+ String receivedMessage = new String(msg.getData());
+ log.debug("Received message: [{}]", receivedMessage);
+ String expectedMessage = "my-message-" + i;
+ testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
+ }
+ // Acknowledge the consumption of all messages at once
+ consumer.acknowledgeCumulative(msg);
+ consumer.close();
+ log.info("-- Exiting {} test --", methodName);
+ }
+
+ /**
+ * It verifies that redelivery-of-specific messages: that redelivers all those messages even when consumer gets
+ * blocked due to unacked messsages
+ *
+ * Usecase: produce message with 10ms interval: so, consumer can consume only 10 messages without acking
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testBlockUnackedConsumerRedeliverySpecificMessagesProduceWithPause() throws Exception {
+ log.info("-- Starting {} test --", methodName);
+
+ int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
+ try {
+ final int unAckedMessagesBufferSize = 10;
+ final int receiverQueueSize = 20;
+ final int totalProducedMsgs = 20;
+
+ pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessagesBufferSize);
+ ConsumerConfiguration conf = new ConsumerConfiguration();
+ conf.setReceiverQueueSize(receiverQueueSize);
+ conf.setSubscriptionType(SubscriptionType.Shared);
+ ConsumerImpl consumer = (ConsumerImpl) pulsarClient
+ .subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
+
+ ProducerConfiguration producerConf = new ProducerConfiguration();
+
+ Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic",
+ producerConf);
+
+ // (1) Produced Messages
+ for (int i = 0; i < totalProducedMsgs; i++) {
+ String message = "my-message-" + i;
+ producer.send(message.getBytes());
+ Thread.sleep(10);
+ }
+
+ // (2) try to consume messages: but will be able to consume number of messages = unAckedMessagesBufferSize
+ Message msg = null;
+ List<Message> messages1 = Lists.newArrayList();
+ for (int i = 0; i < totalProducedMsgs; i++) {
+ msg = consumer.receive(1, TimeUnit.SECONDS);
+ if (msg != null) {
+ messages1.add(msg);
+ log.info("Received message: " + new String(msg.getData()));
+ } else {
+ break;
+ }
+ }
+
+ // client should not receive all produced messages and should be blocked due to unack-messages
+ assertEquals(messages1.size(), unAckedMessagesBufferSize);
+ Set<MessageIdImpl> redeliveryMessages = messages1.stream().map(m -> {
+ return (MessageIdImpl) m.getMessageId();
+ }).collect(Collectors.toSet());
+
+ // (3) redeliver all consumed messages
+ consumer.redeliverUnacknowledgedMessages(Sets.newHashSet(redeliveryMessages));
+ Thread.sleep(1000);
+
+ Set<MessageIdImpl> messages2 = Sets.newHashSet();
+ for (int i = 0; i < totalProducedMsgs; i++) {
+ msg = consumer.receive(1, TimeUnit.SECONDS);
+ if (msg != null) {
+ messages2.add((MessageIdImpl) msg.getMessageId());
+ log.info("Received message: " + new String(msg.getData()));
+ } else {
+ break;
+ }
+ }
+
+ assertEquals(messages1.size(), messages2.size());
+ // (4) Verify: redelivered all previous unacked-consumed messages
+ messages2.removeAll(redeliveryMessages);
+ assertEquals(messages2.size(), 0);
+ producer.close();
+ consumer.close();
+ log.info("-- Exiting {} test --", methodName);
+ } catch (Exception e) {
+ fail();
+ } finally {
+ pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
+ }
+ }
+
+ /**
+ * It verifies that redelivery-of-specific messages: that redelivers all those messages even when consumer gets
+ * blocked due to unacked messsages
+ *
+ * Usecase: Consumer starts consuming only after all messages have been produced. So, consumer consumes total
+ * receiver-queue-size number messages => ask for redelivery and receives all messages again.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testBlockUnackedConsumerRedeliverySpecificMessagesCloseConsumerWhileProduce() throws Exception {
+ log.info("-- Starting {} test --", methodName);
+
+ int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
+ try {
+ final int unAckedMessagesBufferSize = 10;
+ final int receiverQueueSize = 20;
+ final int totalProducedMsgs = 50;
+
+ pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessagesBufferSize);
+ ConsumerConfiguration conf = new ConsumerConfiguration();
+ conf.setReceiverQueueSize(receiverQueueSize);
+ conf.setSubscriptionType(SubscriptionType.Shared);
+ // Only subscribe consumer
+ ConsumerImpl consumer = (ConsumerImpl) pulsarClient
+ .subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
+ consumer.close();
+
+ ProducerConfiguration producerConf = new ProducerConfiguration();
+
+ Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic",
+ producerConf);
+
+ // (1) Produced Messages
+ for (int i = 0; i < totalProducedMsgs; i++) {
+ String message = "my-message-" + i;
+ producer.send(message.getBytes());
+ Thread.sleep(10);
+ }
+
+ // (1.a) start consumer again
+ consumer = (ConsumerImpl) pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic",
+ "subscriber-1", conf);
+
+ // (2) try to consume messages: but will be able to consume number of messages = unAckedMessagesBufferSize
+ Message msg = null;
+ List<Message> messages1 = Lists.newArrayList();
+ for (int i = 0; i < totalProducedMsgs; i++) {
+ msg = consumer.receive(1, TimeUnit.SECONDS);
+ if (msg != null) {
+ messages1.add(msg);
+ log.info("Received message: " + new String(msg.getData()));
+ } else {
+ break;
+ }
+ }
+
+ // client should not receive all produced messages and should be blocked due to unack-messages
+ assertEquals(messages1.size(), receiverQueueSize);
+ Set<MessageIdImpl> redeliveryMessages = messages1.stream().map(m -> {
+ return (MessageIdImpl) m.getMessageId();
+ }).collect(Collectors.toSet());
+
+ // (3) redeliver all consumed messages
+ consumer.redeliverUnacknowledgedMessages(Sets.newHashSet(redeliveryMessages));
+ Thread.sleep(1000);
+
+ Set<MessageIdImpl> messages2 = Sets.newHashSet();
+ for (int i = 0; i < totalProducedMsgs; i++) {
+ msg = consumer.receive(1, TimeUnit.SECONDS);
+ if (msg != null) {
+ messages2.add((MessageIdImpl) msg.getMessageId());
+ log.info("Received message: " + new String(msg.getData()));
+ } else {
+ break;
+ }
+ }
+
+ assertEquals(messages1.size(), messages2.size());
+ // (4) Verify: redelivered all previous unacked-consumed messages
+ messages2.removeAll(redeliveryMessages);
+ assertEquals(messages2.size(), 0);
+ producer.close();
+ consumer.close();
+ log.info("-- Exiting {} test --", methodName);
+ } catch (Exception e) {
+ fail();
+ } finally {
+ pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
+ }
+ }
+
+ @Test
+ public void testPriorityConsumer() throws Exception {
+ log.info("-- Starting {} test --", methodName);
+ ConsumerConfiguration conf1 = new ConsumerConfiguration();
+ conf1.setSubscriptionType(SubscriptionType.Shared);
+ conf1.setPriorityLevel(1);
+ conf1.setReceiverQueueSize(5);
+ ConsumerConfiguration conf4 = new ConsumerConfiguration();
+ conf4.setSubscriptionType(SubscriptionType.Shared);
+ conf4.setPriorityLevel(2);
+ conf4.setReceiverQueueSize(5);
+ Consumer consumer1 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic2",
+ "my-subscriber-name", conf1);
+ Consumer consumer2 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic2",
+ "my-subscriber-name", conf1);
+ Consumer consumer3 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic2",
+ "my-subscriber-name", conf1);
+ Consumer consumer4 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic2",
+ "my-subscriber-name", conf4);
+ ProducerConfiguration producerConf = new ProducerConfiguration();
+ Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic2", producerConf);
+ List<Future<MessageId>> futures = Lists.newArrayList();
+
+ // Asynchronously produce messages
+ for (int i = 0; i < 15; i++) {
+ final String message = "my-message-" + i;
+ Future<MessageId> future = producer.sendAsync(message.getBytes());
+ futures.add(future);
+ }
+
+ log.info("Waiting for async publish to complete");
+ for (Future<MessageId> future : futures) {
+ future.get();
+ }
+
+ for (int i = 0; i < 20; i++) {
+ consumer1.receive(100, TimeUnit.MILLISECONDS);
+ consumer2.receive(100, TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * a. consumer1 and consumer2 now has more permits (as received and sent more permits) b. try to produce more
+ * messages: which will again distribute among consumer1 and consumer2 and should not dispatch to consumer4
+ *
+ */
+ for (int i = 0; i < 5; i++) {
+ final String message = "my-message-" + i;
+ Future<MessageId> future = producer.sendAsync(message.getBytes());
+ futures.add(future);
+ }
+
+ Assert.assertNull(consumer4.receive(100, TimeUnit.MILLISECONDS));
+
+ // Asynchronously acknowledge upto and including the last message
+ producer.close();
+ consumer1.close();
+ consumer2.close();
+ consumer3.close();
+ consumer4.close();
+ log.info("-- Exiting {} test --", methodName);
+ }
+
+ /**
+ * <pre>
+ * Verifies Dispatcher dispatches messages properly with shared-subscription consumers with combination of blocked
+ * and unblocked consumers.
+ *
+ * 1. Dispatcher will have 5 consumers : c1, c2, c3, c4, c5.
+ * Out of which : c1,c2,c4,c5 will be blocked due to MaxUnackedMessages limit.
+ * 2. So, dispatcher should moves round-robin and make sure it delivers unblocked consumer : c3
+ * </pre>
+ *
+ * @throws Exception
+ */
+ @Test(timeOut = 5000)
+ public void testSharedSamePriorityConsumer() throws Exception {
+ log.info("-- Starting {} test --", methodName);
+ ConsumerConfiguration conf1 = new ConsumerConfiguration();
+ conf1.setSubscriptionType(SubscriptionType.Shared);
+ final int queueSize = 5;
+ conf1.setReceiverQueueSize(queueSize);
+ int maxUnAckMsgs = pulsar.getConfiguration().getMaxConcurrentLookupRequest();
+ pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(queueSize);
+ Consumer c1 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic2", "my-subscriber-name",
+ conf1);
+ Consumer c2 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic2", "my-subscriber-name",
+ conf1);
+ ProducerConfiguration producerConf = new ProducerConfiguration();
+ Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic2", producerConf);
+ List<Future<MessageId>> futures = Lists.newArrayList();
+
+ // Asynchronously produce messages
+ final int totalPublishMessages = 500;
+ for (int i = 0; i < totalPublishMessages; i++) {
+ final String message = "my-message-" + i;
+ Future<MessageId> future = producer.sendAsync(message.getBytes());
+ futures.add(future);
+ }
+
+ log.info("Waiting for async publish to complete");
+ for (Future<MessageId> future : futures) {
+ future.get();
+ }
+
+ List<Message> messages = Lists.newArrayList();
+
+ // let consumer1 and consumer2 cosume messages up to the queue will be full
+ for (int i = 0; i < totalPublishMessages; i++) {
+ Message msg = c1.receive(500, TimeUnit.MILLISECONDS);
+ if (msg != null) {
+ messages.add(msg);
+ } else {
+ break;
+ }
+ }
+ for (int i = 0; i < totalPublishMessages; i++) {
+ Message msg = c2.receive(500, TimeUnit.MILLISECONDS);
+ if (msg != null) {
+ messages.add(msg);
+ } else {
+ break;
+ }
+ }
+
+ Assert.assertEquals(queueSize * 2, messages.size());
+
+ // create new consumers with the same priority
+ Consumer c3 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic2", "my-subscriber-name",
+ conf1);
+ Consumer c4 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic2", "my-subscriber-name",
+ conf1);
+ Consumer c5 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic2", "my-subscriber-name",
+ conf1);
+
+ // c1 and c2 are blocked: so, let c3, c4 and c5 consume rest of the messages
+
+ for (int i = 0; i < totalPublishMessages; i++) {
+ Message msg = c4.receive(500, TimeUnit.MILLISECONDS);
+ if (msg != null) {
+ messages.add(msg);
+ } else {
+ break;
+ }
+ }
+
+ for (int i = 0; i < totalPublishMessages; i++) {
+ Message msg = c5.receive(500, TimeUnit.MILLISECONDS);
+ if (msg != null) {
+ messages.add(msg);
+ } else {
+ break;
+ }
+ }
+
+ for (int i = 0; i < totalPublishMessages; i++) {
+ Message msg = c3.receive(500, TimeUnit.MILLISECONDS);
+ if (msg != null) {
+ messages.add(msg);
+ c3.acknowledge(msg);
+ } else {
+ break;
+ }
+ }
+
+ // total messages must be consumed by all consumers
+ Assert.assertEquals(messages.size(), totalPublishMessages);
+
+ // Asynchronously acknowledge upto and including the last message
+ producer.close();
+ c1.close();
+ c2.close();
+ c3.close();
+ c4.close();
+ c5.close();
+ pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(maxUnAckMsgs);
+ log.info("-- Exiting {} test --", methodName);
+ }
+
+ @Test
+ public void testRedeliveryFailOverConsumer() throws Exception {
+ log.info("-- Starting {} test --", methodName);
+
+ final int receiverQueueSize = 10;
+
+ ConsumerConfiguration conf = new ConsumerConfiguration();
+ conf.setReceiverQueueSize(receiverQueueSize);
+ conf.setSubscriptionType(SubscriptionType.Failover);
+ // Only subscribe consumer
+ ConsumerImpl consumer = (ConsumerImpl) pulsarClient
+ .subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
+
+ ProducerConfiguration producerConf = new ProducerConfiguration();
+
+ Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic",
+ producerConf);
+
+ // (1) First round to produce-consume messages
+ int consumeMsgInParts = 4;
+ for (int i = 0; i < receiverQueueSize; i++) {
+ String message = "my-message-" + i;
+ producer.send(message.getBytes());
+ Thread.sleep(10);
+ }
+ // (1.a) consume first consumeMsgInParts msgs and trigger redeliver
+ Message msg = null;
+ List<Message> messages1 = Lists.newArrayList();
+ for (int i = 0; i < consumeMsgInParts; i++) {
+ msg = consumer.receive(1, TimeUnit.SECONDS);
+ if (msg != null) {
+ messages1.add(msg);
+ consumer.acknowledge(msg);
+ log.info("Received message: " + new String(msg.getData()));
+ } else {
+ break;
+ }
+ }
+ assertEquals(messages1.size(), consumeMsgInParts);
+ consumer.redeliverUnacknowledgedMessages();
+
+ // (1.b) consume second consumeMsgInParts msgs and trigger redeliver
+ messages1.clear();
+ for (int i = 0; i < consumeMsgInParts; i++) {
+ msg = consumer.receive(1, TimeUnit.SECONDS);
+ if (msg != null) {
+ messages1.add(msg);
+ consumer.acknowledge(msg);
+ log.info("Received message: " + new String(msg.getData()));
+ } else {
+ break;
+ }
+ }
+ assertEquals(messages1.size(), consumeMsgInParts);
+ consumer.redeliverUnacknowledgedMessages();
+
+ // (2) Second round to produce-consume messages
+ for (int i = 0; i < receiverQueueSize; i++) {
+ String message = "my-message-" + i;
+ producer.send(message.getBytes());
+ Thread.sleep(100);
+ }
+
+ int remainingMsgs = (2 * receiverQueueSize) - (2 * consumeMsgInParts);
+ messages1.clear();
+ for (int i = 0; i < remainingMsgs; i++) {
+ msg = consumer.receive(1, TimeUnit.SECONDS);
+ if (msg != null) {
+ messages1.add(msg);
+ consumer.acknowledge(msg);
+ log.info("Received message: " + new String(msg.getData()));
+ } else {
+ break;
+ }
+ }
+ assertEquals(messages1.size(), remainingMsgs);
+
+ producer.close();
+ consumer.close();
+ log.info("-- Exiting {} test --", methodName);
+
+ }
+
+ @Test(timeOut = 5000)
+ public void testFailReceiveAsyncOnConsumerClose() throws Exception {
+ log.info("-- Starting {} test --", methodName);
+
+ // (1) simple consumers
+ Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/failAsyncReceive",
+ "my-subscriber-name", new ConsumerConfiguration());
+ consumer.close();
+ // receive messages
+ try {
+ consumer.receiveAsync().get(1, TimeUnit.SECONDS);
+ fail("it should have failed because consumer is already closed");
+ } catch (ExecutionException e) {
+ assertTrue(e.getCause() instanceof PulsarClientException.AlreadyClosedException);
+ }
+
+ // (2) Partitioned-consumer
+ int numPartitions = 4;
+ TopicName topicName = TopicName.get("persistent://my-property/use/my-ns/failAsyncReceive");
+ admin.persistentTopics().createPartitionedTopic(topicName.toString(), numPartitions);
+ Consumer partitionedConsumer = pulsarClient.subscribe(topicName.toString(), "my-partitioned-subscriber",
+ new ConsumerConfiguration());
+ partitionedConsumer.close();
+ // receive messages
+ try {
+ partitionedConsumer.receiveAsync().get(1, TimeUnit.SECONDS);
+ fail("it should have failed because consumer is already closed");
+ } catch (ExecutionException e) {
+ assertTrue(e.getCause() instanceof PulsarClientException.AlreadyClosedException);
+ }
+
+ log.info("-- Exiting {} test --", methodName);
+ }
+
+ @Test(groups = "encryption")
+ public void testECDSAEncryption() throws Exception {
+ log.info("-- Starting {} test --", methodName);
+
+ class EncKeyReader implements CryptoKeyReader {
+
+ EncryptionKeyInfo keyInfo = new EncryptionKeyInfo();
+
+ @Override
+ public EncryptionKeyInfo getPublicKey(String keyName, Map<String, String> keyMeta) {
+ String CERT_FILE_PATH = "./src/test/resources/certificate/public-key." + keyName;
+ if (Files.isReadable(Paths.get(CERT_FILE_PATH))) {
+ try {
+ keyInfo.setKey(Files.readAllBytes(Paths.get(CERT_FILE_PATH)));
+ return keyInfo;
+ } catch (IOException e) {
+ Assert.fail("Failed to read certificate from " + CERT_FILE_PATH);
+ }
+ } else {
+ Assert.fail("Certificate file " + CERT_FILE_PATH + " is not present or not readable.");
+ }
+ return null;
+ }
+
+ @Override
+ public EncryptionKeyInfo getPrivateKey(String keyName, Map<String, String> keyMeta) {
+ String CERT_FILE_PATH = "./src/test/resources/certificate/private-key." + keyName;
+ if (Files.isReadable(Paths.get(CERT_FILE_PATH))) {
+ try {
+ keyInfo.setKey(Files.readAllBytes(Paths.get(CERT_FILE_PATH)));
+ return keyInfo;
+ } catch (IOException e) {
+ Assert.fail("Failed to read certificate from " + CERT_FILE_PATH);
+ }
+ } else {
+ Assert.fail("Certificate file " + CERT_FILE_PATH + " is not present or not readable.");
+ }
+ return null;
+ }
+ }
+
+ final int totalMsg = 10;
+
+ Set<String> messageSet = Sets.newHashSet();
+ ConsumerConfiguration conf = new ConsumerConfiguration();
+ conf.setSubscriptionType(SubscriptionType.Exclusive);
+ conf.setCryptoKeyReader(new EncKeyReader());
+ Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/myecdsa-topic1",
+ "my-subscriber-name", conf);
+
+ ProducerConfiguration producerConf = new ProducerConfiguration();
+ producerConf.addEncryptionKey("client-ecdsa.pem");
+ producerConf.setCryptoKeyReader(new EncKeyReader());
+
+ Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/myecdsa-topic1",
+ producerConf);
+ for (int i = 0; i < totalMsg; i++) {
+ String message = "my-message-" + i;
+ producer.send(message.getBytes());
+ }
+
+ Message msg = null;
+
+ for (int i = 0; i < totalMsg; i++) {
+ msg = consumer.receive(5, TimeUnit.SECONDS);
+ String receivedMessage = new String(msg.getData());
+ log.debug("Received message: [{}]", receivedMessage);
+ String expectedMessage = "my-message-" + i;
+ testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
+ }
+ // Acknowledge the consumption of all messages at once
+ consumer.acknowledgeCumulative(msg);
+ consumer.close();
+ log.info("-- Exiting {} test --", methodName);
+ }
+
+ @Test(groups = "encryption")
+ public void testRSAEncryption() throws Exception {
+ log.info("-- Starting {} test --", methodName);
+
+ class EncKeyReader implements CryptoKeyReader {
+
+ EncryptionKeyInfo keyInfo = new EncryptionKeyInfo();
+
+ @Override
+ public EncryptionKeyInfo getPublicKey(String keyName, Map<String, String> keyMeta) {
+ String CERT_FILE_PATH = "./src/test/resources/certificate/public-key." + keyName;
+ if (Files.isReadable(Paths.get(CERT_FILE_PATH))) {
+ try {
+ keyInfo.setKey(Files.readAllBytes(Paths.get(CERT_FILE_PATH)));
+ return keyInfo;
+ } catch (IOException e) {
+ Assert.fail("Failed to read certificate from " + CERT_FILE_PATH);
+ }
+ } else {
+ Assert.fail("Certificate file " + CERT_FILE_PATH + " is not present or not readable.");
+ }
+ return null;
+ }
+
+ @Override
+ public EncryptionKeyInfo getPrivateKey(String keyName, Map<String, String> keyMeta) {
+ String CERT_FILE_PATH = "./src/test/resources/certificate/private-key." + keyName;
+ if (Files.isReadable(Paths.get(CERT_FILE_PATH))) {
+ try {
+ keyInfo.setKey(Files.readAllBytes(Paths.get(CERT_FILE_PATH)));
+ return keyInfo;
+ } catch (IOException e) {
+ Assert.fail("Failed to read certificate from " + CERT_FILE_PATH);
+ }
+ } else {
+ Assert.fail("Certificate file " + CERT_FILE_PATH + " is not present or not readable.");
+ }
+ return null;
+ }
+ }
+
+ final int totalMsg = 10;
+
+ Set<String> messageSet = Sets.newHashSet();
+ ConsumerConfiguration conf = new ConsumerConfiguration();
+ conf.setSubscriptionType(SubscriptionType.Exclusive);
+ conf.setCryptoKeyReader(new EncKeyReader());
+ Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/myrsa-topic1",
+ "my-subscriber-name", conf);
+
+ ProducerConfiguration producerConf = new ProducerConfiguration();
+ producerConf.addEncryptionKey("client-rsa.pem");
+ producerConf.setCryptoKeyReader(new EncKeyReader());
+
+ Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/myrsa-topic1",
+ producerConf);
+ Producer producer2 = pulsarClient.createProducer("persistent://my-property/use/my-ns/myrsa-topic1",
+ producerConf);
+ for (int i = 0; i < totalMsg; i++) {
+ String message = "my-message-" + i;
+ producer.send(message.getBytes());
+ }
+ for (int i = totalMsg; i < totalMsg * 2; i++) {
+ String message = "my-message-" + i;
+ producer2.send(message.getBytes());
+ }
+
+ Message msg = null;
+
+ for (int i = 0; i < totalMsg * 2; i++) {
+ msg = consumer.receive(5, TimeUnit.SECONDS);
+ String receivedMessage = new String(msg.getData());
+ log.debug("Received message: [{}]", receivedMessage);
+ String expectedMessage = "my-message-" + i;
+ testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
+ }
+ // Acknowledge the consumption of all messages at once
+ consumer.acknowledgeCumulative(msg);
+ consumer.close();
+ log.info("-- Exiting {} test --", methodName);
+ }
+
+ @Test(groups = "encryption")
+ public void testEncryptionFailure() throws Exception {
+ log.info("-- Starting {} test --", methodName);
+
+ class EncKeyReader implements CryptoKeyReader {
+
+ EncryptionKeyInfo keyInfo = new EncryptionKeyInfo();
+
+ @Override
+ public EncryptionKeyInfo getPublicKey(String keyName, Map<String, String> keyMeta) {
+ String CERT_FILE_PATH = "./src/test/resources/certificate/public-key." + keyName;
+ if (Files.isReadable(Paths.get(CERT_FILE_PATH))) {
+ try {
+ keyInfo.setKey(Files.readAllBytes(Paths.get(CERT_FILE_PATH)));
+ return keyInfo;
+ } catch (IOException e) {
+ log.error("Failed to read certificate from {}", CERT_FILE_PATH);
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public EncryptionKeyInfo getPrivateKey(String keyName, Map<String, String> keyMeta) {
+ String CERT_FILE_PATH = "./src/test/resources/certificate/private-key." + keyName;
+ if (Files.isReadable(Paths.get(CERT_FILE_PATH))) {
+ try {
+ keyInfo.setKey(Files.readAllBytes(Paths.get(CERT_FILE_PATH)));
+ return keyInfo;
+ } catch (IOException e) {
+ log.error("Failed to read certificate from {}", CERT_FILE_PATH);
+ }
+ }
+ return null;
+ }
+ }
+
+ final int totalMsg = 10;
+
+ ProducerConfiguration producerConf = new ProducerConfiguration();
+
+ Message msg = null;
+ Set<String> messageSet = Sets.newHashSet();
+ ConsumerConfiguration conf = new ConsumerConfiguration();
+ conf.setSubscriptionType(SubscriptionType.Exclusive);
+ Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/myenc-topic1",
+ "my-subscriber-name", conf);
+
+ // 1. Invalid key name
+ producerConf.addEncryptionKey("client-non-existant-rsa.pem");
+ producerConf.setCryptoKeyReader(new EncKeyReader());
+
+ try {
+ pulsarClient.createProducer("persistent://my-property/use/myenc-ns/myenc-topic1", producerConf);
+ Assert.fail("Producer creation should not suceed if failing to read key");
+ } catch (Exception e) {
+ // ok
+ }
+
+ // 2. Producer with valid key name
+ producerConf = new ProducerConfiguration();
+ producerConf.setCryptoKeyReader(new EncKeyReader());
+ producerConf.addEncryptionKey("client-rsa.pem");
+ Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/myenc-topic1",
+ producerConf);
+
+ for (int i = 0; i < totalMsg; i++) {
+ String message = "my-message-" + i;
+ producer.send(message.getBytes());
+ }
+
+ // 3. KeyReder is not set by consumer
+ // Receive should fail since key reader is not setup
+ msg = consumer.receive(5, TimeUnit.SECONDS);
+ Assert.assertNull(msg, "Receive should have failed with no keyreader");
+
+ // 4. Set consumer config to consume even if decryption fails
+ conf.setCryptoFailureAction(ConsumerCryptoFailureAction.CONSUME);
+ consumer.close();
+ consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/myenc-topic1", "my-subscriber-name",
+ conf);
+
+ int msgNum = 0;
+ try {
+ // Receive should proceed and deliver encrypted message
+ msg = consumer.receive(5, TimeUnit.SECONDS);
+ String receivedMessage = new String(msg.getData());
+ String expectedMessage = "my-message-" + msgNum++;
+ Assert.assertNotEquals(receivedMessage, expectedMessage, "Received encrypted message " + receivedMessage
+ + " should not match the expected message " + expectedMessage);
+ consumer.acknowledgeCumulative(msg);
+ } catch (Exception e) {
+ Assert.fail("Failed to receive message even aftet ConsumerCryptoFailureAction.CONSUME is set.");
+ }
+
+ // 5. Set keyreader and failure action
+ conf.setCryptoFailureAction(ConsumerCryptoFailureAction.FAIL);
+ consumer.close();
+ // Set keyreader
+ conf.setCryptoKeyReader(new EncKeyReader());
+ consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/myenc-topic1", "my-subscriber-name",
+ conf);
+
+ for (int i = msgNum; i < totalMsg - 1; i++) {
+ msg = consumer.receive(5, TimeUnit.SECONDS);
+ String receivedMessage = new String(msg.getData());
+ log.debug("Received message: [{}]", receivedMessage);
+ String expectedMessage = "my-message-" + i;
+ testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
+ }
+ // Acknowledge the consumption of all messages at once
+ consumer.acknowledgeCumulative(msg);
+ consumer.close();
+
+ // 6. Set consumer config to discard if decryption fails
+ consumer.close();
+ ConsumerConfiguration conf2 = new ConsumerConfiguration();
+ conf2.setSubscriptionType(SubscriptionType.Exclusive);
+ conf2.setCryptoFailureAction(ConsumerCryptoFailureAction.DISCARD);
+ consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/myenc-topic1", "my-subscriber-name",
+ conf2);
+
+ // Receive should proceed and discard encrypted messages
+ msg = consumer.receive(5, TimeUnit.SECONDS);
+ Assert.assertNull(msg, "Message received even aftet ConsumerCryptoFailureAction.DISCARD is set.");
+
+ log.info("-- Exiting {} test --", methodName);
+ }
+
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/BrokerClientIntegrationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/BrokerClientIntegrationTest.java
index 8cf5618ba..d7d0d73b3 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/BrokerClientIntegrationTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/BrokerClientIntegrationTest.java
@@ -51,21 +51,18 @@
import org.apache.pulsar.broker.namespace.OwnershipCache;
import org.apache.pulsar.broker.service.Topic;
import org.apache.pulsar.client.admin.PulsarAdminException;
-import org.apache.pulsar.client.api.ClientConfiguration;
import org.apache.pulsar.client.api.Consumer;
-import org.apache.pulsar.client.api.ConsumerConfiguration;
+import org.apache.pulsar.client.api.ConsumerBuilder;
import org.apache.pulsar.client.api.Message;
-import org.apache.pulsar.client.api.MessageListener;
import org.apache.pulsar.client.api.Producer;
-import org.apache.pulsar.client.api.ProducerConfiguration;
import org.apache.pulsar.client.api.ProducerConsumerBase;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.client.api.SubscriptionType;
import org.apache.pulsar.client.impl.HandlerBase.State;
import org.apache.pulsar.common.api.PulsarHandler;
-import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.naming.NamespaceBundle;
+import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.policies.data.ClusterData;
import org.apache.pulsar.common.policies.data.RetentionPolicies;
import org.apache.pulsar.common.util.FutureUtil;
@@ -79,7 +76,6 @@
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
-import org.testng.collections.Maps;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -107,6 +103,7 @@ protected void cleanup() throws Exception {
/**
* Verifies unload namespace-bundle doesn't close shared connection used by other namespace-bundle.
+ *
* <pre>
* 1. after disabling broker fron loadbalancer
* 2. unload namespace-bundle "my-ns1" which disconnects client (producer/consumer) connected on that namespacebundle
@@ -118,6 +115,7 @@ protected void cleanup() throws Exception {
*
* @throws Exception
*/
+ @SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testDisconnectClientWithoutClosingConnection() throws Exception {
@@ -128,11 +126,12 @@ public void testDisconnectClientWithoutClosingConnection() throws Exception {
final String topic1 = "persistent://" + ns1 + "/my-topic";
final String topic2 = "persistent://" + ns2 + "/my-topic";
- ConsumerImpl cons1 = (ConsumerImpl) pulsarClient.subscribe(topic1, "my-subscriber-name",
- new ConsumerConfiguration());
- ProducerImpl prod1 = (ProducerImpl) pulsarClient.createProducer(topic1, new ProducerConfiguration());
- ProducerImpl prod2 = (ProducerImpl) pulsarClient.createProducer(topic2, new ProducerConfiguration());
- ConsumerImpl consumer1 = spy(cons1);
+ ConsumerImpl<byte[]> cons1 = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topic1)
+ .subscriptionName("my-subscriber-name").subscribe();
+
+ ProducerImpl<byte[]> prod1 = (ProducerImpl<byte[]>) pulsarClient.newProducer().topic(topic1).create();
+ ProducerImpl<byte[]> prod2 = (ProducerImpl<byte[]>) pulsarClient.newProducer().topic(topic2).create();
+ ConsumerImpl<byte[]> consumer1 = spy(cons1);
doAnswer(invocationOnMock -> cons1.getState()).when(consumer1).getState();
doAnswer(invocationOnMock -> cons1.getClientCnx()).when(consumer1).getClientCnx();
doAnswer(invocationOnMock -> cons1.cnx()).when(consumer1).cnx();
@@ -140,7 +139,7 @@ public void testDisconnectClientWithoutClosingConnection() throws Exception {
cons1.connectionClosed((ClientCnx) invocationOnMock.getArguments()[0]);
return null;
}).when(consumer1).connectionClosed(anyObject());
- ProducerImpl producer1 = spy(prod1);
+ ProducerImpl<byte[]> producer1 = spy(prod1);
doAnswer(invocationOnMock -> prod1.getState()).when(producer1).getState();
doAnswer(invocationOnMock -> prod1.getClientCnx()).when(producer1).getClientCnx();
doAnswer(invocationOnMock -> prod1.cnx()).when(producer1).cnx();
@@ -148,7 +147,7 @@ public void testDisconnectClientWithoutClosingConnection() throws Exception {
prod1.connectionClosed((ClientCnx) invocationOnMock.getArguments()[0]);
return null;
}).when(producer1).connectionClosed(anyObject());
- ProducerImpl producer2 = spy(prod2);
+ ProducerImpl<byte[]> producer2 = spy(prod2);
doAnswer(invocationOnMock -> prod2.getState()).when(producer2).getState();
doAnswer(invocationOnMock -> prod2.getClientCnx()).when(producer2).getClientCnx();
doAnswer(invocationOnMock -> prod2.cnx()).when(producer2).cnx();
@@ -164,8 +163,8 @@ public void testDisconnectClientWithoutClosingConnection() throws Exception {
Field cfield = ClientCnx.class.getDeclaredField("consumers");
cfield.setAccessible(true);
- ConcurrentLongHashMap<ProducerImpl> producers = (ConcurrentLongHashMap<ProducerImpl>) pfield.get(clientCnx);
- ConcurrentLongHashMap<ConsumerImpl> consumers = (ConcurrentLongHashMap<ConsumerImpl>) cfield.get(clientCnx);
+ ConcurrentLongHashMap<ProducerImpl<byte[]>> producers = (ConcurrentLongHashMap) pfield.get(clientCnx);
+ ConcurrentLongHashMap<ConsumerImpl<byte[]>> consumers = (ConcurrentLongHashMap) cfield.get(clientCnx);
producers.put(2, producers.get(0));
producers.put(3, producers.get(1));
@@ -248,10 +247,10 @@ public void testCloseBrokerService() throws Exception {
final String topic1 = "persistent://" + ns1 + "/my-topic";
final String topic2 = "persistent://" + ns2 + "/my-topic";
- ConsumerImpl consumer1 = (ConsumerImpl) pulsarClient.subscribe(topic1, "my-subscriber-name",
- new ConsumerConfiguration());
- ProducerImpl producer1 = (ProducerImpl) pulsarClient.createProducer(topic1, new ProducerConfiguration());
- ProducerImpl producer2 = (ProducerImpl) pulsarClient.createProducer(topic2, new ProducerConfiguration());
+ ConsumerImpl<byte[]> consumer1 = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topic1)
+ .subscriptionName("my-subscriber-name").subscribe();
+ ProducerImpl<byte[]> producer1 = (ProducerImpl<byte[]>) pulsarClient.newProducer().topic(topic1).create();
+ ProducerImpl<byte[]> producer2 = (ProducerImpl<byte[]>) pulsarClient.newProducer().topic(topic2).create();
// unload all other namespace
pulsar.getBrokerService().close();
@@ -297,20 +296,12 @@ public void testUnsupportedBatchMessageConsumer(SubscriptionType subType) throws
final String topicName = "persistent://my-property/use/my-ns/my-topic1";
final String subscriptionName = "my-subscriber-name" + subType;
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(subType);
- ConsumerImpl consumer1 = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriptionName, conf);
+ ConsumerImpl<byte[]> consumer1 = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName(subscriptionName).subscriptionType(subType).subscribe();
- ProducerConfiguration producerConf = new ProducerConfiguration();
-
- if (batchMessageDelayMs != 0) {
- producerConf.setBatchingEnabled(true);
- producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
- producerConf.setBatchingMaxMessages(20);
- }
-
- Producer producer = pulsarClient.createProducer(topicName, new ProducerConfiguration());
- Producer batchProducer = pulsarClient.createProducer(topicName, producerConf);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
+ Producer<byte[]> batchProducer = pulsarClient.newProducer().topic(topicName).enableBatching(true)
+ .batchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS).batchingMaxMessages(20).create();
// update consumer's version to incompatible batch-message version = Version.V3
Topic topic = pulsar.getBrokerService().getTopic(topicName).get();
@@ -329,7 +320,7 @@ public void testUnsupportedBatchMessageConsumer(SubscriptionType subType) throws
producer.send(message.getBytes());
}
Set<String> messageSet = Sets.newHashSet();
- Message msg = null;
+ Message<byte[]> msg = null;
for (int i = 0; i < 10; i++) {
msg = consumer1.receive(1, TimeUnit.SECONDS);
String receivedMessage = new String(msg.getData());
@@ -354,8 +345,8 @@ public void testUnsupportedBatchMessageConsumer(SubscriptionType subType) throws
assertNull(msg);
// subscrie consumer2 with supporting batch version
- pulsarClient = PulsarClient.create(brokerUrl.toString());
- Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, conf);
+ Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
+ .subscribe();
messageSet.clear();
for (int i = 0; i < 10; i++) {
@@ -376,11 +367,10 @@ public void testUnsupportedBatchMessageConsumer(SubscriptionType subType) throws
@Test(timeOut = 10000, dataProvider = "subType")
public void testResetCursor(SubscriptionType subType) throws Exception {
final RetentionPolicies policy = new RetentionPolicies(60, 52 * 1024);
- final TopicName destName = TopicName.get("persistent://my-property/use/my-ns/unacked-topic");
+ final TopicName topicName = TopicName.get("persistent://my-property/use/my-ns/unacked-topic");
final int warmup = 20;
final int testSize = 150;
- final List<Message> received = new ArrayList<Message>();
- final ConsumerConfiguration consConfig = new ConsumerConfiguration();
+ final List<Message<byte[]>> received = new ArrayList<>();
final String subsId = "sub";
final NavigableMap<Long, TimestampEntryCount> publishTimeIdMap = new ConcurrentSkipListMap<>();
@@ -389,30 +379,29 @@ public void testResetCursor(SubscriptionType subType) throws Exception {
conf.setActiveConsumerFailoverDelayTimeMillis(500);
restartBroker();
- consConfig.setSubscriptionType(subType);
- consConfig.setMessageListener((MessageListener) (Consumer consumer, Message msg) -> {
- try {
- synchronized (received) {
- received.add(msg);
- }
- consumer.acknowledge(msg);
- long publishTime = ((MessageImpl) msg).getPublishTime();
- log.info(" publish time is " + publishTime + "," + msg.getMessageId());
- TimestampEntryCount timestampEntryCount = publishTimeIdMap.computeIfAbsent(publishTime,
- (k) -> new TimestampEntryCount(publishTime));
- timestampEntryCount.incrementAndGet();
- } catch (final PulsarClientException e) {
- log.warn("Failed to ack!");
- }
- });
+ admin.namespaces().setRetention(topicName.getNamespace(), policy);
- admin.namespaces().setRetention(destName.getNamespace(), policy);
-
- Consumer consumer1 = pulsarClient.subscribe(destName.toString(), subsId, consConfig);
- Consumer consumer2 = pulsarClient.subscribe(destName.toString(), subsId, consConfig);
- final Producer producer = pulsarClient.createProducer(destName.toString());
+ ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName.toString())
+ .subscriptionName(subsId).subscriptionType(subType).messageListener((consumer, msg) -> {
+ try {
+ synchronized (received) {
+ received.add(msg);
+ }
+ consumer.acknowledge(msg);
+ long publishTime = ((MessageImpl<?>) msg).getPublishTime();
+ log.info(" publish time is " + publishTime + "," + msg.getMessageId());
+ TimestampEntryCount timestampEntryCount = publishTimeIdMap.computeIfAbsent(publishTime,
+ (k) -> new TimestampEntryCount(publishTime));
+ timestampEntryCount.incrementAndGet();
+ } catch (final PulsarClientException e) {
+ log.warn("Failed to ack!");
+ }
+ });
+ Consumer<byte[]> consumer1 = consumerBuilder.subscribe();
+ Consumer<byte[]> consumer2 = consumerBuilder.subscribe();
+ final Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName.toString()).create();
- log.info("warm up started for " + destName.toString());
+ log.info("warm up started for " + topicName.toString());
// send warmup msgs
byte[] msgBytes = new byte[1000];
for (Integer i = 0; i < warmup; i++) {
@@ -450,16 +439,16 @@ public void testResetCursor(SubscriptionType subType) throws Exception {
received.clear();
- log.info("reset cursor to " + timestamp + " for topic " + destName.toString() + " for subs " + subsId);
+ log.info("reset cursor to " + timestamp + " for topic " + topicName.toString() + " for subs " + subsId);
log.info("issuing admin operation on " + admin.getServiceUrl().toString());
- List<String> subList = admin.persistentTopics().getSubscriptions(destName.toString());
+ List<String> subList = admin.persistentTopics().getSubscriptions(topicName.toString());
for (String subs : subList) {
log.info("got sub " + subs);
}
publishTimeIdMap.clear();
// reset the cursor to this timestamp
Assert.assertTrue(subList.contains(subsId));
- admin.persistentTopics().resetCursor(destName.toString(), subsId, timestamp);
+ admin.persistentTopics().resetCursor(topicName.toString(), subsId, timestamp);
Thread.sleep(3000);
int totalExpected = 0;
@@ -503,22 +492,17 @@ public void testCloseConnectionOnBrokerRejectedRequest() throws Exception {
final int maxConccurentLookupRequest = pulsar.getConfiguration().getMaxConcurrentLookupRequest();
try {
final int concurrentLookupRequests = 20;
- ClientConfiguration clientConf = new ClientConfiguration();
- clientConf.setStatsInterval(0, TimeUnit.SECONDS);
- clientConf.setMaxNumberOfRejectedRequestPerConnection(0);
stopBroker();
pulsar.getConfiguration().setMaxConcurrentLookupRequest(1);
startBroker();
String lookupUrl = new URI("pulsar://localhost:" + BROKER_PORT).toString();
- pulsarClient = PulsarClient.create(lookupUrl, clientConf);
+ pulsarClient = PulsarClient.builder().serviceUrl(lookupUrl).statsInterval(0, TimeUnit.SECONDS)
+ .maxNumberOfRejectedRequestPerConnection(0).build();
- ClientConfiguration clientConf2 = new ClientConfiguration();
- clientConf2.setStatsInterval(0, TimeUnit.SECONDS);
- clientConf2.setIoThreads(concurrentLookupRequests);
- clientConf2.setConnectionsPerBroker(20);
- pulsarClient2 = PulsarClient.create(lookupUrl, clientConf2);
+ pulsarClient2 = PulsarClient.builder().serviceUrl(lookupUrl).statsInterval(0, TimeUnit.SECONDS)
+ .ioThreads(concurrentLookupRequests).connectionsPerBroker(20).build();
- ProducerImpl producer = (ProducerImpl) pulsarClient.createProducer(topicName);
+ ProducerImpl<byte[]> producer = (ProducerImpl<byte[]>) pulsarClient.newProducer().topic(topicName).create();
ClientCnx cnx = producer.cnx();
assertTrue(cnx.channel().isActive());
ExecutorService executor = Executors.newFixedThreadPool(concurrentLookupRequests);
@@ -527,14 +511,14 @@ public void testCloseConnectionOnBrokerRejectedRequest() throws Exception {
AtomicInteger failed = new AtomicInteger(0);
for (int i = 0; i < totalProducer; i++) {
executor.submit(() -> {
- pulsarClient2.createProducerAsync(topicName).handle((ok, e) -> {
+ pulsarClient2.newProducer().topic(topicName).createAsync().handle((ok, e) -> {
if (e != null) {
failed.set(1);
}
latch.countDown();
return null;
});
- pulsarClient.createProducerAsync(topicName).handle((ok, e) -> {
+ pulsarClient.newProducer().topic(topicName).createAsync().handle((ok, e) -> {
if (e != null) {
failed.set(1);
}
@@ -583,36 +567,32 @@ public void testMaxConcurrentTopicLoading() throws Exception {
try {
pulsar.getConfiguration().setAuthorizationEnabled(false);
final int concurrentLookupRequests = 20;
- ClientConfiguration clientConf = new ClientConfiguration();
- clientConf.setMaxNumberOfRejectedRequestPerConnection(0);
- clientConf.setStatsInterval(0, TimeUnit.SECONDS);
stopBroker();
pulsar.getConfiguration().setMaxConcurrentTopicLoadRequest(1);
startBroker();
String lookupUrl = new URI("pulsar://localhost:" + BROKER_PORT).toString();
- pulsarClient = (PulsarClientImpl) PulsarClient.create(lookupUrl, clientConf);
- ClientConfiguration clientConf2 = new ClientConfiguration();
- clientConf2.setStatsInterval(0, TimeUnit.SECONDS);
- clientConf2.setIoThreads(concurrentLookupRequests);
- clientConf2.setConnectionsPerBroker(20);
- pulsarClient2 = (PulsarClientImpl) PulsarClient.create(lookupUrl, clientConf2);
+ pulsarClient = (PulsarClientImpl) PulsarClient.builder().serviceUrl(lookupUrl)
+ .statsInterval(0, TimeUnit.SECONDS).maxNumberOfRejectedRequestPerConnection(0).build();
+
+ pulsarClient2 = (PulsarClientImpl) PulsarClient.builder().serviceUrl(lookupUrl)
+ .statsInterval(0, TimeUnit.SECONDS).ioThreads(concurrentLookupRequests).connectionsPerBroker(20)
+ .build();
- ProducerImpl producer = (ProducerImpl) pulsarClient.createProducer(topicName);
+ ProducerImpl<byte[]> producer = (ProducerImpl<byte[]>) pulsarClient.newProducer().topic(topicName).create();
ClientCnx cnx = producer.cnx();
assertTrue(cnx.channel().isActive());
ExecutorService executor = Executors.newFixedThreadPool(concurrentLookupRequests);
- List<CompletableFuture<Producer>> futures = Lists.newArrayList();
+ List<CompletableFuture<Producer<byte[]>>> futures = Lists.newArrayList();
final int totalProducers = 10;
CountDownLatch latch = new CountDownLatch(totalProducers);
- final ProducerConfiguration config1 = new ProducerConfiguration();
for (int i = 0; i < totalProducers; i++) {
executor.submit(() -> {
final String randomTopicName1 = topicName + randomUUID().toString();
final String randomTopicName2 = topicName + randomUUID().toString();
// pass producer-name to avoid exception: producer is already connected to topic
- futures.add(pulsarClient2.createProducerAsync(randomTopicName1, config1));
- futures.add(pulsarClient.createProducerAsync(randomTopicName2, config1));
+ futures.add(pulsarClient2.newProducer().topic(randomTopicName1).createAsync());
+ futures.add(pulsarClient.newProducer().topic(randomTopicName2).createAsync());
latch.countDown();
});
}
@@ -626,6 +606,7 @@ public void testMaxConcurrentTopicLoading() throws Exception {
pulsar.getConfiguration().setMaxConcurrentTopicLoadRequest(concurrentTopic);
}
}
+
/**
* It verifies that client closes the connection on internalSerevrError which is "ServiceNotReady" from Broker-side
*
@@ -638,22 +619,20 @@ public void testCloseConnectionOnInternalServerError() throws Exception {
final String topicName = "persistent://prop/usw/my-ns/newTopic";
- ClientConfiguration clientConf = new ClientConfiguration();
- clientConf.setStatsInterval(0, TimeUnit.SECONDS);
String lookupUrl = new URI("pulsar://localhost:" + BROKER_PORT).toString();
- pulsarClient = PulsarClient.create(lookupUrl, clientConf);
+ pulsarClient = PulsarClient.builder().serviceUrl(lookupUrl).statsInterval(0, TimeUnit.SECONDS).build();
- ProducerImpl producer = (ProducerImpl) pulsarClient.createProducer(topicName);
+ ProducerImpl<byte[]> producer = (ProducerImpl<byte[]>) pulsarClient.newProducer().topic(topicName).create();
ClientCnx cnx = producer.cnx();
assertTrue(cnx.channel().isActive());
-
+
// Need broker to throw InternalServerError. so, make global-zk unavailable
Field globalZkCacheField = PulsarService.class.getDeclaredField("globalZkCache");
globalZkCacheField.setAccessible(true);
globalZkCacheField.set(pulsar, null);
-
+
try {
- pulsarClient.createProducer(topicName);
+ pulsarClient.newProducer().topic(topicName).create();
fail("it should have fail with lookup-exception:");
} catch (Exception e) {
// ok
@@ -668,8 +647,7 @@ public void testInvalidDynamicConfiguration() throws Exception {
// (1) try to update invalid loadManagerClass name
try {
- admin.brokers().updateDynamicConfiguration("loadManagerClassName",
- "org.apache.pulsar.invalid.loadmanager");
+ admin.brokers().updateDynamicConfiguration("loadManagerClassName", "org.apache.pulsar.invalid.loadmanager");
fail("it should have failed due to invalid argument");
} catch (PulsarAdminException e) {
// Ok: should have failed due to invalid config value
@@ -687,8 +665,7 @@ public void testInvalidDynamicConfiguration() throws Exception {
ZooKeeperDataCache<Map<String, String>> dynamicConfigurationCache = pulsar.getBrokerService()
.getDynamicConfigurationCache();
- Map<String, String> configurationMap = dynamicConfigurationCache.get(BROKER_SERVICE_CONFIGURATION_PATH)
- .get();
+ Map<String, String> configurationMap = dynamicConfigurationCache.get(BROKER_SERVICE_CONFIGURATION_PATH).get();
configurationMap.put("loadManagerClassName", "org.apache.pulsar.invalid.loadmanager");
byte[] content = ObjectMapperFactory.getThreadLocal().writeValueAsBytes(configurationMap);
dynamicConfigurationCache.invalidate(BROKER_SERVICE_CONFIGURATION_PATH);
@@ -703,7 +680,7 @@ public void testInvalidDynamicConfiguration() throws Exception {
}
}
- private static class TimestampEntryCount {
+ static class TimestampEntryCount {
private final long timestamp;
private int numMessages;
@@ -725,29 +702,24 @@ public long getTimestamp() {
public void testCleanProducer() throws Exception {
log.info("-- Starting {} test --", methodName);
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setSubscriptionType(SubscriptionType.Exclusive);
- ProducerConfiguration producerConf = new ProducerConfiguration();
admin.clusters().createCluster("global", new ClusterData());
admin.namespaces().createNamespace("my-property/global/lookup");
- ClientConfiguration clientConf = new ClientConfiguration();
final int operationTimeOut = 500;
- clientConf.setStatsInterval(0, TimeUnit.SECONDS);
- clientConf.setOperationTimeout(operationTimeOut, TimeUnit.MILLISECONDS);
- PulsarClient pulsarClient = PulsarClient.create(lookupUrl.toString(), clientConf);
+ PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(lookupUrl.toString())
+ .statsInterval(0, TimeUnit.SECONDS).operationTimeout(operationTimeOut, TimeUnit.MILLISECONDS).build();
CountDownLatch latch = new CountDownLatch(1);
- pulsarClient.createProducerAsync("persistent://my-property/global/lookup/my-topic1", producerConf)
+ pulsarClient.newProducer().topic("persistent://my-property/global/lookup/my-topic1").createAsync()
.handle((producer, e) -> {
latch.countDown();
return null;
});
- latch.await(operationTimeOut+1000, TimeUnit.MILLISECONDS);
+ latch.await(operationTimeOut + 1000, TimeUnit.MILLISECONDS);
Field prodField = PulsarClientImpl.class.getDeclaredField("producers");
prodField.setAccessible(true);
@SuppressWarnings("unchecked")
- IdentityHashMap<ProducerBase, Boolean> producers = (IdentityHashMap<ProducerBase, Boolean>) prodField
+ IdentityHashMap<ProducerBase<byte[]>, Boolean> producers = (IdentityHashMap<ProducerBase<byte[]>, Boolean>) prodField
.get(pulsarClient);
assertTrue(producers.isEmpty());
pulsarClient.close();
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConsumeBaseExceptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConsumeBaseExceptionTest.java
index fafc541c1..a71fc2019 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConsumeBaseExceptionTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConsumeBaseExceptionTest.java
@@ -19,8 +19,6 @@
package org.apache.pulsar.client.impl;
import org.apache.pulsar.client.api.Consumer;
-import org.apache.pulsar.client.api.ConsumerConfiguration;
-import org.apache.pulsar.client.api.Message;
import org.apache.pulsar.client.api.ProducerConsumerBase;
import org.apache.pulsar.client.api.PulsarClientException;
import org.testng.Assert;
@@ -45,8 +43,8 @@ protected void cleanup() throws Exception {
@Test
public void testClosedConsumer() throws PulsarClientException {
- Consumer consumer = null;
- consumer = pulsarClient.subscribe("persistent://prop/cluster/ns/topicName", "my-subscription");
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://prop/cluster/ns/topicName")
+ .subscriptionName("my-subscription").subscribe();
consumer.close();
Assert.assertTrue(consumer.receiveAsync().isCompletedExceptionally());
@@ -62,11 +60,11 @@ public void testClosedConsumer() throws PulsarClientException {
@Test
public void testListener() throws PulsarClientException {
- Consumer consumer = null;
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setMessageListener((Consumer c, Message msg) -> {
- });
- consumer = pulsarClient.subscribe("persistent://prop/cluster/ns/topicName", "my-subscription", conf);
+
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://prop/cluster/ns/topicName")
+ .subscriptionName("my-subscription").messageListener((consumer1, msg) -> {
+
+ }).subscribe();
Assert.assertTrue(consumer.receiveAsync().isCompletedExceptionally());
try {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConsumerConfigurationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConsumerConfigurationTest.java
index 4b85a08ee..e070bf906 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConsumerConfigurationTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConsumerConfigurationTest.java
@@ -18,26 +18,19 @@
*/
package org.apache.pulsar.client.impl;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
-import org.apache.pulsar.broker.service.persistent.PersistentTopic;
-import org.apache.pulsar.client.api.ConsumerConfiguration;
import org.apache.pulsar.client.api.PulsarClientException.InvalidConfigurationException;
import org.apache.pulsar.client.api.SubscriptionType;
-
import org.apache.pulsar.common.policies.data.ClusterData;
import org.apache.pulsar.common.policies.data.PropertyAdmin;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
public class ConsumerConfigurationTest extends MockedPulsarServiceBaseTest {
- private static final Logger log = LoggerFactory.getLogger(ConsumerConfigurationTest.class);
private static String persistentTopic = "persistent://my-property/use/my-ns/persist";
private static String nonPersistentTopic = "non-persistent://my-property/use/my-ns/nopersist";
@@ -46,8 +39,7 @@
public void setup() throws Exception {
super.internalSetup();
- admin.clusters().createCluster("use",
- new ClusterData("http://127.0.0.1:" + BROKER_WEBSERVICE_PORT));
+ admin.clusters().createCluster("use", new ClusterData("http://127.0.0.1:" + BROKER_WEBSERVICE_PORT));
admin.properties().createProperty("my-property",
new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet("use")));
admin.namespaces().createNamespace("my-property/use/my-ns");
@@ -61,29 +53,25 @@ public void cleanup() throws Exception {
@Test
public void testReadCompactPersistentExclusive() throws Exception {
- ConsumerConfiguration conf = new ConsumerConfiguration().setReadCompacted(true);
- conf.setSubscriptionType(SubscriptionType.Exclusive);
- pulsarClient.subscribe(persistentTopic, "sub1", conf).close();
+ pulsarClient.newConsumer().topic(persistentTopic).subscriptionName("sub1").readCompacted(true)
+ .subscriptionType(SubscriptionType.Exclusive).subscribe().close();
}
@Test
public void testReadCompactPersistentFailover() throws Exception {
- ConsumerConfiguration conf = new ConsumerConfiguration().setReadCompacted(true);
- conf.setSubscriptionType(SubscriptionType.Failover);
- pulsarClient.subscribe(persistentTopic, "sub1", conf).close();
+ pulsarClient.newConsumer().topic(persistentTopic).subscriptionName("sub1").readCompacted(true)
+ .subscriptionType(SubscriptionType.Failover).subscribe().close();
}
- @Test(expectedExceptions=InvalidConfigurationException.class)
+ @Test(expectedExceptions = InvalidConfigurationException.class)
public void testReadCompactPersistentShared() throws Exception {
- ConsumerConfiguration conf = new ConsumerConfiguration().setReadCompacted(true);
- conf.setSubscriptionType(SubscriptionType.Shared);
- pulsarClient.subscribe(persistentTopic, "sub1", conf).close();
+ pulsarClient.newConsumer().topic(persistentTopic).subscriptionName("sub1").readCompacted(true)
+ .subscriptionType(SubscriptionType.Shared).subscribe().close();
}
- @Test(expectedExceptions=InvalidConfigurationException.class)
+ @Test(expectedExceptions = InvalidConfigurationException.class)
public void testReadCompactNonPersistentExclusive() throws Exception {
- ConsumerConfiguration conf = new ConsumerConfiguration().setReadCompacted(true);
- conf.setSubscriptionType(SubscriptionType.Exclusive);
- pulsarClient.subscribe(nonPersistentTopic, "sub1", conf).close();
+ pulsarClient.newConsumer().topic(nonPersistentTopic).subscriptionName("sub1").readCompacted(true)
+ .subscriptionType(SubscriptionType.Exclusive).subscribe().close();
}
}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageIdTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageIdTest.java
index 30e54a51b..323000622 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageIdTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageIdTest.java
@@ -42,13 +42,11 @@
import org.apache.pulsar.broker.service.BrokerTestBase;
import org.apache.pulsar.client.admin.PulsarAdminException;
-import org.apache.pulsar.client.api.ClientConfiguration;
import org.apache.pulsar.client.api.Consumer;
import org.apache.pulsar.client.api.Message;
import org.apache.pulsar.client.api.MessageBuilder;
import org.apache.pulsar.client.api.MessageId;
import org.apache.pulsar.client.api.Producer;
-import org.apache.pulsar.client.api.ProducerConfiguration;
import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.client.impl.ProducerImpl.OpSendMsg;
import org.apache.pulsar.client.impl.conf.ClientConfigurationData;
@@ -91,14 +89,15 @@ public void producerSendAsync() throws PulsarClientException {
final int numberOfMessages = 30;
// 2. Create Producer
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
// 3. Create Consumer
- Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
+ .subscribe();
// 4. Publish message and get message id
- Set<MessageId> messageIds = new HashSet();
- List<Future<MessageId>> futures = new ArrayList();
+ Set<MessageId> messageIds = new HashSet<>();
+ List<Future<MessageId>> futures = new ArrayList<>();
for (int i = 0; i < numberOfMessages; i++) {
String message = messagePredicate + i;
futures.add(producer.sendAsync(message.getBytes()));
@@ -124,7 +123,7 @@ public void producerSendAsync() throws PulsarClientException {
Assert.assertEquals(messageIds.size(), numberOfMessages, "Not all messages published successfully");
for (int i = 0; i < numberOfMessages; i++) {
- Message message = consumer.receive();
+ Message<byte[]> message = consumer.receive();
Assert.assertEquals(new String(message.getData()), messagePredicate + i);
MessageId messageId = message.getMessageId();
Assert.assertTrue(messageIds.remove(messageId), "Failed to receive message");
@@ -144,13 +143,14 @@ public void producerSend() throws PulsarClientException {
final int numberOfMessages = 30;
// 2. Create Producer
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
// 3. Create Consumer
- Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
+ .subscribe();
// 4. Publish message and get message id
- Set<MessageId> messageIds = new HashSet();
+ Set<MessageId> messageIds = new HashSet<>();
for (int i = 0; i < numberOfMessages; i++) {
String message = messagePredicate + i;
messageIds.add(producer.send(message.getBytes()));
@@ -181,14 +181,15 @@ public void partitionedProducerSendAsync() throws PulsarClientException, PulsarA
admin.persistentTopics().createPartitionedTopic(topicName, numberOfPartitions);
// 2. Create Producer
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
// 3. Create Consumer
- Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
+ .subscribe();
// 4. Publish message and get message id
- Set<MessageId> messageIds = new HashSet();
- Set<Future<MessageId>> futures = new HashSet();
+ Set<MessageId> messageIds = new HashSet<>();
+ Set<Future<MessageId>> futures = new HashSet<>();
for (int i = 0; i < numberOfMessages; i++) {
String message = messagePredicate + i;
futures.add(producer.sendAsync(message.getBytes()));
@@ -228,13 +229,14 @@ public void partitionedProducerSend() throws PulsarClientException, PulsarAdminE
admin.persistentTopics().createPartitionedTopic(topicName, numberOfPartitions);
// 2. Create Producer
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
// 3. Create Consumer
- Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
+ .subscribe();
// 4. Publish message and get message id
- Set<MessageId> messageIds = new HashSet();
+ Set<MessageId> messageIds = new HashSet<>();
for (int i = 0; i < numberOfMessages; i++) {
String message = messagePredicate + i;
messageIds.add(producer.send(message.getBytes()));
@@ -255,19 +257,17 @@ public void partitionedProducerSend() throws PulsarClientException, PulsarAdminE
}
/**
- * Verifies: different versions of broker-deployment (one broker understands Checksum and other
- * doesn't in that case remove checksum before sending to broker-2)
+ * Verifies: different versions of broker-deployment (one broker understands Checksum and other doesn't in that case
+ * remove checksum before sending to broker-2)
*
- * client first produce message with checksum and then retries to send message due to connection unavailable. But this time, if
- * broker doesn't understand checksum: then client should remove checksum from the message before sending to broker.
+ * client first produce message with checksum and then retries to send message due to connection unavailable. But
+ * this time, if broker doesn't understand checksum: then client should remove checksum from the message before
+ * sending to broker.
*
- * 1. stop broker
- * 2. client compute checksum and add into message
- * 3. produce 2 messages and corrupt 1 message
- * 4. start broker with lower version (which doesn't support checksum)
- * 5. client reconnects to broker and due to incompatibility of version: removes checksum from message
- * 6. broker doesn't do checksum validation and persist message
- * 7. client receives ack
+ * 1. stop broker 2. client compute checksum and add into message 3. produce 2 messages and corrupt 1 message 4.
+ * start broker with lower version (which doesn't support checksum) 5. client reconnects to broker and due to
+ * incompatibility of version: removes checksum from message 6. broker doesn't do checksum validation and persist
+ * message 7. client receives ack
*
* @throws Exception
*/
@@ -276,15 +276,15 @@ public void testChecksumVersionComptability() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/topic1";
// 1. producer connect
- ProducerImpl prod = (ProducerImpl) pulsarClient.createProducer(topicName);
- ProducerImpl producer = spy(prod);
+ ProducerImpl<byte[]> prod = (ProducerImpl<byte[]>) pulsarClient.newProducer().topic(topicName).create();
+ ProducerImpl<byte[]> producer = spy(prod);
// return higher version compare to broker : so, it forces client-producer to remove checksum from payload
doReturn(producer.brokerChecksumSupportedVersion() + 1).when(producer).brokerChecksumSupportedVersion();
doAnswer(invocationOnMock -> prod.getState()).when(producer).getState();
doAnswer(invocationOnMock -> prod.getClientCnx()).when(producer).getClientCnx();
doAnswer(invocationOnMock -> prod.cnx()).when(producer).cnx();
- Consumer consumer = pulsarClient.subscribe(topicName, "my-sub");
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").subscribe();
// Stop the broker, and publishes messages. Messages are accumulated in the producer queue and they're checksums
// would have already been computed. If we change the message content at that point, it should result in a
@@ -300,10 +300,10 @@ public void testChecksumVersionComptability() throws Exception {
doReturn(producer.brokerChecksumSupportedVersion() - 1).when(mockClientCnx).getRemoteEndpointProtocolVersion();
prod.setClientCnx(mockClientCnx);
- Message msg1 = MessageBuilder.create().setContent("message-1".getBytes()).build();
+ Message<byte[]> msg1 = MessageBuilder.create().setContent("message-1".getBytes()).build();
CompletableFuture<MessageId> future1 = producer.sendAsync(msg1);
- Message msg2 = MessageBuilder.create().setContent("message-2".getBytes()).build();
+ Message<byte[]> msg2 = MessageBuilder.create().setContent("message-2".getBytes()).build();
CompletableFuture<MessageId> future2 = producer.sendAsync(msg2);
// corrupt the message
@@ -327,9 +327,9 @@ public void testChecksumVersionComptability() throws Exception {
fail("Broker shouldn't verify checksum for corrupted message and it shouldn't fail");
}
- ((ConsumerImpl) consumer).grabCnx();
+ ((ConsumerImpl<byte[]>) consumer).grabCnx();
// We should only receive msg1
- Message msg = consumer.receive(1, TimeUnit.SECONDS);
+ Message<byte[]> msg = consumer.receive(1, TimeUnit.SECONDS);
assertEquals(new String(msg.getData()), "message-1");
msg = consumer.receive(1, TimeUnit.SECONDS);
assertEquals(new String(msg.getData()), "message-3");
@@ -341,8 +341,8 @@ public void testChecksumReconnection() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/topic1";
// 1. producer connect
- ProducerImpl prod = (ProducerImpl) pulsarClient.createProducer(topicName);
- ProducerImpl producer = spy(prod);
+ ProducerImpl<byte[]> prod = (ProducerImpl<byte[]>) pulsarClient.newProducer().topic(topicName).create();
+ ProducerImpl<byte[]> producer = spy(prod);
// mock: broker-doesn't support checksum (remote_version < brokerChecksumSupportedVersion) so, it forces
// client-producer to perform checksum-strip from msg at reconnection
doReturn(producer.brokerChecksumSupportedVersion() + 1).when(producer).brokerChecksumSupportedVersion();
@@ -350,7 +350,7 @@ public void testChecksumReconnection() throws Exception {
doAnswer(invocationOnMock -> prod.getClientCnx()).when(producer).getClientCnx();
doAnswer(invocationOnMock -> prod.cnx()).when(producer).cnx();
- Consumer consumer = pulsarClient.subscribe(topicName, "my-sub");
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").subscribe();
stopBroker();
@@ -365,10 +365,10 @@ public void testChecksumReconnection() throws Exception {
doReturn(producer.brokerChecksumSupportedVersion() - 1).when(mockClientCnx).getRemoteEndpointProtocolVersion();
prod.setClientCnx(mockClientCnx);
- Message msg1 = MessageBuilder.create().setContent("message-1".getBytes()).build();
+ Message<byte[]> msg1 = MessageBuilder.create().setContent("message-1".getBytes()).build();
CompletableFuture<MessageId> future1 = producer.sendAsync(msg1);
- Message msg2 = MessageBuilder.create().setContent("message-2".getBytes()).build();
+ Message<byte[]> msg2 = MessageBuilder.create().setContent("message-2".getBytes()).build();
CompletableFuture<MessageId> future2 = producer.sendAsync(msg2);
// corrupt the message
@@ -395,22 +395,19 @@ public void testChecksumReconnection() throws Exception {
fail("Broker shouldn't verify checksum for corrupted message and it shouldn't fail");
}
- ((ConsumerImpl) consumer).grabCnx();
+ ((ConsumerImpl<byte[]>) consumer).grabCnx();
// We should only receive msg1
- Message msg = consumer.receive(1, TimeUnit.SECONDS);
+ Message<byte[]> msg = consumer.receive(1, TimeUnit.SECONDS);
assertEquals(new String(msg.getData()), "message-1");
msg = consumer.receive(1, TimeUnit.SECONDS);
assertEquals(new String(msg.getData()), "message-3");
}
-
/**
- * Verifies: if message is corrupted before sending to broker and if broker gives checksum error: then
- * 1. Client-Producer recomputes checksum with modified data
- * 2. Retry message-send again
- * 3. Broker verifies checksum
- * 4. client receives send-ack success
+ * Verifies: if message is corrupted before sending to broker and if broker gives checksum error: then 1.
+ * Client-Producer recomputes checksum with modified data 2. Retry message-send again 3. Broker verifies checksum 4.
+ * client receives send-ack success
*
* @throws Exception
*/
@@ -419,16 +416,15 @@ public void testCorruptMessageRemove() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/retry-topic";
- ProducerConfiguration config = new ProducerConfiguration();
- config.setSendTimeout(10, TimeUnit.MINUTES);
// 1. producer connect
- Producer prod = pulsarClient.createProducer(topicName, config);
- ProducerImpl producer = spy((ProducerImpl) prod);
+ ProducerImpl<byte[]> prod = (ProducerImpl<byte[]>) pulsarClient.newProducer().topic(topicName)
+ .sendTimeout(10, TimeUnit.MINUTES).create();
+ ProducerImpl<byte[]> producer = spy(prod);
Field producerIdField = ProducerImpl.class.getDeclaredField("producerId");
producerIdField.setAccessible(true);
long producerId = (long) producerIdField.get(producer);
producer.cnx().registerProducer(producerId, producer); // registered spy ProducerImpl
- Consumer consumer = pulsarClient.subscribe(topicName, "my-sub");
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").subscribe();
// 2. Stop the broker, and publishes messages. Messages are accumulated in the producer queue and they're
// checksums
@@ -437,7 +433,7 @@ public void testCorruptMessageRemove() throws Exception {
// enable checksum at producer
stopBroker();
- Message msg = MessageBuilder.create().setContent("message-1".getBytes()).build();
+ Message<byte[]> msg = MessageBuilder.create().setContent("message-1".getBytes()).build();
CompletableFuture<MessageId> future = producer.sendAsync(msg);
// 3. corrupt the message
@@ -451,7 +447,7 @@ public void testCorruptMessageRemove() throws Exception {
fail("send message should have failed with checksum excetion");
} catch (Exception e) {
if (e.getCause() instanceof PulsarClientException.ChecksumException) {
- //ok (callback should get checksum exception as message was modified and corrupt)
+ // ok (callback should get checksum exception as message was modified and corrupt)
} else {
fail("Callback should have only failed with ChecksumException", e);
}
@@ -463,19 +459,18 @@ public void testCorruptMessageRemove() throws Exception {
verify(producer, times(1)).recoverChecksumError(any(), anyLong());
verify(producer, times(1)).verifyLocalBufferIsNotCorrupted(any());
-
/**
- * (5.3) verify: ProducerImpl.verifyLocalBufferIsNotCorrupted() => validates if message
- * is corrupt
+ * (5.3) verify: ProducerImpl.verifyLocalBufferIsNotCorrupted() => validates if message is corrupt
*/
- MessageImpl msg2 = (MessageImpl) MessageBuilder.create().setContent("message-1".getBytes()).build();
+ MessageImpl<byte[]> msg2 = (MessageImpl<byte[]>) MessageBuilder.create().setContent("message-1".getBytes())
+ .build();
ByteBuf payload = msg2.getDataBuffer();
- Builder metadataBuilder = ((MessageImpl) msg).getMessageBuilder();
+ Builder metadataBuilder = ((MessageImpl<byte[]>) msg).getMessageBuilder();
MessageMetadata msgMetadata = metadataBuilder.setProducerName("test").setSequenceId(1).setPublishTime(10L)
.build();
ByteBufPair cmd = Commands.newSend(producerId, 1, 1, ChecksumType.Crc32c, msgMetadata, payload);
// (a) create OpSendMsg with message-data : "message-1"
- OpSendMsg op = OpSendMsg.create(((MessageImpl) msg), cmd, 1, null);
+ OpSendMsg op = OpSendMsg.create(((MessageImpl<byte[]>) msg), cmd, 1, null);
// a.verify: as message is not corrupt: no need to update checksum
assertTrue(producer.verifyLocalBufferIsNotCorrupted(op));
// (b) corrupt message
@@ -487,7 +482,8 @@ public void testCorruptMessageRemove() throws Exception {
// [2] test-recoverChecksumError functionality
stopBroker();
- MessageImpl msg1 = (MessageImpl) MessageBuilder.create().setContent("message-1".getBytes()).build();
+ MessageImpl<byte[]> msg1 = (MessageImpl<byte[]>) MessageBuilder.create().setContent("message-1".getBytes())
+ .build();
future = producer.sendAsync(msg1);
ClientCnx cnx = spy(
new ClientCnx(new ClientConfigurationData(), ((PulsarClientImpl) pulsarClient).eventLoopGroup()));
@@ -497,7 +493,7 @@ public void testCorruptMessageRemove() throws Exception {
try {
producer.recoverChecksumError(cnx, 1);
fail("it should call : resendMessages() => which should throw above mocked exception");
- }catch(IllegalStateException e) {
+ } catch (IllegalStateException e) {
assertEquals(exc, e.getMessage());
}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplTest.java
index c82c52587..4f9cc7002 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplTest.java
@@ -24,11 +24,11 @@
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
-import com.google.common.collect.Lists;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
import java.util.stream.IntStream;
+
import org.apache.pulsar.broker.namespace.NamespaceService;
import org.apache.pulsar.client.api.Consumer;
import org.apache.pulsar.client.api.Message;
@@ -44,6 +44,8 @@
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
+import com.google.common.collect.Lists;
+
public class PatternTopicsConsumerImplTest extends ProducerConsumerBase {
private static final long testTimeout = 90000; // 1.5 min
private static final Logger log = LoggerFactory.getLogger(PatternTopicsConsumerImplTest.class);
@@ -80,7 +82,7 @@ public void testPatternTopicsSubscribeWithBuilderFail() throws Exception {
// test failing builder with pattern and topic should fail
try {
- Consumer consumer1 = pulsarClient.newConsumer()
+ pulsarClient.newConsumer()
.topicsPattern(pattern)
.topic(topicName1)
.subscriptionName(subscriptionName)
@@ -94,7 +96,7 @@ public void testPatternTopicsSubscribeWithBuilderFail() throws Exception {
// test failing builder with pattern and topics should fail
try {
- Consumer consumer2 = pulsarClient.newConsumer()
+ pulsarClient.newConsumer()
.topicsPattern(pattern)
.topics(topicNames)
.subscriptionName(subscriptionName)
@@ -108,7 +110,7 @@ public void testPatternTopicsSubscribeWithBuilderFail() throws Exception {
// test failing builder with pattern and patternString should fail
try {
- Consumer consumer3 = pulsarClient.newConsumer()
+ pulsarClient.newConsumer()
.topicsPattern(pattern)
.topicsPattern(patternString)
.subscriptionName(subscriptionName)
@@ -140,16 +142,16 @@ public void testBinaryProtoToGetTopicsOfNamespace() throws Exception {
String messagePredicate = "my-message-" + key + "-";
int totalMessages = 30;
- Producer producer1 = pulsarClient.newProducer().topic(topicName1)
+ Producer<byte[]> producer1 = pulsarClient.newProducer().topic(topicName1)
.create();
- Producer producer2 = pulsarClient.newProducer().topic(topicName2)
+ Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topicName2)
.messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition)
.create();
- Producer producer3 = pulsarClient.newProducer().topic(topicName3)
+ Producer<byte[]> producer3 = pulsarClient.newProducer().topic(topicName3)
.messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition)
.create();
- Consumer consumer = pulsarClient.newConsumer()
+ Consumer<byte[]> consumer = pulsarClient.newConsumer()
.topicsPattern(pattern)
.patternAutoDiscoveryPeriod(2)
.subscriptionName(subscriptionName)
@@ -159,13 +161,13 @@ public void testBinaryProtoToGetTopicsOfNamespace() throws Exception {
.subscribe();
// 4. verify consumer get methods, to get right number of partitions and topics.
- assertSame(pattern, ((PatternTopicsConsumerImpl) consumer).getPattern());
- List<String> topics = ((PatternTopicsConsumerImpl) consumer).getPartitionedTopics();
- List<ConsumerImpl> consumers = ((PatternTopicsConsumerImpl) consumer).getConsumers();
+ assertSame(pattern, ((PatternTopicsConsumerImpl<?>) consumer).getPattern());
+ List<String> topics = ((PatternTopicsConsumerImpl<?>) consumer).getPartitionedTopics();
+ List<ConsumerImpl<byte[]>> consumers = ((PatternTopicsConsumerImpl<byte[]>) consumer).getConsumers();
assertEquals(topics.size(), 6);
assertEquals(consumers.size(), 6);
- assertEquals(((PatternTopicsConsumerImpl) consumer).getTopics().size(), 3);
+ assertEquals(((PatternTopicsConsumerImpl<?>) consumer).getTopics().size(), 3);
topics.forEach(topic -> log.debug("topic: {}", topic));
consumers.forEach(c -> log.debug("consumer: {}", c.getTopic()));
@@ -173,7 +175,7 @@ public void testBinaryProtoToGetTopicsOfNamespace() throws Exception {
IntStream.range(0, topics.size()).forEach(index ->
assertTrue(topics.get(index).equals(consumers.get(index).getTopic())));
- ((PatternTopicsConsumerImpl) consumer).getTopics().forEach(topic -> log.debug("getTopics topic: {}", topic));
+ ((PatternTopicsConsumerImpl<?>) consumer).getTopics().forEach(topic -> log.debug("getTopics topic: {}", topic));
// 5. produce data
for (int i = 0; i < totalMessages / 3; i++) {
@@ -184,7 +186,7 @@ public void testBinaryProtoToGetTopicsOfNamespace() throws Exception {
// 6. should receive all the message
int messageSet = 0;
- Message message = consumer.receive();
+ Message<byte[]> message = consumer.receive();
do {
assertTrue(message instanceof TopicMessageImpl);
messageSet ++;
@@ -278,7 +280,7 @@ public void testStartEmptyPatternConsumer() throws Exception {
admin.persistentTopics().createPartitionedTopic(topicName3, 3);
// 2. Create consumer, this should success, but with empty sub-consumser internal
- Consumer consumer = pulsarClient.newConsumer()
+ Consumer<byte[]> consumer = pulsarClient.newConsumer()
.topicsPattern(pattern)
.patternAutoDiscoveryPeriod(2)
.subscriptionName(subscriptionName)
@@ -288,35 +290,35 @@ public void testStartEmptyPatternConsumer() throws Exception {
.subscribe();
// 3. verify consumer get methods, to get 0 number of partitions and topics.
- assertSame(pattern, ((PatternTopicsConsumerImpl) consumer).getPattern());
- assertEquals(((PatternTopicsConsumerImpl) consumer).getPartitionedTopics().size(), 0);
- assertEquals(((PatternTopicsConsumerImpl) consumer).getConsumers().size(), 0);
- assertEquals(((PatternTopicsConsumerImpl) consumer).getTopics().size(), 0);
+ assertSame(pattern, ((PatternTopicsConsumerImpl<?>) consumer).getPattern());
+ assertEquals(((PatternTopicsConsumerImpl<?>) consumer).getPartitionedTopics().size(), 0);
+ assertEquals(((PatternTopicsConsumerImpl<?>) consumer).getConsumers().size(), 0);
+ assertEquals(((PatternTopicsConsumerImpl<?>) consumer).getTopics().size(), 0);
// 4. create producer
String messagePredicate = "my-message-" + key + "-";
int totalMessages = 30;
- Producer producer1 = pulsarClient.newProducer().topic(topicName1)
+ Producer<byte[]> producer1 = pulsarClient.newProducer().topic(topicName1)
.create();
- Producer producer2 = pulsarClient.newProducer().topic(topicName2)
+ Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topicName2)
.messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition)
.create();
- Producer producer3 = pulsarClient.newProducer().topic(topicName3)
+ Producer<byte[]> producer3 = pulsarClient.newProducer().topic(topicName3)
.messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition)
.create();
// 5. call recheckTopics to subscribe each added topics above
log.debug("recheck topics change");
- PatternTopicsConsumerImpl consumer1 = ((PatternTopicsConsumerImpl) consumer);
+ PatternTopicsConsumerImpl<byte[]> consumer1 = ((PatternTopicsConsumerImpl<byte[]>) consumer);
consumer1.run(consumer1.getRecheckPatternTimeout());
Thread.sleep(100);
// 6. verify consumer get methods, to get number of partitions and topics, value 6=1+2+3.
- assertSame(pattern, ((PatternTopicsConsumerImpl) consumer).getPattern());
- assertEquals(((PatternTopicsConsumerImpl) consumer).getPartitionedTopics().size(), 6);
- assertEquals(((PatternTopicsConsumerImpl) consumer).getConsumers().size(), 6);
- assertEquals(((PatternTopicsConsumerImpl) consumer).getTopics().size(), 3);
+ assertSame(pattern, ((PatternTopicsConsumerImpl<?>) consumer).getPattern());
+ assertEquals(((PatternTopicsConsumerImpl<?>) consumer).getPartitionedTopics().size(), 6);
+ assertEquals(((PatternTopicsConsumerImpl<?>) consumer).getConsumers().size(), 6);
+ assertEquals(((PatternTopicsConsumerImpl<?>) consumer).getTopics().size(), 3);
// 7. produce data
@@ -328,7 +330,7 @@ public void testStartEmptyPatternConsumer() throws Exception {
// 8. should receive all the message
int messageSet = 0;
- Message message = consumer.receive();
+ Message<byte[]> message = consumer.receive();
do {
assertTrue(message instanceof TopicMessageImpl);
messageSet ++;
@@ -364,16 +366,16 @@ public void testAutoSubscribePatternConsumer() throws Exception {
String messagePredicate = "my-message-" + key + "-";
int totalMessages = 30;
- Producer producer1 = pulsarClient.newProducer().topic(topicName1)
+ Producer<byte[]> producer1 = pulsarClient.newProducer().topic(topicName1)
.create();
- Producer producer2 = pulsarClient.newProducer().topic(topicName2)
+ Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topicName2)
.messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition)
.create();
- Producer producer3 = pulsarClient.newProducer().topic(topicName3)
+ Producer<byte[]> producer3 = pulsarClient.newProducer().topic(topicName3)
.messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition)
.create();
- Consumer consumer = pulsarClient.newConsumer()
+ Consumer<byte[]> consumer = pulsarClient.newConsumer()
.topicsPattern(pattern)
.patternAutoDiscoveryPeriod(2)
.subscriptionName(subscriptionName)
@@ -385,10 +387,10 @@ public void testAutoSubscribePatternConsumer() throws Exception {
assertTrue(consumer instanceof PatternTopicsConsumerImpl);
// 4. verify consumer get methods, to get 6 number of partitions and topics: 6=1+2+3
- assertSame(pattern, ((PatternTopicsConsumerImpl) consumer).getPattern());
- assertEquals(((PatternTopicsConsumerImpl) consumer).getPartitionedTopics().size(), 6);
- assertEquals(((PatternTopicsConsumerImpl) consumer).getConsumers().size(), 6);
- assertEquals(((PatternTopicsConsumerImpl) consumer).getTopics().size(), 3);
+ assertSame(pattern, ((PatternTopicsConsumerImpl<?>) consumer).getPattern());
+ assertEquals(((PatternTopicsConsumerImpl<?>) consumer).getPartitionedTopics().size(), 6);
+ assertEquals(((PatternTopicsConsumerImpl<?>) consumer).getConsumers().size(), 6);
+ assertEquals(((PatternTopicsConsumerImpl<?>) consumer).getTopics().size(), 3);
// 5. produce data to topic 1,2,3; verify should receive all the message
for (int i = 0; i < totalMessages / 3; i++) {
@@ -398,7 +400,7 @@ public void testAutoSubscribePatternConsumer() throws Exception {
}
int messageSet = 0;
- Message message = consumer.receive();
+ Message<byte[]> message = consumer.receive();
do {
assertTrue(message instanceof TopicMessageImpl);
messageSet ++;
@@ -411,18 +413,18 @@ public void testAutoSubscribePatternConsumer() throws Exception {
// 6. create another producer with 4 partitions
String topicName4 = "persistent://prop/use/ns-abc/pattern-topic-4-" + key;
admin.persistentTopics().createPartitionedTopic(topicName4, 4);
- Producer producer4 = pulsarClient.newProducer().topic(topicName4)
+ Producer<byte[]> producer4 = pulsarClient.newProducer().topic(topicName4)
.messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition)
.create();
// 7. call recheckTopics to subscribe each added topics above, verify topics number: 10=1+2+3+4
log.debug("recheck topics change");
- PatternTopicsConsumerImpl consumer1 = ((PatternTopicsConsumerImpl) consumer);
+ PatternTopicsConsumerImpl<byte[]> consumer1 = ((PatternTopicsConsumerImpl<byte[]>) consumer);
consumer1.run(consumer1.getRecheckPatternTimeout());
Thread.sleep(100);
- assertEquals(((PatternTopicsConsumerImpl) consumer).getPartitionedTopics().size(), 10);
- assertEquals(((PatternTopicsConsumerImpl) consumer).getConsumers().size(), 10);
- assertEquals(((PatternTopicsConsumerImpl) consumer).getTopics().size(), 4);
+ assertEquals(((PatternTopicsConsumerImpl<?>) consumer).getPartitionedTopics().size(), 10);
+ assertEquals(((PatternTopicsConsumerImpl<?>) consumer).getConsumers().size(), 10);
+ assertEquals(((PatternTopicsConsumerImpl<?>) consumer).getTopics().size(), 4);
// 8. produce data to topic3 and topic4, verify should receive all the message
for (int i = 0; i < totalMessages / 2; i++) {
@@ -467,16 +469,16 @@ public void testAutoUnbubscribePatternConsumer() throws Exception {
String messagePredicate = "my-message-" + key + "-";
int totalMessages = 30;
- Producer producer1 = pulsarClient.newProducer().topic(topicName1)
+ Producer<byte[]> producer1 = pulsarClient.newProducer().topic(topicName1)
.create();
- Producer producer2 = pulsarClient.newProducer().topic(topicName2)
+ Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topicName2)
.messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition)
.create();
- Producer producer3 = pulsarClient.newProducer().topic(topicName3)
+ Producer<byte[]> producer3 = pulsarClient.newProducer().topic(topicName3)
.messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition)
.create();
- Consumer consumer = pulsarClient.newConsumer()
+ Consumer<byte[]> consumer = pulsarClient.newConsumer()
.topicsPattern(pattern)
.patternAutoDiscoveryPeriod(2)
.subscriptionName(subscriptionName)
@@ -488,10 +490,10 @@ public void testAutoUnbubscribePatternConsumer() throws Exception {
assertTrue(consumer instanceof PatternTopicsConsumerImpl);
// 4. verify consumer get methods, to get 0 number of partitions and topics: 6=1+2+3
- assertSame(pattern, ((PatternTopicsConsumerImpl) consumer).getPattern());
- assertEquals(((PatternTopicsConsumerImpl) consumer).getPartitionedTopics().size(), 6);
- assertEquals(((PatternTopicsConsumerImpl) consumer).getConsumers().size(), 6);
- assertEquals(((PatternTopicsConsumerImpl) consumer).getTopics().size(), 3);
+ assertSame(pattern, ((PatternTopicsConsumerImpl<?>) consumer).getPattern());
+ assertEquals(((PatternTopicsConsumerImpl<?>) consumer).getPartitionedTopics().size(), 6);
+ assertEquals(((PatternTopicsConsumerImpl<?>) consumer).getConsumers().size(), 6);
+ assertEquals(((PatternTopicsConsumerImpl<?>) consumer).getTopics().size(), 3);
// 5. produce data to topic 1,2,3; verify should receive all the message
for (int i = 0; i < totalMessages / 3; i++) {
@@ -501,7 +503,7 @@ public void testAutoUnbubscribePatternConsumer() throws Exception {
}
int messageSet = 0;
- Message message = consumer.receive();
+ Message<byte[]> message = consumer.receive();
do {
assertTrue(message instanceof TopicMessageImpl);
messageSet ++;
@@ -519,12 +521,12 @@ public void testAutoUnbubscribePatternConsumer() throws Exception {
// 7. call recheckTopics to unsubscribe topic 1,3 , verify topics number: 2=6-1-3
log.debug("recheck topics change");
- PatternTopicsConsumerImpl consumer1 = ((PatternTopicsConsumerImpl) consumer);
+ PatternTopicsConsumerImpl<byte[]> consumer1 = ((PatternTopicsConsumerImpl<byte[]>) consumer);
consumer1.run(consumer1.getRecheckPatternTimeout());
Thread.sleep(100);
- assertEquals(((PatternTopicsConsumerImpl) consumer).getPartitionedTopics().size(), 2);
- assertEquals(((PatternTopicsConsumerImpl) consumer).getConsumers().size(), 2);
- assertEquals(((PatternTopicsConsumerImpl) consumer).getTopics().size(), 1);
+ assertEquals(((PatternTopicsConsumerImpl<byte[]>) consumer).getPartitionedTopics().size(), 2);
+ assertEquals(((PatternTopicsConsumerImpl<byte[]>) consumer).getConsumers().size(), 2);
+ assertEquals(((PatternTopicsConsumerImpl<byte[]>) consumer).getTopics().size(), 1);
// 8. produce data to topic2, verify should receive all the message
for (int i = 0; i < totalMessages; i++) {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PerMessageUnAcknowledgedRedeliveryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PerMessageUnAcknowledgedRedeliveryTest.java
index e9cecbb1a..7a94cc26e 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PerMessageUnAcknowledgedRedeliveryTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PerMessageUnAcknowledgedRedeliveryTest.java
@@ -24,14 +24,10 @@
import org.apache.pulsar.broker.service.BrokerTestBase;
import org.apache.pulsar.client.api.Consumer;
-import org.apache.pulsar.client.api.ConsumerConfiguration;
import org.apache.pulsar.client.api.Message;
+import org.apache.pulsar.client.api.MessageRoutingMode;
import org.apache.pulsar.client.api.Producer;
-import org.apache.pulsar.client.api.ProducerConfiguration;
import org.apache.pulsar.client.api.SubscriptionType;
-import org.apache.pulsar.client.api.ProducerConfiguration.MessageRoutingMode;
-import org.apache.pulsar.client.impl.ConsumerImpl;
-import org.apache.pulsar.client.impl.PartitionedConsumerImpl;
import org.apache.pulsar.common.policies.data.PropertyAdmin;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -65,14 +61,12 @@ public void testSharedAckedNormalTopic() throws Exception {
final int totalMessages = 15;
// 1. producer connect
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
// 2. Create consumer
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setReceiverQueueSize(50);
- conf.setAckTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS);
- conf.setSubscriptionType(SubscriptionType.Shared);
- Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName, conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
+ .receiverQueueSize(50).ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS)
+ .subscriptionType(SubscriptionType.Shared).subscribe();
// 3. producer publish messages
for (int i = 0; i < totalMessages / 3; i++) {
@@ -82,13 +76,13 @@ public void testSharedAckedNormalTopic() throws Exception {
}
// 4. Receiver receives the message, doesn't ack
- Message message = consumer.receive();
+ Message<byte[]> message = consumer.receive();
while (message != null) {
String data = new String(message.getData());
log.info("Consumer received : " + data);
message = consumer.receive(100, TimeUnit.MILLISECONDS);
}
- long size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
+ long size = ((ConsumerImpl<byte[]>) consumer).getUnAckedMessageTracker().size();
log.info(key + " Unacked Message Tracker size is " + size);
assertEquals(size, 5);
@@ -109,13 +103,13 @@ public void testSharedAckedNormalTopic() throws Exception {
consumer.acknowledge(message);
message = consumer.receive(100, TimeUnit.MILLISECONDS);
}
- size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
+ size = ((ConsumerImpl<byte[]>) consumer).getUnAckedMessageTracker().size();
log.info(key + " Unacked Message Tracker size is " + size);
assertEquals(size, 5);
assertEquals(received, 5);
// 7. Simulate ackTimeout
- ((ConsumerImpl) consumer).getUnAckedMessageTracker().toggle();
+ ((ConsumerImpl<byte[]>) consumer).getUnAckedMessageTracker().toggle();
// 8. producer publish more messages
for (int i = 0; i < totalMessages / 3; i++) {
@@ -131,7 +125,7 @@ public void testSharedAckedNormalTopic() throws Exception {
log.info("Consumer received : " + data);
message = consumer.receive(100, TimeUnit.MILLISECONDS);
}
- size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
+ size = ((ConsumerImpl<byte[]>) consumer).getUnAckedMessageTracker().size();
log.info(key + " Unacked Message Tracker size is " + size);
assertEquals(size, 10);
@@ -148,7 +142,7 @@ public void testSharedAckedNormalTopic() throws Exception {
message = consumer.receive(100, TimeUnit.MILLISECONDS);
}
assertEquals(redelivered, 5);
- size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
+ size = ((ConsumerImpl<byte[]>) consumer).getUnAckedMessageTracker().size();
log.info(key + " Unacked Message Tracker size is " + size);
assertEquals(size, 5);
}
@@ -162,14 +156,12 @@ public void testExclusiveAckedNormalTopic() throws Exception {
final int totalMessages = 15;
// 1. producer connect
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
// 2. Create consumer
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setReceiverQueueSize(50);
- conf.setAckTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS);
- conf.setSubscriptionType(SubscriptionType.Exclusive);
- Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName, conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
+ .receiverQueueSize(50).ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS)
+ .subscriptionType(SubscriptionType.Exclusive).subscribe();
// 3. producer publish messages
for (int i = 0; i < totalMessages / 3; i++) {
@@ -179,13 +171,13 @@ public void testExclusiveAckedNormalTopic() throws Exception {
}
// 4. Receiver receives the message, doesn't ack
- Message message = consumer.receive();
+ Message<byte[]> message = consumer.receive();
while (message != null) {
String data = new String(message.getData());
log.info("Consumer received : " + data);
message = consumer.receive(100, TimeUnit.MILLISECONDS);
}
- long size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
+ long size = ((ConsumerImpl<byte[]>) consumer).getUnAckedMessageTracker().size();
log.info(key + " Unacked Message Tracker size is " + size);
assertEquals(size, 5);
@@ -206,13 +198,13 @@ public void testExclusiveAckedNormalTopic() throws Exception {
consumer.acknowledge(message);
message = consumer.receive(100, TimeUnit.MILLISECONDS);
}
- size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
+ size = ((ConsumerImpl<byte[]>) consumer).getUnAckedMessageTracker().size();
log.info(key + " Unacked Message Tracker size is " + size);
assertEquals(size, 5);
assertEquals(received, 5);
// 7. Simulate ackTimeout
- ((ConsumerImpl) consumer).getUnAckedMessageTracker().toggle();
+ ((ConsumerImpl<byte[]>) consumer).getUnAckedMessageTracker().toggle();
// 8. producer publish more messages
for (int i = 0; i < totalMessages / 3; i++) {
@@ -228,7 +220,7 @@ public void testExclusiveAckedNormalTopic() throws Exception {
log.info("Consumer received : " + data);
message = consumer.receive(100, TimeUnit.MILLISECONDS);
}
- size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
+ size = ((ConsumerImpl<byte[]>) consumer).getUnAckedMessageTracker().size();
log.info(key + " Unacked Message Tracker size is " + size);
assertEquals(size, 10);
@@ -245,7 +237,7 @@ public void testExclusiveAckedNormalTopic() throws Exception {
message = consumer.receive(100, TimeUnit.MILLISECONDS);
}
assertEquals(redelivered, 10);
- size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
+ size = ((ConsumerImpl<byte[]>) consumer).getUnAckedMessageTracker().size();
log.info(key + " Unacked Message Tracker size is " + size);
assertEquals(size, 0);
}
@@ -259,14 +251,12 @@ public void testFailoverAckedNormalTopic() throws Exception {
final int totalMessages = 15;
// 1. producer connect
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
// 2. Create consumer
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setReceiverQueueSize(50);
- conf.setAckTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS);
- conf.setSubscriptionType(SubscriptionType.Failover);
- Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName, conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
+ .receiverQueueSize(50).ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS)
+ .subscriptionType(SubscriptionType.Failover).subscribe();
// 3. producer publish messages
for (int i = 0; i < totalMessages / 3; i++) {
@@ -276,13 +266,13 @@ public void testFailoverAckedNormalTopic() throws Exception {
}
// 4. Receiver receives the message, doesn't ack
- Message message = consumer.receive();
+ Message<byte[]> message = consumer.receive();
while (message != null) {
String data = new String(message.getData());
log.info("Consumer received : " + data);
message = consumer.receive(100, TimeUnit.MILLISECONDS);
}
- long size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
+ long size = ((ConsumerImpl<byte[]>) consumer).getUnAckedMessageTracker().size();
log.info(key + " Unacked Message Tracker size is " + size);
assertEquals(size, 5);
@@ -303,13 +293,13 @@ public void testFailoverAckedNormalTopic() throws Exception {
consumer.acknowledge(message);
message = consumer.receive(100, TimeUnit.MILLISECONDS);
}
- size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
+ size = ((ConsumerImpl<byte[]>) consumer).getUnAckedMessageTracker().size();
log.info(key + " Unacked Message Tracker size is " + size);
assertEquals(size, 5);
assertEquals(received, 5);
// 7. Simulate ackTimeout
- ((ConsumerImpl) consumer).getUnAckedMessageTracker().toggle();
+ ((ConsumerImpl<byte[]>) consumer).getUnAckedMessageTracker().toggle();
// 8. producer publish more messages
for (int i = 0; i < totalMessages / 3; i++) {
@@ -325,7 +315,7 @@ public void testFailoverAckedNormalTopic() throws Exception {
log.info("Consumer received : " + data);
message = consumer.receive(100, TimeUnit.MILLISECONDS);
}
- size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
+ size = ((ConsumerImpl<byte[]>) consumer).getUnAckedMessageTracker().size();
log.info(key + " Unacked Message Tracker size is " + size);
assertEquals(size, 10);
@@ -342,15 +332,15 @@ public void testFailoverAckedNormalTopic() throws Exception {
message = consumer.receive(100, TimeUnit.MILLISECONDS);
}
assertEquals(redelivered, 10);
- size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
+ size = ((ConsumerImpl<byte[]>) consumer).getUnAckedMessageTracker().size();
log.info(key + " Unacked Message Tracker size is " + size);
assertEquals(size, 0);
}
- private static long getUnackedMessagesCountInPartitionedConsumer(Consumer c) {
- PartitionedConsumerImpl pc = (PartitionedConsumerImpl) c;
- return pc.getUnAckedMessageTracker().size() + pc.getConsumers().stream()
- .mapToLong(consumer -> consumer.getUnAckedMessageTracker().size()).sum();
+ private static long getUnackedMessagesCountInPartitionedConsumer(Consumer<byte[]> c) {
+ PartitionedConsumerImpl<byte[]> pc = (PartitionedConsumerImpl<byte[]>) c;
+ return pc.getUnAckedMessageTracker().size()
+ + pc.getConsumers().stream().mapToLong(consumer -> consumer.getUnAckedMessageTracker().size()).sum();
}
@Test(timeOut = testTimeout)
@@ -365,16 +355,13 @@ public void testSharedAckedPartitionedTopic() throws Exception {
admin.persistentTopics().createPartitionedTopic(topicName, numberOfPartitions);
// 1. producer connect
- ProducerConfiguration prodConfig = new ProducerConfiguration();
- prodConfig.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
- Producer producer = pulsarClient.createProducer(topicName, prodConfig);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
+ .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();
// 2. Create consumer
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setReceiverQueueSize(50);
- conf.setAckTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS);
- conf.setSubscriptionType(SubscriptionType.Shared);
- Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName, conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
+ .receiverQueueSize(50).ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS)
+ .subscriptionType(SubscriptionType.Shared).subscribe();
// 3. producer publish messages
for (int i = 0; i < totalMessages / 3; i++) {
@@ -384,7 +371,7 @@ public void testSharedAckedPartitionedTopic() throws Exception {
}
// 4. Receiver receives the message, doesn't ack
- Message message = consumer.receive();
+ Message<byte[]> message = consumer.receive();
while (message != null) {
String data = new String(message.getData());
log.info("Consumer received : " + data);
@@ -418,8 +405,8 @@ public void testSharedAckedPartitionedTopic() throws Exception {
assertEquals(received, 5);
// 7. Simulate ackTimeout
- ((PartitionedConsumerImpl) consumer).getUnAckedMessageTracker().toggle();
- ((PartitionedConsumerImpl) consumer).getConsumers().forEach(c -> c.getUnAckedMessageTracker().toggle());
+ ((PartitionedConsumerImpl<byte[]>) consumer).getUnAckedMessageTracker().toggle();
+ ((PartitionedConsumerImpl<byte[]>) consumer).getConsumers().forEach(c -> c.getUnAckedMessageTracker().toggle());
// 8. producer publish more messages
for (int i = 0; i < totalMessages / 3; i++) {
@@ -435,7 +422,7 @@ public void testSharedAckedPartitionedTopic() throws Exception {
log.info("Consumer received : " + data);
message = consumer.receive(100, TimeUnit.MILLISECONDS);
}
- size = getUnackedMessagesCountInPartitionedConsumer(consumer);
+ size = getUnackedMessagesCountInPartitionedConsumer(consumer);
log.info(key + " Unacked Message Tracker size is " + size);
assertEquals(size, 10);
@@ -452,7 +439,7 @@ public void testSharedAckedPartitionedTopic() throws Exception {
message = consumer.receive(100, TimeUnit.MILLISECONDS);
}
assertEquals(redelivered, 5);
- size = getUnackedMessagesCountInPartitionedConsumer(consumer);
+ size = getUnackedMessagesCountInPartitionedConsumer(consumer);
log.info(key + " Unacked Message Tracker size is " + size);
assertEquals(size, 5);
}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawMessageSerDeserTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawMessageSerDeserTest.java
index 79ed79465..d634a0090 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawMessageSerDeserTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawMessageSerDeserTest.java
@@ -20,7 +20,6 @@
import org.apache.pulsar.client.api.RawMessage;
-import org.apache.pulsar.common.api.ByteBufPair;
import org.apache.pulsar.common.api.proto.PulsarApi.MessageIdData;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -29,6 +28,7 @@
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
+import lombok.Cleanup;
public class RawMessageSerDeserTest {
static final Logger log = LoggerFactory.getLogger(RawMessageSerDeserTest.class);
@@ -43,6 +43,7 @@ public void testSerializationAndDeserialization() throws Exception {
.setLedgerId(0xf00).setEntryId(0xbaa)
.setPartition(10).setBatchIndex(20).build();
+ @Cleanup
RawMessage m = new RawMessageImpl(id, headersAndPayload);
ByteBuf serialized = m.serialize();
byte[] bytes = new byte[serialized.readableBytes()];
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java
index 53da2eefa..b63d9c442 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java
@@ -18,13 +18,6 @@
*/
package org.apache.pulsar.client.impl;
-import static org.testng.Assert.assertEquals;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-import io.netty.buffer.ByteBuf;
-
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -33,37 +26,33 @@
import java.util.Set;
import java.util.concurrent.CancellationException;
import java.util.concurrent.Future;
-import java.util.concurrent.TimeoutException;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.function.Consumer;
import org.apache.bookkeeper.mledger.ManagedLedger;
-
import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
import org.apache.pulsar.broker.service.persistent.PersistentTopic;
-import org.apache.pulsar.common.api.Commands;
import org.apache.pulsar.client.api.MessageBuilder;
import org.apache.pulsar.client.api.MessageId;
import org.apache.pulsar.client.api.Producer;
-import org.apache.pulsar.client.api.ProducerConfiguration;
import org.apache.pulsar.client.api.RawMessage;
import org.apache.pulsar.client.api.RawReader;
+import org.apache.pulsar.common.api.Commands;
import org.apache.pulsar.common.api.proto.PulsarApi.MessageMetadata;
-import org.apache.pulsar.common.api.proto.PulsarApi.SingleMessageMetadata;
-import org.apache.pulsar.client.impl.BatchMessageIdImpl;
-
import org.apache.pulsar.common.policies.data.ClusterData;
import org.apache.pulsar.common.policies.data.PropertyAdmin;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+import io.netty.buffer.ByteBuf;
+
public class RawReaderTest extends MockedPulsarServiceBaseTest {
- private static final Logger log = LoggerFactory.getLogger(RawReaderTest.class);
private static final int BATCH_MAX_MESSAGES = 10;
private static final String subscription = "foobar-sub";
@@ -87,13 +76,10 @@ public void cleanup() throws Exception {
private Set<String> publishMessagesBase(String topic, int count, boolean batching) throws Exception {
Set<String> keys = new HashSet<>();
- ProducerConfiguration producerConf = new ProducerConfiguration();
- producerConf.setMaxPendingMessages(count);
- producerConf.setBatchingEnabled(batching);
- producerConf.setBatchingMaxMessages(BATCH_MAX_MESSAGES);
- producerConf.setBatchingMaxPublishDelay(Long.MAX_VALUE, TimeUnit.DAYS);
- try (Producer producer = pulsarClient.createProducer(topic, producerConf)) {
+ try (Producer<byte[]> producer = pulsarClient.newProducer().topic(topic).maxPendingMessages(count)
+ .enableBatching(batching).batchingMaxMessages(BATCH_MAX_MESSAGES)
+ .batchingMaxPublishDelay(Long.MAX_VALUE, TimeUnit.DAYS).create()) {
Future<?> lastFuture = null;
for (int i = 0; i < count; i++) {
String key = "key"+i;
@@ -260,7 +246,6 @@ public void testBatching() throws Exception {
Set<String> keys = publishMessagesInBatches(topic, numMessages);
RawReader reader = RawReader.create(pulsarClient, topic, subscription).get();
- List<Future<RawMessage>> futures = new ArrayList<>();
Consumer<RawMessage> consumer = new Consumer<RawMessage>() {
BatchMessageIdImpl lastId = new BatchMessageIdImpl(-1, -1, -1, -1);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java
index 2aa386e71..952dfac22 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java
@@ -46,6 +46,7 @@
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
+@SuppressWarnings({ "unchecked", "rawtypes" })
public class TopicsConsumerImplTest extends ProducerConsumerBase {
private static final long testTimeout = 90000; // 1.5 min
private static final Logger log = LoggerFactory.getLogger(TopicsConsumerImplTest.class);
@@ -80,7 +81,7 @@ public void testDifferentTopicsNameSubscribe() throws Exception {
// 2. Create consumer
try {
- Consumer consumer = pulsarClient.newConsumer()
+ pulsarClient.newConsumer()
.topics(topicNames)
.subscriptionName(subscriptionName)
.subscriptionType(SubscriptionType.Shared)
@@ -92,7 +93,6 @@ public void testDifferentTopicsNameSubscribe() throws Exception {
}
}
-
@Test(timeOut = testTimeout)
public void testGetConsumersAndGetTopics() throws Exception {
String key = "TopicsConsumerGet";
@@ -108,7 +108,7 @@ public void testGetConsumersAndGetTopics() throws Exception {
admin.persistentTopics().createPartitionedTopic(topicName3, 3);
// 2. Create consumer
- Consumer consumer = pulsarClient.newConsumer()
+ Consumer<byte[]> consumer = pulsarClient.newConsumer()
.topics(topicNames)
.topic(topicName3)
.subscriptionName(subscriptionName)
@@ -118,8 +118,8 @@ public void testGetConsumersAndGetTopics() throws Exception {
.subscribe();
assertTrue(consumer instanceof TopicsConsumerImpl);
- List<String> topics = ((TopicsConsumerImpl) consumer).getPartitionedTopics();
- List<ConsumerImpl> consumers = ((TopicsConsumerImpl) consumer).getConsumers();
+ List<String> topics = ((TopicsConsumerImpl<byte[]>) consumer).getPartitionedTopics();
+ List<ConsumerImpl<byte[]>> consumers = ((TopicsConsumerImpl) consumer).getConsumers();
topics.forEach(topic -> log.info("topic: {}", topic));
consumers.forEach(c -> log.info("consumer: {}", c.getTopic()));
@@ -127,7 +127,7 @@ public void testGetConsumersAndGetTopics() throws Exception {
IntStream.range(0, 6).forEach(index ->
assertTrue(topics.get(index).equals(consumers.get(index).getTopic())));
- assertTrue(((TopicsConsumerImpl) consumer).getTopics().size() == 3);
+ assertTrue(((TopicsConsumerImpl<byte[]>) consumer).getTopics().size() == 3);
consumer.unsubscribe();
consumer.close();
@@ -150,17 +150,17 @@ public void testSyncProducerAndConsumer() throws Exception {
admin.persistentTopics().createPartitionedTopic(topicName3, 3);
// 1. producer connect
- Producer producer1 = pulsarClient.newProducer().topic(topicName1)
+ Producer<byte[]> producer1 = pulsarClient.newProducer().topic(topicName1)
.create();
- Producer producer2 = pulsarClient.newProducer().topic(topicName2)
+ Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topicName2)
.messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition)
.create();
- Producer producer3 = pulsarClient.newProducer().topic(topicName3)
+ Producer<byte[]> producer3 = pulsarClient.newProducer().topic(topicName3)
.messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition)
.create();
// 2. Create consumer
- Consumer consumer = pulsarClient.newConsumer()
+ Consumer<byte[]> consumer = pulsarClient.newConsumer()
.topics(topicNames)
.subscriptionName(subscriptionName)
.subscriptionType(SubscriptionType.Shared)
@@ -177,7 +177,7 @@ public void testSyncProducerAndConsumer() throws Exception {
}
int messageSet = 0;
- Message message = consumer.receive();
+ Message<byte[]> message = consumer.receive();
do {
assertTrue(message instanceof TopicMessageImpl);
messageSet ++;
@@ -211,17 +211,17 @@ public void testAsyncConsumer() throws Exception {
admin.persistentTopics().createPartitionedTopic(topicName3, 3);
// 1. producer connect
- Producer producer1 = pulsarClient.newProducer().topic(topicName1)
+ Producer<byte[]> producer1 = pulsarClient.newProducer().topic(topicName1)
.create();
- Producer producer2 = pulsarClient.newProducer().topic(topicName2)
+ Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topicName2)
.messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition)
.create();
- Producer producer3 = pulsarClient.newProducer().topic(topicName3)
+ Producer<byte[]> producer3 = pulsarClient.newProducer().topic(topicName3)
.messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition)
.create();
// 2. Create consumer
- Consumer consumer = pulsarClient.newConsumer()
+ Consumer<byte[]> consumer = pulsarClient.newConsumer()
.topics(topicNames)
.subscriptionName(subscriptionName)
.subscriptionType(SubscriptionType.Shared)
@@ -290,17 +290,17 @@ public void testConsumerUnackedRedelivery() throws Exception {
admin.persistentTopics().createPartitionedTopic(topicName3, 3);
// 1. producer connect
- Producer producer1 = pulsarClient.newProducer().topic(topicName1)
+ Producer<byte[]> producer1 = pulsarClient.newProducer().topic(topicName1)
.create();
- Producer producer2 = pulsarClient.newProducer().topic(topicName2)
+ Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topicName2)
.messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition)
.create();
- Producer producer3 = pulsarClient.newProducer().topic(topicName3)
+ Producer<byte[]> producer3 = pulsarClient.newProducer().topic(topicName3)
.messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition)
.create();
// 2. Create consumer
- Consumer consumer = pulsarClient.newConsumer()
+ Consumer<byte[]> consumer = pulsarClient.newConsumer()
.topics(topicNames)
.subscriptionName(subscriptionName)
.subscriptionType(SubscriptionType.Shared)
@@ -317,13 +317,13 @@ public void testConsumerUnackedRedelivery() throws Exception {
}
// 4. Receiver receives the message, not ack, Unacked Message Tracker size should be totalMessages.
- Message message = consumer.receive();
+ Message<byte[]> message = consumer.receive();
while (message != null) {
assertTrue(message instanceof TopicMessageImpl);
log.debug("Consumer received : " + new String(message.getData()));
message = consumer.receive(500, TimeUnit.MILLISECONDS);
}
- long size = ((TopicsConsumerImpl) consumer).getUnAckedMessageTracker().size();
+ long size = ((TopicsConsumerImpl<byte[]>) consumer).getUnAckedMessageTracker().size();
log.debug(key + " Unacked Message Tracker size is " + size);
assertEquals(size, totalMessages);
@@ -338,7 +338,7 @@ public void testConsumerUnackedRedelivery() throws Exception {
message = consumer.receive(500, TimeUnit.MILLISECONDS);
} while (message != null);
- size = ((TopicsConsumerImpl) consumer).getUnAckedMessageTracker().size();
+ size = ((TopicsConsumerImpl<byte[]>) consumer).getUnAckedMessageTracker().size();
log.debug(key + " Unacked Message Tracker size is " + size);
assertEquals(size, 0);
assertEquals(hSet.size(), totalMessages);
@@ -361,14 +361,14 @@ public void testConsumerUnackedRedelivery() throws Exception {
consumer.acknowledge(message);
message = consumer.receive(100, TimeUnit.MILLISECONDS);
}
- size = ((TopicsConsumerImpl) consumer).getUnAckedMessageTracker().size();
+ size = ((TopicsConsumerImpl<byte[]>) consumer).getUnAckedMessageTracker().size();
log.debug(key + " Unacked Message Tracker size is " + size);
assertEquals(size, 0);
assertEquals(received, totalMessages);
// 8. Simulate ackTimeout
- ((TopicsConsumerImpl) consumer).getUnAckedMessageTracker().toggle();
- ((TopicsConsumerImpl) consumer).getConsumers().forEach(c -> c.getUnAckedMessageTracker().toggle());
+ ((TopicsConsumerImpl<byte[]>) consumer).getUnAckedMessageTracker().toggle();
+ ((TopicsConsumerImpl<byte[]>) consumer).getConsumers().forEach(c -> c.getUnAckedMessageTracker().toggle());
// 9. producer publish more messages
for (int i = 0; i < totalMessages / 3; i++) {
@@ -384,7 +384,7 @@ public void testConsumerUnackedRedelivery() throws Exception {
log.debug("Consumer received : " + data);
message = consumer.receive(100, TimeUnit.MILLISECONDS);
}
- size = ((TopicsConsumerImpl) consumer).getUnAckedMessageTracker().size();
+ size = ((TopicsConsumerImpl<byte[]>) consumer).getUnAckedMessageTracker().size();
log.debug(key + " Unacked Message Tracker size is " + size);
assertEquals(size, 30);
@@ -402,7 +402,7 @@ public void testConsumerUnackedRedelivery() throws Exception {
message = consumer.receive(100, TimeUnit.MILLISECONDS);
}
assertEquals(redelivered, 30);
- size = ((TopicsConsumerImpl) consumer).getUnAckedMessageTracker().size();
+ size = ((TopicsConsumerImpl<byte[]>) consumer).getUnAckedMessageTracker().size();
log.info(key + " Unacked Message Tracker size is " + size);
assertEquals(size, 0);
@@ -430,17 +430,17 @@ public void testSubscribeUnsubscribeSingleTopic() throws Exception {
admin.persistentTopics().createPartitionedTopic(topicName3, 3);
// 1. producer connect
- Producer producer1 = pulsarClient.newProducer().topic(topicName1)
+ Producer<byte[]> producer1 = pulsarClient.newProducer().topic(topicName1)
.create();
- Producer producer2 = pulsarClient.newProducer().topic(topicName2)
+ Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topicName2)
.messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition)
.create();
- Producer producer3 = pulsarClient.newProducer().topic(topicName3)
+ Producer<byte[]> producer3 = pulsarClient.newProducer().topic(topicName3)
.messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition)
.create();
// 2. Create consumer
- Consumer consumer = pulsarClient.newConsumer()
+ Consumer<byte[]> consumer = pulsarClient.newConsumer()
.topics(topicNames)
.subscriptionName(subscriptionName)
.subscriptionType(SubscriptionType.Shared)
@@ -457,7 +457,7 @@ public void testSubscribeUnsubscribeSingleTopic() throws Exception {
}
int messageSet = 0;
- Message message = consumer.receive();
+ Message<byte[]> message = consumer.receive();
do {
assertTrue(message instanceof TopicMessageImpl);
messageSet ++;
@@ -468,7 +468,7 @@ public void testSubscribeUnsubscribeSingleTopic() throws Exception {
assertEquals(messageSet, totalMessages);
// 4, unsubscribe topic3
- CompletableFuture<Void> unsubFuture = ((TopicsConsumerImpl)consumer).unsubscribeAsync(topicName3);
+ CompletableFuture<Void> unsubFuture = ((TopicsConsumerImpl<byte[]>) consumer).unsubscribeAsync(topicName3);
unsubFuture.get();
// 5. producer publish messages
@@ -491,15 +491,15 @@ public void testSubscribeUnsubscribeSingleTopic() throws Exception {
assertEquals(messageSet, totalMessages * 2 / 3);
// 7. use getter to verify internal topics number after un-subscribe topic3
- List<String> topics = ((TopicsConsumerImpl) consumer).getPartitionedTopics();
- List<ConsumerImpl> consumers = ((TopicsConsumerImpl) consumer).getConsumers();
+ List<String> topics = ((TopicsConsumerImpl<byte[]>) consumer).getPartitionedTopics();
+ List<ConsumerImpl<byte[]>> consumers = ((TopicsConsumerImpl) consumer).getConsumers();
assertEquals(topics.size(), 3);
assertEquals(consumers.size(), 3);
- assertTrue(((TopicsConsumerImpl) consumer).getTopics().size() == 2);
+ assertTrue(((TopicsConsumerImpl<byte[]>) consumer).getTopics().size() == 2);
// 8. re-subscribe topic3
- CompletableFuture<Void> subFuture = ((TopicsConsumerImpl)consumer).subscribeAsync(topicName3);
+ CompletableFuture<Void> subFuture = ((TopicsConsumerImpl<byte[]>)consumer).subscribeAsync(topicName3);
subFuture.get();
// 9. producer publish messages
@@ -522,12 +522,12 @@ public void testSubscribeUnsubscribeSingleTopic() throws Exception {
assertEquals(messageSet, totalMessages);
// 11. use getter to verify internal topics number after subscribe topic3
- topics = ((TopicsConsumerImpl) consumer).getPartitionedTopics();
+ topics = ((TopicsConsumerImpl<byte[]>) consumer).getPartitionedTopics();
consumers = ((TopicsConsumerImpl) consumer).getConsumers();
assertEquals(topics.size(), 6);
assertEquals(consumers.size(), 6);
- assertTrue(((TopicsConsumerImpl) consumer).getTopics().size() == 3);
+ assertTrue(((TopicsConsumerImpl<byte[]>) consumer).getTopics().size() == 3);
consumer.unsubscribe();
consumer.close();
@@ -542,10 +542,8 @@ public void testTopicsNameSubscribeWithBuilderFail() throws Exception {
String key = "TopicsNameSubscribeWithBuilder";
final String subscriptionName = "my-ex-subscription-" + key;
- final String topicName1 = "persistent://prop/use/ns-abc/topic-1-" + key;
final String topicName2 = "persistent://prop/use/ns-abc/topic-2-" + key;
final String topicName3 = "persistent://prop/use/ns-abc/topic-3-" + key;
- List<String> topicNames = Lists.newArrayList(topicName1, topicName2, topicName3);
admin.properties().createProperty("prop", new PropertyAdmin());
admin.persistentTopics().createPartitionedTopic(topicName2, 2);
@@ -553,7 +551,7 @@ public void testTopicsNameSubscribeWithBuilderFail() throws Exception {
// test failing builder with empty topics
try {
- Consumer consumer1 = pulsarClient.newConsumer()
+ pulsarClient.newConsumer()
.subscriptionName(subscriptionName)
.subscriptionType(SubscriptionType.Shared)
.ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS)
@@ -564,7 +562,7 @@ public void testTopicsNameSubscribeWithBuilderFail() throws Exception {
}
try {
- Consumer consumer2 = pulsarClient.newConsumer()
+ pulsarClient.newConsumer()
.topic()
.subscriptionName(subscriptionName)
.subscriptionType(SubscriptionType.Shared)
@@ -576,7 +574,7 @@ public void testTopicsNameSubscribeWithBuilderFail() throws Exception {
}
try {
- Consumer consumer3 = pulsarClient.newConsumer()
+ pulsarClient.newConsumer()
.topics(null)
.subscriptionName(subscriptionName)
.subscriptionType(SubscriptionType.Shared)
@@ -588,7 +586,7 @@ public void testTopicsNameSubscribeWithBuilderFail() throws Exception {
}
try {
- Consumer consumer4 = pulsarClient.newConsumer()
+ pulsarClient.newConsumer()
.topics(Lists.newArrayList())
.subscriptionName(subscriptionName)
.subscriptionType(SubscriptionType.Shared)
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/UnAcknowledgedMessagesTimeoutTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/UnAcknowledgedMessagesTimeoutTest.java
index 133570e66..23306ba03 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/UnAcknowledgedMessagesTimeoutTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/UnAcknowledgedMessagesTimeoutTest.java
@@ -25,16 +25,12 @@
import org.apache.pulsar.broker.service.BrokerTestBase;
import org.apache.pulsar.client.api.Consumer;
-import org.apache.pulsar.client.api.ConsumerConfiguration;
import org.apache.pulsar.client.api.Message;
import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageRoutingMode;
import org.apache.pulsar.client.api.Producer;
-import org.apache.pulsar.client.api.ProducerConfiguration;
import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.client.api.SubscriptionType;
-import org.apache.pulsar.client.api.ProducerConfiguration.MessageRoutingMode;
-import org.apache.pulsar.client.impl.ConsumerImpl;
-import org.apache.pulsar.client.impl.MessageIdImpl;
import org.apache.pulsar.common.policies.data.PropertyAdmin;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -69,13 +65,11 @@ public void testExclusiveSingleAckedNormalTopic() throws Exception {
final int totalMessages = 10;
// 1. producer connect
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
// 2. Create consumer
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setReceiverQueueSize(7);
- conf.setAckTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS);
- Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName, conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
+ .receiverQueueSize(7).ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS).subscribe();
// 3. producer publish messages
for (int i = 0; i < totalMessages / 2; i++) {
@@ -85,12 +79,12 @@ public void testExclusiveSingleAckedNormalTopic() throws Exception {
}
// 4. Receiver receives the message
- Message message = consumer.receive();
+ Message<byte[]> message = consumer.receive();
while (message != null) {
log.info("Consumer received : " + new String(message.getData()));
message = consumer.receive(500, TimeUnit.MILLISECONDS);
}
- long size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
+ long size = ((ConsumerImpl<?>) consumer).getUnAckedMessageTracker().size();
log.info(key + " Unacked Message Tracker size is " + size);
assertEquals(size, totalMessages / 2);
@@ -111,7 +105,7 @@ public void testExclusiveSingleAckedNormalTopic() throws Exception {
message = consumer.receive(500, TimeUnit.MILLISECONDS);
} while (message != null);
- size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
+ size = ((ConsumerImpl<?>) consumer).getUnAckedMessageTracker().size();
log.info(key + " Unacked Message Tracker size is " + size);
assertEquals(size, 0);
assertEquals(hSet.size(), totalMessages);
@@ -126,13 +120,11 @@ public void testExclusiveCumulativeAckedNormalTopic() throws Exception {
final int totalMessages = 10;
// 1. producer connect
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
// 2. Create consumer
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setReceiverQueueSize(7);
- conf.setAckTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS);
- Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName, conf);
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
+ .receiverQueueSize(7).ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS).subscribe();
// 3. producer publish messages
for (int i = 0; i < totalMessages; i++) {
@@ -142,8 +134,8 @@ public void testExclusiveCumulativeAckedNormalTopic() throws Exception {
// 4. Receiver receives the message
HashSet<String> hSet = new HashSet<>();
- Message message = consumer.receive();
- Message lastMessage = message;
+ Message<byte[]> message = consumer.receive();
+ Message<byte[]> lastMessage = message;
while (message != null) {
lastMessage = message;
hSet.add(new String(message.getData()));
@@ -151,12 +143,12 @@ public void testExclusiveCumulativeAckedNormalTopic() throws Exception {
log.info("Message ID details " + ((MessageIdImpl) message.getMessageId()).toString());
message = consumer.receive(500, TimeUnit.MILLISECONDS);
}
- long size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
+ long size = ((ConsumerImpl<?>) consumer).getUnAckedMessageTracker().size();
assertEquals(size, totalMessages);
log.info("Comulative Ack sent for " + new String(lastMessage.getData()));
log.info("Message ID details " + ((MessageIdImpl) lastMessage.getMessageId()).toString());
consumer.acknowledgeCumulative(lastMessage);
- size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
+ size = ((ConsumerImpl<?>) consumer).getUnAckedMessageTracker().size();
assertEquals(size, 0);
message = consumer.receive((int) (2 * ackTimeOutMillis), TimeUnit.MILLISECONDS);
assertEquals(message, null);
@@ -175,19 +167,16 @@ public void testSharedSingleAckedPartitionedTopic() throws Exception {
// Special step to create partitioned topic
// 1. producer connect
- ProducerConfiguration prodConfig = new ProducerConfiguration();
- prodConfig.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
- Producer producer = pulsarClient.createProducer(topicName, prodConfig);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
+ .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();
// 2. Create consumer
- ConsumerConfiguration consumerConfig = new ConsumerConfiguration();
- consumerConfig.setReceiverQueueSize(100);
- consumerConfig.setSubscriptionType(SubscriptionType.Shared);
- consumerConfig.setAckTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS);
- consumerConfig.setConsumerName("Consumer-1");
- Consumer consumer1 = pulsarClient.subscribe(topicName, subscriptionName, consumerConfig);
- consumerConfig.setConsumerName("Consumer-2");
- Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, consumerConfig);
+ Consumer<byte[]> consumer1 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
+ .receiverQueueSize(100).subscriptionType(SubscriptionType.Shared)
+ .ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS).consumerName("Consumer-1").subscribe();
+ Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
+ .receiverQueueSize(100).subscriptionType(SubscriptionType.Shared)
+ .ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS).consumerName("Consumer-2").subscribe();
// 3. producer publish messages
for (int i = 0; i < totalMessages; i++) {
@@ -235,9 +224,9 @@ public void testSharedSingleAckedPartitionedTopic() throws Exception {
assertEquals(ackCount1 + ackCount2, totalMessages);
}
- private static int receiveAllMessage(Consumer consumer, boolean ackMessages) throws Exception {
+ private static int receiveAllMessage(Consumer<?> consumer, boolean ackMessages) throws Exception {
int messagesReceived = 0;
- Message msg = consumer.receive(1, TimeUnit.SECONDS);
+ Message<?> msg = consumer.receive(1, TimeUnit.SECONDS);
while (msg != null) {
++messagesReceived;
log.info("Consumer received {}", new String(msg.getData()));
@@ -265,19 +254,16 @@ public void testFailoverSingleAckedPartitionedTopic() throws Exception {
// Special step to create partitioned topic
// 1. producer connect
- ProducerConfiguration prodConfig = new ProducerConfiguration();
- prodConfig.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
- Producer producer = pulsarClient.createProducer(topicName, prodConfig);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
+ .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();
// 2. Create consumer
- ConsumerConfiguration consumerConfig = new ConsumerConfiguration();
- consumerConfig.setReceiverQueueSize(7);
- consumerConfig.setSubscriptionType(SubscriptionType.Failover);
- consumerConfig.setAckTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS);
- consumerConfig.setConsumerName("Consumer-1");
- Consumer consumer1 = pulsarClient.subscribe(topicName, subscriptionName, consumerConfig);
- consumerConfig.setConsumerName("Consumer-2");
- Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, consumerConfig);
+ Consumer<byte[]> consumer1 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
+ .receiverQueueSize(7).subscriptionType(SubscriptionType.Shared)
+ .ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS).consumerName("Consumer-1").subscribe();
+ Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
+ .receiverQueueSize(7).subscriptionType(SubscriptionType.Shared)
+ .ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS).consumerName("Consumer-2").subscribe();
// 3. producer publish messages
for (int i = 0; i < totalMessages; i++) {
@@ -287,8 +273,8 @@ public void testFailoverSingleAckedPartitionedTopic() throws Exception {
}
// 4. Receive messages
- Message message1 = consumer1.receive();
- Message message2 = consumer2.receive();
+ Message<byte[]> message1 = consumer1.receive();
+ Message<byte[]> message2 = consumer2.receive();
int messageCount1 = 0;
int messageCount2 = 0;
int ackCount1 = 0;
@@ -341,11 +327,8 @@ public void testFailoverSingleAckedPartitionedTopic() throws Exception {
@Test
public void testAckTimeoutMinValue() throws PulsarClientException {
- ConsumerConfiguration consumerConfig = new ConsumerConfiguration();
- consumerConfig.setReceiverQueueSize(7);
- consumerConfig.setSubscriptionType(SubscriptionType.Failover);
try {
- consumerConfig.setAckTimeout(999, TimeUnit.MILLISECONDS);
+ pulsarClient.newConsumer().ackTimeout(999, TimeUnit.MILLISECONDS);
Assert.fail("Exception should have been thrown since the set timeout is less than min timeout.");
} catch (Exception ex) {
// Ok
@@ -361,13 +344,12 @@ public void testCheckUnAcknowledgedMessageTimer() throws PulsarClientException,
final int totalMessages = 3;
// 1. producer connect
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
// 2. Create consumer
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setReceiverQueueSize(7);
- conf.setAckTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS);
- ConsumerImpl consumer = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriptionName, conf);
+ ConsumerImpl<byte[]> consumer = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName(subscriptionName).receiverQueueSize(7).subscriptionType(SubscriptionType.Shared)
+ .ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS).subscribe();
// 3. producer publish messages
for (int i = 0; i < totalMessages; i++) {
@@ -379,13 +361,13 @@ public void testCheckUnAcknowledgedMessageTimer() throws PulsarClientException,
Thread.sleep((long) (ackTimeOutMillis * 1.1));
for (int i = 0; i < totalMessages - 1; i++) {
- Message msg = consumer.receive();
+ Message<byte[]> msg = consumer.receive();
consumer.acknowledge(msg);
}
assertEquals(consumer.getUnAckedMessageTracker().size(), 1);
- Message msg = consumer.receive();
+ Message<byte[]> msg = consumer.receive();
consumer.acknowledge(msg);
assertEquals(consumer.getUnAckedMessageTracker().size(), 0);
@@ -394,18 +376,4 @@ public void testCheckUnAcknowledgedMessageTimer() throws PulsarClientException,
assertEquals(consumer.getUnAckedMessageTracker().size(), 0);
}
- @Test()
- public void testConfiguration() {
- ConsumerConfiguration conf = new ConsumerConfiguration();
- conf.setAckTimeout(10, TimeUnit.MINUTES);
- assertEquals(conf.getAckTimeoutMillis(), 10 * 60 * 1000);
- conf.setAckTimeout(11, TimeUnit.SECONDS);
- assertEquals(conf.getAckTimeoutMillis(), 11 * 1000);
- conf.setAckTimeout(15000000, TimeUnit.MICROSECONDS);
- assertEquals(conf.getAckTimeoutMillis(), 15000);
- conf.setAckTimeout(17000, TimeUnit.MILLISECONDS);
- assertEquals(conf.getAckTimeoutMillis(), 17000);
- conf.setAckTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS);
- assertEquals(conf.getAckTimeoutMillis(), ackTimeOutMillis);
- }
}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ZeroQueueSizeTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ZeroQueueSizeTest.java
index c1101752e..6cdc5069b 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ZeroQueueSizeTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ZeroQueueSizeTest.java
@@ -25,13 +25,11 @@
import org.apache.pulsar.broker.service.BrokerTestBase;
import org.apache.pulsar.client.admin.PulsarAdminException;
import org.apache.pulsar.client.api.Consumer;
-import org.apache.pulsar.client.api.ConsumerConfiguration;
import org.apache.pulsar.client.api.Message;
import org.apache.pulsar.client.api.Producer;
-import org.apache.pulsar.client.api.ProducerConfiguration;
+import org.apache.pulsar.client.api.ProducerBuilder;
import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.client.api.SubscriptionType;
-import org.apache.pulsar.client.impl.ConsumerImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
@@ -57,18 +55,12 @@ protected void cleanup() throws Exception {
@Test
public void validQueueSizeConfig() {
- try {
- ConsumerConfiguration configuration = new ConsumerConfiguration();
- configuration.setReceiverQueueSize(0);
- } catch (Exception ex) {
- Assert.fail();
- }
+ pulsarClient.newConsumer().receiverQueueSize(0);
}
@Test(expectedExceptions = IllegalArgumentException.class)
public void InvalidQueueSizeConfig() {
- ConsumerConfiguration configuration = new ConsumerConfiguration();
- configuration.setReceiverQueueSize(-1);
+ pulsarClient.newConsumer().receiverQueueSize(-1);
}
@Test(expectedExceptions = PulsarClientException.InvalidConfigurationException.class)
@@ -76,9 +68,9 @@ public void zeroQueueSizeReceieveAsyncInCompatibility() throws PulsarClientExcep
String key = "zeroQueueSizeReceieveAsyncInCompatibility";
final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
final String subscriptionName = "my-ex-subscription-" + key;
- ConsumerConfiguration configuration = new ConsumerConfiguration();
- configuration.setReceiverQueueSize(0);
- Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName, configuration);
+
+ Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
+ .receiverQueueSize(0).subscribe();
consumer.receive(10, TimeUnit.SECONDS);
}
@@ -89,9 +81,7 @@ public void zeroQueueSizePartitionedTopicInCompatibility() throws PulsarClientEx
final String subscriptionName = "my-ex-subscription-" + key;
int numberOfPartitions = 3;
admin.persistentTopics().createPartitionedTopic(topicName, numberOfPartitions);
- ConsumerConfiguration configuration = new ConsumerConfiguration();
- configuration.setReceiverQueueSize(0);
- Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName, configuration);
+ pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).receiverQueueSize(0).subscribe();
}
@Test()
@@ -104,12 +94,11 @@ public void zeroQueueSizeNormalConsumer() throws PulsarClientException {
final String messagePredicate = "my-message-" + key + "-";
// 2. Create Producer
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
// 3. Create Consumer
- ConsumerConfiguration configuration = new ConsumerConfiguration();
- configuration.setReceiverQueueSize(0);
- ConsumerImpl consumer = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriptionName, configuration);
+ ConsumerImpl<byte[]> consumer = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName(subscriptionName).receiverQueueSize(0).subscribe();
// 3. producer publish messages
for (int i = 0; i < totalMessages; i++) {
@@ -119,7 +108,7 @@ public void zeroQueueSizeNormalConsumer() throws PulsarClientException {
}
// 4. Receiver receives the message
- Message message;
+ Message<byte[]> message;
for (int i = 0; i < totalMessages; i++) {
assertEquals(consumer.numMessagesInQueue(), 0);
message = consumer.receive();
@@ -139,16 +128,15 @@ public void zeroQueueSizeSharedSubscription() throws PulsarClientException {
final String messagePredicate = "my-message-" + key + "-";
// 2. Create Producer
- Producer producer = pulsarClient.createProducer(topicName);
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
// 3. Create Consumer
int numOfSubscribers = 4;
- ConsumerConfiguration configuration = new ConsumerConfiguration();
- configuration.setReceiverQueueSize(0);
- configuration.setSubscriptionType(SubscriptionType.Shared);
- ConsumerImpl[] consumers = new ConsumerImpl[numOfSubscribers];
+ ConsumerImpl<?>[] consumers = new ConsumerImpl[numOfSubscribers];
for (int i = 0; i < numOfSubscribers; i++) {
- consumers[i] = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriptionName, configuration);
+ consumers[i] = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topicName)
+ .subscriptionName(subscriptionName).receiverQueueSize(0).subscriptionType(SubscriptionType.Shared)
+ .subscribe();
}
// 4. Produce Messages
@@ -158,7 +146,7 @@ public void zeroQueueSizeSharedSubscription() throws PulsarClientException {
}
// 5. Consume messages
- Message message;
+ Message<?> message;
for (int i = 0; i < totalMessages; i++) {
assertEquals(consumers[i % numOfSubscribers].numMessagesInQueue(), 0);
message = consumers[i % numOfSubscribers].receive();
(This diff was longer than 20,000 lines, and has been truncated...)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services