You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by la...@apache.org on 2019/05/12 23:19:25 UTC
[phoenix] branch 4.x-HBase-1.3 updated: PHOENIX-5225 Update Omid to
1.0.1.
This is an automated email from the ASF dual-hosted git repository.
larsh pushed a commit to branch 4.x-HBase-1.3
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.x-HBase-1.3 by this push:
new 2197d7b PHOENIX-5225 Update Omid to 1.0.1.
2197d7b is described below
commit 2197d7b35f2fd2b38b3d77b6f28f3a33134679ce
Author: Lars Hofhansl <la...@apache.org>
AuthorDate: Sun May 12 16:19:32 2019 -0700
PHOENIX-5225 Update Omid to 1.0.1.
---
bin/hbase-omid-client-config.yml | 18 +++
bin/omid-server-configuration.yml | 159 +++++++++++++++++++--
bin/omid.sh | 13 +-
.../build/components/all-common-dependencies.xml | 2 +
phoenix-core/pom.xml | 35 +++++
phoenix-server/pom.xml | 2 +
pom.xml | 2 +-
7 files changed, 211 insertions(+), 20 deletions(-)
diff --git a/bin/hbase-omid-client-config.yml b/bin/hbase-omid-client-config.yml
new file mode 100644
index 0000000..0c286c2
--- /dev/null
+++ b/bin/hbase-omid-client-config.yml
@@ -0,0 +1,18 @@
+
+#Omid TSO connection
+connectionType: !!org.apache.omid.tso.client.OmidClientConfiguration$ConnType DIRECT
+connectionString: "localhost:54758"
+
+# When Omid is working in High Availability mode, two or more replicas of the TSO server are running in primary/backup
+# mode. When a TSO server replica is elected as master, it publishes its address through ZK. In order to configure
+# the Omid client to access the TSO server in HA mode:
+# 1) set 'connectionType' to !!org.apache.omid.tso.client.OmidClientConfiguration$ConnType HA
+# 2) set 'connectionString' to the ZK cluster connection string where the server is publishing its address
+zkConnectionTimeoutInSecs: 10
+# In HA mode, make sure that the next settings match same settings on the TSO server side
+zkNamespace: "omid"
+zkCurrentTsoPath: "/current-tso"
+
+# Configure whether the TM performs the post-commit actions for a tx (update shadow cells and clean commit table entry)
+# before returning to the control to the client (SYNC) or in parallel (ASYNC)
+postCommitMode: !!org.apache.omid.tso.client.OmidClientConfiguration$PostCommitMode ASYNC
diff --git a/bin/omid-server-configuration.yml b/bin/omid-server-configuration.yml
index 470d791..82577b2 100644
--- a/bin/omid-server-configuration.yml
+++ b/bin/omid-server-configuration.yml
@@ -24,18 +24,155 @@
# =====================================================================================================================
-timestampStoreModule: !!org.apache.omid.timestamp.storage.HBaseTimestampStorageModule [ ]
-commitTableStoreModule: !!org.apache.omid.committable.hbase.HBaseCommitTableStorageModule [ ]
+# ---------------------------------------------------------------------------------------------------------------------
+# Basic configuration parameters
+# ---------------------------------------------------------------------------------------------------------------------
-metrics: !!org.apache.omid.metrics.CodahaleMetricsProvider [
-!!org.apache.omid.metrics.CodahaleMetricsConfig {
- outputFreqInSecs: 10,
- reporters: !!set {
- !!org.apache.omid.metrics.CodahaleMetricsConfig$Reporter CSV
- },
- csvDir: "csvMetrics",
-}
-]
+# Network interface for TSO server communication. Uncomment the following line to use a specific interface
+# networkIfaceName: eth0
+# If a network interface in the configuration, the TSO will attempt to guess default network interface.
+# See org.apache.omid.tso.TSOServerConfig.getDefaultNetworkInterface for more information.
+# Port reserved by the Status Oracle
+port: 54758
+# Wait strategy for the Disruptor processors in TSO pipeline. Options:
+# 1) HIGH_THROUGHPUT - [Default] Use this in production deployments for maximum performance
+# 2) LOW_CPU - Use this option when testing or in deployments where saving CPU cycles is more important than throughput
+waitStrategy: LOW_CPU
+# The number of elements reserved in the conflict map to perform conflict resolution
+conflictMapSize: 100000000
+# The number of Commit Table writers that persist data concurrently to the datastore. It has to be at least 2.
+numConcurrentCTWriters: 2
+# The size of the batch of operations that each Commit Table writes has. The maximum number of operations that can be
+# batched in the system at a certain point in time is: numConcurrentCTWriters * batchSizePerCTWriter
+batchSizePerCTWriter: 25
+# When this timeout expires, the contents of the batch are flushed to the datastore
+batchPersistTimeoutInMs: 10
+# Timestamp generation strategy
+# INCREMENTAL - [Default] regular counter
+# WORLD_TIME - world time based counter
timestampType: WORLD_TIME
lowLatency: false
+# Default module configuration (No TSO High Availability & in-memory storage for timestamp and commit tables)
+timestampStoreModule: !!org.apache.omid.timestamp.storage.HBaseTimestampStorageModule [ ]
+commitTableStoreModule: !!org.apache.omid.committable.hbase.HBaseCommitTableStorageModule [ ]
+
+leaseModule: !!org.apache.omid.tso.VoidLeaseManagementModule [ ]
+
+# Default stats/metrics configuration
+metrics: !!org.apache.omid.metrics.NullMetricsProvider [ ]
+
+monitorContext: false
+
+# ---------------------------------------------------------------------------------------------------------------------
+# Timestamp storage configuration options
+# ---------------------------------------------------------------------------------------------------------------------
+# Could be any guava module that binds org.apache.omid.timestamp.storage.TimestampStorage
+# Current available Timestamp stores:
+# org.apache.omid.tso.InMemoryTimestampStorageModule
+# org.apache.omid.timestamp.storage.HBaseTimestampStorageModule
+# org.apache.omid.timestamp.storage.ZKTimestampStorageModule
+
+# ---------------------------------------------------------------------------------------------------------------------
+# Commit Table storage configuration options
+# ---------------------------------------------------------------------------------------------------------------------
+# Could be any guava module that binds org.apache.omid.committable.CommitTable
+# Available CommitTable stores:
+# org.apache.omid.committable.hbase.HBaseCommitTableStorageModule
+# org.apache.omid.tso.InMemoryCommitTableStorageModule
+
+# ---------------------------------------------------------------------------------------------------------------------
+# Metrics configuration options
+# ---------------------------------------------------------------------------------------------------------------------
+# Metrics could be anything that is org.apache.omid.metrics.MetricsRegistry
+# There are 4 built-in reporters: CSV, SLF4J, GRAPHITE, CONSOLE
+# Please see org.apache.omid.metrics.CodahaleMetricsConfig for details.
+
+# Example configuration for reporting statistics to the console every minute:
+#
+# metrics: !!org.apache.omid.metrics.CodahaleMetricsProvider [
+# !!org.apache.omid.metrics.CodahaleMetricsConfig {
+# outputFreqInSecs: 60,
+# reporters: !!set {
+# !!org.apache.omid.metrics.CodahaleMetricsConfig$Reporter CONSOLE
+# },
+# }
+# ]
+
+# Example of multiple reporter configuration (to CSV files and console)
+#
+# metrics: !!org.apache.omid.metrics.CodahaleMetricsProvider [
+# !!org.apache.omid.metrics.CodahaleMetricsConfig {
+# outputFreqInSecs: 60,
+# reporters: !!set {
+# !!org.apache.omid.metrics.CodahaleMetricsConfig$Reporter CSV,
+# !!org.apache.omid.metrics.CodahaleMetricsConfig$Reporter CONSOLE
+# },
+# csvDir: "csvMetrics",
+# prefix: "somePrefix",
+# }
+# ]
+
+# =====================================================================================================================
+# Some example configurations
+# =====================================================================================================================
+
+# ---------------------------------------------------------------------------------------------------------------------
+# Configuration WITHOUT High Availability using HBase for all required storage & reporting metrics to CSV files
+# ---------------------------------------------------------------------------------------------------------------------
+#
+# commitTableStoreModule: !!org.apache.omid.tso.DefaultHBaseCommitTableStorageModule [ ]
+# See optional params
+# - tableName
+# - familyName
+# - principal
+# - keytab
+# timestampStoreModule: !!org.apache.omid.tso.DefaultHBaseTimestampStorageModule [ ]
+# See optional params
+# - tableName
+# - familyName
+# - principal
+# - keytab
+# leaseModule: !!org.apache.omid.tso.VoidLeaseManagementModule [ ]
+# metrics: !!org.apache.omid.metrics.CodahaleMetricsProvider [
+# !!org.apache.omid.metrics.CodahaleMetricsConfig {
+# reporters: !!set {
+# !!org.apache.omid.metrics.CodahaleMetricsConfig$Reporter CSV
+# },
+# csvDir: "myCSVMetricsDir",
+# prefix: "myAppPrefixForMetrics",
+# }
+# ]
+
+# ---------------------------------------------------------------------------------------------------------------------
+# Configuration WITHOUT High Availability using ZK to store the timestamps & reporting metrics to console every 30 secs
+# ---------------------------------------------------------------------------------------------------------------------
+#
+# commitTableStoreModule: !!org.apache.omid.tso.DefaultHBaseCommitTableStorageModule [ ]
+# timestampStoreModule: !!org.apache.omid.tso.DefaultZKTimestampStorageModule
+# zkCluster: "localhost:2181"
+# namespace: "omid"
+# leaseModule: !!org.apache.omid.tso.VoidLeaseManagementModule [ ]
+# metrics: !!org.apache.omid.metrics.CodahaleMetricsProvider [
+# !!org.apache.omid.metrics.CodahaleMetricsConfig {
+# outputFreqInSecs: 30,
+# reporters: !!set {
+# !!org.apache.omid.metrics.CodahaleMetricsConfig$Reporter CONSOLE
+# },
+# }
+# ]
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+# Configuration WITH High Availability using HBase for all required storage and no metrics reports
+# ---------------------------------------------------------------------------------------------------------------------
+#
+# commitTableStoreModule: !!org.apache.omid.tso.DefaultHBaseCommitTableStorageModule [ ]
+# timestampStoreModule: !!org.apache.omid.tso.DefaultHBaseTimestampStorageModule [ ]
+# leaseModule: !!org.apache.omid.tso.HALeaseManagementModule
+# leasePeriodInMs: 10000
+# tsoLeasePath: "/tso-lease"
+# currentTsoPath: "/current-tso"
+# zkCluster: "localhost:2181"
+# zkNamespace: "omid"
+# metrics: !!org.apache.omid.metrics.NullMetricsProvider [ ]
diff --git a/bin/omid.sh b/bin/omid.sh
index 5b33ed5..1ecd05d 100755
--- a/bin/omid.sh
+++ b/bin/omid.sh
@@ -67,18 +67,12 @@ usage() {
echo " create-hbase-timestamp-table Creates the hbase timestamp table."
}
-# if no args specified, show usage
-if [ $# = 0 ]; then
- usage;
- exit 1
-fi
COMMAND=$1
shift
if [ "$COMMAND" = "tso" ]; then
- createHBaseTimestampTable $@;
- createHBaseCommitTable $@;
+ echo "Running Omid TSO."
tso $@;
elif [ "$COMMAND" = "tso-relauncher" ]; then
tsoRelauncher $@;
@@ -87,7 +81,10 @@ elif [ "$COMMAND" = "create-hbase-commit-table" ]; then
elif [ "$COMMAND" = "create-hbase-timestamp-table" ]; then
createHBaseTimestampTable $@;
else
- exec java -cp $CLASSPATH $COMMAND $@
+ createHBaseTimestampTable $@;
+ createHBaseCommitTable $@;
+ echo "Running Omid TSO."
+ tso $@;
fi
diff --git a/phoenix-assembly/src/build/components/all-common-dependencies.xml b/phoenix-assembly/src/build/components/all-common-dependencies.xml
index 84388cd..d52ce70 100644
--- a/phoenix-assembly/src/build/components/all-common-dependencies.xml
+++ b/phoenix-assembly/src/build/components/all-common-dependencies.xml
@@ -38,6 +38,8 @@
<include>com.google.protobuf:protobuf-java</include>
<include>org.slf4j:slf4j-api</include>
<include>org.apache.zookeeper:zookeeper</include>
+ <include>org.apache.curator:curator-framework</include>
+ <include>org.apache.curator:curator-client</include>
<include>log4j:log4j</include>
<include>org.apache.hbase:hbase*</include>
<include>org.antlr:antlr-runtime</include>
diff --git a/phoenix-core/pom.xml b/phoenix-core/pom.xml
index e54c98b..e16449b 100644
--- a/phoenix-core/pom.xml
+++ b/phoenix-core/pom.xml
@@ -201,6 +201,10 @@
<version>${omid.version}</version>
<exclusions>
<exclusion>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
</exclusion>
@@ -223,6 +227,18 @@
<version>${omid.version}</version>
<exclusions>
<exclusion>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-recipes</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
</exclusion>
@@ -235,6 +251,18 @@
<type>test-jar</type>
<exclusions>
<exclusion>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-recipes</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
</exclusion>
@@ -487,6 +515,13 @@
<groupId>com.lmax</groupId>
<artifactId>disruptor</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ <version>${curator.version}</version>
+ </dependency>
+
+
</dependencies>
<reporting>
diff --git a/phoenix-server/pom.xml b/phoenix-server/pom.xml
index b4a90e9..5cf04c3 100644
--- a/phoenix-server/pom.xml
+++ b/phoenix-server/pom.xml
@@ -123,6 +123,8 @@
<include>org.antlr:antlr*</include>
<include>org.apache.tephra:tephra*</include>
<include>org.apache.omid:omid*</include>
+ <include>org.apache.commons:commons-collections4</include>
+ <include>org.jboss.netty:netty</include>
<include>com.google.code.gson:gson</include>
<include>org.jruby.joni:joni</include>
<include>org.jruby.jcodings:jcodings</include>
diff --git a/pom.xml b/pom.xml
index 4ac9e29..cde97f9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -93,7 +93,7 @@
<avatica.version>1.12.0</avatica.version>
<jettyVersion>8.1.7.v20120910</jettyVersion>
<tephra.version>0.15.0-incubating</tephra.version>
- <omid.version>1.0.0</omid.version>
+ <omid.version>1.0.1</omid.version>
<spark.version>2.4.0</spark.version>
<scala.version>2.11.8</scala.version>
<scala.binary.version>2.11</scala.binary.version>