You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by la...@apache.org on 2019/05/12 23:48:24 UTC
[phoenix] branch master updated: PHOENIX-5225 Update Omid to 1.0.1.
This is an automated email from the ASF dual-hosted git repository.
larsh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/master by this push:
new ad4b6d0 PHOENIX-5225 Update Omid to 1.0.1.
ad4b6d0 is described below
commit ad4b6d04cc739537903ca2bd8f3fc2b0b1741f4b
Author: Lars Hofhansl <la...@apache.org>
AuthorDate: Sun May 12 16:48:33 2019 -0700
PHOENIX-5225 Update Omid to 1.0.1.
---
bin/hbase-omid-client-config.yml | 18 +++
bin/omid-server-configuration.yml | 159 +++++++++++++++++++--
bin/omid.sh | 13 +-
.../build/components/all-common-dependencies.xml | 2 +
phoenix-core/pom.xml | 33 ++++-
phoenix-server/pom.xml | 8 ++
pom.xml | 2 +-
7 files changed, 213 insertions(+), 22 deletions(-)
diff --git a/bin/hbase-omid-client-config.yml b/bin/hbase-omid-client-config.yml
new file mode 100644
index 0000000..0c286c2
--- /dev/null
+++ b/bin/hbase-omid-client-config.yml
@@ -0,0 +1,18 @@
+
+#Omid TSO connection
+connectionType: !!org.apache.omid.tso.client.OmidClientConfiguration$ConnType DIRECT
+connectionString: "localhost:54758"
+
+# When Omid is working in High Availability mode, two or more replicas of the TSO server are running in primary/backup
+# mode. When a TSO server replica is elected as master, it publishes its address through ZK. In order to configure
+# the Omid client to access the TSO server in HA mode:
+# 1) set 'connectionType' to !!org.apache.omid.tso.client.OmidClientConfiguration$ConnType HA
+# 2) set 'connectionString' to the ZK cluster connection string where the server is publishing its address
+zkConnectionTimeoutInSecs: 10
+# In HA mode, make sure that the next settings match same settings on the TSO server side
+zkNamespace: "omid"
+zkCurrentTsoPath: "/current-tso"
+
+# Configure whether the TM performs the post-commit actions for a tx (update shadow cells and clean commit table entry)
+# before returning to the control to the client (SYNC) or in parallel (ASYNC)
+postCommitMode: !!org.apache.omid.tso.client.OmidClientConfiguration$PostCommitMode ASYNC
diff --git a/bin/omid-server-configuration.yml b/bin/omid-server-configuration.yml
index 470d791..82577b2 100644
--- a/bin/omid-server-configuration.yml
+++ b/bin/omid-server-configuration.yml
@@ -24,18 +24,155 @@
# =====================================================================================================================
-timestampStoreModule: !!org.apache.omid.timestamp.storage.HBaseTimestampStorageModule [ ]
-commitTableStoreModule: !!org.apache.omid.committable.hbase.HBaseCommitTableStorageModule [ ]
+# ---------------------------------------------------------------------------------------------------------------------
+# Basic configuration parameters
+# ---------------------------------------------------------------------------------------------------------------------
-metrics: !!org.apache.omid.metrics.CodahaleMetricsProvider [
-!!org.apache.omid.metrics.CodahaleMetricsConfig {
- outputFreqInSecs: 10,
- reporters: !!set {
- !!org.apache.omid.metrics.CodahaleMetricsConfig$Reporter CSV
- },
- csvDir: "csvMetrics",
-}
-]
+# Network interface for TSO server communication. Uncomment the following line to use a specific interface
+# networkIfaceName: eth0
+# If a network interface in the configuration, the TSO will attempt to guess default network interface.
+# See org.apache.omid.tso.TSOServerConfig.getDefaultNetworkInterface for more information.
+# Port reserved by the Status Oracle
+port: 54758
+# Wait strategy for the Disruptor processors in TSO pipeline. Options:
+# 1) HIGH_THROUGHPUT - [Default] Use this in production deployments for maximum performance
+# 2) LOW_CPU - Use this option when testing or in deployments where saving CPU cycles is more important than throughput
+waitStrategy: LOW_CPU
+# The number of elements reserved in the conflict map to perform conflict resolution
+conflictMapSize: 100000000
+# The number of Commit Table writers that persist data concurrently to the datastore. It has to be at least 2.
+numConcurrentCTWriters: 2
+# The size of the batch of operations that each Commit Table writes has. The maximum number of operations that can be
+# batched in the system at a certain point in time is: numConcurrentCTWriters * batchSizePerCTWriter
+batchSizePerCTWriter: 25
+# When this timeout expires, the contents of the batch are flushed to the datastore
+batchPersistTimeoutInMs: 10
+# Timestamp generation strategy
+# INCREMENTAL - [Default] regular counter
+# WORLD_TIME - world time based counter
timestampType: WORLD_TIME
lowLatency: false
+# Default module configuration (No TSO High Availability & in-memory storage for timestamp and commit tables)
+timestampStoreModule: !!org.apache.omid.timestamp.storage.HBaseTimestampStorageModule [ ]
+commitTableStoreModule: !!org.apache.omid.committable.hbase.HBaseCommitTableStorageModule [ ]
+
+leaseModule: !!org.apache.omid.tso.VoidLeaseManagementModule [ ]
+
+# Default stats/metrics configuration
+metrics: !!org.apache.omid.metrics.NullMetricsProvider [ ]
+
+monitorContext: false
+
+# ---------------------------------------------------------------------------------------------------------------------
+# Timestamp storage configuration options
+# ---------------------------------------------------------------------------------------------------------------------
+# Could be any guava module that binds org.apache.omid.timestamp.storage.TimestampStorage
+# Current available Timestamp stores:
+# org.apache.omid.tso.InMemoryTimestampStorageModule
+# org.apache.omid.timestamp.storage.HBaseTimestampStorageModule
+# org.apache.omid.timestamp.storage.ZKTimestampStorageModule
+
+# ---------------------------------------------------------------------------------------------------------------------
+# Commit Table storage configuration options
+# ---------------------------------------------------------------------------------------------------------------------
+# Could be any guava module that binds org.apache.omid.committable.CommitTable
+# Available CommitTable stores:
+# org.apache.omid.committable.hbase.HBaseCommitTableStorageModule
+# org.apache.omid.tso.InMemoryCommitTableStorageModule
+
+# ---------------------------------------------------------------------------------------------------------------------
+# Metrics configuration options
+# ---------------------------------------------------------------------------------------------------------------------
+# Metrics could be anything that is org.apache.omid.metrics.MetricsRegistry
+# There are 4 built-in reporters: CSV, SLF4J, GRAPHITE, CONSOLE
+# Please see org.apache.omid.metrics.CodahaleMetricsConfig for details.
+
+# Example configuration for reporting statistics to the console every minute:
+#
+# metrics: !!org.apache.omid.metrics.CodahaleMetricsProvider [
+# !!org.apache.omid.metrics.CodahaleMetricsConfig {
+# outputFreqInSecs: 60,
+# reporters: !!set {
+# !!org.apache.omid.metrics.CodahaleMetricsConfig$Reporter CONSOLE
+# },
+# }
+# ]
+
+# Example of multiple reporter configuration (to CSV files and console)
+#
+# metrics: !!org.apache.omid.metrics.CodahaleMetricsProvider [
+# !!org.apache.omid.metrics.CodahaleMetricsConfig {
+# outputFreqInSecs: 60,
+# reporters: !!set {
+# !!org.apache.omid.metrics.CodahaleMetricsConfig$Reporter CSV,
+# !!org.apache.omid.metrics.CodahaleMetricsConfig$Reporter CONSOLE
+# },
+# csvDir: "csvMetrics",
+# prefix: "somePrefix",
+# }
+# ]
+
+# =====================================================================================================================
+# Some example configurations
+# =====================================================================================================================
+
+# ---------------------------------------------------------------------------------------------------------------------
+# Configuration WITHOUT High Availability using HBase for all required storage & reporting metrics to CSV files
+# ---------------------------------------------------------------------------------------------------------------------
+#
+# commitTableStoreModule: !!org.apache.omid.tso.DefaultHBaseCommitTableStorageModule [ ]
+# See optional params
+# - tableName
+# - familyName
+# - principal
+# - keytab
+# timestampStoreModule: !!org.apache.omid.tso.DefaultHBaseTimestampStorageModule [ ]
+# See optional params
+# - tableName
+# - familyName
+# - principal
+# - keytab
+# leaseModule: !!org.apache.omid.tso.VoidLeaseManagementModule [ ]
+# metrics: !!org.apache.omid.metrics.CodahaleMetricsProvider [
+# !!org.apache.omid.metrics.CodahaleMetricsConfig {
+# reporters: !!set {
+# !!org.apache.omid.metrics.CodahaleMetricsConfig$Reporter CSV
+# },
+# csvDir: "myCSVMetricsDir",
+# prefix: "myAppPrefixForMetrics",
+# }
+# ]
+
+# ---------------------------------------------------------------------------------------------------------------------
+# Configuration WITHOUT High Availability using ZK to store the timestamps & reporting metrics to console every 30 secs
+# ---------------------------------------------------------------------------------------------------------------------
+#
+# commitTableStoreModule: !!org.apache.omid.tso.DefaultHBaseCommitTableStorageModule [ ]
+# timestampStoreModule: !!org.apache.omid.tso.DefaultZKTimestampStorageModule
+# zkCluster: "localhost:2181"
+# namespace: "omid"
+# leaseModule: !!org.apache.omid.tso.VoidLeaseManagementModule [ ]
+# metrics: !!org.apache.omid.metrics.CodahaleMetricsProvider [
+# !!org.apache.omid.metrics.CodahaleMetricsConfig {
+# outputFreqInSecs: 30,
+# reporters: !!set {
+# !!org.apache.omid.metrics.CodahaleMetricsConfig$Reporter CONSOLE
+# },
+# }
+# ]
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+# Configuration WITH High Availability using HBase for all required storage and no metrics reports
+# ---------------------------------------------------------------------------------------------------------------------
+#
+# commitTableStoreModule: !!org.apache.omid.tso.DefaultHBaseCommitTableStorageModule [ ]
+# timestampStoreModule: !!org.apache.omid.tso.DefaultHBaseTimestampStorageModule [ ]
+# leaseModule: !!org.apache.omid.tso.HALeaseManagementModule
+# leasePeriodInMs: 10000
+# tsoLeasePath: "/tso-lease"
+# currentTsoPath: "/current-tso"
+# zkCluster: "localhost:2181"
+# zkNamespace: "omid"
+# metrics: !!org.apache.omid.metrics.NullMetricsProvider [ ]
diff --git a/bin/omid.sh b/bin/omid.sh
index 5b33ed5..1ecd05d 100644
--- a/bin/omid.sh
+++ b/bin/omid.sh
@@ -67,18 +67,12 @@ usage() {
echo " create-hbase-timestamp-table Creates the hbase timestamp table."
}
-# if no args specified, show usage
-if [ $# = 0 ]; then
- usage;
- exit 1
-fi
COMMAND=$1
shift
if [ "$COMMAND" = "tso" ]; then
- createHBaseTimestampTable $@;
- createHBaseCommitTable $@;
+ echo "Running Omid TSO."
tso $@;
elif [ "$COMMAND" = "tso-relauncher" ]; then
tsoRelauncher $@;
@@ -87,7 +81,10 @@ elif [ "$COMMAND" = "create-hbase-commit-table" ]; then
elif [ "$COMMAND" = "create-hbase-timestamp-table" ]; then
createHBaseTimestampTable $@;
else
- exec java -cp $CLASSPATH $COMMAND $@
+ createHBaseTimestampTable $@;
+ createHBaseCommitTable $@;
+ echo "Running Omid TSO."
+ tso $@;
fi
diff --git a/phoenix-assembly/src/build/components/all-common-dependencies.xml b/phoenix-assembly/src/build/components/all-common-dependencies.xml
index b2f5abd..c506bbb 100644
--- a/phoenix-assembly/src/build/components/all-common-dependencies.xml
+++ b/phoenix-assembly/src/build/components/all-common-dependencies.xml
@@ -38,6 +38,8 @@
<include>com.google.protobuf:protobuf-java</include>
<include>org.slf4j:slf4j-api</include>
<include>org.apache.zookeeper:zookeeper</include>
+ <include>org.apache.curator:curator-framework</include>
+ <include>org.apache.curator:curator-client</include>
<include>log4j:log4j</include>
<include>org.apache.hbase:hbase*</include>
<include>org.antlr:antlr-runtime</include>
diff --git a/phoenix-core/pom.xml b/phoenix-core/pom.xml
index 92fe0fb..aa58901 100644
--- a/phoenix-core/pom.xml
+++ b/phoenix-core/pom.xml
@@ -200,6 +200,18 @@
<artifactId>omid-hbase-client-hbase2.x</artifactId>
<version>${omid.version}</version>
<exclusions>
+ <exclusion>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-recipes</artifactId>
+ </exclusion>
<exclusion>
<groupId>org.apache.omid</groupId>
@@ -214,8 +226,6 @@
<artifactId>omid-hbase-shims-hbase1.x</artifactId>
</exclusion>
-
-
<exclusion>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
@@ -340,6 +350,18 @@
<type>test-jar</type>
<exclusions>
<exclusion>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-recipes</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>org.apache.omid</groupId>
<artifactId>omid-hbase-common-hbase1.x</artifactId>
</exclusion>
@@ -642,6 +664,13 @@
<groupId>com.lmax</groupId>
<artifactId>disruptor</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ <version>${curator.version}</version>
+ </dependency>
+
+
</dependencies>
<reporting>
diff --git a/phoenix-server/pom.xml b/phoenix-server/pom.xml
index 579ee5d..bf6ea8d 100644
--- a/phoenix-server/pom.xml
+++ b/phoenix-server/pom.xml
@@ -200,6 +200,14 @@
<shadedPattern>${shaded.package}.org.apache.omid</shadedPattern>
</relocation>
<relocation>
+ <pattern>org.apache.commons-collections4</pattern>
+ <shadedPattern>${shaded.package}.org.apache.commons-collections4</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.jboss.netty:netty</pattern>
+ <shadedPattern>${shaded.package}.org.jboss.netty:netty</shadedPattern>
+ </relocation>
+ <relocation>
<pattern>org.apache.twill</pattern>
<shadedPattern>${shaded.package}.org.apache.twill</shadedPattern>
</relocation>
diff --git a/pom.xml b/pom.xml
index 77a800e..937b487 100644
--- a/pom.xml
+++ b/pom.xml
@@ -92,7 +92,7 @@
<avatica.version>1.12.0</avatica.version>
<jetty.version>9.3.19.v20170502</jetty.version>
<tephra.version>0.15.0-incubating</tephra.version>
- <omid.version>1.0.0</omid.version>
+ <omid.version>1.0.1</omid.version>
<spark.version>2.4.0</spark.version>
<scala.version>2.11.8</scala.version>
<scala.binary.version>2.11</scala.binary.version>