You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by se...@apache.org on 2021/03/19 00:08:24 UTC
[bigtop] branch master updated: BIGTOP-3521. Drop QFS. (#757)
This is an automated email from the ASF dual-hosted git repository.
sekikn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new 8c9fca7 BIGTOP-3521. Drop QFS. (#757)
8c9fca7 is described below
commit 8c9fca7e0637cd34499fdc7769a42d24751d90dc
Author: Masatake Iwasaki <iw...@apache.org>
AuthorDate: Fri Mar 19 09:08:13 2021 +0900
BIGTOP-3521. Drop QFS. (#757)
---
bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml | 9 +-
bigtop-deploy/puppet/hieradata/site.yaml | 1 -
bigtop-deploy/puppet/manifests/cluster.pp | 6 -
.../puppet/modules/hadoop/templates/core-site.xml | 12 -
bigtop-deploy/puppet/modules/qfs/README.md | 80 --
bigtop-deploy/puppet/modules/qfs/manifests/init.pp | 176 ---
.../puppet/modules/qfs/templates/ChunkServer.prp | 280 -----
.../puppet/modules/qfs/templates/MetaServer.prp | 1179 --------------------
.../puppet/modules/qfs/templates/QfsClient.prp | 137 ---
.../puppet/modules/qfs/templates/hadoop-qfs | 79 --
bigtop-packages/src/common/qfs/do-component-build | 129 ---
bigtop-packages/src/common/qfs/install_qfs.sh | 238 ----
bigtop-packages/src/common/qfs/patch0-gf_cpu.diff | 16 -
.../src/common/qfs/patch1-maven-surefire.diff | 29 -
.../common/qfs/patch2-specify-python-version.diff | 20 -
bigtop-packages/src/common/qfs/qfs-chunkserver.cfg | 24 -
bigtop-packages/src/common/qfs/qfs-chunkserver.svc | 94 --
bigtop-packages/src/common/qfs/qfs-metaserver.cfg | 24 -
bigtop-packages/src/common/qfs/qfs-metaserver.svc | 94 --
bigtop-packages/src/common/qfs/qfs-webui.cfg | 24 -
bigtop-packages/src/common/qfs/qfs-webui.svc | 94 --
bigtop-packages/src/deb/qfs/changelog | 1 -
bigtop-packages/src/deb/qfs/compat | 1 -
bigtop-packages/src/deb/qfs/control | 99 --
bigtop-packages/src/deb/qfs/copyright | 15 -
bigtop-packages/src/deb/qfs/qfs-chunkserver.dirs | 2 -
.../src/deb/qfs/qfs-chunkserver.install | 3 -
.../src/deb/qfs/qfs-chunkserver.lintian-overrides | 2 -
.../src/deb/qfs/qfs-chunkserver.postinst | 34 -
bigtop-packages/src/deb/qfs/qfs-chunkserver.postrm | 32 -
.../src/deb/qfs/qfs-chunkserver.preinst | 34 -
bigtop-packages/src/deb/qfs/qfs-client.install | 31 -
.../src/deb/qfs/qfs-client.lintian-overrides | 3 -
bigtop-packages/src/deb/qfs/qfs-dev.install | 27 -
.../src/deb/qfs/qfs-dev.lintian-overrides | 2 -
bigtop-packages/src/deb/qfs/qfs-fuse.install | 1 -
.../src/deb/qfs/qfs-fuse.lintian-overrides | 2 -
bigtop-packages/src/deb/qfs/qfs-hadoop.install | 1 -
.../src/deb/qfs/qfs-hadoop.lintian-overrides | 2 -
bigtop-packages/src/deb/qfs/qfs-java.install | 1 -
.../src/deb/qfs/qfs-java.lintian-overrides | 1 -
bigtop-packages/src/deb/qfs/qfs-metaserver.dirs | 2 -
bigtop-packages/src/deb/qfs/qfs-metaserver.install | 4 -
.../src/deb/qfs/qfs-metaserver.lintian-overrides | 2 -
.../src/deb/qfs/qfs-metaserver.postinst | 34 -
bigtop-packages/src/deb/qfs/qfs-metaserver.postrm | 32 -
bigtop-packages/src/deb/qfs/qfs-metaserver.preinst | 34 -
bigtop-packages/src/deb/qfs/qfs-python.install | 1 -
.../src/deb/qfs/qfs-python.lintian-overrides | 1 -
bigtop-packages/src/deb/qfs/qfs-webui.dirs | 2 -
bigtop-packages/src/deb/qfs/qfs-webui.install | 2 -
.../src/deb/qfs/qfs-webui.lintian-overrides | 1 -
bigtop-packages/src/deb/qfs/qfs-webui.postinst | 34 -
bigtop-packages/src/deb/qfs/qfs-webui.postrm | 32 -
bigtop-packages/src/deb/qfs/qfs-webui.preinst | 34 -
bigtop-packages/src/deb/qfs/qfs.lintian-overrides | 2 -
bigtop-packages/src/deb/qfs/rules | 84 --
bigtop-packages/src/deb/qfs/source/format | 1 -
bigtop-packages/src/rpm/qfs/SPECS/qfs.spec | 353 ------
bigtop-tests/smoke-tests/qfs/build.gradle | 38 -
.../hadoop/mapreduce/TestHadoopExamples.groovy | 34 +-
.../itest/hadoop/mapreduce/TestHadoopSmoke.groovy | 16 +-
bigtop.bom | 11 +-
build.gradle | 1 -
provisioner/utils/smoke-tests.sh | 25 +-
65 files changed, 24 insertions(+), 3795 deletions(-)
diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
index 3a71fab..81e51bd 100644
--- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
+++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
@@ -17,7 +17,7 @@
# be installed as usual. Otherwise only a specified list will be set
# Possible elements:
# hadoop,yarn,hbase,alluxio,flink,solrcloud,spark,oozie,hcat,sqoop,httpfs,
-# pig,hive,zookeeper,ycsb,qfs
+# pig,hive,zookeeper,ycsb
# Example (to deploy only HDFS and YARN server and gateway parts)
# This can be a comma-separated list or an array.
#hadoop_cluster_node::cluster_components:
@@ -153,13 +153,6 @@ spark::common::master_host: "%{hiera('bigtop::hadoop_head_node')}"
alluxio::common::master_host: "%{hiera('bigtop::hadoop_head_node')}"
-# qfs
-qfs::common::metaserver_host: "%{hiera('bigtop::hadoop_head_node')}"
-qfs::common::metaserver_port: "30000"
-qfs::common::chunkserver_port: "30000"
-qfs::common::metaserver_client_port: "20000"
-qfs::common::chunkserver_client_port: "22000"
-
hadoop_zookeeper::server::myid: "0"
hadoop_zookeeper::server::ensemble:
- ["0", "%{hiera('bigtop::hadoop_head_node')}:2888:3888"]
diff --git a/bigtop-deploy/puppet/hieradata/site.yaml b/bigtop-deploy/puppet/hieradata/site.yaml
index 4081b2e..cb861c4 100644
--- a/bigtop-deploy/puppet/hieradata/site.yaml
+++ b/bigtop-deploy/puppet/hieradata/site.yaml
@@ -22,7 +22,6 @@ hadoop::hadoop_storage_dirs:
# - httpfs
# - mapred-app
# - oozie
-# - qfs
# - solrcloud
# - spark
# - spark-standalone
diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp
index a65fb9e..51115a5 100644
--- a/bigtop-deploy/puppet/manifests/cluster.pp
+++ b/bigtop-deploy/puppet/manifests/cluster.pp
@@ -117,11 +117,6 @@ $roles_map = {
zeppelin => {
master => ["zeppelin-server"],
},
- qfs => {
- master => ["qfs-metaserver"],
- worker => ["qfs-chunkserver", "qfs-client"],
- client => ["qfs-client"],
- },
gpdb => {
master => ["gpdb-master"],
worker => ["gpdb-segment"],
@@ -205,7 +200,6 @@ class node_with_roles ($roles = hiera("bigtop::roles")) inherits hadoop_cluster_
"livy",
"solr",
"spark",
- "qfs",
"tez",
"ycsb",
"kerberos",
diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml b/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml
index 72f0bc2..1cae18d 100644
--- a/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml
+++ b/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml
@@ -158,18 +158,6 @@
</property>
<% end -%>
-<% if (scope['::hadoop_cluster_node::cluster_components']).include? 'qfs' -%>
- <!-- run mapreduce over QFS -->
- <property>
- <name>fs.AbstractFileSystem.qfs.impl</name>
- <value>com.quantcast.qfs.hadoop.Qfs</value>
- </property>
- <property>
- <name>fs.qfs.impl</name>
- <value>com.quantcast.qfs.hadoop.QuantcastFileSystem</value>
- </property>
-<% end -%>
-
<% if @kms_host %>
<property>
<name>hadoop.security.key.provider.path</name>
diff --git a/bigtop-deploy/puppet/modules/qfs/README.md b/bigtop-deploy/puppet/modules/qfs/README.md
deleted file mode 100644
index 2225420..0000000
--- a/bigtop-deploy/puppet/modules/qfs/README.md
+++ /dev/null
@@ -1,80 +0,0 @@
-QFS puppet module
-=================
-This module contains puppet recipes for various qfs components, e.g. metaserver,
-chunkserver, and the webui. It is responsible for the following items:
- - Installing all required packages, configuration, init scripts, etc
- - Wiring everything up so that each service can talk to the other
-
-hadoop-qfs
-==========
-Furthermore, this module also installs a compatibility wrapper script called
-hadoop-qfs to make the hadoop and qfs integration and interaction easier.
-
-In order to tell the main hadoop command to use qfs as the underlying
-filesystem, extra options must be specified. For example, to issue a `hadoop fs`
-command, the full command line would look like this:
-
- $ JAVA_LIBRARY_PATH=/usr/lib/qfs hadoop fs
- -Dfs.qfs.impl=com.quantcast.qfs.hadoop.QuantcastFileSystem \
- -Dfs.default.name=qfs://localhost:20000 \
- -Dfs.qfs.metaServerHost=localhost \
- -Dfs.qfs.metaServerPort=20000 \
- -ls /
-
-This (a) is cumbersome and (b) exposes low level details, e.g. metaserver port
-numbers, to the user who likely doesn't care. In order to avoid a poor user
-experience, we provide a wrapper script around the main hadoop command called
-`hadoop-qfs` which handles all the boilerplate for you. Using `hadoop-qfs`, the
-command above is reduced to the following:
-
- $ hadoop-qfs fs -ls /
-
-The `hadoop-qfs` command also supports submitting jobs to hadoop which will use
-qfs as the underlying filesystem instead of the default HDFS. The process is the
-exact same as the standard `hadoop` command, except with using `hadoop-qfs`.
-
- $ hadoop-qfs jar hadoop-mapreduce-examples.jar pi 100 100
-
-Any output data the job writes will be stored in qfs instead of HDFS. Note that
-when submitting jobs through the `hadoop-qfs` command, both the path to the jar
-file as well as the main class are required options.
-
-See the usage for `hadoop-qfs` (`hadoop-qfs --help`) for more information.
-
-Python
-======
-To use the qfs bindings in python, you must set the LD_LIBRARY_PATH to where the
-qfs shared objects are stored so that they can be found.
-
- [root@bigtop1 /]# LD_LIBRARY_PATH=/usr/lib/qfs python
- Python 2.6.6 (r266:84292, Jul 23 2015, 15:22:56)
- [GCC 4.4.7 20120313 (Red Hat 4.4.7-11)] on linux2
- Type "help", "copyright", "credits" or "license" for more information.
- >>> import qfs
- >>>
-
-Why Not Update Hadoop's `core-site.xml`?
-========================================
-The following is a little discussion on the rationale behind why this script has
-been implemented the way it has and some of the other options considered.
-
-One option is to include these configuration options in the core-site.xml file
-in a hadoop configuration directory. If we modify the core-site.xml file that
-the hadoop component installs, qfs silently takes precedence over hdfs. This
-results in a bad user experience since users should have a clear way to choose
-which filesystem they want to interact with.
-
-Another option is to provide our own core-site.xml specific to qfs in an
-alternate location and use the --config option to the hadoop command. This works
-but all of the other configuration in place for hadoop will either be missing or
-have to be symlinked in order to be carried over since we are overriding *all*
-configuration. This is annoying and brittle -- new configuration will have to be
-carried over whenever it is added. Having two places where configuration needs
-to live is usually a bad idea.
-
-The final option considered, and the one used, is to provide a wrapper around
-the hadoop command, don't touch any hadoop configuration, and set the necessary
-override parameters on the command line here. This seemed the cleanest solution
-and also made sure that a qfs installation is as non-confrontational as
-possible. Unfortunately, the hadoop command is very picky about where the
-parameter overrides go depending on the subcommand used.
diff --git a/bigtop-deploy/puppet/modules/qfs/manifests/init.pp b/bigtop-deploy/puppet/modules/qfs/manifests/init.pp
deleted file mode 100644
index aecf186..0000000
--- a/bigtop-deploy/puppet/modules/qfs/manifests/init.pp
+++ /dev/null
@@ -1,176 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-class qfs {
- class deploy($roles) {
- if ("qfs-metaserver" in $roles) {
- include qfs::metaserver
- }
-
- if ("qfs-chunkserver" in $roles) {
- include qfs::chunkserver
- }
-
- if ("qfs-client" in $roles) {
- include qfs::client
- }
- }
-
- class common($metaserver_host, $metaserver_port, $chunkserver_port,
- $metaserver_client_port, $chunkserver_client_port) {
- $cluster_key = sha1($metaserver_host)
- $storage_dirs = suffix($hadoop::hadoop_storage_dirs, "/qfs")
-
- hadoop::create_storage_dir { $qfs::common::storage_dirs: } ->
- file { $qfs::common::storage_dirs:
- ensure => directory,
- owner => root,
- group => root,
- mode => '0755',
- }
-
- file { "/etc/qfs":
- ensure => directory,
- mode => '0755',
- }
- }
-
- class metaserver {
- include qfs::common
-
- package { "qfs-metaserver":
- ensure => latest,
- }
-
- $metaserver_conf = "/etc/qfs/MetaServer.prp"
- file { $metaserver_conf:
- content => template("qfs/MetaServer.prp"),
- require => Package["qfs-metaserver"],
- }
-
- file { [
- "${qfs::common::storage_dirs[0]}/metaserver",
- "${qfs::common::storage_dirs[0]}/metaserver/transaction_logs",
- "${qfs::common::storage_dirs[0]}/metaserver/checkpoint",
- ]:
- ensure => directory,
- owner => qfs,
- group => qfs,
- mode => '0755',
- before => Service['qfs-metaserver'],
- require => [
- File[$qfs::common::storage_dirs[0]],
- Package['qfs-metaserver']
- ],
- }
-
- exec { "mkfs":
- command => "/usr/bin/metaserver -c $metaserver_conf",
- creates => "${qfs::common::storage_dirs[0]}/metaserver/checkpoint/latest",
- # BIGTOP-3126: qfs init script requires to run under a permitted directory
- cwd => "/tmp",
- user => qfs,
- group => qfs,
- require => [
- Package["qfs-metaserver"],
- File[$metaserver_conf],
- File["${qfs::common::storage_dirs[0]}/metaserver/checkpoint"],
- ],
- }
-
- if ($fqdn == $qfs::common::metaserver_host) {
- service { "qfs-metaserver":
- ensure => running,
- require => [
- Package["qfs-metaserver"],
- File[$metaserver_conf],
- Exec["mkfs"],
- ],
- hasrestart => true,
- hasstatus => true,
- }
- }
- }
-
- class chunkserver {
- include qfs::common
-
- package { "qfs-chunkserver":
- ensure => latest,
- }
-
- $chunkserver_conf = "/etc/qfs/ChunkServer.prp"
- file { $chunkserver_conf:
- content => template("qfs/ChunkServer.prp"),
- require => [Package["qfs-chunkserver"], File["/etc/qfs"]],
- }
-
- $cs_dirs = suffix($hadoop::hadoop_storage_dirs, "/qfs/chunkserver")
- $cs_chunks_dirs = suffix($hadoop::hadoop_storage_dirs, "/qfs/chunkserver/chunks")
- $storage_dirs = concat($cs_dirs, $cs_chunks_dirs)
-
- file { $storage_dirs:
- ensure => directory,
- owner => qfs,
- group => qfs,
- mode => '0755',
- before => Service['qfs-chunkserver'],
- require => [
- File[$qfs::common::storage_dirs],
- Package['qfs-chunkserver']
- ],
- }
-
- service { "qfs-chunkserver":
- ensure => running,
- require => [
- Package["qfs-chunkserver"],
- File[$chunkserver_conf]
- ],
- hasrestart => true,
- hasstatus => true,
- }
- }
-
- class client {
- include qfs::common
-
- package { [
- "qfs-client",
- "qfs-hadoop",
- "qfs-java",
- ]:
- ensure => latest,
- }
-
- file { "/etc/qfs/QfsClient.prp":
- content => template("qfs/QfsClient.prp"),
- require => [Package["qfs-client"], File["/etc/qfs"]],
- }
-
- file { "/usr/bin/hadoop-qfs":
- content => template("qfs/hadoop-qfs"),
- mode => '0755',
- }
-
- # Add QFS native lib into Hadoop native lib dir
- exec { "add_qfs_native_lib":
- path => ['/bin','/sbin','/usr/bin','/usr/sbin'],
- command => 'find /usr/lib/qfs/ -name "lib*" -exec ln -s {} /usr/lib/hadoop/lib/native \;',
- require => [ Package["hadoop-yarn-nodemanager"], Package["qfs-client"] ],
- notify => [ Service["hadoop-yarn-nodemanager"] ],
- }
- }
-}
diff --git a/bigtop-deploy/puppet/modules/qfs/templates/ChunkServer.prp b/bigtop-deploy/puppet/modules/qfs/templates/ChunkServer.prp
deleted file mode 100644
index 84875ae..0000000
--- a/bigtop-deploy/puppet/modules/qfs/templates/ChunkServer.prp
+++ /dev/null
@@ -1,280 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# The chunk server configuration.
-
-# The following parameters set must be specified at startup time. Other
-# parameters can be changed at runtime, and it usually more convenient to
-# specify these in meta server configuration. The meta server broadcasts the
-# corresponding chunk server parameters to all connected chunk chunk servers.
-
-# The meta server location.
-chunkServer.metaServer.hostname = <%= scope['qfs::common::metaserver_host'] %>
-chunkServer.metaServer.port = <%= scope['qfs::common::metaserver_port'] %>
-
-# Client connection listener ip address to bind to.
-# Use :: to bind to ipv6 address any.
-# Default is empty, treaated as 0.0.0.0 ipv4 address any, unless the following
-# parameter chunkServer.clientIpV6Only set to 1
-# chunkServer.clientIp =
-
-# Accept ipv4 chunk servers connections.
-# Default is 0, enable acception ipv4 connection. Only has effect if
-# chunkServer.clientIp left empty, or set to ::
-# chunkServer.clientIpV6Only = 0
-
-# Port to open for client connections
-chunkServer.clientPort = <%= scope['qfs::common::chunkserver_client_port'] %>
-
-# Chunk server's rack id. Has effect only if meta server rack prefixes
-# (metaServer.rackPrefixes) are nots set, or chunk server's ip do not match
-# any of the prefix.
-# Default is no rack assigned.
-# chunkServer.rackId = -1
-
-# Space separated list of directories to store chunks (blocks).
-# Usually one directory per physical disk. More than one directory can
-# be used in the cases where the host file system has problems / limitations
-# large directories.
-# The directories that are "not available" (don't exists, io errors,
-# "evacuate.done" file exists, etc.) at a given moment directories are
-# periodically scanned.
-# If the directory becomes available while chunk server is running, chunk server
-# deletes all chunk files in this directory (if any), and starts using this
-# directory.
-# All available directories are periodically scanned, if the directory becomes
-# "unavailable" all chunks in this directory are declared lost, and the
-# gets added to "not available" directories which are periodically scanned as
-# described the above.
-chunkServer.chunkDir = <%= scope['qfs::common::storage_dirs'][0] %>/chunkserver/chunks
-
-# Number of io threads (max. number of disk io requests in flight) per host file
-# system.
-# The typical setup is to have one host file system per physical disk.
-# Even if raid controller is available jbod configuration is recommended,
-# leaving the failure handling and striping to the distributed file system.
-# The default is 2.
-# With large requests (~1MB) two io requests in flight should be sufficient.
-# chunkServer.diskQueue.threadCount = 2
-
-# Number of "client" / network io threads used to service "client" requests,
-# including requests from other chunk servers, handle synchronous replication,
-# chunk re-replication, and chunk RS recovery. Client threads allow to use more
-# than one cpu to perform network io, encryption / decryption, request parsing,
-# checksum and RS recovery calculation.
-# Multiple "client" threads might help to increase network throughput. With
-# modern Intel CPUs, maximum single cpu throughput can be expected roughly
-# 300MB/sec (under 3Gbps network) with 1MB average write request size, and
-# approximately 150MB/sec with network encryption enabled.
-# This parameter has effect only on startup. Please see meta server
-# configuration for related parameters description. The parameters described in
-# the meta server configuration can be changed at any time, and the changes
-# will have effect without chunk server restart.
-# Default is 0. All network io, replication and recovery is handled by the
-# "main" thread.
-# chunkServer.clientThreadCount = 0
-
-# Set client thread affinity to CPU, starting from the specified CPU index. The
-# first cpu index is 0.
-# If the number of CPUs is less than start index plus the number of threads, the
-# the threads affinity at start index plus CPU count will be set to the last
-# CPU.
-# Setting affinity might help to reduce processor ram cache misses.
-# The parameter has effect only on startup, and has effect only on Linux OS.
-# Default is -1, no cpu affinity set.
-# chunkServer.clientThreadFirstCpuIndex = -1
-
-# Set the cluster / fs key, to protect against data loss and "data corruption"
-# due to connecting to a meta server hosting different file system.
-chunkServer.clusterKey = <%= scope['qfs::common::cluster_key'] %>
-
-# Redirect stderror and out into /dev/null to handle the case where one or both
-# are written into a file and the host file system / disk where the file resides
-# exhibiting noticeable io stalls, or completely unavailable..
-# Normally all the log message output performed by the message writer (thread)
-# that deals with log io stall by dropping log messages. This redirection is for
-# extra safety to handle the case if some library function attempting to write
-# into stdout / stderror.
-chunkServer.stdout = /dev/null
-chunkServer.stderr = /dev/null
-
-# The following parameter defines max size of io buffer memory used by the chunk
-# server.
-# The value set here, 128K means 128K * 4K buffer = 512M of buffers
-# The default values are 64K (128MB) for 32 bit build, and 192K (768MB) for 64
-# bit build.
-# The optimal amount of memory depends on the number of disks in the system, and
-# the io (read, write) concurrency -- the number of concurrent "clients". The
-# memory should be increased if large number of concurrent write appenders is
-# expected. Ideally the disk io request should be around 1MB, thus for each
-# chunk opened for append at least 1MB of io buffers is recommended.
-chunkServer.ioBufferPool.partitionBufferCount = 131072
-
-# Buffer manager portion of all io buffers.
-# This value defines max amount of io buffers that can be used for servicing
-# "client" requests, chunk re-replication, and recovery.
-# The remaining (1 - chunkServer.bufferManager.maxRatio) used for write append
-# buffering and other "internal" purposes.
-# Default is 0.4 or 40%
-# chunkServer.bufferManager.maxRatio = 0.4
-
-# Set the following to 1 if no backward compatibility with the previous kfs
-# releases required. 0 is the default.
-# When set to 0 the 0 header checksum (all 8 bytes must be 0) is treated as
-# no checksum and therefore no chunk file header checksum verification
-# performed.
-# The downside of the compatibility mode is that chunk server might not detect
-# the cases where the host os zero fills the data during the host file system
-# recovery / journal / transaction log replay,
-# thus the data loss / corruption problem might not be detected.
-# chunkServer.requireChunkHeaderChecksum = 0
-
-# If set to a value greater than 0 then locked memory limit will be set to the
-# specified value, and mlock(MCL_CURRENT|MCL_FUTURE) invoked.
-# On linux running under non root user setting locked memory "hard" limit
-# greater or equal to the specified value required. ulimit -l can be used for
-# example.
-# Default is 0 -- no memory locking.
-# chunkServer.maxLockedMemory = 0
-
-# Mlock io buffers memory at startup, if set to non 0.
-# Default is 0 -- no io buffer memory locking.
-# chunkServer.ioBufferPool.lockMemory = 0
-
-# ---------------------------------- Message log. ------------------------------
-
-# Set reasonable log level, and other message log parameter to handle the case
-# when meta server not available, or doesn't accept this chunk server for any
-# reason.
-# The chunk servers message log configuration parameters including log level
-# level can be changed in the meta server configuration file.
-chunkServer.msgLogWriter.logLevel = INFO
-
-# Colon (:) separated file name prefixes to store log segments.
-# Default is empty list. The default is to use file name from the command line
-# or if none specified write into file descriptor 2 -- stderror.
-# chunkServer.msgLogWriter.logFilePrefixes =
-
-# Maximum log segment size.
-# Default is -1 -- unlimited.
-# chunkServer.msgLogWriter.maxLogFileSize = -1
-
-# Maximum number of log segments.
-# Default is -1 -- unlimited.
-# chunkServer.msgLogWriter.maxLogFiles = -1
-
-# Max. time to wait for the log buffer to become available.
-# When wait is enabled the request processing thread will wait for the log
-# buffer disk io to complete. If the disk subsystem cannot handle the
-# amount of logging it will slow down the request processing.
-# For chunk servers keeping the default is strongly recommended to minimize
-# dependency on the host's disk subsystem reliability and performance.
-# Default is -1. Do not wait, drop log record instead.
-# chunkServer.msgLogWriter.waitMicroSec = -1
-
-# -------------------- Chunk and meta server authentication. -------------------
-# By default chunk and meta server authentication is off.
-#
-# If any of the following meta authentication methods is configured then chunk
-# server requires QFS client connection to be authenticated. In other words, the
-# QFS client, and other chunk server acting as a client, must obtain from the
-# meta server chunk server access token and corresponding key and use this token
-# and the key to authenticate with the chunk server.
-
-# ================= X509 authentication ========================================
-#
-# Chunk server's X509 certificate file in PEM format.
-# chunkserver.meta.auth.X509.X509PemFile =
-
-# Password if X509 PEM file is encrypted.
-# chunkserver.meta.auth.X509.X509Password =
-
-# Chunk server's private key file.
-# chunkserver.meta.auth.X509.PKeyPemFile =
-
-# Password if private key PEM file is encrypted.
-# chunkserver.meta.auth.X509.PKeyPassword =
-
-# Certificate authorities file. Used for both meta server certificate
-# validation and to create certificate chain with chunk server's X509
-# certificate.
-# chunkserver.meta.auth.X509.CAFile =
-
-# Certificate authorities directory can be used in addition to CAFile.
-# For more detailed information please see SSL_CTX_load_verify_locations manual
-# page. CAFile/CADir corresponds to CAfile/CApath in the man page.
-# chunkserver.meta.auth.X509.CADir =
-
-# If set (the default) verify peer certificate, and declare error if peer, i.e.
-# meta server, does not preset "trusted" valid X509 certificate.
-# Default is on.
-# chunkserver.meta.auth.X509.verifyPeer = 1
-
-# OpenSSL cipher configuration.
-# chunkserver.meta.auth.X509.cipher = !ADH:!AECDH:!MD5:HIGH:@STRENGTH
-
-# The long integer value passed to SSL_CTX_set_options() call.
-# See open ssl documentation for details.
-# Default is the integer value that corresponds to SSL_OP_NO_COMPRESSION
-# chunkserver.meta.auth.X509.options =
-
-# ================= Kerberos authentication ====================================
-#
-# Kerberos principal: service/host@realm
-
-# Meta server's Kerberos principal [service/host@realm] service name part.
-# chunkserver.meta.auth.krb5.service =
-
-# Meta server's Kerberos principal [service/host@realm] host name part.
-# chunkserver.meta.auth.krb5.host =
-
-# Kerberos keytab file with the key(s) that corresponds to the chunk server's
-# principal.
-# chunkserver.meta.auth.krb5.keytab =
-
-# Chunk server's kerberos principal. krb5_parse_name() used to convert the name
-# into the Kerberos 5 internal principal representation.
-# chunkserver.meta.auth.krb5.clientName =
-
-# Force Kerberos client cache initialization during intialization.
-# Default is off.
-# chunkserver.meta.auth.krb5.initClientCache = 0
-
-# OpenSSL cipher configuration for TLS-PSK authentication method. This method
-# is used with delegation and with Kerberos authentication.
-# chunkserver.meta.auth.psk.cipherpsk = !ADH:!AECDH:!MD5:!3DES:PSK:@STRENGTH
-
-# The long integer value passed to SSL_CTX_set_options() call.
-# See open ssl documentation for details.
-# Default is the integer value that corresponds to the logical OR of
-# SSL_OP_NO_COMPRESSION and SSL_OP_NO_TICKET
-# metaServer.clientAuthentication.psk.options =
-
-# ================= PSK authentication =========================================
-#
-# PSK chunk server authentication is intended only for testing and possibly for
-# small [test] clusters with very few chunk servers, where the same
-# authentication credentials [PSK "key"] are used for for all chunk servers.
-
-# Chunk server's PSK key, Must be identical to the key the meta server
-# configured with. See metaServer.CSAuthentication.psk.key parameter description
-# in the annotated meta server configuration file.
-# chunkserver.meta.auth.psk.key =
-
-# Chunk server's PSK key id. See metaServer.CSAuthentication.psk.keyId parameter
-# description in the annotated meta server configuration file.
-# chunkserver.meta.auth.psk.keyId =
-
-#-------------------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/qfs/templates/MetaServer.prp b/bigtop-deploy/puppet/modules/qfs/templates/MetaServer.prp
deleted file mode 100644
index 0bcba74..0000000
--- a/bigtop-deploy/puppet/modules/qfs/templates/MetaServer.prp
+++ /dev/null
@@ -1,1179 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# The meta server configuration.
-
-# Client listener port.
-metaServer.clientPort = <%= scope['qfs::common::metaserver_client_port'] %>
-
-# Client connection listener ip address to bind to.
-# Use :: to bind to ipv6 address any.
-# Default is empty, treaated as 0.0.0.0 ipv4 address any, unless the following
-# parameter metaServer.clientIpV6Only set to 1
-# metaServer.clientIp =
-
-# Accept ipv4 client connections.
-# Default is 0, enable acception ipv4 connection. Only has effect if
-# metaServer.clientIp left empty, or set to ::
-# metaServer.clientIpV6Only = 0
-
-# Chunk server connection listener ip address to bind to.
-# Use :: to bind to ipv6 address any.
-# Default is empty, treaated as 0.0.0.0 ipv4 address any, unless the following
-# parameter metaServer.clientIpV6Only set to 1
-# metaServer.clientIp =
-
-# Accept ipv4 chunk servers connections.
-# Default is 0, enable acception ipv4 connection. Only has effect if
-# metaServer.clientIp left empty, or set to ::
-# metaServer.clientIpV6Only = 0
-
-# Chunk server listener port.
-metaServer.chunkServerPort = <%= scope['qfs::common::chunkserver_port'] %>
-
-# Meta serve transactions log directory.
-metaServer.logDir = <%= scope['qfs::common::storage_dirs'][0] %>/metaserver/transaction_logs
-
-# Meta server checkpoint directory.
-metaServer.cpDir = <%= scope['qfs::common::storage_dirs'][0] %>/metaserver/checkpoint
-
-# Allow to automatically create an empty file system if checkpoint file does
-# not exist.
-# The default is 0, as under the normal circumstances where the file system
-# content is of value, completely losing checkpoint, transaction log, and
-# automatically creating an empty fs will have the same effect as conventional
-# "mkfs". All chunks (blocks) will get deleted, and restoring the checkpoint
-# and logs later won't be sufficient to recover the data.
-# Use "-c" command line option to create new empty file system. For exampe:
-# metaserver -c MetaServer.prp
-# metaServer.createEmptyFs = 0
-
-# Root directory permissions -- used only when the new file system created.
-# metaServer.rootDirUser = 0
-# metaServer.rootDirGroup = 0
-# metaServer.rootDirMode = 0755
-
-# Defaults for checkpoint and transaction log without permissions conversion on
-# startup.
-# metaServer.defaultLoadUser = 0
-# metaServer.defaultLoadGroup = 0
-# metaServer.defaultLoadFileMode = 0644
-# metaServer.defaultLoadDirMode = 0755
-
-# The size of the "client" thread pool.
-# When set to greater than 0, dedicated threads to do client network io, request
-# parsing, and response assembly are created. The thread pool size should
-# usually be (at least one) less than the number of cpus. "Client" threads help
-# with processing large amount of ["short"] requests where more cpu used for
-# context switch, network io, request parsing, and response assembly, than
-# the cpu for the request processing itself. For example i-node attribute
-# lookup, or write append chunk allocations that can be satisfied from the write
-# append allocation cache.
-# Default is 0 -- no dedicated "client" threads.
-# metaServer.clientThreadCount = 0
-
-# Meta server threads affinity.
-# Presently only supported on linux.
-# The first cpu index to set thread affinity to.
-# The main thread will be assigned to the cpu at the specified index, then the
-# next "client" thread will be assigned to the cpu index plus one and so on.
-# For example with 2 client threads and start cpu index 0 the threads affinity
-# would be 0 1 2 respectively.
-# Useful on machines with more than one multi-core processor with shared dram
-# cache. Assigning the threads to the same processor might help minimize dram
-# cache misses.
-# Default is off (start index less than 0) no thread affinity set.
-# metaServer.clientThreadStartCpuAffinity = -1
-
-# Meta server process max. locked memory.
-# If set to a value greater than 0 then locked memory limit will be set to the
-# specified value, and mlock(MCL_CURRENT|MCL_FUTURE) invoked.
-# On linux running under non root user setting locked memory "hard" limit
-# greater or equal to the specified value required. ulimit -l can be used for
-# example.
-# Default is 0 -- no memory locking.
-# metaServer.maxLockedMemory = 0
-
-# Size of [network] io buffer pool.
-# The default buffer size is 4K, therefore the amount of memory is
-# 4K * metaServer.bufferPool.partionBuffers.
-# All io buffers are allocated at startup.
-# If memory locking enabled io buffers are locked in memory at startup.
-# Default is 256K or 1GB on 64 bit system, and 32K or 128MB on 32 bit system.
-# metaServer.bufferPool.partionBuffers = 262144
-
-# ==============================================================================
-# The parameters below this line can be changed at runtime by editing the
-# configuration file and sending meta server process HUP signal.
-# Note that to restore parameter to default at run time the default value must
-# be explicitly specified in the configuration file. In other words commenting
-# out the parameter will not have any effect until restart.
-
-# WORM mode.
-# Write once, read many mode.
-# In this mode only modification of files ".tmp" (without quotes) suffix is
-# allowed.
-# Typically the application would create and write the file with ".tmp" suffix,
-# and then rename it so the destination file name will not have ".tmp " suffix.
-# To delete a file without ".tmp" suffix the mode can be temporary turned off
-# by the administrator. "qfstoggleworm" utility, or temporary configuration
-# modification can be used to do that.
-# Default is 0.
-# metaServer.wormMode = 0
-
-# Mininum number of connected / functional chunk servers before the file system
-# can be used.
-# Default is 1.
-# metaServer.minChunkservers = 1
-
-# Wait 30 sec for chunk servers to connect back after restarting, before file
-# system considered fully functional.
-metaServer.recoveryInterval = 30
-
-# Ignore master/slave chunk server assignment for write append.
-# Master/slave assignment can help with append replication 2, so avoid
-# theoretically possible IO buffers resource deadlock when chunk server A is a
-# in one "AB" synchronous append replication chain, and chunk server B is a
-# master in another chunk "BA" synchronous replication.
-# In practice such deadlocks should be rare enough to matter, and, if occur,
-# are resolve with replication timeout mechanism.
-# The downside of using master/slave assignment is that presently it only works
-# with replication 3, and only half of the chunk server population will be
-# accepting client's append requests.
-# Default is "on" -- ignore.
-# metaServer.appendPlacementIgnoreMasterSlave = 1
-
-# For write append use the low order bit of the IP address for the chunk servers
-# master/slave assignment. This scheme is works well if least significant bit of
-# ip address uniformly distributes masters and slaves withing the rack,
-# especially with "in rack" placement for append.
-# Default is 0. Assign master / slave to keep number of masters and slaves
-# equal. The obvious downside of this is that the assignment depends on the
-# chunk servers connection order.
-# metaServer.assignMasterByIp = 0
-
-# Chunk server executables md5 sums white list.
-# The chunk server sends its executable md5sum when it connects to the meta
-# server. If the following space separated list is not empty and does not
-# does not contain the chunk server executable md5 sum then the chunk server
-# is instructed to exit or restart itself.
-# This might be useful for upgrade or versions control.
-# While the chunk server is connected to the meta server no md5 sum verification
-# performed.
-# Default is empty list.
-# metaServer.chunkServerMd5sums =
-
-# Unique file system id -- some name that uniquely identifies distributed file
-# system instance.
-# This is used to protect data loss / and or corruption in the case where chunk
-# server(s) connect to the "wrong" meta server.
-# The meta server will not accept connections from the chunk servers with a
-# different "cluster key".
-# Default is empty string.
-metaServer.clusterKey = <%= scope['qfs::common::cluster_key'] %>
-
-# Assign rack id by ip prefix -- ip address treated as string.
-# The prefix can be positioned with trailing ??
-# For example: 10.6.34.2?
-# The rack id assigned on chunk server connect, and will not change until the
-# chunk server re-connect. Therefore the configuration file changes will not
-# have any effect until the chunk servers re-connect.
-# Default is empty -- use rack id assigned in the chunk server config.
-# metaServer.rackPrefixes =
-# Example:
-# 10.6.1.* -- rack 1, 10.6.2.* -- rack 2, 10.6.4.1? -- rack 4 etc.
-# metaServer.rackPrefixes = 10.6.1. 1 10.6.2. 2 10.6.4.1? 4 10.6.4.1 5
-
-# "Static" placement weights of the racks. The more weight and more chunk
-# servers are in the rack the more likely the rack will be chosen for chunk
-# allocation.
-# Default is empty -- all weight are default to 1.
-# metaServer.rackWeights =
-# Example: Racks 1 and 2 have weight 1, rack 3 -- 0.9, rack 4 weight 1.2,
-# rack 5 weight 1.5. All other rack weights are 1.
-# metaServer.rackWeights = 1 1 2 1 3 0.9 4 1.2 5 1.5
-
-# Various timeout settings.
-
-# Extend write lease expiration time by 30 sec. in the case of the write master
-# disconnect, to give it a chance to re-connect.
-# Default is 30 sec. Production value is 60 sec.
-# metaServer.leaseOwnerDownExpireDelay = 30
-
-# Re-replication or recovery delay in seconds on chunk server down, to give
-# chunk server a chance to re-connect.
-# Default is 120 sec.
-# metaServer.serverDownReplicationDelay = 120
-
-# Chunk server heartbeat interval.
-# Default is 30 sec.
-# metaServer.chunkServer.heartbeatInterval = 30
-
-# Chunk server operations timeouts.
-# Heartbeat timeout results in declaring chunk server non operational, and
-# closing connection.
-# All other operations timeout are interpreted as the operation failure.
-# The values are in seconds.
-# The defaults:
-# metaServer.chunkServer.heartbeatTimeout = 60
-# metaServer.chunkServer.chunkReallocTimeout = 75
-# metaServer.chunkServer.chunkAllocTimeout = 40
-# metaServer.chunkServer.chunkReallocTimeout = 75
-# metaServer.chunkServer.makeStableTimeout = 330
-# metaServer.chunkServer.replicationTimeout = 330
-
-# The current production values.
-# metaServer.chunkServer.heartbeatInterval = 18
-# metaServer.chunkServer.heartbeatTimeout = 30
-# metaServer.chunkServer.chunkReallocTimeout = 18
-# metaServer.chunkServer.chunkAllocTimeout = 18
-# metaServer.chunkServer.makeStableTimeout = 60
-
-# Other chunk server operations timeout.
-# metaServer.chunkServer.requestTimeout = 600
-
-# Chunk server space utilization placement threshold.
-# Chunk servers with space utilization over this threshold are not considered
-# as candidates for the chunk placement.
-# Default is 0.95 or 95%.
-# metaServer.maxSpaceUtilizationThreshold = 0.95
-
-# Unix style permissions
-# Space separated list of ip addresses of hosts where root user is allowed.
-# Empty list means that root user is allowed on any host.
-# Default is empty.
-# metaServer.rootHosts =
-
-# File modification time update resolution. Increasing the value will reduce the
-# transaction log writes with large files.
-# Default is 1 sec.
-# metaServer.MTimeUpdateResolution = 1
-
-# --------------- File create limits. ------------------------------------------
-#
-# Disallow specific file types. The list is space separate file type ids.
-# Default is empty list. All valid file types are allowed.
-# metaServer.createFileTypeExclude =
-
-# Limit number of data stripes for all file types.If create attempt exceeds
-# the limit the meta server returns "permission denied".
-# Default is the max supported by the compile time constants.
-# metaServer.maxDataStripeCount = 511
-
-# Limit number of recovery stripes for all file types. If create attempt exceeds
-# the limit the meta server returns "permission denied".
-# Default is 32.
-# Max supported by the compile time constants in common/kfstypes.h is 127.
-# metaServer.maxRecoveryStripeCount = 32
-
-# Limit number of data stripes for files with recovery.
-# Default is 64.
-# Max supported by the compile time constants in common/kfstypes.h is 511.
-# metaServer.maxRSDataStripeCount = 64
-
-# Max number of replicas for "regular / replicated" file with no recovery.
-# If create, or change replication requests exceeds this limit then the meta
-# server replaces the value with the value specified.
-# metaServer.maxReplicasPerFile = 64
-
-# Max number of replicas for RS (file with recovery).
-# If create, or change replication requests exceeds this limit then the meta
-# server replaces the value with the value specified.
-# metaServer.maxReplicasPerRSFile = 64
-
-# Force effective user to root. This effectively turns off all permissions
-# control.
-# Default is off.
-# metaServer.forceEUserToRoot = 0
-
-# Client backward compatibility.
-# Defaults are no user and no group -- no backward compatibility.
-# metaServer.defaultUser = 0xFFFFFFFF
-# metaServer.defaultGroup = 0xFFFFFFFF
-# metaServer.defaultFileMode = 0644
-# metaServer.defaultDirMode = 0755
-
-# The chunk server disconnects history size. Useful for monitoring.
-# Default is 4096 slots / disconnect events.
-# metaServer.maxDownServersHistorySize = 4096
-
-# Space and placement re-balancing.
-# Space re-balancing is controlled by the next two parameters (thresholds) below.
-# Re-balancing constantly scans all chunks in the system and checks chunk
-# placement within the replication or RS groups, and moves chunks from chunk
-# servers that are above metaServer.maxRebalanceSpaceUtilThreshold to the chunk
-# servers that are below metaServer.minRebalanceSpaceUtilThreshold.
-# Default is 1 -- on.
-# metaServer.rebalancingEnabled = 1
-
-# Space re-balancing thresholds.
-# Move chunk from the servers that exceed the
-# metaServer.maxRebalanceSpaceUtilThreshold
-# Default is 0.82
-# metaServer.maxRebalanceSpaceUtilThreshold = 0.82
-
-# Move chunks to server below metaServer.minRebalanceSpaceUtilThreshold.
-# Default is 0.72.
-# metaServer.minRebalanceSpaceUtilThreshold = 0.72
-
-# Time interval in seconds between replication queues scans.
-# The more often the scan is scheduled the more cpu can potentially use.
-# Default is 5 sec.
-# metaServer.replicationCheckInterval = 5
-
-# Re-balance scan depth.
-# Max number of chunks to scan in one partial scan. The more chunks are scanned
-# the more cpu re-balance will use, and the "faster" it will scan the chunks.
-# metaServer.maxRebalanceScan = 1024
-
-# Single re-balance partial scan time limit.
-# Default is 0.03 sec.
-# metaServer.maxRebalanceRunTime = 0.03
-
-# Minimum time between two consecutive re-balance partial scans.
-# Default is 0.512 sec.
-# metaServer.rebalanceRunInterval = 0.512
-
-# Max. number of a single client connection requests in flight.
-# The higher value might reduce cpu and alleviate "head of the line blocking"
-# when single client connection shared between multiple concurrent file readers
-# and writers, potentially at the cost of reducing "fairness" between the client
-# connections. Increasing the value could also reduce number of context
-# switches, and os scheduling overhead with the "client" threads enabled.
-# Default is 16 if the "client" threads are enabled, and 1 otherwise.
-# metaServer.clientSM.maxPendingOps = 16
-
-# ------------------ Chunk placement parameters --------------------------------
-
-# The metaServer.sortCandidatesByLoadAvg and
-# metaServer.sortCandidatesBySpaceUtilization are mutially exclusive.
-# metaServer.sortCandidatesBySpaceUtilization takes precedence over
-# metaServer.sortCandidatesByLoadAvg if both set to 1
-
-# When allocating (placing) a chunk prefer chunk servers with lower "load"
-# metric over the chunk servers with the higher "load" metric.
-# For the write intensive file systems turning this mode on is
-# recommended.
-# Default is 0. Do not take chunk server "load" metric into the account.
-# metaServer.sortCandidatesByLoadAvg = 0
-
-# When allocating (placing) a chunk prefer chunk servers with lower disk space
-# utilizaiton.
-# Default is 0. Do not take space utilization into the account.
-# metaServer.sortCandidatesBySpaceUtilization = 0
-
-# When allocating (placing) a chunk do not consider chunk server with the "load"
-# exceeding average load multiplied by metaServer.maxGoodCandidateLoadRatio.
-# Default is 4.
-# metaServer.maxGoodCandidateLoadRatio = 4
-
-# When allocating (placing) a chunk do not consider chunk server with the "load"
-# exceeding average "master" chunk server load multiplied by
-# metaServer.maxGoodMasterLoadRatio if the chunk server is used as master (head
-# or synchronous replication chain).
-# Default is 4.
-# metaServer.maxGoodMasterLoadRatio = 4
-
-# When allocating (placing) a chunk do not consider chunk server with the "load"
-# exceeding average "slave" load multiplied by metaServer.maxGoodSlaveLoadRatio
-# if the chunk server is used as slave.
-# Default is 4.
-# metaServer.maxGoodSlaveLoadRatio = 4
-
-# When allocating (placing) a chunk do not consider chunk server with the
-# average number of chunks opened for write per drive (disk) exceeding average
-# number of chunks opened for write across all disks / chunks servers multiplied
-# by metaServer.maxWritesPerDriveRatio.
-# Default is 1.5.
-# metaServer.maxWritesPerDriveRatio = 1.5
-
-# When allocating (placing) a chunk do not consider chunk server running on the
-# same host as writer if the average number of chunks opened for write per drive
-# (disk) exceeding average number of chunks opened for write across all disks /
-# chunks servers multiplied by metaServer.maxLocalPlacementWeight.
-# Default is 1.0.
-# metaServer.maxLocalPlacementWeight = 1.0
-
-# "In rack" placement for append and non append chunk allocations.
-# Place chunk replicas on the same rack to save cross rack bandwidth at the cost
-# of reduced reliability. Useful for temporary / scratch file systems.
-# Default is 0.
-# metaServer.inRackPlacementForAppend = 0
-
-# "In rack" placement for non append files.
-# Default is 0 - place replicas and chunks from the same RS blocks on different
-# racks.
-# metaServer.inRackPlacement = 0
-
-# Limit number of re-replications (this does not include RS chunk recovery),
-# that the given chunk server can be used as replication "source".
-# Default is 10.
-# metaServer.maxConcurrentReadReplicationsPerNode = 10
-
-# Limit max concurrent chunk re-replications and RS recoveries per chunk server.
-# Default is 5.
-# metaServer.maxConcurrentWriteReplicationsPerNode = 5
-
-#-------------------------------------------------------------------------------
-
-# Order chunk replicas locations by the chunk "load average" metric in "get
-# alloc" responses. The read client logic attempts to use replicas in this
-# order.
-# Default is 0. The replicas locations are shuffled randomly.
-# metaServer.getAllocOrderServersByLoad = 0
-
-# Delay recovery for the chunks that are past the logical end of file in files
-# with Reed-Solomon redundant encoding.
-# The delay is required to avoid starting recovery while the file is being
-# written into, and the chunk sizes aren't known / final. The writer can stop
-# writing into a file, and the corresponding chunks write leases might timed
-# out, and will be automatically revoked. The existing writer logic sets logical
-# EOF when it closes the file, before that the logical file size remains 0
-# during write. (Unless it is re-write which is currently for all practical
-# purposes not supported with RS files). The timeout below should be set to
-# at least the max. practical file "write" time.
-# Setting the timeout to a very large value will prevent processing the chunks
-# sitting in the replication delayed queue from the "abandoned" files, i.e.
-# files that the writer wrote something and then exited without closing the
-# file.
-# The parameter and the corresponding "delay" logic will likely be removed in
-# future releases, and replaced with the write lease renew logic.
-# Default is 6 hours or 21600 seconds.
-# metaServer.pastEofRecoveryDelay = 21600
-
-# Periodic checkpointing.
-# If set to -1 checkpoint is disabled. In such case "logcompactor" can be used
-# periodically create new checkpoint from the transaction logs.
-# Default is 3600 sec.
-# metaServer.checkpoint.interval = 3600
-
-# Checkpoint lock file name. Can be used to serialize checkpoint write and load
-# with external programs, for example logcompactor.
-# Default is empty -- no lock file used.
-# metaServer.checkpoint.lockFileName =
-
-# Max consecutive checkpoint write failures.
-# Meta server will exit if checkpoint write fails
-# metaServer.checkpoint.maxFailedCount times in the row for any reason (not
-# enough disk space for example).
-# Default is 2.
-# metaServer.checkpoint.maxFailedCount = 2
-
-# Checkpoint write timeout. Max time the checkpoint write can take before
-# declaring write failure.
-# Default is 3600 sec.
-# metaServer.checkpoint.writeTimeoutSec = 3600
-
-# Use synchronous mode to write checkpoint, i.e. tell host os to flush all data
-# to disk prior to write system call return.
-# The main purpose is to reduce the number of "dirty" / unwritten pages in the
-# host os vm subsystem / file system buffer cache, therefore reducing memory
-# contention and lowering the chances of paging out meta server and other
-# processes with no memory locking.
-# Default is on.
-# metaServer.checkpoint.writeSync = 1
-
-# Checkpoint write buffer size.
-# The buffer size should be adequate with synchrounous write mode enabled,
-# especially if journal and data of host's file system are on the same spinning
-# media device, in order to minimize the number of seeks.
-# Default is 16MB.
-# metaServer.checkpoint.writeBufferSize = 16777216
-
-# ---------------------------------- Audit log. --------------------------------
-
-# All request headers and response status are logged.
-# The audit log records are null ('\0') separated.
-# The log could be useful for debugging and audit purposes.
-# The logging require some cpu, but the main resource consumption is disk io.
-# Default is off.
-# metaServer.clientSM.auditLogging = 0
-
-# Colon (:) separated file name prefixes to store log segments.
-# Default is empty list.
-# metaServer.auditLogWriter.logFilePrefixes =
-
-# Maximum log segment size.
-# Default is -1 -- unlimited.
-# metaServer.auditLogWriter.maxLogFileSize = -1
-
-# Maximum number of log segments.
-# Default is -1 -- unlimited.
-# metaServer.auditLogWriter.maxLogFiles = -1
-
-# Max. time to wait for the log buffer to become available.
-# When wait is enabled the request processing thread will wait for the log
-# buffer disk io to complete. If the disk subsystem cannot keep up with the
-# logging it will slow down the meta server request processing.
-# Default is -1. Do not wait, drop log record instead.
-# metaServer.auditLogWriter.waitMicroSec = -1
-
-#-------------------------------------------------------------------------------
-
-# ---------------------------------- Message log. ------------------------------
-
-# Message log level FATAL, ALERT, CRIT, ERROR, WARN, NOTICE, INFO, DEBUG
-# Defaul is DEBUG, except for non debug builds with NDEBUG defined INFO is
-# default.
-metaServer.msgLogWriter.logLevel = INFO
-
-# Colon (:) separated file name prefixes to store log segments.
-# Default is empty list. The default is to use file name from the command line
-# or if none specified write into file descriptor 2 -- stderror.
-# metaServer.msgLogWriter.logFilePrefixes =
-
-# Maximum log segment size.
-# Default is -1 -- unlimited.
-# metaServer.msgLogWriter.maxLogFileSize = -1
-
-# Maximum number of log segments.
-# Default is -1 -- unlimited.
-# metaServer.msgLogWriter.maxLogFiles = -1
-
-# Max. time to wait for the log buffer to become available.
-# When wait is enabled the request processing thread will wait for the log
-# buffer disk io to complete. If the disk subsystem cannot keep up with the
-# logging it will slow down the meta server request processing.
-# Default is -1. Do not wait, drop log record instead.
-# metaServer.msgLogWriter.waitMicroSec = -1
-
-#-------------------------------------------------------------------------------
-
-# -------------------- Chunk servers authentication. ---------------------------
-#
-# Authentication is off by default. Both X509 (ssl) and Kerberos authentication
-# methods can be enabled at the same time. Chunk server can negotiate
-# authentication method. If both Kerberos and X509 are configured on the chunk
-# server and meta server then Kerberos authentication is used.
-# Chunk and meta servers perform mutual authentication with authentication
-# enabled.
-#
-# Use of X509 authentication is recommended in order to avoid KDC dependency.
-# Chunk servers have to periodically request Kerberos tickets from KDC. The meta
-# server enforces Kerberos ticket expiration time, by asking chunk server to
-# re-authenticate when its ticket expires. Therefore KDC unavailability for any
-# reason, including network communication outage, might result in chunk servers
-# disconnects. Long enough KDC unavailability might result in unrecoverable data
-# loss, due to the file system unability to perform replication and recovery
-# in response to disk and node failures.
-#
-# Please see OpenSSL documentation for detailed description about X509
-# authentication configuration.
-# src/test-scripts/qfsmkcerts.sh might be used as a simple example how to create
-# and use certificate authority, and X509 certificates.
-
-# Maximum authenticated session lifetime. This limits authenticated session time
-# for all authentication methods. In other words, the session [connection] must
-# be re-authenticated if the authentication token (Kerberos ticket, or x509
-# certificate) "end time" is reached or authenticated session exists longer than
-# the value of this parameter.
-# Default is 24 hours.
-# metaServer.clientAuthentication.maxAuthenticationValidTimeSec = 86400
-
-# Check chunk server authenticated name against the user and group database.
-# If enabled then the authenticated name must be present in the user database in
-# order for chunk server to be accepted.
-# Default is 0 (off), use only black and white lists, if configured, see below.
-# metaServer.CSAuthentication.useUserAndGrupDb = 0
-
-# ================= X509 authentication ========================================
-# Meta server's X509 certificate file in PEM format.
-# metaServer.CSAuthentication.X509.X509PemFile =
-
-# Password if X509 PEM file is encrypted.
-# metaServer.CSAuthentication.X509.X509Password =
-
-# Meta server's private key file.
-# metaServer.CSAuthentication.X509.PKeyPemFile =
-
-# Password if private key PEM file is encrypted.
-# metaServer.CSAuthentication.X509.PKeyPassword =
-
-# Certificate authorities file. Used for both chunk server certificate
-# validation and to create certificate chain with meta server's X509
-# certificate.
-# metaServer.CSAuthentication.X509.CAFile =
-
-# Certificate authorities directory can be used in addition to CAFile.
-# For more detailed information please see SSL_CTX_load_verify_locations manual
-# page. CAFile/CADir corresponds to CAfile/CApath in the man page.
-# metaServer.CSAuthentication.X509.CADir =
-
-# If set (the default) verify peer certificate, and declare error if peer, i.e.
-# chunk server, does not preset "trusted" valid X509 certificate.
-# Default is on.
-# metaServer.CSAuthentication.X509.verifyPeer = 1
-
-# OpenSSL cipher configuration.
-# metaServer.CSAuthentication.X509.cipher = !ADH:!AECDH:!MD5:HIGH:@STRENGTH
-
-# SSL/TLS session cache timeout. Session cache is only used with X509
-# authentication method, with non default client or server side openssl options
-# that turns off use of tls session tickets.
-# Default is 4 hours.
-# metaServer.CSAuthentication.X509.session.timeout = 14400
-
-# The long integer value passed to SSL_CTX_set_options() call.
-# See open ssl documentation for details.
-# Default is the integer value that corresponds to SSL_OP_NO_COMPRESSION
-# metaServer.clientAuthentication.X509.options =
-
-# ================= Kerberos authentication =====================================
-# Kerberos principal: service/host@realm
-
-# Meta server's Kerberos principal [service/host@realm] service name part.
-# metaServer.CSAuthentication.krb5.service =
-
-# Meta server's Kerberos principal [service/host@realm] host name part.
-# metaServer.CSAuthentication.krb5.host =
-
-# Kerberos keytab file with the key(s) that corresponds to the meta server's
-# principal.
-# metaServer.CSAuthentication.krb5.keytab =
-
-# Copy keytab into memory keytab, if supported by the kerberos versions, to
-# improve performance, and avoid disk access.
-# Default is on.
-# metaServer.CSAuthentication.krb5.copyToMemKeytab = 1
-
-# Client's (chunk server) principal "unparse" mode.
-# Can be set to space separated combination of the following modes:
-# short noRealm display
-# The result of the principal conversion to string is used as client's
-# (chunk server's) "authenticated name".
-# The default is fullly qualified principal name. For chunk servers it
-# would typically be in the form of service/host@realm.
-# The "unparsed" chunk server name is checked against "black" and "white" chunk
-# server list names as described below.
-# metaServer.CSAuthentication.krb5.princUnparseMode =
-
-# OpenSSL cipher configuration for TLS-PSK authentication method. This method
-# is used with TLS-PSK and with Kerberos authentication.
-# metaServer.CSAuthentication.psk.cipherpsk = !ADH:!AECDH:!MD5:!3DES:PSK:@STRENGTH
-
-# The long integer value passed to SSL_CTX_set_options() call.
-# See open ssl documentation for details.
-# Default is the integer value that corresponds to the logical OR of
-# SSL_OP_NO_COMPRESSION and SSL_OP_NO_TICKET
-# metaServer.CSAuthentication.psk.options =
-
-# ================= PSK authentication =========================================
-# PSK chunk server authentication is intended only for testing and possibly for
-# small [test] clusters with very few chunk servers, where the same
-# authentication credentials [PSK "key"] are used for for all chunk servers.
-
-# Chunk server PSK key id. This string sent to the chunk as TLS PSK "hint", and
-# also used as chunk server "authenticated name".
-# This effectively overrides chunk server key id.
-# If chunk server key id set to non empty string, then it can be left empty.
-# In such case chunk server key id is used as authenticated name. The chunk
-# server key id sent as "clear text" as part of ssl handshake, and is not
-# "tied" in any way known to the meta server logic to the key id, therefore any
-# "name" can be used. In other words the key is the only real security
-# "credential" with this authentication scheme.
-# The resulting chunk server name must not be empty, and pass "black" and
-# "white" list check, see below.
-# metaServer.CSAuthentication.psk.keyId =
-
-# Chunk server PSK key (the "pre-shared-key"). The same key must be used on the
-# chunk server side in order for psk authentication to work.
-# The default is empty key -- PSK authentication is not enabled.
-# The key must be base 64 encoded, i.e. it must be valid base 64 sequence.
-# metaServer.CSAuthentication.psk.key =
-
-# ================= Chunk servers's "black" and "white" lists ==================
-
-# Chunk server's X509 common names and/or kerberos names, "black" ("revocation")
-# list. If chunk server's authenticated name matches one of the name in this
-# list the authentication will fail. The names in the list are must be
-# separated by spaces. Names with white space symbols are not supported.
-# metaServer.CSAuthentication.blackList =
-
-# Chunk server's X509 common names and/or kerberos names, "white list". Unless
-# the list is empty the chunk server's authenticated name must match one of the
-# names in the list.
-# metaServer.CSAuthentication.whiteList =
-
-#-------------------------------------------------------------------------------
-
-# -------------------- User / "client" authentication. -------------------------
-# Client X509 and kerberos authentication parameters only differ from chunk
-# server's authentication parameters by metaServer.clientAuthentication prefix.
-# The defaults are identical to chunk server authentication.
-
-# Maximum authenticated session lifetime. This limits authenticated session time
-# for all authentication methods. In other words, the session [connection] must
-# be re-authenticated if the authentication token (delegation token, Kerberos
-# ticket, or x509 certificate) "end time" is reached or authenticated session
-# exists longer than the value of this parameter.
-# Default is 24 hours.
-# metaServer.clientAuthentication.maxAuthenticationValidTimeSec = 86400
-
-# ================= X509 authentication ========================================
-# Meta server's X509 certificate file in PEM format.
-# metaServer.clientAuthentication.X509.X509PemFile =
-
-# Password if X509 PEM file is encrypted.
-# metaServer.clientAuthentication.X509.X509Password =
-
-# Meta server's private key file.
-# metaServer.clientAuthentication.X509.PKeyPemFile =
-
-# Password if private key PEM file is encrypted.
-# metaServer.clientAuthentication.X509.PKeyPassword =
-
-# Certificate authorities file. Used for both chunk server certificate
-# validation and to create certificate chain with meta server's X509
-# certificate.
-# metaServer.clientAuthentication.X509.CAFile =
-
-# Certificate authorities directory can be used in addition to CAFile.
-# For more detailed information please see SSL_CTX_load_verify_locations manual
-# page. CAFile/CADir corresponds to CAfile/CApath in the manual page.
-# metaServer.clientAuthentication.X509.CADir =
-
-# If set (the default) verify peer certificate, and declare error if peer, i.e.
-# QFS client, does not preset certificate.
-# Default is on.
-# metaServer.clientAuthentication.X509.verifyPeer = 1
-
-# OpenSSL cipher configuration for X509 authentication method.
-# metaServer.clientAuthentication.X509.cipher = !ADH:!AECDH:!MD5:HIGH:@STRENGTH
-
-# SSL/TLS session cache timeout. Session cache is only used with X509
-# authentication method, with non default client or server side openssl options
-# that turns off use of tls session tickets.
-# Default is 4 hours.
-# metaServer.clientAuthentication.X509.session.timeout = 14400
-
-# The long integer value passed to SSL_CTX_set_options() call.
-# See open ssl documentation for details.
-# Default is the integer value that corresponds to SSL_OP_NO_COMPRESSION
-# metaServer.clientAuthentication.X509.options =
-
-# ================= Kerberos authentication =====================================
-# Kerberos principal: service/host@realm
-
-# Meta server's Kerberos principal [service/host@realm] service name part.
-# metaServer.clientAuthentication.krb5.service =
-
-# Meta server's Kerberos principal [service/host@realm] host name part.
-# metaServer.clientAuthentication.krb5.host =
-
-# Kerberos keytab file with the key(s) that corresponds to the meta server's
-# principal.
-# metaServer.clientAuthentication.krb5.keytab =
-
-# Copy keytab into memory keytab, if supported by the kerberos versions, to
-# improve performance, and avoid disk access.
-# Default is on.
-# metaServer.clientAuthentication.krb5.copyToMemKeytab = 1
-
-# Client's principal "unparse" mode.
-# Can be set to space separated combination of the following modes:
-# short noRealm display
-# The result of the principal conversion to string is used as client's
-# (client's) "authenticated name".
-# The default is fully qualified principal name. For users this typically would
-# it would be in the form user@realm
-# The resulting authentication name should match password database, the meta
-# server host uses. The recommended value is "short', discard realm if it
-# matches the default the kerberos configuration's default realm.
-# metaServer.clientAuthentication.krb5.princUnparseMode =
-
-# OpenSSL cipher configuration for TLS-PSK authentication method. This method
-# is used with delegation and with Kerberos authentication.
-# metaServer.clientAuthentication.psk.cipherpsk = !ADH:!AECDH:!MD5:!3DES:PSK:@STRENGTH
-
-# The long integer value passed to SSL_CTX_set_options() call.
-# See open ssl documentation for details.
-# Default is the integer value that corresponds to the logical OR of
-# SSL_OP_NO_COMPRESSION and SSL_OP_NO_TICKET
-# metaServer.clientAuthentication.psk.options =
-
-# The following two parameters and respective defaults are intended to allow
-# non authenticated access for the meta server web UI from the local host.
-
-# Space separated list of host ips that RPC listed in the next parameter are
-# permitted with no authentication.
-# Default is 127.0.0.1
-# metaServer.clientAuthentication.noAuthOpsHostIps = 127.0.0.1
-
-# Space separated list of RPC names that allowed with no authentication, if the
-# client's host ip obtained with getpeername() call matches one of the ips in
-# the preceding list.
-# Default is RPCs used by the meta server web UI.
-# metaServer.clientAuthentication.noAuthOps = PING GET_CHUNK_SERVERS_COUNTERS GET_CHUNK_SERVER_DIRS_COUNTERS GET_REQUEST_COUNTERS DISCONNECT
-
-# ================= Client's "black" and "white" lists =========================
-
-# Client's (user) X509 common names and/or kerberos names, "black"
-# ("revocation") list. If client's authenticated name matches one of the name in
-# this list the authentication will fail. The names in the list are must be
-# separated by spaces. Names with white space symbols are not supported.
-# metaServer.clientAuthentication.blackList =
-
-# Client's X509 common names and/or kerberos names, "white list". Unless
-# the list is empty the client's authenticated name must match one of the
-# names in the list.
-# metaServer.clientAuthentication.whiteList =
-
-# ================== Delegation ================================================
-#
-# Delegation token expiration time limit.
-# Default is 24 hours.
-# metaServer.clientAuthentication.maxDelegationValidForTimeSec = 86400
-
-# Do not limit delegation token end time to the meta server session credentials
-# (Kerberos ticket or X509 certificate) end time.
-# Default is 0.
-# metaServer.clientAuthentication.delegationIgnoreCredEndTime = 0
-
-================================================================================
-
-# Allow to use "clear text" communication mode by performing SSL/TLS shutdown
-# immediately after successful authentication completion. If enabled, the QFS
-# client's corresponding setting defines the communication mode between the
-# client and the write master. The "clear text" communication mode between chunk
-# servers (synchronous replication, re-replication, and chunk recovery) will be
-# used if this parameter set to "on".
-# Using this mode might make sense in order reduce chunk server CPU utilization
-# and/or possibly increase IO throughput, in the cases where chunk server
-# communication channel is considered to have adequate security for the purpose
-# at hands.
-# Default if "off" / "no"
-# metaServer.clientCSAllowClearText = 0
-
-# Chunk server access token maximum lifetime.
-# Chunk server access token time defines chunk access time limit.
-# Chunk access tokens have 10 min time limit -- twice chunk lease time. The
-# chunk server access token effectively defines maximum client and chunk server
-# to chunk server connections lifetimes. The client and chunk servers attempt
-# to obtain and use a new chunk server access token before the current token
-# expires, and re-open connection with the newly obtained token.
-# Default is 2 hours.
-# metaServer.CSAccessValidForTimeSec = 7200
-
-# The meta server limits the write lease end time to the max of the current time
-# plus the value of the following parameter, and the authentication end time.
-# The parameter is intended primarily for testing, to avoid spurious write
-# retries with authentication maximum life time set to very small value -- 5sec.
-# (The short authentication lifetime is used in order to test the
-# re-authentication logic.)
-# metaServer.minWriteLeaseTimeSec = 600
-
-#-------------------------------------------------------------------------------
-
-# -------------------- User and group configuration. ---------------------------
-# User and group database parameters.
-# The meta server host's user and group configuration used for QFS file system
-# when QFS authentication is enabled. The user and group database is used to map
-# "authenticated names" obtained with Kerberos and X509 authentication methods
-# to user and group ids, and to establish group membership. Authenticated names
-# that have no corresponding user id, or user id that have no corresponding
-# "user name" are considered invalid, and as the result the authentication
-# fails.
-# User and group id with value 4294967295 have special treatment. Access always
-# denied for users with such id.
-# Root user entry with name "root" and id 0 added if not present in the the user
-# database, unless expliclity excluded with metaServer.userAndGroup.excludeUser
-# parameter.
-# With authentication enabled QFS client library does not use host's local user
-# and group database, the meta server's host database is effectively used by all
-# QFS clients.
-
-# Minimal user id to include in user name to id mapping.
-# Default is 0.
-# metaServer.userAndGroup.minUserId = 0
-
-# Maximum user id to include in user name to id mapping.
-# Default is 4294967295.
-# metaServer.userAndGroup.maxUserId = 4294967295
-
-# Minimal group id to include in group name to group id mapping.
-# Default is 0.
-# metaServer.userAndGroup.minGroupId = 0
-
-# Maximum group id to include in group name to group id mapping.
-# Default is 4294967295.
-# metaServer.userAndGroup.maxGroupId = 4294967295
-
-# Omit entries with user names if it has one of the specified prefixes.
-# metaServer.userAndGroup.omitUserPrefix =
-
-# Omit entries with group names if it has one of the specified prefixes.
-# Default is empty list.
-# metaServer.userAndGroup.omitGroupPrefix =
-
-# Update / re-read user and group to id mappings with every N seconds.
-# By default period updates are effectively disabled. The parameter reload with
-# HUP signal can be used to trigger user and group information update.
-# Default is 315360000.
-# metaServer.userAndGroup.updatePeriodSec = 315360000
-
-# Disable user and group initial loading and/or reloading.
-# Default is enabled.
-# metaServer.userAndGroup.disable = 0
-
-# Space separated list of the user names to exclude when loading or updating
-# user database.
-# Default is empty list.
-# metaServer.userAndGroup.excludeUser =
-
-# Space separated list of the group names to exclude when loading or updating
-# group database.
-# Default is empty list.
-# metaServer.userAndGroup.excludeGroup =
-
-# Space separated list of the group names, where members of these groups
-# have effective user id 0 -- root.
-# Default is empty list.
-# metaServer.userAndGroup.rootGroups =
-
-# Space separated list of the user names, where such users have effective user
-# id 0 -- root.
-# User with name root and id 0 always added, even if it isn't presnet or
-# excluded from the user database.
-# Default is empty list.
-# metaServer.userAndGroup.rootUsers =
-
-# Space separated list of the user names. Specified users are allowed to
-# perform meta server administrative requests: fsck, chunk server retire,
-# toggle worm, recompute directory sizes, dump to chunk to servers map,
-# dump replication candidates, check chunk leases, list open files.
-# Default is root user.
-# metaServer.userAndGroup.metaServerAdminUsers = root
-
-# Space separated list of group names. Members of these groups are allowed to
-# perform meta server aministration described in the previous parameter's
-# section.
-# Default is empty list.
-# metaServer.userAndGroup.metaServerAdminGroups =
-
-# Space separated list of the user names. Specified users are allowed to
-# perform meta server status inquiry requests: ping, up servers, meta stats, get
-# chunk servers counters, get chunk directory counters, get meta server request
-# counters.
-# Default is root user.
-# metaServer.userAndGroup.metaServerStatsUsers = root
-
-# Space separated list of group names. Members of these groups are allowed to
-# perform meta server status requests described in the previous parameter's
-# section.
-# Default is empty list.
-# metaServer.userAndGroup.metaServerStatsGroups =
-
-# Space separated list of the user names. Specified users are allowed to
-# renew and cancel delegation tokens that belong to other users. No delegation
-# can be used, to perform delegation renew or cancel, i.e. Kerberos or X509
-# authentication methods must be used.
-# Default is empty list.
-# metaServer.userAndGroup.delegationRenewAndCancelUsers =
-
-# Space separated list of group names. Members of these groups are allowed to
-# renew and cancel delegation tokens that belong to other users, as described
-# in the previous parameter's section.
-# metaServer.userAndGroup.delegationRenewAndCancelGruops =
-
-# -------------------- Meta server cryptographic keys. -------------------------
-
-# Key lifetime. The value defines maximum time before the delegation tokens
-# issued by the meta server expire, and have to be renewed.
-# Default is 4 hours.
-# metaServer.cryptoKeys.keyValidTimeSec = 14400
-
-# Keys change period.
-# Key lifetime minus key change period is the minim time before delegation token
-# must be renewed.
-# metaServer.cryptoKeys.keyChangePeriodSec = 7200
-
-# Meta server crypto keys file name.
-# File name to save meta server keys.
-# Specify file name to save keys, in order to ensure that delegation tokens are
-# persistent across meta server restarts.
-# Default is none, no keys are not persistent across meta server restarts.
-# metaServer.cryptoKeys.keysFileName =
-
-# ------------------- Meta server authentication override ----------------------
-#
-# Setting the following 3 parameters to the values specified below, will
-# effectively disable client authentication. The chunk server authentication can
-# still be enabled.
-# Like the other parameters, removing or commenting out these parameters will not
-# turn back QFS client authentication on, until chunk and meta servers restart.
-# To turn back the QFS client authentication the parameters should be explicitly
-# set back to the original / default values.
-#
-# Overriding the default behavior might be useful for initial authentication
-# setup and/or debugging. Only two sets of values, the default and the set of
-# inverted values work, any other combinations will not though could be useful
-# for testing, i.e. 1 0 0 0 and 0 1 1 1
-#
-# Default is 1 if QFS client authentication *not* configured, 0 otherwise.
-# metaServer.clientAuthentication.authNone = 1
-#
-# Default is 0 if QFS client authentication *not* configured, 1 otherwise.
-# metaServer.clientCSAuthRequired = 0
-#
-# Default is 0 if chunk and meta server server authentication is *not*
-# configured, 1 otherwise.
-# chunkServer.client.auth.enabled = 0
-#
-# Default is 0 if chunk and meta server server authentication is *not*
-# configured, 1 otherwise.
-# chunkServer.remoteSync.auth.enabled = 0
-
-# -------------------- File system ID ------------------------------------------
-#
-# File system id is 64 bit file system identifier generated by the meta server
-# at the time the file system is created. The file system id is used to protect
-# against accidental use of chunk files that belong to a different file system.
-# For example file system id should prevent use of stale chunk inventory that
-# belongs to "old"/different file system, in the case where a new file file
-# file system was created, and the same "cluster key" is used.
-
-# Require file system id in the chunk server hello.
-# Default is 0, to maintain backward compatibility with file systems crated with
-# no file system id.
-# metaServer.fileSystemIdRequired = 0
-
-# The following parameter might be used for temporary file systems, that might
-# be intentionally re-created from scratch on meta server restart, or in the
-# case of loss of transaction log and/or checkpoint. Use with extra caution.
-# Default is 0, Do not delete chunks on file system id mismatch. Do not use
-# chunk directories if chunk directory file system does not match.
-# metaServer.deleteChunkOnFsIdMismatch = 0
-
-#-------------------------------------------------------------------------------
-
-# ===================== Chunk servers configuration parameters. ================
-# Configuration parameters in the meta server configuration file take precedence
-# over the chunk server configuration files.
-
-# ---------------------------------- Message log. ------------------------------
-
-# Chunk server log level.
-chunkServer.msgLogWriter.logLevel = NOTICE
-
-# Colon (:) separated file name prefixes to store log segments.
-# Default is empty list. The default is to use file name from the command line
-# or if none specified write into file descriptor 2 -- stderror.
-# chunkServer.msgLogWriter.logFilePrefixes =
-
-# Maximum log segment size.
-# Default is -1 -- unlimited.
-# chunkServer.msgLogWriter.maxLogFileSize = -1
-
-# Maximum number of log segments.
-# Default is -1 -- unlimited.
-# chunkServer.msgLogWriter.maxLogFiles = -1
-
-# Max. time to wait for the log buffer to become available.
-# When wait is enabled the request processing thread will wait for the log
-# buffer disk io to complete. If the disk subsystem cannot keep up with the
-# logging it will slow down the request processing.
-# For chunk servers keeping the default is strongly recommended to minimize
-# dependency on the host's disk subsystem reliability and performance.
-# Default is -1. Do not wait, drop log record instead.
-# chunkServer.msgLogWriter.waitMicroSec = -1
-
-#-------------------------------------------------------------------------------
-
-# Disk io request timeout.
-# Default is 270 sec. Production value is 40 sec.
-# chunkServer.diskIo.maxIoTimeSec = 270
-
-# Synchronous replication timeouts.
-# Record append synchrounous replicaiton timeout.
-# Default is 180 sec. Production value is 20 sec.
-# chunkServer.recAppender.replicationTimeoutSec = 180
-# Write replication timeout.
-# Default is 300 sec. Production value is 20 sec.
-# chunkServer.remoteSync.responseTimeoutSec = 300
-
-# Controls buffered io -- use os file system cache, instead of direct io on the
-# os / file systems that support direct io (most file systems on linux).
-# Default is off.
-# It is conceivable that enabling buffered io might help with short reads for
-# the "broadcast" / "web server" type of loads. For the "typical" large io (1MB)
-# requests sequential type loads enabling caching will likely lower cluster
-# performance due to higher system (os) cpu overhead, and memory contention.
-# Default is off.
-# chunkServer.bufferedIo = 0
-
-# If sparse files, and in particular chunks aren't used (sequential write only
-# for example) the following parameter can be set to 0.
-# Default is 1 -- enabled.
-# chunkServer.allowSparseChunks = 1
-
-# The minimal amount of space in bytes that must be available in order for the
-# chunk directory to be used for chunk placement (considered as "writable").
-# Default is chunk size -- 64MB plus chunk header size 16KB.
-# chunkServer.minFsAvailableSpace = 67125248
-
-# The minimal amount of space that must be available in order for the chunk
-# directory to be used for chunk placement (considered as "writable"), expressed
-# as part total host file system space.
-# Default is 0.05 or 5%, or in other words stop using chunk directory when the
-# host file system where the chunk directory resides reaches 95% space
-# utilization.
-# chunkServer.maxSpaceUtilizationThreshold = 0.05
-
-# The "weight" of pending disk io in chunk placement.
-# If set to 0 or less the pending io (number of io bytes in the disk queue) has
-# no no effect on the placement (choosing chunk directory where to create chunk).
-# If weight set to greater than 0, then the average pending io per chunk
-# directory (host file system / disk) is calculated, as
-# (total_pending_read_bytes * total_pending_read_weight +
-# total_pending_write_bytes * total_pending_write_weight) / chunk_directory_count
-# Chunk directories with pending_read + pending_write that exceed the value the
-# above are taken out of the consideration for placement.
-# Default is 0. Typical production value is 1.3
-# chunkServer.chunkPlacementPendingReadWeight = 0
-# chunkServer.chunkPlacementPendingWriteWeight = 0
-
-# Averaging interval for calculating the average time the incoming "client's"
-# requests spend in io buffer wait queue. The "average wait time" value used by
-# the meta server for chunk placement. The average exponentially decays (IIR
-# filter).
-# Default is 20 sec. Typical production value is 8.
-# chunkServer.bufferManager.waitingAvgInterval = 20
-
-# "Not available" directories rescan interval in seconds. Default is 180 sec.
-# (see comment in chunk server configuration file).
-# chunkServer.dirRecheckInterval = 60
-
-# The following parameter has effect only if client threads enabled, i.e. if
-# chunkServer.clientThreadCount parameter set to a value greater than 0 in the
-# chunk server configuration.
-# If the value is less than chunkServer.clientThreadCount, then
-# threads in range
-# [chunkServer.client.firstClientThreadIndex, chunkServer.clientThreadCount)
-# will be used to service all requests, except RS chunk recovery, otherwise the
-# "main" thread will be used.
-# chunkServer.client.firstClientThreadIndex = 0
-
-# The following parameter has effect only if client threads enabled, i.e. if
-# chunkServer.clientThreadCount parameter set to a value greater than 0 in the
-# chunk server configuration.
-# Limit number of client threads used for RS recovery. Each thread uses
-# single dedicated connection to the meta server to perform RS recovery.
-# If set to 0 or less, then the "main" thread is used to perform RS recovery.
-# The client threads in the range
-# [0, min(chunkServer.rsReader.maxRecoveryThreads, chunkServer.clientThreadCount)
-# are used to preform RS recovery.
-# Default is 5. The same value as
-# metaServer.maxConcurrentWriteReplicationsPerNode default.
-# chunkServer.rsReader.maxRecoveryThreads = 5
diff --git a/bigtop-deploy/puppet/modules/qfs/templates/QfsClient.prp b/bigtop-deploy/puppet/modules/qfs/templates/QfsClient.prp
deleted file mode 100644
index 94ae962..0000000
--- a/bigtop-deploy/puppet/modules/qfs/templates/QfsClient.prp
+++ /dev/null
@@ -1,137 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# The meta server configuration.
-
-# Where is the metaserver
-metaServer.name = <%= scope['qfs::common::metaserver_host'] %>
-metaServer.port = <%= scope['qfs::common::metaserver_port'] %>
-
-# -------------------- Client and meta server authentication. ------------------
-# By default QFS client and meta server authentication (client and chunk server
-# authentication as a consequence) is off.
-#
-# If any of the following meta authentication method is configured then QFS
-# client and the meta server perform mutual authentication.
-#
-# The QFS client configuration parameters can be specified also via environment
-# variables: QFS_CLIENT_CONFIG and QFS_CLIENT_CONFIG and
-# QFS_CLIENT_CONFIG_meta_server_ip_port. The later variable takes precedence.
-# The dots in the meta server ip (or host name) are replaced with _ (underscore)
-# symbols. The underscore symbol also used to separate meta server ip and port.
-# The later, longer form allows to use configuration specific to a
-# particular meta server, and mainly intended to be used with the QFS
-# delegation where both the delegation token and the key can be passed via
-# environment variables (see PSK authentication section below)),
-#
-# The two from environment vairable values are supported:
-# 1. FILE:configuration_file_name
-# 2. parameter_name1=parameter_value1 parameter_name2=parameter_value2...
-# The second space separated key value pairs can be used to pass delegation
-# token and the corresponding key. Both these must be obtained from the meta
-# server via "delegate" request. See qfs tool help.
-# For example:
-# QFS_CLIENT_CONFIG_127_0_0_1_20000='client.auth.psk.keyId=AAAB9dYIWfKBXhXCI1jJ9gAAU0XunwAAAACMoK0z30ztT5S7k9slRuRdzy9CXmi1 client.auth.psk.keyId=P+4XRIBLLBvkICXWO+1aXBPUTMghEakkTk1T+RVsifR9NQ71E32KVd27y+2DbyC2'
-# export QFS_CLIENT_CONFIG_127_0_0_1_20000
-
-
-# ================= X509 authentication ========================================
-#
-# QFS client's X509 certificate file in PEM format.
-# client.auth.X509.X509PemFile =
-
-# Password if X509 PEM file is encrypted.
-# client.auth.X509.X509Password =
-
-# QFS client's private key file.
-# client.auth.X509.PKeyPemFile =
-
-# Password if private key PEM file is encrypted.
-# client.auth.X509.PKeyPassword =
-
-# Certificate authorities file. Used for both meta server certificate
-# validation and to create certificate chain with QFS client's X509
-# certificate.
-# client.auth.X509.CAFile =
-
-# Certificate authorities directory can be used in addition to CAFile.
-# For more detailed information please see SSL_CTX_load_verify_locations manual
-# page. CAFile/CADir corresponds to CAfile/CApath in the man page.
-# client.auth.X509.CADir =
-
-# If set (the default) verify peer certificate, and declare error if peer, i.e.
-# meta server, does not preset "trusted" valid X509 certificate.
-# Default is on.
-# client.auth.X509.verifyPeer = 1
-
-# OpenSSL cipher configuration.
-# client.auth.X509.cipher = !ADH:!AECDH:!MD5:HIGH:@STRENGTH
-
-# The long integer value passed to SSL_CTX_set_options() call.
-# See open ssl documentation for details.
-# Default is the integer value that corresponds to SSL_OP_NO_COMPRESSION
-# client.auth.X509.options =
-
-# ================= Kerberos authentication ====================================
-#
-# Kerberos service principal: service/host@realm
-
-# Meta server's Kerberos principal [service/host@realm] service name part.
-# client.auth.krb5.service =
-
-# Meta server's Kerberos principal [service/host@realm] host name part.
-# client.auth.krb5.host =
-
-# Normally kinit is sufficient for the user authentication.
-# The following Kerberos parameters might be used in the case when another
-# "service" acts as QFS client.
-
-# Kerberos keytab file with the key(s) that corresponds to the QFS client's
-# principal, if used. Key table is typically used for service.
-# client.auth.krb5.keytab =
-
-# QFS client's kerberos principal. krb5_parse_name() used to convert the name
-# into the Kerberos 5 internal principal representation.
-# client.auth.krb5.clientName =
-
-# Force Kerberos client cache initialization during intialization.
-# Default is off.
-# client.auth.krb5.initClientCache = 0
-
-# OpenSSL cipher configuration for TLS-PSK authentication method. This method
-# is used with delegation and with Kerberos authentication.
-# client.auth.psk.cipherpsk = !ADH:!AECDH:!MD5:!3DES:PSK:@STRENGTH
-
-# The long integer value passed to SSL_CTX_set_options() call.
-# See open ssl documentation for details.
-# Default is the integer value that corresponds to the logical OR of
-# SSL_OP_NO_COMPRESSION and SSL_OP_NO_TICKET
-# metaServer.clientAuthentication.psk.options =
-
-# ================= PSK / delegation authentication ============================
-#
-# Both delegation token and delegation key are expected to be valid base 64
-# encoded binary blobs -- the exact string representation returned by the
-# delegation request.
-
-# QFS client delegation token, The token must be obtained via delegation request
-# the meta server. Both the token and the corresponding key must be specified.
-# client.auth.psk.keyId =
-
-# QFS client delegation key, The key must be obtained via delegation request to
-# the meta server.
-# client.auth.psk.key =
-
-#-------------------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/qfs/templates/hadoop-qfs b/bigtop-deploy/puppet/modules/qfs/templates/hadoop-qfs
deleted file mode 100644
index 7041daf..0000000
--- a/bigtop-deploy/puppet/modules/qfs/templates/hadoop-qfs
+++ /dev/null
@@ -1,79 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# A simple wrapper script around hadoop which enables the use of qfs easily.
-# Using the hadoop command with qfs requires users to specify overrides for
-# various parameters (e.g. fs.defaultFS) which can become cumbersome to do on
-# the command line every time. The goal here is to provide a clean user
-# experience in using hadoop and qfs.
-
-export JAVA_LIBRARY_PATH=/usr/lib/qfs
-
-QFS_OPTS=" -Dfs.qfs.impl=com.quantcast.qfs.hadoop.QuantcastFileSystem"
-QFS_OPTS="$QFS_OPTS -Dfs.defaultFS=qfs://<%= scope['qfs::common::metaserver_host'] %>:<%= scope['qfs::common::metaserver_client_port'] %>"
-QFS_OPTS="$QFS_OPTS -Dfs.qfs.metaServerHost=<%= scope['qfs::common::metaserver_host'] %>"
-QFS_OPTS="$QFS_OPTS -Dfs.qfs.metaServerPort=<%= scope['qfs::common::metaserver_port'] %>"
-
-usage() {
- echo "
-usage: $0 <options>
- $0 is a simple wrapper around the standard hadoop command that allows for an
- easy interface with qfs. A subset of the standard hadoop commands are
- supported (where it makes sense). For all other commands, use the hadoop
- command directly.
-
- Commands:
- fs query the qfs filesystem
- jar <jar> <main class> submit a job to hadoop that uses qfs
- Note: the main class is required
- distcp <srcurl> <desturl> copy file or directories recursively
- Note: supports copy into qfs from hdfs
- "
- exit 1
-}
-
-if [ $# == 0 ]; then
- usage
-fi
-
-COMMAND=$1
-shift
-
-case $COMMAND in
- # usage flags
- --help|-h)
- usage
- ;;
- fs|distcp)
- hadoop $COMMAND $QFS_OPTS "$@"
- ;;
- jar)
- if [ $# -lt 2 ]; then
- echo "$COMMAND: jar file and main class are required"
- usage
- fi
-
- JAR_FILE=$1
- MAIN_CLASS=$2
- shift 2
- hadoop $COMMAND $JAR_FILE $MAIN_CLASS $QFS_OPTS "$@"
- ;;
- *)
- echo "$COMMAND: unsupported -- try using the hadoop command directly"
- usage
- ;;
-esac
diff --git a/bigtop-packages/src/common/qfs/do-component-build b/bigtop-packages/src/common/qfs/do-component-build
deleted file mode 100644
index 738f011..0000000
--- a/bigtop-packages/src/common/qfs/do-component-build
+++ /dev/null
@@ -1,129 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-. `dirname $0`/bigtop.bom
-
-usage() {
- echo "
-usage: $0 <options>
- Required not-so-options:
- --qfs-version=VERSION version of qfs we are installing
- --python=PATH path to python
-
- Optional options:
- --python3=PATH path to python 3
- "
- exit 1
-}
-
-OPTS=$(getopt \
- -n $0 \
- -o '' \
- -l 'qfs-version:' \
- -l 'python:' \
- -l 'python3:' \
- -- "$@")
-
-if [ $? != 0 ] ; then
- usage
-fi
-
-eval set -- "$OPTS"
-while true ; do
- case "$1" in
- --qfs-version)
- QFS_VERSION=$2 ; shift 2
- ;;
- --python)
- PYTHON_PATH=$2 ; shift 2
- ;;
- --python3)
- PYTHON3_PATH=$2 ; shift 2
- ;;
- --)
- shift ; break
- ;;
- *)
- echo "Unknown option: $1"
- usage
- exit 1
- ;;
- esac
-done
-
-for var in QFS_VERSION PYTHON_PATH ; do
- if [ -z "$(eval "echo \$$var")" ]; then
- echo Missing required param: $var
- usage
- fi
-done
-
-# bigtop installs maven on its own and not through the package manager
-export PATH=/usr/local/maven/bin:$PATH
-command -v mvn > /dev/null 2>&1 || {
- echo >&2 "Maven is required but not found in PATH. Check your installation."
- exit 1
-}
-
-PYTHON3_PATH=${PYTHON3_PATH:-}
-
-# Workaround for BIGTOP-2710
-sed -ie '/which/!s#mvn #mvn -Duser.home=${HOME} #g' src/java/javabuild.sh
-
-# Build and install QFS into the build/release directory
-make
-
-# Build QFS Java library JAR and Hadoop plugin JAR.
-# NOTE(fsareshwala): we have src/java/javabuild.sh to do this for us, but that
-# script does some git magic to determine the release numbers and hashes from
-# git tags. However, in bigtop, we have an entirely different set of git objects
-# and will get the wrong tag name and release hash if we use javabuild.sh as it
-# currently is. Until qfs can update its build process upstream, we replicate
-# what javabuild.sh would do.
-get_hadoop_qfs_profile() {
- myversion="`echo "$HADOOP_VERSION" | cut -d. -f 1-2`"
- myversionmaj="`echo "$HADOOP_VERSION" | cut -d. -f 1`"
-
- if [ x"$myversion" = x"1.0" -o x"$myversion" = x"1.1" ]; then
- echo "hadoop_branch1_profile"
- elif [ x"$myversion" = x"0.23" ]; then
- echo "hadoop_trunk_profile"
- elif [ x"$myversionmaj" = x"2" ]; then
- echo "hadoop_trunk_profile,hadoop_trunk_profile_2"
- else
- echo "Unsupported Hadoop release version."
- exit 1
- fi
-}
-
-HADOOP_QFS_PROFILE=$(get_hadoop_qfs_profile)
-cd src/java
-mvn -Duser.home=${HOME} -Dqfs.release.version=${QFS_VERSION} --projects qfs-access package
-mvn -P $HADOOP_QFS_PROFILE -Duser.home=${HOME} -Dqfs.release.version=${QFS_VERSION} \
- -Dhadoop.release.version=${HADOOP_VERSION} -Dtest.build.data=/tmp \
- package
-cd ../..
-
-# Build QFS Python libraries.
-cd build/release
-$PYTHON_PATH ../../src/cc/access/kfs_setup.py build
-if [ ! -z "$PYTHON3_PATH" ]; then
- $PYTHON3_PATH ../../src/cc/access/kfs_setup.py build
-fi
-cd ..
diff --git a/bigtop-packages/src/common/qfs/install_qfs.sh b/bigtop-packages/src/common/qfs/install_qfs.sh
deleted file mode 100644
index 1b4249b..0000000
--- a/bigtop-packages/src/common/qfs/install_qfs.sh
+++ /dev/null
@@ -1,238 +0,0 @@
-#!/bin/bash -x
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-usage() {
- echo "
-usage: $0 <options>
- Required not-so-options:
- --prefix=PREFIX path to install into
- --source-dir=DIR path to package shared files dir
- --qfs-version=VERSION version of qfs we are installing
- --python=PYTHON the path to python
-
- Optional options:
- --bin-dir=DIR path to install binaries (default: /usr/bin)
- --lib-dir=DIR path to install libraries (default: /usr/lib)
- --etc-dir=DIR path to install configuration (default: /etc/qfs)
- --include-dir=DIR path to install devel headers (default: /usr/include)
- --data-dir=DIR path to install various data (default: /usr/share/qfs)
- --var-dir=DIR path to keep variable data (default: /var/qfs)
- --hadoop-home=DIR path where hadoop lives (default: /usr/lib/hadoop)
- --python3=PYTHON3 path to python 3
- --python-extra=PYTHON_EXTRA extra arguments to pass to python when installing
-
- Note: you will have to change the init scripts with the new paths you use
- above if you deviate from the defaults.
- "
- exit 1
-}
-
-OPTS=$(getopt \
- -n $0 \
- -o '' \
- -l 'source-dir:' \
- -l 'prefix:' \
- -l 'bin-dir:' \
- -l 'lib-dir:' \
- -l 'etc-dir:' \
- -l 'include-dir:' \
- -l 'data-dir:' \
- -l 'var-dir:' \
- -l 'hadoop-home:' \
- -l 'qfs-version:' \
- -l 'python:' \
- -l 'python3:' \
- -l 'python-extra:' \
- -- "$@")
-
-if [ $? != 0 ] ; then
- usage
-fi
-
-eval set -- "$OPTS"
-while true ; do
- case "$1" in
- --source-dir)
- SOURCE_DIR=$2 ; shift 2
- ;;
- --prefix)
- PREFIX=$2 ; shift 2
- ;;
- --bin-dir)
- BIN_DIR=$2 ; shift 2
- ;;
- --lib-dir)
- LIB_DIR=$2 ; shift 2
- ;;
- --etc-dir)
- ETC_DIR=$2 ; shift 2
- ;;
- --include-dir)
- INCLUDE_DIR=$2 ; shift 2
- ;;
- --data-dir)
- DATA_DIR=$2 ; shift 2
- ;;
- --var-dir)
- VAR_DIR=$2 ; shift 2
- ;;
- --hadoop-home)
- HADOOP_HOME=$2 ; shift 2
- ;;
- --qfs-version)
- QFS_VERSION=$2 ; shift 2
- ;;
- --python)
- PYTHON_PATH=$2 ; shift 2
- ;;
- --python3)
- PYTHON3_PATH=$2 ; shift 2
- ;;
- --python-extra)
- PYTHON_EXTRA=$2 ; shift 2
- ;;
- --)
- shift ; break
- ;;
- *)
- echo "Unknown option: $1"
- usage
- exit 1
- ;;
- esac
-done
-
-for var in SOURCE_DIR PREFIX QFS_VERSION PYTHON_PATH ; do
- if [ -z "$(eval "echo \$$var")" ]; then
- echo Missing required param: $var
- usage
- fi
-done
-
-if [ -f "$SOURCE_DIR/bigtop.bom" ]; then
- . $SOURCE_DIR/bigtop.bom
-fi
-
-PREFIX=$(readlink -f $PREFIX)
-BIN_DIR=$PREFIX/${BIN_DIR:-/usr/bin/qfs}
-LIB_DIR=$PREFIX/${LIB_DIR:-/usr/lib/qfs}
-ETC_DIR=$PREFIX/${ETC_DIR:-/etc/qfs}
-INCLUDE_DIR=$PREFIX/${INCLUDE_DIR:-/usr/include}
-DATA_DIR=$PREFIX/${DATA_DIR:-/usr/share/qfs}
-VAR_DIR=$PREFIX/${VAR_DIR:-/var/qfs}
-HADOOP_HOME=$PREFIX/${HADOOP_HOME:-/usr/lib/hadoop}
-PYTHON_EXTRA=${PYTHON_EXTRA:-}
-
-mkdir -p $BIN_DIR
-install -m 755 build/release/bin/filelister $BIN_DIR
-install -m 755 build/release/bin/logcompactor $BIN_DIR
-install -m 755 build/release/bin/metaserver $BIN_DIR
-install -m 755 build/release/bin/qfsfsck $BIN_DIR
-install -m 755 build/release/bin/qfs_fuse $BIN_DIR
-install -m 755 build/release/bin/chunkserver $BIN_DIR
-install -m 755 build/release/bin/chunkscrubber $BIN_DIR
-
-install -m 755 build/release/bin/devtools/checksum $BIN_DIR
-install -m 755 build/release/bin/devtools/dirtree_creator $BIN_DIR
-install -m 755 build/release/bin/devtools/dtokentest $BIN_DIR
-install -m 755 build/release/bin/devtools/logger $BIN_DIR/qfslogger
-install -m 755 build/release/bin/devtools/rand-sfmt $BIN_DIR
-install -m 755 build/release/bin/devtools/requestparser $BIN_DIR
-install -m 755 build/release/bin/devtools/sortedhash $BIN_DIR
-install -m 755 build/release/bin/devtools/sslfiltertest $BIN_DIR
-install -m 755 build/release/bin/devtools/stlset $BIN_DIR
-
-install -m 755 build/release/bin/emulator/rebalanceexecutor $BIN_DIR
-install -m 755 build/release/bin/emulator/rebalanceplanner $BIN_DIR
-install -m 755 build/release/bin/emulator/replicachecker $BIN_DIR
-
-install -m 755 build/release/bin/examples/qfssample $BIN_DIR
-
-install -m 755 build/release/bin/tools/cpfromqfs $BIN_DIR
-install -m 755 build/release/bin/tools/cptoqfs $BIN_DIR
-install -m 755 build/release/bin/tools/qfs $BIN_DIR
-install -m 755 build/release/bin/tools/qfsadmin $BIN_DIR
-install -m 755 build/release/bin/tools/qfscat $BIN_DIR
-install -m 755 build/release/bin/tools/qfsdataverify $BIN_DIR
-install -m 755 build/release/bin/tools/qfsfileenum $BIN_DIR
-install -m 755 build/release/bin/tools/qfshibernate $BIN_DIR
-install -m 755 build/release/bin/tools/qfsping $BIN_DIR
-install -m 755 build/release/bin/tools/qfsput $BIN_DIR
-install -m 755 build/release/bin/tools/qfsshell $BIN_DIR
-install -m 755 build/release/bin/tools/qfsstats $BIN_DIR
-install -m 755 build/release/bin/tools/qfstoggleworm $BIN_DIR
-
-install -m 755 scripts/qfs_backup $BIN_DIR
-
-mkdir -p $LIB_DIR
-install -m 644 build/release/lib/libgf_complete.so $LIB_DIR
-install -m 644 build/release/lib/libgf_complete.so.1 $LIB_DIR
-install -m 644 build/release/lib/libgf_complete.so.1.0.0 $LIB_DIR
-install -m 644 build/release/lib/libJerasure.so $LIB_DIR
-install -m 644 build/release/lib/libJerasure.so.2 $LIB_DIR
-install -m 644 build/release/lib/libJerasure.so.2.0.0 $LIB_DIR
-install -m 644 build/release/lib/libqfs_access.so $LIB_DIR
-install -m 644 build/release/lib/libqfs_client.so $LIB_DIR
-install -m 644 build/release/lib/libqfs_common.so $LIB_DIR
-install -m 644 build/release/lib/libqfsc.so $LIB_DIR
-install -m 644 build/release/lib/libqfs_io.so $LIB_DIR
-install -m 644 build/release/lib/libqfskrb.so $LIB_DIR
-install -m 644 build/release/lib/libqfs_qcdio.so $LIB_DIR
-install -m 644 build/release/lib/libqfs_qcrs.so $LIB_DIR
-
-install -m 644 build/release/lib/static/libgf_complete.a $LIB_DIR
-install -m 644 build/release/lib/static/libJerasure.a $LIB_DIR
-install -m 644 build/release/lib/static/libqfsc.a $LIB_DIR
-install -m 644 build/release/lib/static/libqfs_client.a $LIB_DIR
-install -m 644 build/release/lib/static/libqfs_common.a $LIB_DIR
-install -m 644 build/release/lib/static/libqfs_emulator.a $LIB_DIR
-install -m 644 build/release/lib/static/libqfs_io.a $LIB_DIR
-install -m 644 build/release/lib/static/libqfskrb.a $LIB_DIR
-install -m 644 build/release/lib/static/libqfs_meta.a $LIB_DIR
-install -m 644 build/release/lib/static/libqfs_qcdio.a $LIB_DIR
-install -m 644 build/release/lib/static/libqfs_qcrs.a $LIB_DIR
-install -m 644 build/release/lib/static/libqfs_tools.a $LIB_DIR
-
-mkdir -p $ETC_DIR/logrotate.d
-cp $SOURCE_DIR/qfs-chunkserver.cfg $ETC_DIR/logrotate.d/qfs-chunkserver
-cp $SOURCE_DIR/qfs-metaserver.cfg $ETC_DIR/logrotate.d/qfs-metaserver
-cp $SOURCE_DIR/qfs-webui.cfg $ETC_DIR/logrotate.d/qfs-webui
-
-mkdir -p $INCLUDE_DIR
-cp -a build/release/include/* $INCLUDE_DIR
-
-mkdir -p $DATA_DIR/webui
-cp -rp webui/* $DATA_DIR/webui
-
-mkdir -p $DATA_DIR/java
-install -m 644 build/java/qfs-access/qfs-access-${QFS_VERSION}.jar $DATA_DIR/java
-
-mkdir -p $HADOOP_HOME/lib
-install -m 644 build/java/hadoop-qfs/hadoop-${HADOOP_VERSION}-qfs-${QFS_VERSION}.jar ${HADOOP_HOME}/lib
-
-cd build/release
-$PYTHON_PATH ../../src/cc/access/kfs_setup.py install --skip-build --root=$PREFIX --prefix=/usr $PYTHON_EXTRA
-if [ ! -z "$PYTHON3_PATH" ]; then
- $PYTHON3_PATH ../../src/cc/access/kfs_setup.py install --skip-build --root=$PREFIX --prefix=/usr $PYTHON_EXTRA
-fi
-cd ..
-
-mkdir -p $VAR_DIR/metaserver/checkpoint
-mkdir -p $VAR_DIR/log/qfs
-mkdir -p $VAR_DIR/lib/qfs
diff --git a/bigtop-packages/src/common/qfs/patch0-gf_cpu.diff b/bigtop-packages/src/common/qfs/patch0-gf_cpu.diff
deleted file mode 100644
index 8dcc9c2..0000000
--- a/bigtop-packages/src/common/qfs/patch0-gf_cpu.diff
+++ /dev/null
@@ -1,16 +0,0 @@
-diff --git a/ext/gf-complete/src/gf_cpu.c b/ext/gf-complete/src/gf_cpu.c
-index f1afa2b..a0c9b10 100644
---- a/ext/gf-complete/src/gf_cpu.c
-+++ b/ext/gf-complete/src/gf_cpu.c
-@@ -206,10 +206,9 @@ void gf_cpu_identify(void)
-
- #else // defined(__arm__) || defined(__aarch64__)
-
--int gf_cpu_identify(void)
-+void gf_cpu_identify(void)
- {
- gf_cpu_identified = 1;
-- return 0;
- }
-
- #endif
diff --git a/bigtop-packages/src/common/qfs/patch1-maven-surefire.diff b/bigtop-packages/src/common/qfs/patch1-maven-surefire.diff
deleted file mode 100644
index 1dd9400..0000000
--- a/bigtop-packages/src/common/qfs/patch1-maven-surefire.diff
+++ /dev/null
@@ -1,29 +0,0 @@
-From 7af25985e802eafec2307307e1703f9191b49b9f Mon Sep 17 00:00:00 2001
-From: Mike Ovsiannikov <mo...@quantcast.com>
-Date: Thu, 1 Nov 2018 17:50:30 -0700
-Subject: [PATCH] Java build: work around new oracle SDK problem by setting use
- system class loader property for sure fire plugin.
-
----
- src/java/pom.xml | 8 ++++++++
- 1 file changed, 8 insertions(+)
-
-diff --git a/src/java/pom.xml b/src/java/pom.xml
-index ca4a7f4e..ae28ff72 100644
---- a/src/java/pom.xml
-+++ b/src/java/pom.xml
-@@ -42,6 +42,14 @@ permissions and limitations under the License.
- <artifactId>maven-compiler-plugin</artifactId>
- <version>2.3</version>
- </plugin>
-+ <plugin>
-+ <groupId>org.apache.maven.plugins</groupId>
-+ <artifactId>maven-surefire-plugin</artifactId>
-+ <version>2.22.1</version>
-+ <configuration>
-+ <useSystemClassLoader>false</useSystemClassLoader>
-+ </configuration>
-+ </plugin>
- </plugins>
- </build>
-
diff --git a/bigtop-packages/src/common/qfs/patch2-specify-python-version.diff b/bigtop-packages/src/common/qfs/patch2-specify-python-version.diff
deleted file mode 100644
index 7b33b7f..0000000
--- a/bigtop-packages/src/common/qfs/patch2-specify-python-version.diff
+++ /dev/null
@@ -1,20 +0,0 @@
-diff --git a/scripts/qfs_backup b/scripts/qfs_backup
-index d3743611..96be5fe7 100755
---- a/scripts/qfs_backup
-+++ b/scripts/qfs_backup
-@@ -1,4 +1,4 @@
--#!/usr/bin/env python
-+#!/usr/bin/env python2
- #
- # Author: Christopher Zimmerman
- #
-diff --git a/webui/qfsstatus.py b/webui/qfsstatus.py
-index c2178b23..c4525985 100755
---- a/webui/qfsstatus.py
-+++ b/webui/qfsstatus.py
-@@ -1,4 +1,4 @@
--#!/usr/bin/env python
-+#!/usr/bin/env python2
- #
- # $Id$
- #
diff --git a/bigtop-packages/src/common/qfs/qfs-chunkserver.cfg b/bigtop-packages/src/common/qfs/qfs-chunkserver.cfg
deleted file mode 100644
index 8ee0761..0000000
--- a/bigtop-packages/src/common/qfs/qfs-chunkserver.cfg
+++ /dev/null
@@ -1,24 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-/var/log/qfs/chunkserver.log {
- daily
- dateext
- rotate 25
- missingok
- notifempty
- compress
- copytruncate
-}
-
diff --git a/bigtop-packages/src/common/qfs/qfs-chunkserver.svc b/bigtop-packages/src/common/qfs/qfs-chunkserver.svc
deleted file mode 100644
index 7cb55f3..0000000
--- a/bigtop-packages/src/common/qfs/qfs-chunkserver.svc
+++ /dev/null
@@ -1,94 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-DAEMON="chunkserver"
-DESC="QFS Chunkserver"
-PROC_NAME="chunkserver"
-EXEC_PATH="/usr/bin/${PROC_NAME}"
-SVC_USER="qfs"
-WORKING_DIR="/var/lib/qfs"
-CONF_DIR="/etc/qfs"
-CONF_FILE="ChunkServer.prp"
-LOGFILE="/var/log/qfs/${DAEMON}.log"
-PIDFILE="/var/run/qfs/${DAEMON}.pid"
-
-generate_start() {
-
-cat <<'__EOT__'
-start() {
- [ -x $EXEC_PATH ] || exit $ERROR_PROGRAM_NOT_INSTALLED
- log_success_msg "Starting $DESC ($DAEMON): "
-
- checkstatusofproc
- status=$?
- if [ "$status" -eq "$STATUS_RUNNING" ]; then
- log_success_msg "$DESC is running"
- exit 0
- fi
-
- echo "Starting $DESC: "
-
- # Debian doesn't like us to package anything inside /var/run. We create the
- # directory here and set permissions as necessary for both debian and rpm
- mkdir -p /var/run/qfs
- chown qfs:qfs -R /var/run/qfs
- chmod 755 /var/run/qfs
-
- runuser -s /bin/bash $SVC_USER -c "cd $WORKING_DIR && nohup $EXEC_PATH $CONF_DIR/$CONF_FILE > $LOGFILE 2>&1 < /dev/null & "'echo $! '"> $PIDFILE"
- sleep 3
-
- checkstatusofproc
- RETVAL=$?
- if [ $RETVAL -eq $STATUS_RUNNING ]; then
- touch $LOCKFILE
- log_success_msg "Starting $DESC (${DAEMON}): "
- else
- log_failure_msg "Failure to start $DESC (${DAEMON}). Return value: $RETVAL"
- fi
- return $RETVAL
-}
-__EOT__
-
-}
-
-generate_stop() {
-
-cat <<'__EOT__'
-stop() {
- checkstatusofproc
- if [ "$?" = "$STATUS_RUNNING" ] ; then
- if [ -f $PIDFILE ]; then
- PID=`cat $PIDFILE`
- if [ -n $PID ]; then
- kill -TERM $PID &>/dev/null
- fi
- fi
- RETVAL=$?
- else
- RETVAL=$RETVAL_SUCCESS
- fi
-
- if [ $RETVAL -eq $RETVAL_SUCCESS ]; then
- rm -f $LOCKFILE $PIDFILE
- log_success_msg "Stopped ${DESC}: "
- else
- log_failure_msg "Failed to stop ${DESC}. Return value: $RETVAL"
- fi
- return $RETVAL
-}
-
-__EOT__
-
-}
diff --git a/bigtop-packages/src/common/qfs/qfs-metaserver.cfg b/bigtop-packages/src/common/qfs/qfs-metaserver.cfg
deleted file mode 100644
index fe994c8..0000000
--- a/bigtop-packages/src/common/qfs/qfs-metaserver.cfg
+++ /dev/null
@@ -1,24 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-/var/log/qfs/metaserver.log {
- daily
- dateext
- rotate 25
- missingok
- notifempty
- compress
- copytruncate
-}
-
diff --git a/bigtop-packages/src/common/qfs/qfs-metaserver.svc b/bigtop-packages/src/common/qfs/qfs-metaserver.svc
deleted file mode 100644
index d2dffd8..0000000
--- a/bigtop-packages/src/common/qfs/qfs-metaserver.svc
+++ /dev/null
@@ -1,94 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-DAEMON="metaserver"
-DESC="QFS Metaserver"
-PROC_NAME="metaserver"
-EXEC_PATH="/usr/bin/${PROC_NAME}"
-SVC_USER="qfs"
-WORKING_DIR="/var/lib/qfs"
-CONF_DIR="/etc/qfs"
-CONF_FILE="MetaServer.prp"
-LOGFILE="/var/log/qfs/${DAEMON}.log"
-PIDFILE="/var/run/qfs/${DAEMON}.pid"
-
-generate_start() {
-
-cat <<'__EOT__'
-start() {
- [ -x $EXEC_PATH ] || exit $ERROR_PROGRAM_NOT_INSTALLED
- log_success_msg "Starting $DESC ($DAEMON): "
-
- checkstatusofproc
- status=$?
- if [ "$status" -eq "$STATUS_RUNNING" ]; then
- log_success_msg "$DESC is running"
- exit 0
- fi
-
- echo "Starting $DESC: "
-
- # Debian doesn't like us to package anything inside /var/run. We create the
- # directory here and set permissions as necessary for both debian and rpm
- mkdir -p /var/run/qfs
- chown qfs:qfs -R /var/run/qfs
- chmod 755 /var/run/qfs
-
- runuser -s /bin/bash $SVC_USER -c "cd $WORKING_DIR && nohup $EXEC_PATH $CONF_DIR/$CONF_FILE > $LOGFILE 2>&1 < /dev/null & "'echo $! '"> $PIDFILE"
- sleep 3
-
- checkstatusofproc
- RETVAL=$?
- if [ $RETVAL -eq $STATUS_RUNNING ]; then
- touch $LOCKFILE
- log_success_msg "Starting $DESC (${DAEMON}): "
- else
- log_failure_msg "Failure to start $DESC (${DAEMON}). Return value: $RETVAL"
- fi
- return $RETVAL
-}
-__EOT__
-
-}
-
-generate_stop() {
-
-cat <<'__EOT__'
-stop() {
- checkstatusofproc
- if [ "$?" = "$STATUS_RUNNING" ] ; then
- if [ -f $PIDFILE ]; then
- PID=`cat $PIDFILE`
- if [ -n $PID ]; then
- kill -TERM $PID &>/dev/null
- fi
- fi
- RETVAL=$?
- else
- RETVAL=$RETVAL_SUCCESS
- fi
-
- if [ $RETVAL -eq $RETVAL_SUCCESS ]; then
- rm -f $LOCKFILE $PIDFILE
- log_success_msg "Stopped ${DESC}: "
- else
- log_failure_msg "Failed to stop ${DESC}. Return value: $RETVAL"
- fi
- return $RETVAL
-}
-
-__EOT__
-
-}
diff --git a/bigtop-packages/src/common/qfs/qfs-webui.cfg b/bigtop-packages/src/common/qfs/qfs-webui.cfg
deleted file mode 100644
index 63638c7..0000000
--- a/bigtop-packages/src/common/qfs/qfs-webui.cfg
+++ /dev/null
@@ -1,24 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-/var/log/qfs/webui.log {
- daily
- dateext
- rotate 25
- missingok
- notifempty
- compress
- copytruncate
-}
-
diff --git a/bigtop-packages/src/common/qfs/qfs-webui.svc b/bigtop-packages/src/common/qfs/qfs-webui.svc
deleted file mode 100644
index caf9638..0000000
--- a/bigtop-packages/src/common/qfs/qfs-webui.svc
+++ /dev/null
@@ -1,94 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-DAEMON="webui"
-DESC="QFS WebUI"
-PROC_NAME="python"
-EXEC_PATH="/usr/bin/${PROC_NAME} /usr/share/qfs/webui/qfsstatus.py"
-SVC_USER="qfs"
-WORKING_DIR="/var/lib/qfs"
-CONF_DIR="/usr/share/qfs/webui"
-CONF_FILE="server.conf"
-LOGFILE="/var/log/qfs/${DAEMON}.log"
-PIDFILE="/var/run/qfs/${DAEMON}.pid"
-
-generate_start() {
-
-cat <<'__EOT__'
-start() {
- [ -x $EXEC_PATH ] || exit $ERROR_PROGRAM_NOT_INSTALLED
- log_success_msg "Starting $DESC ($DAEMON): "
-
- checkstatusofproc
- status=$?
- if [ "$status" -eq "$STATUS_RUNNING" ]; then
- log_success_msg "$DESC is running"
- exit 0
- fi
-
- echo "Starting $DESC: "
-
- # Debian doesn't like us to package anything inside /var/run. We create the
- # directory here and set permissions as necessary for both debian and rpm
- mkdir -p /var/run/qfs
- chown qfs:qfs -R /var/run/qfs
- chmod 755 /var/run/qfs
-
- runuser -s /bin/bash $SVC_USER -c "cd $WORKING_DIR && nohup $EXEC_PATH $CONF_DIR/$CONF_FILE > $LOGFILE 2>&1 < /dev/null & "'echo $! '"> $PIDFILE"
- sleep 3
-
- checkstatusofproc
- RETVAL=$?
- if [ $RETVAL -eq $STATUS_RUNNING ]; then
- touch $LOCKFILE
- log_success_msg "Starting $DESC (${DAEMON}): "
- else
- log_failure_msg "Failure to start $DESC (${DAEMON}). Return value: $RETVAL"
- fi
- return $RETVAL
-}
-__EOT__
-
-}
-
-generate_stop() {
-
-cat <<'__EOT__'
-stop() {
- checkstatusofproc
- if [ "$?" = "$STATUS_RUNNING" ] ; then
- if [ -f $PIDFILE ]; then
- PID=`cat $PIDFILE`
- if [ -n $PID ]; then
- kill -TERM $PID &>/dev/null
- fi
- fi
- RETVAL=$?
- else
- RETVAL=$RETVAL_SUCCESS
- fi
-
- if [ $RETVAL -eq $RETVAL_SUCCESS ]; then
- rm -f $LOCKFILE $PIDFILE
- log_success_msg "Stopped ${DESC}: "
- else
- log_failure_msg "Failed to stop ${DESC}. Return value: $RETVAL"
- fi
- return $RETVAL
-}
-
-__EOT__
-
-}
diff --git a/bigtop-packages/src/deb/qfs/changelog b/bigtop-packages/src/deb/qfs/changelog
deleted file mode 100644
index d4858cd..0000000
--- a/bigtop-packages/src/deb/qfs/changelog
+++ /dev/null
@@ -1 +0,0 @@
---- This is auto-generated
diff --git a/bigtop-packages/src/deb/qfs/compat b/bigtop-packages/src/deb/qfs/compat
deleted file mode 100644
index ec63514..0000000
--- a/bigtop-packages/src/deb/qfs/compat
+++ /dev/null
@@ -1 +0,0 @@
-9
diff --git a/bigtop-packages/src/deb/qfs/control b/bigtop-packages/src/deb/qfs/control
deleted file mode 100644
index 2bce55d..0000000
--- a/bigtop-packages/src/deb/qfs/control
+++ /dev/null
@@ -1,99 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-Source: qfs
-Section: misc
-Priority: extra
-Maintainer: Apache Bigtop <de...@bigtop.apache.org>
-Build-Depends: cmake, libboost-regex-dev, xfslibs-dev, libssl-dev, python2.7-dev | python3-dev, libfuse-dev, dh-python, debhelper (>= 7.0.50~)
-Standards-Version: 3.9.4
-Homepage: https://quantcast.github.io/qfs
-
-Package: qfs
-Depends: qfs-client (= ${binary:Version}), qfs-chunkserver (= ${binary:Version}), qfs-metaserver (= ${binary:Version}), qfs-dev (= ${binary:Version}), qfs-fuse (= ${binary:Version}), qfs-hadoop (= ${binary:Version}), qfs-java (= ${binary:Version}), qfs-python (= ${binary:Version}), qfs-webui (= ${binary:Version})
-Architecture: any
-Description: Metapackage for all QFS components
- Quantcast File System (QFS) is a high-performance, fault-tolerant, distributed
- file system developed to support MapReduce processing, or other applications
- reading and writing large files sequentially.
-
-Package: qfs-client
-Depends: ${shlibs:Depends}, ${misc:Depends}, python
-Architecture: any
-Description: Client binaries and libraries used to link against qfs
- This package provides base client binaries and libraries used to interact with
- a running qfs cluster and link against qfs APIs to build your own software that
- interacts with a running qfs cluster.
-
-Package: qfs-chunkserver
-Depends: ${shlibs:Depends}, ${misc:Depends}
-Recommends: logrotate
-Architecture: any
-Description: Executables required to run the Quantcast File System chunkserver
- The QFS chunkserver service hosts the binary contents of the QFS distributed
- filesystem. A metaserver coordinates many data nodes running this service,
- replicating data amongst chunkservers in a redundant fashion.
-
-Package: qfs-metaserver
-Depends: ${shlibs:Depends}, ${misc:Depends}
-Recommends: logrotate
-Architecture: any
-Description: Executables required to run the Quantcast File System metaserver
- This package contains the executables required to run the Quantcast File System
- metaserver service, which tracks the location of data chunks distributed across
- QFS chunkservers.
-
-Package: qfs-dev
-Depends: ${shlibs:Depends}, ${misc:Depends}
-Architecture: any
-Description: Files needed for building Quantcast File System-based applications
- The QFS devel package contains the headers, static libraries, and developer
- tool binaries required to develop applications which build against QFS.
-
-Package: qfs-fuse
-Depends: ${shlibs:Depends}, ${misc:Depends}
-Architecture: any
-Description: Support for mounting the Quantcast File System under FUSE
- This package contains the qfs_fuse executable which is required when mounting
- QFS distributed filesystems under FUSE.
-
-Package: qfs-hadoop
-Depends: ${misc:Depends}
-Architecture: any
-Description: Quantcast File System plugin JAR for Hadoop
- This package contains a plugin JAR to enable QFS to serve as a drop-in
- replacement for HDFS under Hadoop.
-
-Package: qfs-java
-Depends: ${misc:Depends}
-Architecture: any
-Description: Java libraries for accessing the Quantcast File System
- This package contains a JAR which enables Java applications to access QFS via
- its JNI interface.
-
-Package: qfs-python
-Depends: ${shlibs:Depends}, ${misc:Depends}, ${python:Depends}
-Architecture: any
-Description: Python libraries for accessing the Quantcast File System
- This package contains the libraries required to access the Quantcast File
- System libraries via Python.
-
-Package: qfs-webui
-Depends: ${misc:Depends}, ${misc:Depends}, python
-Recommends: logrotate
-Architecture: all
-Description: Quantcast File System metaserver/chunkserver web frontend
- This package contains several Python scripts which provide a simple Hadoop-like
- Web UI for viewing Quantcast File Server chunkserver and metaserver status.
diff --git a/bigtop-packages/src/deb/qfs/copyright b/bigtop-packages/src/deb/qfs/copyright
deleted file mode 100644
index 5d02a4e..0000000
--- a/bigtop-packages/src/deb/qfs/copyright
+++ /dev/null
@@ -1,15 +0,0 @@
-Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
-Upstream-Name: qfs
-Source: https://quantcast.github.io/qfs
-
-Files: *
-Copyright: 2010-2011, The Apache Software Foundation
-License: Apache-2.0
-
-Files debian/*
-Copyright: 2011, The Apache Software Foundation
-License: Apache-2.0
-
-License: Apache-2.0
- On Debian systems, the complete text of the Apache 2.0 license
- can be found in "/usr/share/common-licenses/Apache-2.0".
diff --git a/bigtop-packages/src/deb/qfs/qfs-chunkserver.dirs b/bigtop-packages/src/deb/qfs/qfs-chunkserver.dirs
deleted file mode 100644
index 725c19a..0000000
--- a/bigtop-packages/src/deb/qfs/qfs-chunkserver.dirs
+++ /dev/null
@@ -1,2 +0,0 @@
-/var/log/qfs
-/var/lib/qfs
diff --git a/bigtop-packages/src/deb/qfs/qfs-chunkserver.install b/bigtop-packages/src/deb/qfs/qfs-chunkserver.install
deleted file mode 100644
index 9f9390c..0000000
--- a/bigtop-packages/src/deb/qfs/qfs-chunkserver.install
+++ /dev/null
@@ -1,3 +0,0 @@
-/usr/bin/chunkserver
-/usr/bin/chunkscrubber
-/etc/qfs/logrotate.d/qfs-chunkserver
diff --git a/bigtop-packages/src/deb/qfs/qfs-chunkserver.lintian-overrides b/bigtop-packages/src/deb/qfs/qfs-chunkserver.lintian-overrides
deleted file mode 100644
index ca75f34..0000000
--- a/bigtop-packages/src/deb/qfs/qfs-chunkserver.lintian-overrides
+++ /dev/null
@@ -1,2 +0,0 @@
-qfs-chunkserver binary: new-package-should-close-itp-bug
-qfs-chunkserver binary: binary-without-manpage
diff --git a/bigtop-packages/src/deb/qfs/qfs-chunkserver.postinst b/bigtop-packages/src/deb/qfs/qfs-chunkserver.postinst
deleted file mode 100644
index ee7c493..0000000
--- a/bigtop-packages/src/deb/qfs/qfs-chunkserver.postinst
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -e
-
-case "$1" in
- configure)
- update-rc.d qfs-chunkserver defaults >/dev/null || exit 1
- chown qfs:qfs -R /var/log/qfs
- chown qfs:qfs -R /var/lib/qfs
- ;;
- abort-upgrade|abort-remove|abort-deconfigure)
- ;;
- *)
- echo "postinst called with unknown argument \`$1'" >&2
- exit 1
- ;;
-esac
-
-#DEBHELPER#
diff --git a/bigtop-packages/src/deb/qfs/qfs-chunkserver.postrm b/bigtop-packages/src/deb/qfs/qfs-chunkserver.postrm
deleted file mode 100644
index d6098a7..0000000
--- a/bigtop-packages/src/deb/qfs/qfs-chunkserver.postrm
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/bin/sh
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -e
-
-case "$1" in
- purge)
- update-rc.d -f qfs-chunkserver remove > /dev/null || exit 1
- ;;
- remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear)
- ;;
- *)
- echo "postrm called with unknown argument \`$1'" >&2
- exit 1
- ;;
-esac
-
-#DEBHELPER#
diff --git a/bigtop-packages/src/deb/qfs/qfs-chunkserver.preinst b/bigtop-packages/src/deb/qfs/qfs-chunkserver.preinst
deleted file mode 100644
index 7416ecd..0000000
--- a/bigtop-packages/src/deb/qfs/qfs-chunkserver.preinst
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -e
-
-case "$1" in
- install|upgrade)
- getent group qfs >/dev/null || groupadd -r qfs
- getent passwd qfs > /dev/null || useradd -c "qfs" -s /bin/bash -g qfs \
- -d /var/lib/qfs qfs 2> /dev/null || :
- ;;
- abort-upgrade)
- ;;
- *)
- echo "preinst called with unknown argument \`$1'" >&2
- exit 1
- ;;
-esac
-
-#DEBHELPER#
diff --git a/bigtop-packages/src/deb/qfs/qfs-client.install b/bigtop-packages/src/deb/qfs/qfs-client.install
deleted file mode 100644
index 3e6f777..0000000
--- a/bigtop-packages/src/deb/qfs/qfs-client.install
+++ /dev/null
@@ -1,31 +0,0 @@
-/usr/bin/qfs
-/usr/bin/qfsadmin
-/usr/bin/qfscat
-/usr/bin/qfsdataverify
-/usr/bin/qfsfileenum
-/usr/bin/qfsfsck
-/usr/bin/qfshibernate
-/usr/bin/qfsping
-/usr/bin/qfsput
-/usr/bin/qfsshell
-/usr/bin/qfsstats
-/usr/bin/qfstoggleworm
-/usr/bin/qfssample
-/usr/bin/qfs_backup
-/usr/bin/cpfromqfs
-/usr/bin/cptoqfs
-
-/usr/lib/qfs/libgf_complete.so
-/usr/lib/qfs/libgf_complete.so.1
-/usr/lib/qfs/libgf_complete.so.1.0.0
-/usr/lib/qfs/libJerasure.so
-/usr/lib/qfs/libJerasure.so.2
-/usr/lib/qfs/libJerasure.so.2.0.0
-/usr/lib/qfs/libqfs_access.so
-/usr/lib/qfs/libqfs_client.so
-/usr/lib/qfs/libqfs_common.so
-/usr/lib/qfs/libqfsc.so
-/usr/lib/qfs/libqfs_io.so
-/usr/lib/qfs/libqfskrb.so
-/usr/lib/qfs/libqfs_qcdio.so
-/usr/lib/qfs/libqfs_qcrs.so
diff --git a/bigtop-packages/src/deb/qfs/qfs-client.lintian-overrides b/bigtop-packages/src/deb/qfs/qfs-client.lintian-overrides
deleted file mode 100644
index a694309..0000000
--- a/bigtop-packages/src/deb/qfs/qfs-client.lintian-overrides
+++ /dev/null
@@ -1,3 +0,0 @@
-qfs-client binary: new-package-should-close-itp-bug
-qfs-client binary: binary-or-shlib-defines-rpath
-qfs-client binary: binary-without-manpage
diff --git a/bigtop-packages/src/deb/qfs/qfs-dev.install b/bigtop-packages/src/deb/qfs/qfs-dev.install
deleted file mode 100644
index 8371fa0..0000000
--- a/bigtop-packages/src/deb/qfs/qfs-dev.install
+++ /dev/null
@@ -1,27 +0,0 @@
-/usr/bin/checksum
-/usr/bin/dirtree_creator
-/usr/bin/dtokentest
-/usr/bin/qfslogger
-/usr/bin/rand-sfmt
-/usr/bin/rebalanceexecutor
-/usr/bin/rebalanceplanner
-/usr/bin/replicachecker
-/usr/bin/requestparser
-/usr/bin/sortedhash
-/usr/bin/sslfiltertest
-/usr/bin/stlset
-
-/usr/include
-
-/usr/lib/qfs/libJerasure.a
-/usr/lib/qfs/libgf_complete.a
-/usr/lib/qfs/libqfs_client.a
-/usr/lib/qfs/libqfs_common.a
-/usr/lib/qfs/libqfs_emulator.a
-/usr/lib/qfs/libqfs_io.a
-/usr/lib/qfs/libqfs_meta.a
-/usr/lib/qfs/libqfs_qcdio.a
-/usr/lib/qfs/libqfs_qcrs.a
-/usr/lib/qfs/libqfs_tools.a
-/usr/lib/qfs/libqfsc.a
-/usr/lib/qfs/libqfskrb.a
diff --git a/bigtop-packages/src/deb/qfs/qfs-dev.lintian-overrides b/bigtop-packages/src/deb/qfs/qfs-dev.lintian-overrides
deleted file mode 100644
index b1ab344..0000000
--- a/bigtop-packages/src/deb/qfs/qfs-dev.lintian-overrides
+++ /dev/null
@@ -1,2 +0,0 @@
-qfs-dev binary: new-package-should-close-itp-bug
-qfs-dev binary: binary-without-manpage
diff --git a/bigtop-packages/src/deb/qfs/qfs-fuse.install b/bigtop-packages/src/deb/qfs/qfs-fuse.install
deleted file mode 100644
index 35b2e03..0000000
--- a/bigtop-packages/src/deb/qfs/qfs-fuse.install
+++ /dev/null
@@ -1 +0,0 @@
-/usr/bin/qfs_fuse
diff --git a/bigtop-packages/src/deb/qfs/qfs-fuse.lintian-overrides b/bigtop-packages/src/deb/qfs/qfs-fuse.lintian-overrides
deleted file mode 100644
index 3daf409..0000000
--- a/bigtop-packages/src/deb/qfs/qfs-fuse.lintian-overrides
+++ /dev/null
@@ -1,2 +0,0 @@
-qfs-fuse binary: new-package-should-close-itp-bug
-qfs-fuse binary: binary-without-manpage
diff --git a/bigtop-packages/src/deb/qfs/qfs-hadoop.install b/bigtop-packages/src/deb/qfs/qfs-hadoop.install
deleted file mode 100644
index c8f20de..0000000
--- a/bigtop-packages/src/deb/qfs/qfs-hadoop.install
+++ /dev/null
@@ -1 +0,0 @@
-/usr/lib/hadoop
diff --git a/bigtop-packages/src/deb/qfs/qfs-hadoop.lintian-overrides b/bigtop-packages/src/deb/qfs/qfs-hadoop.lintian-overrides
deleted file mode 100644
index 3e40193..0000000
--- a/bigtop-packages/src/deb/qfs/qfs-hadoop.lintian-overrides
+++ /dev/null
@@ -1,2 +0,0 @@
-qfs-hadoop binary: new-package-should-close-itp-bug
-qfs-hadoop binary: jar-not-in-usr-share
diff --git a/bigtop-packages/src/deb/qfs/qfs-java.install b/bigtop-packages/src/deb/qfs/qfs-java.install
deleted file mode 100644
index 0fb7754..0000000
--- a/bigtop-packages/src/deb/qfs/qfs-java.install
+++ /dev/null
@@ -1 +0,0 @@
-/usr/share/qfs/java
diff --git a/bigtop-packages/src/deb/qfs/qfs-java.lintian-overrides b/bigtop-packages/src/deb/qfs/qfs-java.lintian-overrides
deleted file mode 100644
index 64e9d8a..0000000
--- a/bigtop-packages/src/deb/qfs/qfs-java.lintian-overrides
+++ /dev/null
@@ -1 +0,0 @@
-qfs-java binary: new-package-should-close-itp-bug
diff --git a/bigtop-packages/src/deb/qfs/qfs-metaserver.dirs b/bigtop-packages/src/deb/qfs/qfs-metaserver.dirs
deleted file mode 100644
index 725c19a..0000000
--- a/bigtop-packages/src/deb/qfs/qfs-metaserver.dirs
+++ /dev/null
@@ -1,2 +0,0 @@
-/var/log/qfs
-/var/lib/qfs
diff --git a/bigtop-packages/src/deb/qfs/qfs-metaserver.install b/bigtop-packages/src/deb/qfs/qfs-metaserver.install
deleted file mode 100644
index c1a57ee..0000000
--- a/bigtop-packages/src/deb/qfs/qfs-metaserver.install
+++ /dev/null
@@ -1,4 +0,0 @@
-/usr/bin/metaserver
-/usr/bin/filelister
-/usr/bin/logcompactor
-/etc/qfs/logrotate.d/qfs-metaserver
diff --git a/bigtop-packages/src/deb/qfs/qfs-metaserver.lintian-overrides b/bigtop-packages/src/deb/qfs/qfs-metaserver.lintian-overrides
deleted file mode 100644
index a8afec7..0000000
--- a/bigtop-packages/src/deb/qfs/qfs-metaserver.lintian-overrides
+++ /dev/null
@@ -1,2 +0,0 @@
-qfs-metaserver binary: new-package-should-close-itp-bug
-qfs-metaserver binary: binary-without-manpage
diff --git a/bigtop-packages/src/deb/qfs/qfs-metaserver.postinst b/bigtop-packages/src/deb/qfs/qfs-metaserver.postinst
deleted file mode 100644
index 597db81..0000000
--- a/bigtop-packages/src/deb/qfs/qfs-metaserver.postinst
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -e
-
-case "$1" in
- configure)
- update-rc.d qfs-metaserver defaults >/dev/null || exit 1
- chown qfs:qfs -R /var/log/qfs
- chown qfs:qfs -R /var/lib/qfs
- ;;
- abort-upgrade|abort-remove|abort-deconfigure)
- ;;
- *)
- echo "postinst called with unknown argument \`$1'" >&2
- exit 1
- ;;
-esac
-
-#DEBHELPER#
diff --git a/bigtop-packages/src/deb/qfs/qfs-metaserver.postrm b/bigtop-packages/src/deb/qfs/qfs-metaserver.postrm
deleted file mode 100644
index 63ef9c5..0000000
--- a/bigtop-packages/src/deb/qfs/qfs-metaserver.postrm
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/bin/sh
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -e
-
-case "$1" in
- purge)
- update-rc.d -f qfs-metaserver remove > /dev/null || exit 1
- ;;
- remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear)
- ;;
- *)
- echo "postrm called with unknown argument \`$1'" >&2
- exit 1
- ;;
-esac
-
-#DEBHELPER#
diff --git a/bigtop-packages/src/deb/qfs/qfs-metaserver.preinst b/bigtop-packages/src/deb/qfs/qfs-metaserver.preinst
deleted file mode 100644
index 7416ecd..0000000
--- a/bigtop-packages/src/deb/qfs/qfs-metaserver.preinst
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -e
-
-case "$1" in
- install|upgrade)
- getent group qfs >/dev/null || groupadd -r qfs
- getent passwd qfs > /dev/null || useradd -c "qfs" -s /bin/bash -g qfs \
- -d /var/lib/qfs qfs 2> /dev/null || :
- ;;
- abort-upgrade)
- ;;
- *)
- echo "preinst called with unknown argument \`$1'" >&2
- exit 1
- ;;
-esac
-
-#DEBHELPER#
diff --git a/bigtop-packages/src/deb/qfs/qfs-python.install b/bigtop-packages/src/deb/qfs/qfs-python.install
deleted file mode 100644
index 5ad7ef7..0000000
--- a/bigtop-packages/src/deb/qfs/qfs-python.install
+++ /dev/null
@@ -1 +0,0 @@
-/usr/lib/python2.7
diff --git a/bigtop-packages/src/deb/qfs/qfs-python.lintian-overrides b/bigtop-packages/src/deb/qfs/qfs-python.lintian-overrides
deleted file mode 100644
index ea76f7c..0000000
--- a/bigtop-packages/src/deb/qfs/qfs-python.lintian-overrides
+++ /dev/null
@@ -1 +0,0 @@
-qfs-python binary: new-package-should-close-itp-bug
diff --git a/bigtop-packages/src/deb/qfs/qfs-webui.dirs b/bigtop-packages/src/deb/qfs/qfs-webui.dirs
deleted file mode 100644
index 725c19a..0000000
--- a/bigtop-packages/src/deb/qfs/qfs-webui.dirs
+++ /dev/null
@@ -1,2 +0,0 @@
-/var/log/qfs
-/var/lib/qfs
diff --git a/bigtop-packages/src/deb/qfs/qfs-webui.install b/bigtop-packages/src/deb/qfs/qfs-webui.install
deleted file mode 100644
index 767f4fe..0000000
--- a/bigtop-packages/src/deb/qfs/qfs-webui.install
+++ /dev/null
@@ -1,2 +0,0 @@
-/usr/share/qfs/webui
-/etc/qfs/logrotate.d/qfs-webui
diff --git a/bigtop-packages/src/deb/qfs/qfs-webui.lintian-overrides b/bigtop-packages/src/deb/qfs/qfs-webui.lintian-overrides
deleted file mode 100644
index d735945..0000000
--- a/bigtop-packages/src/deb/qfs/qfs-webui.lintian-overrides
+++ /dev/null
@@ -1 +0,0 @@
-qfs-webui binary: new-package-should-close-itp-bug
diff --git a/bigtop-packages/src/deb/qfs/qfs-webui.postinst b/bigtop-packages/src/deb/qfs/qfs-webui.postinst
deleted file mode 100644
index 69e1c90..0000000
--- a/bigtop-packages/src/deb/qfs/qfs-webui.postinst
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -e
-
-case "$1" in
- configure)
- update-rc.d qfs-webui defaults >/dev/null || exit 1
- chown qfs:qfs -R /var/log/qfs
- chown qfs:qfs -R /var/lib/qfs
- ;;
- abort-upgrade|abort-remove|abort-deconfigure)
- ;;
- *)
- echo "postinst called with unknown argument \`$1'" >&2
- exit 1
- ;;
-esac
-
-#DEBHELPER#
diff --git a/bigtop-packages/src/deb/qfs/qfs-webui.postrm b/bigtop-packages/src/deb/qfs/qfs-webui.postrm
deleted file mode 100644
index 10975ff..0000000
--- a/bigtop-packages/src/deb/qfs/qfs-webui.postrm
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/bin/sh
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -e
-
-case "$1" in
- purge)
- update-rc.d -f qfs-webui remove > /dev/null || exit 1
- ;;
- remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear)
- ;;
- *)
- echo "postrm called with unknown argument \`$1'" >&2
- exit 1
- ;;
-esac
-
-#DEBHELPER#
diff --git a/bigtop-packages/src/deb/qfs/qfs-webui.preinst b/bigtop-packages/src/deb/qfs/qfs-webui.preinst
deleted file mode 100644
index 7416ecd..0000000
--- a/bigtop-packages/src/deb/qfs/qfs-webui.preinst
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -e
-
-case "$1" in
- install|upgrade)
- getent group qfs >/dev/null || groupadd -r qfs
- getent passwd qfs > /dev/null || useradd -c "qfs" -s /bin/bash -g qfs \
- -d /var/lib/qfs qfs 2> /dev/null || :
- ;;
- abort-upgrade)
- ;;
- *)
- echo "preinst called with unknown argument \`$1'" >&2
- exit 1
- ;;
-esac
-
-#DEBHELPER#
diff --git a/bigtop-packages/src/deb/qfs/qfs.lintian-overrides b/bigtop-packages/src/deb/qfs/qfs.lintian-overrides
deleted file mode 100644
index 5090488..0000000
--- a/bigtop-packages/src/deb/qfs/qfs.lintian-overrides
+++ /dev/null
@@ -1,2 +0,0 @@
-qfs binary: new-package-should-close-itp-bug
-qfs binary: empty-binary-package
diff --git a/bigtop-packages/src/deb/qfs/rules b/bigtop-packages/src/deb/qfs/rules
deleted file mode 100755
index 2625bae..0000000
--- a/bigtop-packages/src/deb/qfs/rules
+++ /dev/null
@@ -1,84 +0,0 @@
-#!/usr/bin/make -f
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# -*- makefile -*-
-
-# Uncomment this to turn on verbose mode.
-export DH_VERBOSE=1
-
-# This has to be exported to make some magic below work.
-export DH_OPTIONS
-
-# Build configuration -- override with environment variables
-SHELL := /bin/bash
-HADOOP_HOME ?= /usr/lib/hadoop
-PYTHON_PATH ?= $(shell command -v python)
-# This will pick up a python3 dev environment, which is not yet provided by bigtop
-# PYTHON3_PATH ?= $(shell command -v python3)
-
-BIN_DIR ?= /usr/bin
-LIB_DIR ?= /usr/lib/qfs
-ETC_DIR ?= /etc/qfs
-INCLUDE_DIR ?= /usr/include
-DATA_DIR ?= /usr/share/qfs
-VAR_DIR ?= /var
-
-SERVICES ?= metaserver chunkserver webui
-
-%:
- dh $@
-
-override_dh_auto_build:
- bash debian/do-component-build \
- --qfs-version=$(QFS_BASE_VERSION) \
- --python=$(PYTHON_PATH)
-
-override_dh_auto_test:
- @echo Tests are disabled
-
-override_dh_auto_install:
- bash -x debian/install_qfs.sh \
- --source-dir=debian \
- --prefix=debian/tmp \
- --qfs-version=$(QFS_BASE_VERSION) \
- --python=$(PYTHON_PATH) \
- --python-extra='--install-layout=deb' \
- --bin-dir=$(BIN_DIR) \
- --lib-dir=$(LIB_DIR) \
- --etc-dir=$(ETC_DIR) \
- --include-dir=$(INCLUDE_DIR) \
- --data-dir=$(DATA_DIR) \
- --var-dir=$(VAR_DIR)
-
- for service in $(SERVICES); do \
- initd_dir=debian/qfs-$$service/etc/init.d ; \
- mkdir -p $$initd_dir ; \
- bash debian/init.d.tmpl debian/qfs-$$service.svc deb $$initd_dir/qfs-$$service ; \
- done
-
-override_dh_shlibdeps:
- dh_shlibdeps -l$(shell pwd)/build/release/lib
-
-override_dh_fixperms:
- dh_fixperms
- # executable-not-elf-or-script
- # upstream qfs has executable gifs -- later version fixes but hasn't been
- # released yet
- chmod -c a-x debian/qfs-webui/usr/share/qfs/webui/files/sorttable/*.gif
-
-override_dh_strip:
- dh_strip --no-automatic-dbgsym
diff --git a/bigtop-packages/src/deb/qfs/source/format b/bigtop-packages/src/deb/qfs/source/format
deleted file mode 100644
index 163aaf8..0000000
--- a/bigtop-packages/src/deb/qfs/source/format
+++ /dev/null
@@ -1 +0,0 @@
-3.0 (quilt)
diff --git a/bigtop-packages/src/rpm/qfs/SPECS/qfs.spec b/bigtop-packages/src/rpm/qfs/SPECS/qfs.spec
deleted file mode 100644
index 15ae9d4..0000000
--- a/bigtop-packages/src/rpm/qfs/SPECS/qfs.spec
+++ /dev/null
@@ -1,353 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-%define qfs_name qfs
-
-%define bin_dir /usr/bin
-%define lib_dir /usr/lib/qfs
-%define etc_dir %{_sysconfdir}/qfs
-%define include_dir /usr/include
-%define data_dir /usr/share/qfs
-%define var_dir /var
-%define qfs_services chunkserver metaserver webui
-%define __python /usr/bin/python2
-
-%if %{?!HADOOP_HOME:1}0
-%global HADOOP_HOME /usr/lib/hadoop
-%endif
-
-# Required for init.d scripts
-%global initd_dir %{_sysconfdir}/init.d
-%if %{?suse_version:1}0
-Requires: insserv
-%global initd_dir %{_sysconfdir}/rc.d
-%else
-Requires: /lib/lsb/init-functions
-%global initd_dir %{_sysconfdir}/rc.d/init.d
-%endif
-
-Name: qfs
-Summary: Quantcast Filesystem (QFS) Meta Package
-Version: %{qfs_version}
-Release: %{qfs_release}
-URL: https://quantcast.github.io/qfs
-Group: Development/Libraries
-License: ASL 2.0
-
-BuildRequires: boost-devel >= 1.3.4
-BuildRequires: cmake >= 2.4.7
-BuildRequires: fuse-devel
-BuildRequires: gcc-c++
-BuildRequires: krb5-devel
-BuildRequires: libuuid-devel
-BuildRequires: net-tools
-BuildRequires: openssl-devel
-BuildRequires: xfsprogs-devel
-
-Requires: boost >= 1.3.4
-Requires: bigtop-utils
-Requires: openssl
-Requires: qfs-chunkserver
-Requires: qfs-devel
-Requires: qfs-fuse
-Requires: qfs-hadoop
-Requires: qfs-java
-Requires: qfs-metaserver
-Requires: qfs-python
-Requires: qfs-webui
-
-Source0: %{qfs_name}-%{qfs_base_version}.tar.gz
-#BIGTOP_PATCH_FILES
-
-%description
-Quantcast File System (QFS) is a high-performance, fault-tolerant, distributed
-file system developed to support MapReduce processing, or other applications
-reading and writing large files sequentially.
-
-%package client
-Group: Applications/System
-Summary: Client binaries and libraries used to link against qfs
-
-%description client
-This package provides base client binaries and libraries used to interact with a
-running qfs cluster and link against qfs APIs to build your own software that
-interacts with a running qfs cluster.
-
-%package chunkserver
-Group: System Environment/Daemons
-Summary: Executables required to run the Quantcast File System chunkserver
-
-%description chunkserver
-The QFS chunkserver service hosts the binary contents of the QFS distributed
-filesystem. A metaserver coordinates many data nodes running this service,
-replicating data amongst chunkservers in a redundant fashion.
-
-
-%package devel
-Group: Development/Libraries
-Summary: Files needed for building Quantcast File System-based applications
-
-%description devel
-The QFS devel package contains the headers, static libraries, and developer
-tool binaries required to develop applications which build against QFS.
-
-
-%package fuse
-Group: Applications/System
-Summary: Support for mounting the Quantcast File System under FUSE
-
-%description fuse
-This package contains the qfs_fuse executable which is required when mounting
-QFS distributed filesystems under FUSE.
-
-
-%package hadoop
-Group: Development/Libraries
-Summary: Quantcast File System plugin JAR for Hadoop
-
-%description hadoop
-This package contains a plugin JAR to enable QFS to serve as a drop-in
-replacement for HDFS under Hadoop.
-
-
-%package java
-Group: Development/Libraries
-Summary: Java libraries for accessing the Quantcast File System
-
-%description java
-This package contains a JAR which enables Java applications to access QFS via
-its JNI interface.
-
-
-%package metaserver
-Group: System Environment/Daemons
-Summary: Executables required to run the Quantcast File System metaserver
-
-%description metaserver
-This package contains the executables required to run the Quantcast File System
-metaserver service, which tracks the location of data chunks distributed across
-QFS chunkservers.
-
-
-%package python
-Group: Development/Libraries
-Summary: Python libraries for accessing the Quantcast File System
-
-%description python
-This package contains the libraries required to access the Quantcast File
-System libraries via Python.
-
-
-%package webui
-Group: System Environment/Daemons
-Requires: python2
-Summary: Quantcast File System metaserver/chunkserver web frontend
-
-%description webui
-This package contains several Python scripts which provide a simple Hadoop-like
-Web UI for viewing Quantcast File Server chunkserver and metaserver status.
-
-%prep
-%setup -n %{qfs_name}-%{qfs_base_version}
-
-echo $RPM_SOURCE_DIR
-echo $RPM_BUILD_ROOT
-
-#BIGTOP_PATCH_COMMANDS
-%build
-%if 0%{?with_python3}
-PYTHON3_PATH=%{__python3}
-%endif # with_python3
-
-bash $RPM_SOURCE_DIR/do-component-build \
- --qfs-version=%{qfs_version} \
- --python=%{__python} \
- --python3=$PYTHON3_PATH
-
-%install
-rm -rf $RPM_BUILD_ROOT
-mkdir -p $RPM_BUILD_ROOT
-
-sh $RPM_SOURCE_DIR/install_qfs.sh \
- --source-dir=$RPM_SOURCE_DIR \
- --prefix=$RPM_BUILD_ROOT \
- --qfs-version=%{qfs_version} \
- --python=%{__python} \
- --python3=$PYTHON3_PATH \
- --bin-dir=%{bin_dir} \
- --lib-dir=%{lib_dir} \
- --etc-dir=%{etc_dir} \
- --include-dir=%{include_dir} \
- --data-dir=%{data_dir} \
- --var-dir=%{var_dir}
-
-for service in %{qfs_services}
-do
- init_file=$RPM_BUILD_ROOT/%{initd_dir}/%{qfs_name}-${service}
- bash $RPM_SOURCE_DIR/init.d.tmpl $RPM_SOURCE_DIR/%{qfs_name}-${service}.svc rpm $init_file
-done
-
-# NOTE(fsareshwala): %pre sections copy pasted :(
-%pre chunkserver
-getent group qfs >/dev/null || groupadd -r qfs
-getent passwd qfs >/dev/null || useradd -c "qfs" -s /sbin/nologin -g qfs -r \
- -d %{var_dir}/lib/qfs qfs 2> /dev/null || :
-
-%pre metaserver
-getent group qfs >/dev/null || groupadd -r qfs
-getent passwd qfs >/dev/null || useradd -c "qfs" -s /sbin/nologin -g qfs -r \
- -d %{var_dir}/lib/qfs qfs 2> /dev/null || :
-
-%pre webui
-getent group qfs >/dev/null || groupadd -r qfs
-getent passwd qfs >/dev/null || useradd -c "qfs" -s /sbin/nologin -g qfs -r \
- -d %{var_dir}/lib/qfs qfs 2> /dev/null || :
-
-%post chunkserver
-/sbin/chkconfig --add qfs-chunkserver
-
-%preun chunkserver
-if [ $1 -eq 0 ]; then
- /sbin/service qfs-chunkserver stop &>/dev/null || :
- /sbin/chkconfig --del qfs-chunkserver
-fi
-
-%post metaserver
-/sbin/chkconfig --add qfs-metaserver
-
-%preun metaserver
-if [ $1 -eq 0 ]; then
- /sbin/service qfs-metaserver stop &>/dev/null || :
- /sbin/chkconfig --del qfs-metaserver
-fi
-
-%post webui
-/sbin/chkconfig --add qfs-webui
-
-%preun webui
-if [ $1 -eq 0 ]; then
- /sbin/service qfs-webui stop &>/dev/null || :
- /sbin/chkconfig --del qfs-webui
-fi
-
-%clean
-rm -rf $RPM_BUILD_ROOT
-
-%files client
-%defattr(-,root,root,755)
-%{bin_dir}/qfs
-%{bin_dir}/qfsadmin
-%{bin_dir}/qfscat
-%{bin_dir}/qfsdataverify
-%{bin_dir}/qfsfileenum
-%{bin_dir}/qfsfsck
-%{bin_dir}/qfshibernate
-%{bin_dir}/qfsping
-%{bin_dir}/qfsput
-%{bin_dir}/qfsshell
-%{bin_dir}/qfsstats
-%{bin_dir}/qfstoggleworm
-%{bin_dir}/qfssample
-%{bin_dir}/qfs_backup
-%{bin_dir}/cpfromqfs
-%{bin_dir}/cptoqfs
-
-%{lib_dir}/libgf_complete.so
-%{lib_dir}/libgf_complete.so.1
-%{lib_dir}/libgf_complete.so.1.0.0
-%{lib_dir}/libJerasure.so
-%{lib_dir}/libJerasure.so.2
-%{lib_dir}/libJerasure.so.2.0.0
-%{lib_dir}/libqfs_access.so
-%{lib_dir}/libqfs_client.so
-%{lib_dir}/libqfs_common.so
-%{lib_dir}/libqfsc.so
-%{lib_dir}/libqfs_io.so
-%{lib_dir}/libqfskrb.so
-%{lib_dir}/libqfs_qcdio.so
-%{lib_dir}/libqfs_qcrs.so
-
-%files metaserver
-%defattr(-,root,root,-)
-%{bin_dir}/metaserver
-%{bin_dir}/filelister
-%{bin_dir}/logcompactor
-%attr(0755,qfs,qfs) %{var_dir}/log/qfs
-%attr(0755,qfs,qfs) %{var_dir}/lib/qfs
-%{initd_dir}/qfs-metaserver
-%config(noreplace) %{etc_dir}/logrotate.d/qfs-metaserver
-
-%files chunkserver
-%defattr(-,root,root,-)
-%{bin_dir}/chunkserver
-%{bin_dir}/chunkscrubber
-%attr(0755,qfs,qfs) %{var_dir}/log/qfs
-%attr(0755,qfs,qfs) %{var_dir}/lib/qfs
-%{initd_dir}/qfs-chunkserver
-%config(noreplace) %{etc_dir}/logrotate.d/qfs-chunkserver
-
-%files devel
-%defattr(-,root,root,-)
-%{bin_dir}/checksum
-%{bin_dir}/dirtree_creator
-%{bin_dir}/dtokentest
-%{bin_dir}/qfslogger
-%{bin_dir}/rand-sfmt
-%{bin_dir}/rebalanceexecutor
-%{bin_dir}/rebalanceplanner
-%{bin_dir}/replicachecker
-%{bin_dir}/requestparser
-%{bin_dir}/sortedhash
-%{bin_dir}/sslfiltertest
-%{bin_dir}/stlset
-
-%{include_dir}/**
-
-%{lib_dir}/libJerasure.a
-%{lib_dir}/libgf_complete.a
-%{lib_dir}/libqfs_client.a
-%{lib_dir}/libqfs_common.a
-%{lib_dir}/libqfs_emulator.a
-%{lib_dir}/libqfs_io.a
-%{lib_dir}/libqfs_meta.a
-%{lib_dir}/libqfs_qcdio.a
-%{lib_dir}/libqfs_qcrs.a
-%{lib_dir}/libqfs_tools.a
-%{lib_dir}/libqfsc.a
-%{lib_dir}/libqfskrb.a
-
-%files fuse
-%defattr(-,root,root,-)
-%{bin_dir}/qfs_fuse
-
-%files hadoop
-%defattr(-,root,root,-)
-%{HADOOP_HOME}/lib/hadoop-%{hadoop_version}-qfs-%{qfs_version}.jar
-
-%files java
-%defattr(-,root,root,-)
-%{data_dir}/java/qfs-access-%{qfs_version}.jar
-
-%files python
-%defattr(-,root,root,-)
-%{python_sitearch}/**
-
-%files webui
-%defattr(-,root,root,-)
-%{data_dir}/webui
-%attr(0755,qfs,qfs) %{var_dir}/log/qfs
-%attr(0755,qfs,qfs) %{var_dir}/lib/qfs
-%{initd_dir}/qfs-webui
-%config(noreplace) %{etc_dir}/logrotate.d/qfs-webui
diff --git a/bigtop-tests/smoke-tests/qfs/build.gradle b/bigtop-tests/smoke-tests/qfs/build.gradle
deleted file mode 100644
index f03e6e4..0000000
--- a/bigtop-tests/smoke-tests/qfs/build.gradle
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-def tests_to_include() {
- return ["TestHadoopExamples.groovy"];
-}
-
-sourceSets {
- test {
- groovy {
- srcDirs = [
- "${BIGTOP_HOME}/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce"
- ]
- exclude {
- FileTreeElement elem -> (doExclude(elem.getName()))
- }
- }
- }
-}
-
-test.doFirst {
- environment("HADOOP_COMMAND", "hadoop-qfs")
- checkEnv(["HADOOP_CONF_DIR", "HADOOP_MAPRED_HOME"])
-}
diff --git a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopExamples.groovy b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopExamples.groovy
index 43fd4f1..45a4c9c 100644
--- a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopExamples.groovy
+++ b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopExamples.groovy
@@ -43,15 +43,6 @@ class TestHadoopExamples {
private static final String HADOOP_MAPRED_HOME = System.getenv('HADOOP_MAPRED_HOME');
private static final String HADOOP_CONF_DIR = System.getenv('HADOOP_CONF_DIR');
- // The hadoop command is dynamic in order to support both hadoop over hdfs
- // and hadoop over qfs easily.
- private static final String HADOOP_COMMAND = System.getenv('HADOOP_COMMAND') ?: "hadoop";
-
- // BIGTOP-3129: Only yarn can successfully write to staging dir, hence workaround by running as yarn.
- // For normal case, use BIGTOP_TEST_USER variable when it's defined. Otherwise run as current user.
- private static final String BIGTOP_TEST_USER = "${HADOOP_COMMAND}" == "hadoop-qfs" ?
- "yarn" : (System.getenv('BIGTOP_TEST_USER') ?: "")
-
private static String hadoopExamplesJar =
JarContent.getJarName(HADOOP_MAPRED_HOME, 'hadoop.*examples.*.jar');
static {
@@ -83,8 +74,8 @@ class TestHadoopExamples {
@AfterClass
public static void tearDown() {
- sh.exec("su -s /bin/bash $BIGTOP_TEST_USER -c '${HADOOP_COMMAND} fs -rmr -skipTrash ${EXAMPLES}'",
- "su -s /bin/bash $BIGTOP_TEST_USER -c '${HADOOP_COMMAND} fs -rmr -skipTrash ${EXAMPLES_OUT}'");
+ sh.exec("hadoop fs -rmr -skipTrash ${EXAMPLES}",
+ "hadoop fs -rmr -skipTrash ${EXAMPLES_OUT}");
}
@@ -106,12 +97,12 @@ class TestHadoopExamples {
LOG.info("MAKING DIRECTORIES ..................... ${EXAMPLES} ${EXAMPLES_OUT}");
//add the files in resources/
- sh.exec("su -s /bin/bash $BIGTOP_TEST_USER -c '${HADOOP_COMMAND} fs -put ${source}/*.* .'");
+ sh.exec("hadoop fs -put ${source}/*.* .");
//add the directories under resources (like examples/)
- sh.exec("su -s /bin/bash $BIGTOP_TEST_USER -c '${HADOOP_COMMAND} fs -put ${source}/${EXAMPLES} ${EXAMPLES}'");
- sh.exec("su -s /bin/bash $BIGTOP_TEST_USER -c '${HADOOP_COMMAND} fs -mkdir -p ${EXAMPLES_OUT}'");
+ sh.exec("hadoop fs -put ${source}/${EXAMPLES} ${EXAMPLES}");
+ sh.exec("hadoop fs -mkdir -p ${EXAMPLES_OUT}");
}
- sh.exec("su -s /bin/bash $BIGTOP_TEST_USER -c '${HADOOP_COMMAND} fs -ls ${EXAMPLES}'");
+ sh.exec("hadoop fs -ls ${EXAMPLES}");
assertTrue("Failed asserting that 'examples' were created in the DFS", sh.getRet() == 0);
}
@@ -132,13 +123,7 @@ class TestHadoopExamples {
aggregatewordhist: "$EXAMPLES/text $EXAMPLES_OUT/aggregatewordhist 2 textinputformat",
grep: "$EXAMPLES/text $EXAMPLES_OUT/grep '[Cc]uriouser'",
secondarysort: "$EXAMPLES/ints $EXAMPLES_OUT/secondarysort",
- randomtextwriter: "-D $RANDOMTEXTWRITER_TOTALBYTES=1073741824 $EXAMPLES_OUT/randomtextwriter"
- ];
-
- // The following example MR jobs are enabled only when running without QFS,
- // which doesn't seem to work with TeraOutputFormat. See BIGTOP-3413 for details.
- static LinkedHashMap additional_examples =
- [
+ randomtextwriter: "-D $RANDOMTEXTWRITER_TOTALBYTES=1073741824 $EXAMPLES_OUT/randomtextwriter",
teragen: "${terasort_rows} teragen${terasortid}",
terasort: "teragen${terasortid} terasort${terasortid}",
teravalidate: "terasort${terasortid} tervalidate${terasortid}"
@@ -152,9 +137,6 @@ class TestHadoopExamples {
public static LinkedHashMap<String, Object[]> generateTests() {
LinkedHashMap<String, Object[]> res = [:];
examples.each { k, v -> res[k] = [k.toString(), v.toString()] as Object[]; }
- if (HADOOP_COMMAND != "hadoop-qfs") {
- additional_examples.each { k, v -> res[k] = [k.toString(), v.toString()] as Object[]; }
- }
return res;
}
@@ -172,7 +154,7 @@ class TestHadoopExamples {
|| FailureVars.instance.getNetworkShutdown()) {
runFailureThread();
}
- sh.exec("su -s /bin/bash $BIGTOP_TEST_USER -c '${HADOOP_COMMAND} jar $testJar $testName $testArgs'");
+ sh.exec("hadoop jar $testJar $testName $testArgs");
assertTrue("Example $testName $testJar $testName $testArgs failed", sh.getRet() == 0);
}
diff --git a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopSmoke.groovy b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopSmoke.groovy
index 118bbb6..cad3a49 100644
--- a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopSmoke.groovy
+++ b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopSmoke.groovy
@@ -37,10 +37,6 @@ class TestHadoopSmoke {
static String streamingHome = System.getenv('STREAMING_HOME')
static String hadoopMapReduceHome = System.getProperty('HADOOP_MAPRED_HOME', '/usr/lib/hadoop-mapreduce')
- // The hadoop command is dynamic in order to support both hadoop over hdfs
- // and hadoop over qfs easily.
- private static final String HADOOP_COMMAND = System.getProperty('HADOOP_COMMAND', 'hadoop');
-
static final String STREAMING_HOME =
(streamingHome == null) ? hadoopMapReduceHome : streamingHome;
static String streaming_jar =
@@ -52,7 +48,7 @@ class TestHadoopSmoke {
static String testDir = "test.hadoopsmoke." + (new Date().getTime())
static String nn = (new Configuration()).get(DFSConfigKeys.FS_DEFAULT_NAME_KEY)
- String cmd = "${HADOOP_COMMAND} jar ${STREAMING_JAR}" +
+ String cmd = "hadoop jar ${STREAMING_JAR}" +
" -D mapred.map.tasks=1 -D mapred.reduce.tasks=1 -D mapred.job.name=Experiment"
String cmd2 = " -input ${testDir}/cachefile/input.txt -mapper map.sh -file map.sh -reducer cat" +
" -output ${testDir}/cachefile/out -verbose"
@@ -70,15 +66,15 @@ class TestHadoopSmoke {
@AfterClass
static void tearDown() {
- sh.exec("${HADOOP_COMMAND} fs -rmr -skipTrash ${testDir}")
+ sh.exec("hadoop fs -rmr -skipTrash ${testDir}")
}
@Test (timeout = 0x810000l)
void testCacheArchive() {
- sh.exec("${HADOOP_COMMAND} fs -rmr ${testDir}/cachefile/out",
+ sh.exec("hadoop fs -rmr ${testDir}/cachefile/out",
cmd + ' -cacheArchive ' + arg + cmd2)
logError(sh)
- sh.exec("${HADOOP_COMMAND} fs -cat ${testDir}/cachefile/out/part-00000")
+ sh.exec("hadoop fs -cat ${testDir}/cachefile/out/part-00000")
logError(sh)
assertEquals("cache1\t\ncache2\t", sh.out.join('\n'))
@@ -86,10 +82,10 @@ class TestHadoopSmoke {
@Test (timeout = 0x810000l)
void testArchives() {
- sh.exec("${HADOOP_COMMAND} fs -rmr ${testDir}/cachefile/out",
+ sh.exec("hadoop fs -rmr ${testDir}/cachefile/out",
cmd + ' -archives ' + arg + cmd2)
logError(sh)
- sh.exec("${HADOOP_COMMAND} fs -cat ${testDir}/cachefile/out/part-00000")
+ sh.exec("hadoop fs -cat ${testDir}/cachefile/out/part-00000")
logError(sh)
assertEquals("cache1\t\ncache2\t", sh.out.join('\n'))
diff --git a/bigtop.bom b/bigtop.bom
index 3d4720f..ba33871 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -110,7 +110,7 @@ bigtop {
*/
dependencies = [
'bigtop-utils':['ambari', 'bigtop-groovy', 'bigtop-jsvc', 'bigtop-tomcat',
- 'elasticsearch', 'flink', 'logstash', 'qfs', 'zookeeper'],
+ 'elasticsearch', 'flink', 'logstash', 'zookeeper'],
'bigtop-groovy':['hadoop'],
'bigtop-jsvc':['hadoop'],
'bigtop-tomcat':['oozie'],
@@ -328,15 +328,6 @@ bigtop {
site = "${apache.APACHE_MIRROR}/${download_path}"
archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
}
- 'qfs' {
- name = 'qfs'
- relNotes = 'Quantcast Filesystem'
- version { base = '2.0.0'; pkg = base; release = 1 }
- tarball { destination = "$name-${version.base}.tar.gz"
- source = "${version.base}.tar.gz" }
- url { site = "https://github.com/quantcast/qfs/archive"
- archive = site }
- }
'gpdb' {
name = 'gpdb'
relNotes = 'GreenPlum'
diff --git a/build.gradle b/build.gradle
index 4ecb0ea..7570aea 100644
--- a/build.gradle
+++ b/build.gradle
@@ -134,7 +134,6 @@ rat {
"bigtop-deploy/puppet/modules/hadoop/templates/taskcontroller.cfg",
"bigtop-deploy/puppet/modules/hadoop/templates/container-executor.cfg",
"bigtop-deploy/puppet/README.md",
- "bigtop-deploy/puppet/modules/qfs/README.md",
"bigtop-deploy/puppet/config/site.csv.example",
"dl/**",
"bigtop-deploy/puppet/modules/gpdb/templates/**",
diff --git a/provisioner/utils/smoke-tests.sh b/provisioner/utils/smoke-tests.sh
index edc13ea..bb3313b 100755
--- a/provisioner/utils/smoke-tests.sh
+++ b/provisioner/utils/smoke-tests.sh
@@ -59,28 +59,19 @@ export ZOOKEEPER_HOME=${ZOOKEEPER_HOME:-/usr/lib/zookeeper}
echo -e "\n===== START TO RUN SMOKE TESTS: $SMOKE_TESTS =====\n"
-prep() {
- HADOOP_COMMAND=$1
- su -s /bin/bash $HCFS_USER -c "JAVA_LIBRARY_PATH=/usr/lib/qfs $HADOOP_COMMAND fs -mkdir -p /user/vagrant /user/root /user/yarn"
- su -s /bin/bash $HCFS_USER -c "JAVA_LIBRARY_PATH=/usr/lib/qfs $HADOOP_COMMAND fs -chmod 777 /user/vagrant"
- su -s /bin/bash $HCFS_USER -c "JAVA_LIBRARY_PATH=/usr/lib/qfs $HADOOP_COMMAND fs -chmod 777 /user/root"
- su -s /bin/bash $HCFS_USER -c "JAVA_LIBRARY_PATH=/usr/lib/qfs $HADOOP_COMMAND fs -chown yarn:yarn /user/yarn"
-}
-
-prep hadoop
-if [[ $SMOKE_TESTS == *"qfs"* ]]; then
- HCFS_USER=root
- prep hadoop-qfs
-fi
+su -s /bin/bash $HCFS_USER -c "hadoop fs -mkdir -p /user/vagrant /user/root /user/yarn"
+su -s /bin/bash $HCFS_USER -c "hadoop fs -chmod 777 /user/vagrant"
+su -s /bin/bash $HCFS_USER -c "hadoop fs -chmod 777 /user/root"
+su -s /bin/bash $HCFS_USER -c "hadoop fs -chown yarn:yarn /user/yarn"
if [[ $SMOKE_TESTS == *"alluxio"* ]]; then
- su -s /bin/bash $HCFS_USER -c "$HADOOP_COMMAND fs -mkdir /underFSStorage"
- su -s /bin/bash $HCFS_USER -c "$HADOOP_COMMAND fs -chmod 777 /underFSStorage"
+ su -s /bin/bash $HCFS_USER -c "hadoop fs -mkdir /underFSStorage"
+ su -s /bin/bash $HCFS_USER -c "hadoop fs -chmod 777 /underFSStorage"
fi
if [[ $SMOKE_TESTS == *"oozie"* ]]; then
- su -s /bin/bash $HCFS_USER -c "$HADOOP_COMMAND fs -mkdir -p /user/oozie/share/lib"
- su -s /bin/bash $HCFS_USER -c "$HADOOP_COMMAND fs -chown -R oozie:oozie /user/oozie"
+ su -s /bin/bash $HCFS_USER -c "hadoop fs -mkdir -p /user/oozie/share/lib"
+ su -s /bin/bash $HCFS_USER -c "hadoop fs -chown -R oozie:oozie /user/oozie"
oozie-setup sharelib create -fs hdfs://$(hostname -f):8020/
oozie admin -sharelibupdate
fi