You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by al...@apache.org on 2018/11/01 01:56:02 UTC
kudu git commit: [cmake] place all DATA_FILES under bin/testdata
Repository: kudu
Updated Branches:
refs/heads/master 1043f3193 -> fec218bf6
[cmake] place all DATA_FILES under bin/testdata
Updated the convention for the DATA_FILES parameter of the ADD_KUDU_TEST
macro. With this patch, all test data files are placed under the
<build_dir>/bin/testdata directory after running cmake.
The new convention eases the restrictions on the layout of the source
data files. That, in its turn, helps to stop duplicating auxiliary files
needed for various tests.
Change-Id: I75064c6bcf9a569648b922b55f539730769cdf4c
Reviewed-on: http://gerrit.cloudera.org:8080/11841
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Alexey Serbin <as...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/fec218bf
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/fec218bf
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/fec218bf
Branch: refs/heads/master
Commit: fec218bf644bb8f8a4b3edccf9a559600ab38659
Parents: 1043f31
Author: Alexey Serbin <al...@apache.org>
Authored: Wed Oct 31 14:56:42 2018 -0700
Committer: Alexey Serbin <as...@cloudera.com>
Committed: Thu Nov 1 01:54:03 2018 +0000
----------------------------------------------------------------------
CMakeLists.txt | 29 +--
src/kudu/integration-tests/CMakeLists.txt | 4 +-
.../scripts/assign-location.py | 244 -------------------
.../integration-tests/scripts/first_argument.sh | 20 --
.../integration-tests/table_locations-itest.cc | 2 +-
src/kudu/master/CMakeLists.txt | 2 +-
src/kudu/master/testdata/first_argument.sh | 20 --
src/kudu/mini-cluster/external_mini_cluster.cc | 2 +-
src/kudu/scripts/assign-location.py | 244 +++++++++++++++++++
src/kudu/scripts/first_argument.sh | 20 ++
src/kudu/tablet/compaction_policy-test.cc | 2 +-
src/kudu/tools/CMakeLists.txt | 4 +-
src/kudu/tools/testdata/first_argument.sh | 20 --
13 files changed, 283 insertions(+), 330 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/fec218bf/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4ae7190..7b34867 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -778,14 +778,11 @@ endfunction()
# set. See build-support/run-test.sh for more details.
#
# DATA_FILES <file1> <file2> ...
-# Specify data files that should be copied into the build directory
-# next to test executable. These will also be automatically uploaded
-# to the dist-test environment.
-#
-# If the file is in a directory, an equivalent directory path will
-# be made in the build directory. For example, 'testdata/foo' will
-# ensure that an equivalent directory 'testdata/' is made in the
-# build directory.
+# Specify data files that should be copied into the build directory next
+# to test executable into the 'testdata' sub-directory, i.e. into
+# '<build_dir>/bin/testdata' with current layout of the test binaries.
+# The path to the source file should be specified from the location
+# of the corresponding CMakeLists.txt file.
#
# Any other arguments will be passed to set_tests_properties().
function(ADD_KUDU_TEST REL_TEST_NAME)
@@ -832,16 +829,12 @@ function(ADD_KUDU_TEST REL_TEST_NAME)
endif()
# Copy data files into the build directory.
+ set(DATA_FILES_DST_SUBDIR testdata)
+ set(DST_DIR ${EXECUTABLE_OUTPUT_PATH}/${DATA_FILES_DST_SUBDIR})
+ set(DATA_FILES_LIST)
foreach(DATA_FILE ${ARG_DATA_FILES})
- get_filename_component(DST_DIR ${DATA_FILE} DIRECTORY)
- set(DST_DIR ${EXECUTABLE_OUTPUT_PATH}/${DST_DIR})
- # If there is a symlink there, replace it with a copy. This is to
- # work around issues with incremental builds where earlier build
- # setups had used symlinks into the build directory instead of
- # copies.
- if(IS_SYMLINK ${DST_DIR}/${DATA_FILE})
- file(REMOVE ${DST_DIR}/${DATA_FILE})
- endif()
+ get_filename_component(DATA_FILE_NAME ${DATA_FILE} NAME)
+ list(APPEND DATA_FILES_LIST ${DATA_FILES_DST_SUBDIR}/${DATA_FILE_NAME})
file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/${DATA_FILE}
# Copy with read and execute permissions, since tests should not modify
# the data files in place, but data files may be scripts used by tests.
@@ -881,7 +874,7 @@ function(ADD_KUDU_TEST REL_TEST_NAME)
# The only way we can pass information to dist-test is through the environment.
# So, use a comma-delimited environment variable to pass the list of data files
# that need to be uploaded.
- string(REPLACE ";" "," DATA_FILES_ENV "${ARG_DATA_FILES}")
+ string(REPLACE ";" "," DATA_FILES_ENV "${DATA_FILES_LIST}")
list(APPEND CUR_TEST_ENV "KUDU_DATA_FILES=${DATA_FILES_ENV}")
set_tests_properties(${TARGET} PROPERTIES
http://git-wip-us.apache.org/repos/asf/kudu/blob/fec218bf/src/kudu/integration-tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/CMakeLists.txt b/src/kudu/integration-tests/CMakeLists.txt
index 5b8bd81..aca65f8 100644
--- a/src/kudu/integration-tests/CMakeLists.txt
+++ b/src/kudu/integration-tests/CMakeLists.txt
@@ -105,7 +105,7 @@ ADD_KUDU_TEST(security-master-auth-itest)
ADD_KUDU_TEST(security-unknown-tsk-itest PROCESSORS 4)
ADD_KUDU_TEST(stop_tablet-itest PROCESSORS 4)
ADD_KUDU_TEST(table_locations-itest
- DATA_FILES scripts/first_argument.sh)
+ DATA_FILES ../scripts/first_argument.sh)
ADD_KUDU_TEST(tablet_copy-itest NUM_SHARDS 6 PROCESSORS 4)
ADD_KUDU_TEST(tablet_copy_client_session-itest)
ADD_KUDU_TEST(tablet_history_gc-itest)
@@ -116,7 +116,7 @@ ADD_KUDU_TEST(tombstoned_voting-itest)
ADD_KUDU_TEST(tombstoned_voting-stress-test RUN_SERIAL true)
ADD_KUDU_TEST(token_signer-itest)
ADD_KUDU_TEST(ts_location_assignment-itest
- DATA_FILES scripts/assign-location.py)
+ DATA_FILES ../scripts/assign-location.py)
ADD_KUDU_TEST(ts_recovery-itest PROCESSORS 4)
ADD_KUDU_TEST(ts_tablet_manager-itest)
ADD_KUDU_TEST(update_scan_delta_compact-test RUN_SERIAL true)
http://git-wip-us.apache.org/repos/asf/kudu/blob/fec218bf/src/kudu/integration-tests/scripts/assign-location.py
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/scripts/assign-location.py b/src/kudu/integration-tests/scripts/assign-location.py
deleted file mode 100644
index 5714772..0000000
--- a/src/kudu/integration-tests/scripts/assign-location.py
+++ /dev/null
@@ -1,244 +0,0 @@
-#!/usr/bin/env python
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import argparse
-import errno
-import fcntl
-import json
-import time
-import random
-
-# This is a simple sequencer to be run as a location assignment script
-# by a Kudu master. The script can be used in location-aware test scenarios
-# and other cases when location assignment rules are specified simply as the
-# distribution of tablet servers among locations: i.e. how many tablet
-# servers should be in every specified location (see below for an example).
-#
-# The script takes as input location mapping rules and an identifier.
-# On success, the script prints the location assigned to the specified
-# identifier to stdout. The identifier might be any string uniquely identifying
-# a tablet server.
-#
-# Locations are assigned based on:
-# a) Location mapping rules specified in the command line and sequencer's
-# offset persistently stored in a state file.
-# b) Previously established and persisted { id, location } mappings in the
-# state file.
-#
-# Once assigned, the location for the specified identifier is recorded and
-# output again upon next call of the script for the same identifier.
-#
-# It's safe to run multiple instances of the script concurrently with the
-# same set of parameters. The access to the sequencer's state file is
-# serialized and the scripts produces consistent results for all concurrent
-# callers.
-#
-# A location mapping rule is specified as a pair 'loc:num', where the 'num'
-# stands for the number of servers to assign to the location 'loc'. Location
-# mapping rules are provided to the script by --map 'loc:num' command line
-# arguments.
-#
-# Below is an example of invocation of the script for location mapping rules
-# specifying that location 'l0' should have one tablet server, location 'l1'
-# should have two, and location 'l2' should have three. The script is run
-# to assign a location for a tablet server running at IP address 127.1.2.3.
-#
-# assign-location.py --map l0:1 --map l1:2 --map l2:3 127.1.2.3
-#
-
-class LocationAssignmentRule(object):
- def __init__(self, location_mapping_rules):
- # Convert the input location information into an auxiliary array of
- # location strings.
- self.location_mapping_rules = location_mapping_rules
- if self.location_mapping_rules is None:
- self.location_mapping_rules = []
- self.locations = []
- self.total_count = 0
-
- seen_locations = []
- for info in self.location_mapping_rules:
- location, server_num_str = info.split(':')
- seen_locations.append(location)
- server_num = int(server_num_str)
- for i in range(0, server_num):
- self.total_count += 1
- self.locations.append(location)
- assert (len(set(seen_locations)) == len(seen_locations)), \
- 'duplicate locations specified: {}'.format(seen_locations)
-
- def get_location(self, idx):
- """
- Get location for the specified index.
- """
- if self.locations:
- return self.locations[idx % len(self.locations)]
- else:
- return ""
-
-
-def acquire_advisory_lock(fpath):
- """
- Acquire a lock on a special .lock file. Don't block while trying: return
- if failed to acquire a lock in 30 seconds.
- """
- timeout_seconds = 30
- now = time.clock()
- deadline = now + timeout_seconds
- random.seed(int(now))
- fpath_lock_file = fpath + ".lock"
- # Open the lock file; create the file if doesn't exist.
- lock_file = open(fpath_lock_file, 'w+')
- got_lock = False
- while time.clock() < deadline:
- try:
- fcntl.flock(lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB)
- got_lock = True
- break
- except IOError as e:
- if e.errno != errno.EAGAIN:
- raise
- else:
- time.sleep(random.uniform(0.001, 0.100))
-
- if not got_lock:
- raise Exception('could not obtain exclusive lock for {} in {} seconds',
- fpath_lock_file, timeout_seconds)
-
- return lock_file
-
-
-def get_location(fpath, rule, uid, relaxed):
- """
- Return location for the specified identifier 'uid'. To do that, use the
- specified location mapping rules and the information stored
- in the sequencer's state file.
-
- * Obtain advisory lock for the state file (using additional .lock file)
- * If the sequencer's state file exists:
- 1. Open the state file in read-only mode.
- 2. Read the information from the state file and search for location
- assigned to the server with the specified identifier.
- a. If already assigned location found:
- -- Return the location.
- b. If location assigned to the identifier is not found:
- -- Use current sequence number 'seq' to assign next location
- by calling LocationAssignmentRule.get_location(seq).
- -- Add the newly generated location assignment into the
- sequencer's state.
- -- Increment the sequence number.
- -- Reopen the state file for writing (if file exists)
- -- Rewrite the file with the new state of the sequencer.
- -- Return the newly assigned location.
- * If the sequencer's state file does not exist:
- 1. Set sequence number 'seq' to 0.
- 2. Use current sequence number 'seq' to assign next location
- by calling LocationAssignmentRule.get_location(seq).
- 3. Update the sequencer's state accordingly.
- 3. Rewrite the file with the new state of the sequencer.
- 4. Return the newly assigned location.
- """
- lock_file = acquire_advisory_lock(fpath)
- state_file = None
- try:
- state_file = open(fpath)
- except IOError as e:
- if e.errno != errno.ENOENT:
- raise
-
- new_assignment = False
- if state_file is None:
- seq = 0
- state = {}
- state['seq'] = seq
- state['mapping_rules'] = rule.location_mapping_rules
- state['mappings'] = {}
- mappings = state['mappings']
- new_assignment = True
- else:
- # If the file exists, it must have proper content.
- state = json.load(state_file)
- seq = state.get('seq')
- mapping_rules = state.get('mapping_rules')
- # Make sure the stored mapping rule corresponds to the specified in args.
- rule_stored = json.dumps(mapping_rules)
- rule_specified = json.dumps(rule.location_mapping_rules)
- if rule_stored != rule_specified:
- raise Exception('stored and specified mapping rules mismatch: '
- '{} vs {}'.format(rule_stored, rule_specified))
- mappings = state['mappings']
- location = mappings.get(uid, None)
- if location is None:
- seq += 1
- state['seq'] = seq
- new_assignment = True
-
- if not new_assignment:
- return location
-
- if not relaxed and rule.total_count != 0 and rule.total_count <= seq:
- raise Exception('too many unique identifiers ({}) to assign next location '
- 'using mapping rules {}'.format(
- seq + 1, rule.location_mapping_rules))
-
- if relaxed and rule.total_count <= seq:
- return ""
-
- # Get next location and add the { uid, location} binding into the mappings.
- location = rule.get_location(seq)
- mappings[uid] = location
-
- # Rewrite the file with the updated state information.
- if state_file is not None:
- state_file.close()
- state_file = open(fpath, 'w+')
- json.dump(state, state_file)
- state_file.close()
- lock_file.close()
- return location
-
-
-def main():
- parser = argparse.ArgumentParser()
- parser.add_argument("--state_store",
- nargs="?",
- default="/tmp/location-sequencer-state",
- help="path to a file to store the sequencer's state")
- parser.add_argument("--map", "-m",
- action="append",
- dest="location_mapping_rules",
- metavar="RULE",
- help="location mapping rule: number of tablet servers per specified "
- "location in form <location>:<number>; this option may be specified "
- "multiple times")
- parser.add_argument("--relaxed",
- action="store_true",
- help="whether to allow more location assignments than specified "
- "by the specified mapping rules")
- parser.add_argument("uid",
- help="hostname, IP address, or any other unique identifier")
- args = parser.parse_args()
-
- location = get_location(args.state_store,
- LocationAssignmentRule(args.location_mapping_rules), args.uid, args.relaxed)
- print(location)
-
-
-if __name__ == "__main__":
- main()
http://git-wip-us.apache.org/repos/asf/kudu/blob/fec218bf/src/kudu/integration-tests/scripts/first_argument.sh
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/scripts/first_argument.sh b/src/kudu/integration-tests/scripts/first_argument.sh
deleted file mode 100755
index 832908c..0000000
--- a/src/kudu/integration-tests/scripts/first_argument.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# A script that prints the first argument and ignores all the others.
-echo "$1"
http://git-wip-us.apache.org/repos/asf/kudu/blob/fec218bf/src/kudu/integration-tests/table_locations-itest.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/table_locations-itest.cc b/src/kudu/integration-tests/table_locations-itest.cc
index 18acc2b..13a516a 100644
--- a/src/kudu/integration-tests/table_locations-itest.cc
+++ b/src/kudu/integration-tests/table_locations-itest.cc
@@ -165,7 +165,7 @@ class TableLocationsWithTSLocationTest : public TableLocationsTest {
public:
void SetUpConfig() override {
const string location_cmd_path = JoinPathSegments(GetTestExecutableDirectory(),
- "scripts/first_argument.sh");
+ "testdata/first_argument.sh");
const string location = "/foo";
FLAGS_location_mapping_cmd = strings::Substitute("$0 $1", location_cmd_path, location);
}
http://git-wip-us.apache.org/repos/asf/kudu/blob/fec218bf/src/kudu/master/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/kudu/master/CMakeLists.txt b/src/kudu/master/CMakeLists.txt
index 2947a12..1333eaf 100644
--- a/src/kudu/master/CMakeLists.txt
+++ b/src/kudu/master/CMakeLists.txt
@@ -78,7 +78,7 @@ ADD_KUDU_TEST(master-test RESOURCE_LOCK "master-web-port")
ADD_KUDU_TEST(mini_master-test RESOURCE_LOCK "master-web-port")
ADD_KUDU_TEST(placement_policy-test)
ADD_KUDU_TEST(sys_catalog-test RESOURCE_LOCK "master-web-port")
-ADD_KUDU_TEST(ts_descriptor-test DATA_FILES testdata/first_argument.sh)
+ADD_KUDU_TEST(ts_descriptor-test DATA_FILES ../scripts/first_argument.sh)
# Actual master executable
add_executable(kudu-master master_main.cc)
http://git-wip-us.apache.org/repos/asf/kudu/blob/fec218bf/src/kudu/master/testdata/first_argument.sh
----------------------------------------------------------------------
diff --git a/src/kudu/master/testdata/first_argument.sh b/src/kudu/master/testdata/first_argument.sh
deleted file mode 100755
index 832908c..0000000
--- a/src/kudu/master/testdata/first_argument.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# A script that prints the first argument and ignores all the others.
-echo "$1"
http://git-wip-us.apache.org/repos/asf/kudu/blob/fec218bf/src/kudu/mini-cluster/external_mini_cluster.cc
----------------------------------------------------------------------
diff --git a/src/kudu/mini-cluster/external_mini_cluster.cc b/src/kudu/mini-cluster/external_mini_cluster.cc
index 5bf98df..4fbeb04 100644
--- a/src/kudu/mini-cluster/external_mini_cluster.cc
+++ b/src/kudu/mini-cluster/external_mini_cluster.cc
@@ -380,7 +380,7 @@ Status ExternalMiniCluster::StartMasters() {
string bin_path;
RETURN_NOT_OK(DeduceBinRoot(&bin_path));
const auto mapping_script_path =
- JoinPathSegments(bin_path, "scripts/assign-location.py");
+ JoinPathSegments(bin_path, "testdata/assign-location.py");
const auto state_store_fpath =
JoinPathSegments(opts_.cluster_root, "location-assignment.state");
auto location_cmd = Substitute("$0 --state_store=$1",
http://git-wip-us.apache.org/repos/asf/kudu/blob/fec218bf/src/kudu/scripts/assign-location.py
----------------------------------------------------------------------
diff --git a/src/kudu/scripts/assign-location.py b/src/kudu/scripts/assign-location.py
new file mode 100644
index 0000000..5714772
--- /dev/null
+++ b/src/kudu/scripts/assign-location.py
@@ -0,0 +1,244 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import errno
+import fcntl
+import json
+import time
+import random
+
+# This is a simple sequencer to be run as a location assignment script
+# by a Kudu master. The script can be used in location-aware test scenarios
+# and other cases when location assignment rules are specified simply as the
+# distribution of tablet servers among locations: i.e. how many tablet
+# servers should be in every specified location (see below for an example).
+#
+# The script takes as input location mapping rules and an identifier.
+# On success, the script prints the location assigned to the specified
+# identifier to stdout. The identifier might be any string uniquely identifying
+# a tablet server.
+#
+# Locations are assigned based on:
+# a) Location mapping rules specified in the command line and sequencer's
+# offset persistently stored in a state file.
+# b) Previously established and persisted { id, location } mappings in the
+# state file.
+#
+# Once assigned, the location for the specified identifier is recorded and
+# output again upon next call of the script for the same identifier.
+#
+# It's safe to run multiple instances of the script concurrently with the
+# same set of parameters. The access to the sequencer's state file is
+# serialized and the scripts produces consistent results for all concurrent
+# callers.
+#
+# A location mapping rule is specified as a pair 'loc:num', where the 'num'
+# stands for the number of servers to assign to the location 'loc'. Location
+# mapping rules are provided to the script by --map 'loc:num' command line
+# arguments.
+#
+# Below is an example of invocation of the script for location mapping rules
+# specifying that location 'l0' should have one tablet server, location 'l1'
+# should have two, and location 'l2' should have three. The script is run
+# to assign a location for a tablet server running at IP address 127.1.2.3.
+#
+# assign-location.py --map l0:1 --map l1:2 --map l2:3 127.1.2.3
+#
+
+class LocationAssignmentRule(object):
+ def __init__(self, location_mapping_rules):
+ # Convert the input location information into an auxiliary array of
+ # location strings.
+ self.location_mapping_rules = location_mapping_rules
+ if self.location_mapping_rules is None:
+ self.location_mapping_rules = []
+ self.locations = []
+ self.total_count = 0
+
+ seen_locations = []
+ for info in self.location_mapping_rules:
+ location, server_num_str = info.split(':')
+ seen_locations.append(location)
+ server_num = int(server_num_str)
+ for i in range(0, server_num):
+ self.total_count += 1
+ self.locations.append(location)
+ assert (len(set(seen_locations)) == len(seen_locations)), \
+ 'duplicate locations specified: {}'.format(seen_locations)
+
+ def get_location(self, idx):
+ """
+ Get location for the specified index.
+ """
+ if self.locations:
+ return self.locations[idx % len(self.locations)]
+ else:
+ return ""
+
+
+def acquire_advisory_lock(fpath):
+ """
+ Acquire a lock on a special .lock file. Don't block while trying: return
+ if failed to acquire a lock in 30 seconds.
+ """
+ timeout_seconds = 30
+ now = time.clock()
+ deadline = now + timeout_seconds
+ random.seed(int(now))
+ fpath_lock_file = fpath + ".lock"
+ # Open the lock file; create the file if doesn't exist.
+ lock_file = open(fpath_lock_file, 'w+')
+ got_lock = False
+ while time.clock() < deadline:
+ try:
+ fcntl.flock(lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ got_lock = True
+ break
+ except IOError as e:
+ if e.errno != errno.EAGAIN:
+ raise
+ else:
+ time.sleep(random.uniform(0.001, 0.100))
+
+ if not got_lock:
+ raise Exception('could not obtain exclusive lock for {} in {} seconds',
+ fpath_lock_file, timeout_seconds)
+
+ return lock_file
+
+
+def get_location(fpath, rule, uid, relaxed):
+ """
+ Return location for the specified identifier 'uid'. To do that, use the
+ specified location mapping rules and the information stored
+ in the sequencer's state file.
+
+ * Obtain advisory lock for the state file (using additional .lock file)
+ * If the sequencer's state file exists:
+ 1. Open the state file in read-only mode.
+ 2. Read the information from the state file and search for location
+ assigned to the server with the specified identifier.
+ a. If already assigned location found:
+ -- Return the location.
+ b. If location assigned to the identifier is not found:
+ -- Use current sequence number 'seq' to assign next location
+ by calling LocationAssignmentRule.get_location(seq).
+ -- Add the newly generated location assignment into the
+ sequencer's state.
+ -- Increment the sequence number.
+ -- Reopen the state file for writing (if file exists)
+ -- Rewrite the file with the new state of the sequencer.
+ -- Return the newly assigned location.
+ * If the sequencer's state file does not exist:
+ 1. Set sequence number 'seq' to 0.
+ 2. Use current sequence number 'seq' to assign next location
+ by calling LocationAssignmentRule.get_location(seq).
+ 3. Update the sequencer's state accordingly.
+ 3. Rewrite the file with the new state of the sequencer.
+ 4. Return the newly assigned location.
+ """
+ lock_file = acquire_advisory_lock(fpath)
+ state_file = None
+ try:
+ state_file = open(fpath)
+ except IOError as e:
+ if e.errno != errno.ENOENT:
+ raise
+
+ new_assignment = False
+ if state_file is None:
+ seq = 0
+ state = {}
+ state['seq'] = seq
+ state['mapping_rules'] = rule.location_mapping_rules
+ state['mappings'] = {}
+ mappings = state['mappings']
+ new_assignment = True
+ else:
+ # If the file exists, it must have proper content.
+ state = json.load(state_file)
+ seq = state.get('seq')
+ mapping_rules = state.get('mapping_rules')
+ # Make sure the stored mapping rule corresponds to the specified in args.
+ rule_stored = json.dumps(mapping_rules)
+ rule_specified = json.dumps(rule.location_mapping_rules)
+ if rule_stored != rule_specified:
+ raise Exception('stored and specified mapping rules mismatch: '
+ '{} vs {}'.format(rule_stored, rule_specified))
+ mappings = state['mappings']
+ location = mappings.get(uid, None)
+ if location is None:
+ seq += 1
+ state['seq'] = seq
+ new_assignment = True
+
+ if not new_assignment:
+ return location
+
+ if not relaxed and rule.total_count != 0 and rule.total_count <= seq:
+ raise Exception('too many unique identifiers ({}) to assign next location '
+ 'using mapping rules {}'.format(
+ seq + 1, rule.location_mapping_rules))
+
+ if relaxed and rule.total_count <= seq:
+ return ""
+
+ # Get next location and add the { uid, location} binding into the mappings.
+ location = rule.get_location(seq)
+ mappings[uid] = location
+
+ # Rewrite the file with the updated state information.
+ if state_file is not None:
+ state_file.close()
+ state_file = open(fpath, 'w+')
+ json.dump(state, state_file)
+ state_file.close()
+ lock_file.close()
+ return location
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--state_store",
+ nargs="?",
+ default="/tmp/location-sequencer-state",
+ help="path to a file to store the sequencer's state")
+ parser.add_argument("--map", "-m",
+ action="append",
+ dest="location_mapping_rules",
+ metavar="RULE",
+ help="location mapping rule: number of tablet servers per specified "
+ "location in form <location>:<number>; this option may be specified "
+ "multiple times")
+ parser.add_argument("--relaxed",
+ action="store_true",
+ help="whether to allow more location assignments than specified "
+ "by the specified mapping rules")
+ parser.add_argument("uid",
+ help="hostname, IP address, or any other unique identifier")
+ args = parser.parse_args()
+
+ location = get_location(args.state_store,
+ LocationAssignmentRule(args.location_mapping_rules), args.uid, args.relaxed)
+ print(location)
+
+
+if __name__ == "__main__":
+ main()
http://git-wip-us.apache.org/repos/asf/kudu/blob/fec218bf/src/kudu/scripts/first_argument.sh
----------------------------------------------------------------------
diff --git a/src/kudu/scripts/first_argument.sh b/src/kudu/scripts/first_argument.sh
new file mode 100755
index 0000000..832908c
--- /dev/null
+++ b/src/kudu/scripts/first_argument.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# A script that prints the first argument and ignores all the others.
+echo "$1"
http://git-wip-us.apache.org/repos/asf/kudu/blob/fec218bf/src/kudu/tablet/compaction_policy-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/compaction_policy-test.cc b/src/kudu/tablet/compaction_policy-test.cc
index 8ec0829..3583c39 100644
--- a/src/kudu/tablet/compaction_policy-test.cc
+++ b/src/kudu/tablet/compaction_policy-test.cc
@@ -232,7 +232,7 @@ TEST_F(TestCompactionPolicy, TestYcsbCompaction) {
LOG(WARNING) << "test is skipped; set KUDU_ALLOW_SLOW_TESTS=1 to run";
return;
}
- const RowSetVector rowsets = LoadFile("ycsb-test-rowsets.tsv");
+ const RowSetVector rowsets = LoadFile("testdata/ycsb-test-rowsets.tsv");
RowSetTree tree;
ASSERT_OK(tree.Reset(rowsets));
vector<double> qualities;
http://git-wip-us.apache.org/repos/asf/kudu/blob/fec218bf/src/kudu/tools/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/kudu/tools/CMakeLists.txt b/src/kudu/tools/CMakeLists.txt
index 875657c..d4017d6 100644
--- a/src/kudu/tools/CMakeLists.txt
+++ b/src/kudu/tools/CMakeLists.txt
@@ -167,7 +167,7 @@ ADD_KUDU_TEST(diagnostics_log_parser-test)
ADD_KUDU_TEST(ksck-test)
ADD_KUDU_TEST(ksck_remote-test
PROCESSORS 3
- DATA_FILES testdata/first_argument.sh)
+ DATA_FILES ../scripts/first_argument.sh)
ADD_KUDU_TEST(kudu-admin-test
NUM_SHARDS 8 PROCESSORS 3)
ADD_KUDU_TEST_DEPENDENCIES(kudu-admin-test
@@ -175,7 +175,7 @@ ADD_KUDU_TEST_DEPENDENCIES(kudu-admin-test
ADD_KUDU_TEST(kudu-tool-test
NUM_SHARDS 4 PROCESSORS 3
DATA_FILES testdata/sample-diagnostics-log.txt testdata/bad-diagnostics-log.txt
- DATA_FILES testdata/first_argument.sh)
+ DATA_FILES ../scripts/first_argument.sh)
ADD_KUDU_TEST_DEPENDENCIES(kudu-tool-test
kudu)
ADD_KUDU_TEST(kudu-ts-cli-test)
http://git-wip-us.apache.org/repos/asf/kudu/blob/fec218bf/src/kudu/tools/testdata/first_argument.sh
----------------------------------------------------------------------
diff --git a/src/kudu/tools/testdata/first_argument.sh b/src/kudu/tools/testdata/first_argument.sh
deleted file mode 100644
index 832908c..0000000
--- a/src/kudu/tools/testdata/first_argument.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# A script that prints the first argument and ignores all the others.
-echo "$1"