You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by to...@apache.org on 2016/08/10 23:28:13 UTC
[1/2] kudu git commit: test_result_server: improve indexes,
add sorting
Repository: kudu
Updated Branches:
refs/heads/master 88fe33c99 -> 76ec7e66e
test_result_server: improve indexes, add sorting
* adds a new index to the underlying database table.
I applied this by hand in production and page loads that used to take several
seconds are now quite fast.
* uses the 'DataTables' jQuery plugin to make the main list of flaky tests
sortable by flaky rate.
Tested manually by running this locally against the production database.
Change-Id: I4f4321ad01d8e5fe83276d07cd6d6025dab5518e
Reviewed-on: http://gerrit.cloudera.org:8080/3889
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Todd Lipcon <to...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/b9c59cdc
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/b9c59cdc
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/b9c59cdc
Branch: refs/heads/master
Commit: b9c59cdceb3825556759b0305ecfb615ece391e4
Parents: 88fe33c
Author: Todd Lipcon <to...@apache.org>
Authored: Wed Aug 10 13:57:47 2016 -0700
Committer: Todd Lipcon <to...@apache.org>
Committed: Wed Aug 10 21:19:42 2016 +0000
----------------------------------------------------------------------
build-support/test_result_server.py | 42 ++++++++++++++++++++++----------
1 file changed, 29 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/b9c59cdc/build-support/test_result_server.py
----------------------------------------------------------------------
diff --git a/build-support/test_result_server.py b/build-support/test_result_server.py
index dde1839..e636a41 100755
--- a/build-support/test_result_server.py
+++ b/build-support/test_result_server.py
@@ -24,6 +24,7 @@
#
# MySQL config:
# MYSQLHOST - host running mysql
+# MYSQLPORT - port of mysql [optional]
# MYSQLUSER - username
# MYSQLPWD - password
# MYSQLDB - mysql database
@@ -59,6 +60,12 @@ from StringIO import StringIO
import threading
import uuid
+def percent_rate(num, denom):
+ if denom == 0:
+ return 0
+ return num/denom * 100
+
+
class TRServer(object):
def __init__(self):
self.thread_local = threading.local()
@@ -91,10 +98,11 @@ class TRServer(object):
return self.thread_local.db
host = os.environ["MYSQLHOST"]
+ port = int(os.environ.get("MYSQLPORT", "3306"))
user = os.environ["MYSQLUSER"]
pwd = os.environ["MYSQLPWD"]
db = os.environ["MYSQLDB"]
- self.thread_local.db = MySQLdb.connect(host, user, pwd, db)
+ self.thread_local.db = MySQLdb.connect(host, user, pwd, db, port=port)
self.thread_local.db.autocommit(True)
logging.info("Connected to MySQL at %s" % host)
return self.thread_local.db
@@ -135,8 +143,7 @@ class TRServer(object):
test_name varchar(100),
status int,
log_key char(40),
- INDEX (revision),
- INDEX (test_name),
+ INDEX (test_name, timestamp),
INDEX (timestamp)
);""")
@@ -286,30 +293,36 @@ class TRServer(object):
results.append(dict(test_name=test_name,
runs_7day=runs_7day,
failures_7day=failures_7day,
+ rate_7day=percent_rate(failures_7day, runs_7day),
runs_2day=runs_2day,
failures_2day=failures_2day,
+ rate_2day=percent_rate(failures_2day, runs_2day),
sparkline=",".join("%.2f" % p for p in sparkline)))
return Template("""
<h1>Flaky rate over last week</h1>
<table class="table" id="flaky-rate">
- <tr>
- <th>test</th>
- <th>failure rate (7-day)</th>
- <th>failure rate (2-day)</th>
- <th>trend</th>
- </tr>
+ <thead>
+ <tr>
+ <th data-order-sequence='["asc"]'>test</th>
+ <th data-order-sequence='["desc"]'>failure rate (7-day)</th>
+ <th data-order-sequence='["desc"]'>failure rate (2-day)</th>
+ <th data-orderable="false">trend</th>
+ </tr>
+ </thead>
{% for r in results %}
<tr>
<td><a href="/test_drilldown?test_name={{ r.test_name |urlencode }}">
{{ r.test_name |e }}
</a></td>
- <td>{{ r.failures_7day |e }} / {{ r.runs_7day }}
- ({{ "%.2f"|format(r.failures_7day / r.runs_7day * 100) }}%)
+ <td data-order="{{ r.rate_7day }}">
+ {{ r.failures_7day |e }} / {{ r.runs_7day }}
+ ({{ "%.2f"|format(r.rate_7day) }}%)
</td>
- <td>{{ r.failures_2day |e }} / {{ r.runs_2day }}
+ <td data-order="{{ r.rate_2day }}">
+ {{ r.failures_2day |e }} / {{ r.runs_2day }}
{% if r.runs_2day > 0 %}
- ({{ "%.2f"|format(r.failures_2day / r.runs_2day * 100) }}%)
+ ({{ "%.2f"|format(r.rate_2day) }}%)
{% endif %}
</td>
<td><span class="inlinesparkline">{{ r.sparkline |e }}</span></td>
@@ -325,6 +338,7 @@ class TRServer(object):
'tooltipFormatter': function(sparkline, options, fields) {
return String(7 - fields.x) + "d ago: " + fields.y + "%"; }
});
+ $('#flaky-rate').DataTable({ paging: false, searching: false, info: false });
});
</script>
""").render(results=results)
@@ -428,6 +442,7 @@ class TRServer(object):
<html>
<head><title>Kudu test results</title>
<link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/bootstrap/3.2.0/css/bootstrap.min.css" />
+ <link rel="stylesheet" type="text/css" href="//cdn.datatables.net/1.10.12/css/jquery.dataTables.css" />
<style>
.new-date { border-bottom: 2px solid #666; }
#flaky-rate tr :nth-child(1) { width: 70%; }
@@ -446,6 +461,7 @@ class TRServer(object):
<script src="//ajax.googleapis.com/ajax/libs/jquery/1.11.1/jquery.min.js"></script>
<script src="//maxcdn.bootstrapcdn.com/bootstrap/3.2.0/js/bootstrap.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery-sparklines/2.1.2/jquery.sparkline.min.js"></script>
+ <script src="//cdn.datatables.net/1.10.12/js/jquery.dataTables.js"></script>
<div class="container-fluid">
{{ body }}
</div>
[2/2] kudu git commit: WIP: Adding some missing 0.10.0 release notes
Posted by to...@apache.org.
WIP: Adding some missing 0.10.0 release notes
This isn't complete and is missing some information, but it should
give us a head start. Several TODOs need to be addressed before
this is complete.
Change-Id: Ibdc9fd57b05434874845ffa9c0ff905b5b8d0422
Reviewed-on: http://gerrit.cloudera.org:8080/3802
Reviewed-by: Misty Stanley-Jones <mi...@apache.org>
Tested-by: Todd Lipcon <to...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/76ec7e66
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/76ec7e66
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/76ec7e66
Branch: refs/heads/master
Commit: 76ec7e66e9227542a3986799599bd04bc7b51fb9
Parents: b9c59cd
Author: Todd Lipcon <to...@apache.org>
Authored: Mon Aug 8 17:11:59 2016 -0700
Committer: Todd Lipcon <to...@apache.org>
Committed: Wed Aug 10 21:21:50 2016 +0000
----------------------------------------------------------------------
docs/release_notes.adoc | 118 ++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 116 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/76ec7e66/docs/release_notes.adoc
----------------------------------------------------------------------
diff --git a/docs/release_notes.adoc b/docs/release_notes.adoc
index 310d273..ae0b5a2 100644
--- a/docs/release_notes.adoc
+++ b/docs/release_notes.adoc
@@ -54,16 +54,119 @@ applications that are difficult or impossible to implement on current-generation
Hadoop storage technologies.
[[rn_0.10.0]]
-=== Release notes specific to 1.0.0
+=== Release notes specific to 0.10.0
+
+Kudu 0.10.0 delivers TODO what it delivers and how compatible it is.
+
+See also +++<a href="https://issues.apache.org/jira/issues/?jql=project%20%3D%20KUDU%20AND%20status%20%3D%20Resolved
+%20AND%20fixVersion%20%3D%200.10.0">JIRAs resolved
+for Kudu 0.10.0</a>+++ and +++<a href="https://github.com/apache/kudu/compare/0.9.1...0.10.0">Git
+changes between 0.10.0 and 0.10.0</a>+++.
+
+To upgrade to Kudu 0.10.0, see link:installation.html#upgrade[Upgrade from 0.9.x to 0.10.0].
[[rn_0.10.0_incompatible_changes]]
-==== Incompatible changes
+==== Incompatible changes and deprecated APIs
+
+- link:http://gerrit.cloudera.org:8080/3737[Gerrit #3737] The Java client has been repackaged
+ under `org.apache.kudu` instead of `org.kududb`. Import statements for Kudu classes must
+ be modified in order to compile against 0.10.0. Wire compatibility is maintained.
+
+- link:https://gerrit.cloudera.org/#/c/3055/[Gerrit #3055] The Java client's
+ synchronous API methods now throw `KuduException` instead of `Exception`.
+ Existing code that catches `Exception` should still compile, but introspection of an
+ exception's message may be impacted. This change was made to allow thrown exceptions to be
+ queried more easily using `KuduException.getStatus` and calling one of `Status`'s methods.
+ For example, an operation that tries to delete a table that doesn't exist would return a
+ `Status` that returns true when queried on `isNotFound()`.
- The Java client's `KuduTable.getTabletsLocations` set of methods is now
deprecated. Additionally, they now take an exclusive end partition key instead
of an inclusive key. Applications are encouraged to use the scan tokens API
instead of these methods in the future.
+- The C++ API for specifying split points on range-partitioned tables has been improved
+ to make it easier for callers to properly manage the ownership of the provided rows.
+
+ The `TableCreator::split_rows` API took a `vector<const KuduPartialRow*>`, which
+ made it very difficult for the calling application to do proper error handling with
+ cleanup when setting the fields of the `KuduPartialRow`. This API has been now been
+ deprecated and replaced by a new method `TableCreator::add_range_split` which allows
+ easier use of smart pointers for safe memory management.
+
+- TODO(dan): should we list the change in buffer size performance due to
+ aa3d4473d229807fa8ee27c6ffb5b4b6fb2893a4 here? users may need to tweak something
+ to keep their existing performance
+
+- The "remote bootstrap" process used to copy a tablet replica from one host to
+ another has been renamed to "Tablet Copy". This resulted in the renaming of
+ several RPC metrics. Any users previously explicitly fetching or monitoring metrics
+ related to Remote Bootstrap should update their scripts to reflect the new names.
+
+[[rn_0.10.0_new_features]]
+==== New features
+
+- TODO(dan): add a release note for non-covering range partitions
+
+- TODO(adar): add something about multi-master support
+
+- TODO(mpercy): add something about reserved space configuration
+
+
+
+[[rn_0.10.0_improvements]]
+==== Improvements and optimizations
+
+- link:https://issues.apache.org/jira/browse/KUDU-1516[KUDU-1516] The `kudu-ksck` tool
+ has been improved and now detects problems such as when a tablet does not have
+ a majority of replicas on live tablet servers, or if those replicas aren\u2019t in a
+ good state. Users who currently depend on the tool to detect inconsistencies may now see
+ failures when before they wouldn't see any.
+
+- link:https://gerrit.cloudera.org:8080/3477[Gerrit #3477] The way operations are buffered in
+ the Java client has been reworked. Previously, the session's buffer size was set per tablet, meaning that a buffer
+ size of 1,000 for 10 tablets being written to allowed for 10,000 operations to be buffered at the
+ same time. With this change, all the tablets share one buffer, so users might need to set a
+ bigger buffer size in order to reach the same level of performance as before.
+
+- link:https://gerrit.cloudera.org/#/c/3674/[Gerrit #3674] Added LESS and GREATER options for
+ column predicates.
+
+- link:https://issues.apache.org/jira/browse/KUDU-1516[KUDU-1516] added support for passing
+ back basic per-scan metrics (e.g cache hit rate) from the server to the C++ client. See the
+ `KuduScanner::GetResourceMetrics()` API for detailed usage. This feature will be supported
+ in the Java client API in a future release.
+
+- link:https://issues.apache.org/jira/browse/KUDU-1446[KUDU-1446] improved the order in
+ which the tablet server evaluates predicates, so that predicates on smaller columns
+ are evaluated first. This may improve performance on queries which apply predicates
+ on multiple columns of different sizes.
+
+- link:https://issues.apache.org/jira/browse/KUDU-1398[KUDU-1398] improved the storage
+ efficiency of Kudu's internal primary key indexes. This optimization should decrease space
+ usage and improve random access performance, particularly for workloads with lengthy
+ primary keys.
+
+[[rn_0.10.0_fixed_issues]]
+==== Fixed Issues
+
+- link:https://gerrit.cloudera.org/#/c/3541/[Gerrit #3541] Fixed a problem in the Java client
+ whereby an RPC could be dropped when a connection to a tablet server or master was forcefully
+ closed on the server-side while RPCs to that server were in the process of being encoded.
+ The effect was that the RPC would not be sent, and users of the synchronous API would receive
+ a `TimeoutException`. Several other Java client bugs which could cause similar spurious timeouts
+ were also fixed in this release.
+
+- link:https://gerrit.cloudera.org/#/c/3724/[Gerrit #3724] Fixed a problem in the Java client
+ whereby an RPC could be dropped when a socket timeout was fired while that RPC was being sent to
+ a tablet server or master. This would manifest itself in the same way
+ link:https://gerrit.cloudera.org/#/c/3541/[Gerrit #3541].
+
+- link:https://issues.apache.org/jira/browse/KUDU-1538[KUDU-1538] fixed a bug in which recycled
+ block identifiers could cause the tablet server to lose data. Following this bug fix, block
+ identifiers will no longer be reused.
+
+[[rn_0.10.0_changes]]
==== Other noteworthy changes
- This is the first release of Apache Kudu as a top-level (non-incubating)
@@ -76,6 +179,17 @@ Hadoop storage technologies.
improve the performance of random-write workloads at the cost of an
incremental increase in disk space usage.
+- The C++ API now has a published Doxygen-based online documentation, which can be found at
+ TODO ADD LINK TO DOXYGEN DOC.
+
+- Kudu now
+ link:http://kudu.apache.org/2016/06/17/raft-consensus-single-node.html[
+ uses the Raft consensus algorithm even for unreplicated tables].
+ This change simplifies code and will also allow administrators to enable
+ replication on a previously-unreplicated table. This change is internal and
+ should not be visible to users.
+ TODO(mpercy): does this prevent downgrade? does everything upgrade OK on its own?
+
[[rn_0.9.1]]
=== Release notes specific to 0.9.1