You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2015/04/09 03:47:41 UTC
spark git commit: [SPARK-6696] [SQL] Adds HiveContext.refreshTable to
PySpark
Repository: spark
Updated Branches:
refs/heads/master 7d7384c78 -> 891ada5be
[SPARK-6696] [SQL] Adds HiveContext.refreshTable to PySpark
<!-- Reviewable:start -->
[<img src="https://reviewable.io/review_button.png" height=40 alt="Review on Reviewable"/>](https://reviewable.io/reviews/apache/spark/5349)
<!-- Reviewable:end -->
Author: Cheng Lian <li...@databricks.com>
Closes #5349 from liancheng/py-refresh-table and squashes the following commits:
004bec0 [Cheng Lian] Adds HiveContext.refreshTable to PySpark
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/891ada5b
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/891ada5b
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/891ada5b
Branch: refs/heads/master
Commit: 891ada5be1e7fdd796380e2626d80843f2ef6017
Parents: 7d7384c
Author: Cheng Lian <li...@databricks.com>
Authored: Wed Apr 8 18:47:39 2015 -0700
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Wed Apr 8 18:47:39 2015 -0700
----------------------------------------------------------------------
python/pyspark/sql/context.py | 9 +++++++++
1 file changed, 9 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/891ada5b/python/pyspark/sql/context.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/context.py b/python/pyspark/sql/context.py
index 93e2d17..e8529a8 100644
--- a/python/pyspark/sql/context.py
+++ b/python/pyspark/sql/context.py
@@ -574,6 +574,15 @@ class HiveContext(SQLContext):
def _get_hive_ctx(self):
return self._jvm.HiveContext(self._jsc.sc())
+ def refreshTable(self, tableName):
+ """Invalidate and refresh all the cached the metadata of the given
+ table. For performance reasons, Spark SQL or the external data source
+ library it uses might cache certain metadata about a table, such as the
+ location of blocks. When those change outside of Spark SQL, users should
+ call this function to invalidate the cache.
+ """
+ self._ssql_ctx.refreshTable(tableName)
+
class UDFRegistration(object):
"""Wrapper for user-defined function registration."""
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org