You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2022/08/20 00:48:47 UTC
[spark] branch master updated: [SPARK-39310][PS] Change `requires_same_anchor` to `check_same_anchor`
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new c8508fa44c4 [SPARK-39310][PS] Change `requires_same_anchor` to `check_same_anchor`
c8508fa44c4 is described below
commit c8508fa44c45e40d84198187de81b0fc222be149
Author: Yikun Jiang <yi...@gmail.com>
AuthorDate: Sat Aug 20 09:48:27 2022 +0900
[SPARK-39310][PS] Change `requires_same_anchor` to `check_same_anchor`
### What changes were proposed in this pull request?
Change `requires_same_anchor` to `check_same_anchor` for `_update_internal_frame` func
### Why are the changes needed?
There were some discussion in https://github.com/apache/spark/pull/36353#discussion_r882216133 , this parameter is a flag for whether checking the same anchor.
Consider it's a parameter of internal use function, it can be renamed safely.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
CI passed
Closes #37585 from Yikun/SPARK-39310.
Authored-by: Yikun Jiang <yi...@gmail.com>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
python/pyspark/pandas/frame.py | 14 +++++++-------
python/pyspark/pandas/indexing.py | 10 +++++-----
python/pyspark/pandas/series.py | 6 +++---
python/pyspark/pandas/spark/accessors.py | 4 ++--
4 files changed, 17 insertions(+), 17 deletions(-)
diff --git a/python/pyspark/pandas/frame.py b/python/pyspark/pandas/frame.py
index 7ad349a6c37..df2b5fffa62 100644
--- a/python/pyspark/pandas/frame.py
+++ b/python/pyspark/pandas/frame.py
@@ -501,7 +501,7 @@ class DataFrame(Frame, Generic[T]):
def _update_internal_frame(
self,
internal: InternalFrame,
- requires_same_anchor: bool = True,
+ check_same_anchor: bool = True,
anchor_force_disconnect: bool = False,
) -> None:
"""
@@ -511,7 +511,7 @@ class DataFrame(Frame, Generic[T]):
`anchor_force_disconnect` flag is set to True, disconnect the original anchor and create
a new one.
- If `requires_same_anchor` is `False`, checking whether or not the same anchor is ignored
+ If `check_same_anchor` is `False`, checking whether or not the same anchor is ignored
and force to update the InternalFrame, e.g., replacing the internal with the resolved_copy,
updating the underlying Spark DataFrame which need to combine a different Spark DataFrame.
@@ -519,7 +519,7 @@ class DataFrame(Frame, Generic[T]):
----------
internal : InternalFrame
The new InternalFrame
- requires_same_anchor : bool
+ check_same_anchor : bool
Whether checking the same anchor
anchor_force_disconnect : bool
Force to disconnect the original anchor and create a new one
@@ -536,7 +536,7 @@ class DataFrame(Frame, Generic[T]):
psser = self._pssers[old_label]
renamed = old_label != new_label
- not_same_anchor = requires_same_anchor and not same_anchor(internal, psser)
+ not_same_anchor = check_same_anchor and not same_anchor(internal, psser)
if renamed or not_same_anchor or anchor_force_disconnect:
psdf: DataFrame = DataFrame(self._internal.select_column(old_label))
@@ -5676,7 +5676,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
inplace = validate_bool_kwarg(inplace, "inplace")
if inplace:
- self._update_internal_frame(psdf._internal, requires_same_anchor=False)
+ self._update_internal_frame(psdf._internal, check_same_anchor=False)
return None
else:
return psdf
@@ -8614,7 +8614,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
*HIDDEN_COLUMNS,
)
internal = self._internal.with_new_sdf(sdf, data_fields=data_fields)
- self._update_internal_frame(internal, requires_same_anchor=False)
+ self._update_internal_frame(internal, check_same_anchor=False)
# TODO: ddof should be implemented.
def cov(self, min_periods: Optional[int] = None) -> "DataFrame":
@@ -12185,7 +12185,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
if inplace:
# Here, the result is always a frame because the error is thrown during schema inference
# from pandas.
- self._update_internal_frame(result._internal, requires_same_anchor=False)
+ self._update_internal_frame(result._internal, check_same_anchor=False)
return None
elif should_return_series:
return first_series(result).rename(series_name)
diff --git a/python/pyspark/pandas/indexing.py b/python/pyspark/pandas/indexing.py
index 2cda404bdb8..c35fde56232 100644
--- a/python/pyspark/pandas/indexing.py
+++ b/python/pyspark/pandas/indexing.py
@@ -619,7 +619,7 @@ class LocIndexerLike(IndexerLike, metaclass=ABCMeta):
psser._psdf[
self._psdf_or_psser._psdf._internal.column_labels
]._internal.resolved_copy,
- requires_same_anchor=False,
+ check_same_anchor=False,
)
return
@@ -653,7 +653,7 @@ class LocIndexerLike(IndexerLike, metaclass=ABCMeta):
internal = self._internal.with_new_spark_column(
self._psdf_or_psser._column_label, scol # TODO: dtype?
)
- self._psdf_or_psser._psdf._update_internal_frame(internal, requires_same_anchor=False)
+ self._psdf_or_psser._psdf._update_internal_frame(internal, check_same_anchor=False)
else:
assert self._is_df
@@ -705,7 +705,7 @@ class LocIndexerLike(IndexerLike, metaclass=ABCMeta):
self._psdf_or_psser._update_internal_frame(
psdf[list(self._psdf_or_psser.columns)]._internal.resolved_copy,
- requires_same_anchor=False,
+ check_same_anchor=False,
)
return
@@ -772,7 +772,7 @@ class LocIndexerLike(IndexerLike, metaclass=ABCMeta):
internal = self._internal.with_new_columns(
new_data_spark_columns, column_labels=column_labels, data_fields=new_fields
)
- self._psdf_or_psser._update_internal_frame(internal, requires_same_anchor=False)
+ self._psdf_or_psser._update_internal_frame(internal, check_same_anchor=False)
class LocIndexer(LocIndexerLike):
@@ -1817,7 +1817,7 @@ class iLocIndexer(LocIndexerLike):
super().__setitem__(key, value)
# Update again with resolved_copy to drop extra columns.
self._psdf._update_internal_frame(
- self._psdf._internal.resolved_copy, requires_same_anchor=False
+ self._psdf._internal.resolved_copy, check_same_anchor=False
)
# Clean up implicitly cached properties to be able to reuse the indexer.
diff --git a/python/pyspark/pandas/series.py b/python/pyspark/pandas/series.py
index 12dbc4ce30e..e1b4ac3a3e3 100644
--- a/python/pyspark/pandas/series.py
+++ b/python/pyspark/pandas/series.py
@@ -2114,7 +2114,7 @@ class Series(Frame, IndexOpsMixin, Generic[T]):
inplace = validate_bool_kwarg(inplace, "inplace")
if inplace:
- self._psdf._update_internal_frame(psser._psdf._internal, requires_same_anchor=False)
+ self._psdf._update_internal_frame(psser._psdf._internal, check_same_anchor=False)
return None
else:
return psser.copy()
@@ -2418,7 +2418,7 @@ class Series(Frame, IndexOpsMixin, Generic[T]):
data_spark_columns=[scol.alias(self._internal.data_spark_column_names[0])],
data_fields=[self._internal.data_fields[0]],
)
- self._psdf._update_internal_frame(internal, requires_same_anchor=False)
+ self._psdf._update_internal_frame(internal, check_same_anchor=False)
return None
else:
return self._with_new_scol(
@@ -5129,7 +5129,7 @@ class Series(Frame, IndexOpsMixin, Generic[T]):
self._column_label, scol # TODO: dtype?
)
- self._psdf._update_internal_frame(internal.resolved_copy, requires_same_anchor=False)
+ self._psdf._update_internal_frame(internal.resolved_copy, check_same_anchor=False)
def where(self, cond: "Series", other: Any = np.nan) -> "Series":
"""
diff --git a/python/pyspark/pandas/spark/accessors.py b/python/pyspark/pandas/spark/accessors.py
index f13898b6004..c8e7f507c6f 100644
--- a/python/pyspark/pandas/spark/accessors.py
+++ b/python/pyspark/pandas/spark/accessors.py
@@ -555,7 +555,7 @@ class SparkFrameMethods:
from pyspark.pandas.frame import CachedDataFrame
self._psdf._update_internal_frame(
- self._psdf._internal.resolved_copy, requires_same_anchor=False
+ self._psdf._internal.resolved_copy, check_same_anchor=False
)
return CachedDataFrame(self._psdf._internal)
@@ -633,7 +633,7 @@ class SparkFrameMethods:
from pyspark.pandas.frame import CachedDataFrame
self._psdf._update_internal_frame(
- self._psdf._internal.resolved_copy, requires_same_anchor=False
+ self._psdf._internal.resolved_copy, check_same_anchor=False
)
return CachedDataFrame(self._psdf._internal, storage_level=storage_level)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org