You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@beam.apache.org by "ASF GitHub Bot (Jira)" <ji...@apache.org> on 2020/08/03 17:51:00 UTC
[jira] [Work logged] (BEAM-10619) Report ratio of implemented
pandas tests
[ https://issues.apache.org/jira/browse/BEAM-10619?focusedWorklogId=465843&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-465843 ]
ASF GitHub Bot logged work on BEAM-10619:
-----------------------------------------
Author: ASF GitHub Bot
Created on: 03/Aug/20 17:50
Start Date: 03/Aug/20 17:50
Worklog Time Spent: 10m
Work Description: robertwb commented on a change in pull request #12440:
URL: https://github.com/apache/beam/pull/12440#discussion_r464566521
##########
File path: sdks/python/apache_beam/dataframe/doctests.py
##########
@@ -290,34 +298,60 @@ def to_callable(cond):
super(BeamDataframeDoctestRunner, self).__init__(
checker=_DeferrredDataframeOutputChecker(self._test_env, use_beam),
**kwargs)
+ self.skipped = 0
+ self.wont_implement = 0
def run(self, test, **kwargs):
self._checker.reset()
- if test.name in self._skip:
- for example in test.examples:
- if any(should_skip(example) for should_skip in self._skip[test.name]):
- example.source = 'pass'
- example.want = ''
for example in test.examples:
- if example.exc_msg is None:
+ if any(should_skip(example)
+ for should_skip in self._skip.get(test.name, [])):
+ example.source = 'pass'
+ example.want = ''
+ self.skipped += 1
+ elif example.exc_msg is None and any(
+ wont_implement(example)
+ for wont_implement in self._wont_implement.get(test.name, [])):
# Don't fail doctests that raise this error.
example.exc_msg = (
'apache_beam.dataframe.frame_base.WontImplementError: ...')
+ self.wont_implement += 1
with self._test_env.context():
- return super(BeamDataframeDoctestRunner, self).run(test, **kwargs)
+ result = super(BeamDataframeDoctestRunner, self).run(test, **kwargs)
+ return result
def fake_pandas_module(self):
return self._test_env.fake_pandas_module()
+ def summarize(self):
+ super(BeamDataframeDoctestRunner, self).summarize()
+ if self.failures:
+ return
Review comment:
Worth printing the stats regardless of failure?
##########
File path: sdks/python/apache_beam/dataframe/doctests.py
##########
@@ -290,34 +298,60 @@ def to_callable(cond):
super(BeamDataframeDoctestRunner, self).__init__(
checker=_DeferrredDataframeOutputChecker(self._test_env, use_beam),
**kwargs)
+ self.skipped = 0
+ self.wont_implement = 0
def run(self, test, **kwargs):
self._checker.reset()
- if test.name in self._skip:
- for example in test.examples:
- if any(should_skip(example) for should_skip in self._skip[test.name]):
- example.source = 'pass'
- example.want = ''
for example in test.examples:
- if example.exc_msg is None:
+ if any(should_skip(example)
+ for should_skip in self._skip.get(test.name, [])):
+ example.source = 'pass'
+ example.want = ''
+ self.skipped += 1
+ elif example.exc_msg is None and any(
+ wont_implement(example)
+ for wont_implement in self._wont_implement.get(test.name, [])):
# Don't fail doctests that raise this error.
example.exc_msg = (
'apache_beam.dataframe.frame_base.WontImplementError: ...')
+ self.wont_implement += 1
Review comment:
It'd be nice to track how many actually triggered this error (and perhaps track the reasons why).
##########
File path: sdks/python/apache_beam/dataframe/doctests.py
##########
@@ -290,34 +298,60 @@ def to_callable(cond):
super(BeamDataframeDoctestRunner, self).__init__(
checker=_DeferrredDataframeOutputChecker(self._test_env, use_beam),
**kwargs)
+ self.skipped = 0
+ self.wont_implement = 0
def run(self, test, **kwargs):
self._checker.reset()
- if test.name in self._skip:
- for example in test.examples:
- if any(should_skip(example) for should_skip in self._skip[test.name]):
- example.source = 'pass'
- example.want = ''
for example in test.examples:
- if example.exc_msg is None:
+ if any(should_skip(example)
+ for should_skip in self._skip.get(test.name, [])):
+ example.source = 'pass'
+ example.want = ''
+ self.skipped += 1
+ elif example.exc_msg is None and any(
+ wont_implement(example)
+ for wont_implement in self._wont_implement.get(test.name, [])):
# Don't fail doctests that raise this error.
example.exc_msg = (
'apache_beam.dataframe.frame_base.WontImplementError: ...')
+ self.wont_implement += 1
with self._test_env.context():
- return super(BeamDataframeDoctestRunner, self).run(test, **kwargs)
+ result = super(BeamDataframeDoctestRunner, self).run(test, **kwargs)
+ return result
def fake_pandas_module(self):
return self._test_env.fake_pandas_module()
+ def summarize(self):
+ super(BeamDataframeDoctestRunner, self).summarize()
+ if self.failures:
+ return
+ total_test_cases = self.skipped + self.tries
+ will_implement = total_test_cases - self.wont_implement
+ print("%d total test cases." % total_test_cases)
+ print(
+ "%d will implement, %d won't implement." %
Review comment:
Just because they're not won't implement, doesn't mean we will (e.g .they could be skipped, or they're already implemented in which case the future tense is odd). Maybe break this down as
```
Xxx total test cases.
Xxx skipped
Xxx won't implement
Yyy reason A
Yyy reason B
...
Xxx passed
```
##########
File path: sdks/python/apache_beam/dataframe/doctests.py
##########
@@ -290,34 +298,60 @@ def to_callable(cond):
super(BeamDataframeDoctestRunner, self).__init__(
checker=_DeferrredDataframeOutputChecker(self._test_env, use_beam),
**kwargs)
+ self.skipped = 0
+ self.wont_implement = 0
def run(self, test, **kwargs):
self._checker.reset()
- if test.name in self._skip:
- for example in test.examples:
- if any(should_skip(example) for should_skip in self._skip[test.name]):
- example.source = 'pass'
- example.want = ''
for example in test.examples:
- if example.exc_msg is None:
+ if any(should_skip(example)
+ for should_skip in self._skip.get(test.name, [])):
+ example.source = 'pass'
+ example.want = ''
+ self.skipped += 1
+ elif example.exc_msg is None and any(
+ wont_implement(example)
+ for wont_implement in self._wont_implement.get(test.name, [])):
# Don't fail doctests that raise this error.
example.exc_msg = (
'apache_beam.dataframe.frame_base.WontImplementError: ...')
+ self.wont_implement += 1
with self._test_env.context():
- return super(BeamDataframeDoctestRunner, self).run(test, **kwargs)
+ result = super(BeamDataframeDoctestRunner, self).run(test, **kwargs)
+ return result
def fake_pandas_module(self):
return self._test_env.fake_pandas_module()
+ def summarize(self):
+ super(BeamDataframeDoctestRunner, self).summarize()
+ if self.failures:
+ return
+ total_test_cases = self.skipped + self.tries
Review comment:
The way this is implemented, self.tries includes the skipped ones (which get mutated to pass). I suppose we could instead remove them from the examples list altogether.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Issue Time Tracking
-------------------
Worklog Id: (was: 465843)
Time Spent: 40m (was: 0.5h)
> Report ratio of implemented pandas tests
> -----------------------------------------
>
> Key: BEAM-10619
> URL: https://issues.apache.org/jira/browse/BEAM-10619
> Project: Beam
> Issue Type: Improvement
> Components: sdk-py-core
> Reporter: Brian Hulette
> Assignee: Brian Hulette
> Priority: P2
> Time Spent: 40m
> Remaining Estimate: 0h
>
> Measure and report (#implemented + #wont implement) / total # tests
--
This message was sent by Atlassian Jira
(v8.3.4#803005)