You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by bh...@apache.org on 2022/05/26 16:09:29 UTC
[beam] branch master updated: [BEAM-14336] Re-enable `flight_delays_it_test` with `apache-beam-testing` dataset (#17758)
This is an automated email from the ASF dual-hosted git repository.
bhulette pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new f10d5781aa9 [BEAM-14336] Re-enable `flight_delays_it_test` with `apache-beam-testing` dataset (#17758)
f10d5781aa9 is described below
commit f10d5781aa962bda2f3f7ef013eba873d7f16870
Author: Brian Hulette <bh...@google.com>
AuthorDate: Thu May 26 09:09:21 2022 -0700
[BEAM-14336] Re-enable `flight_delays_it_test` with `apache-beam-testing` dataset (#17758)
* Sort expected data by index
* Update flight_delays pipeline to use apache-beam-testing dataset
---
.../examples/dataframe/flight_delays.py | 21 ++++----
.../examples/dataframe/flight_delays_it_test.py | 63 +++++++++++-----------
2 files changed, 42 insertions(+), 42 deletions(-)
diff --git a/sdks/python/apache_beam/examples/dataframe/flight_delays.py b/sdks/python/apache_beam/examples/dataframe/flight_delays.py
index a75e6387814..c2c467b39ea 100644
--- a/sdks/python/apache_beam/examples/dataframe/flight_delays.py
+++ b/sdks/python/apache_beam/examples/dataframe/flight_delays.py
@@ -60,23 +60,24 @@ def run_flight_delay_pipeline(
pipeline, start_date=None, end_date=None, output=None):
query = f"""
SELECT
- date,
- airline,
- departure_airport,
- arrival_airport,
- departure_delay,
- arrival_delay
- FROM `bigquery-samples.airline_ontime_data.flights`
- WHERE date >= '{start_date}' AND date <= '{end_date}'
+ FlightDate AS date,
+ IATA_CODE_Reporting_Airline AS airline,
+ Origin AS departure_airport,
+ Dest AS arrival_airport,
+ DepDelay AS departure_delay,
+ ArrDelay AS arrival_delay
+ FROM `apache-beam-testing.airline_ontime_data.flights`
+ WHERE
+ FlightDate >= '{start_date}' AND FlightDate <= '{end_date}' AND
+ DepDelay IS NOT NULL AND ArrDelay IS NOT NULL
"""
# Import this here to avoid pickling the main session.
import time
- import datetime
from apache_beam import window
def to_unixtime(s):
- return time.mktime(datetime.datetime.strptime(s, "%Y-%m-%d").timetuple())
+ return time.mktime(s.timetuple())
# The pipeline will be run on exiting the with block.
with pipeline as p:
diff --git a/sdks/python/apache_beam/examples/dataframe/flight_delays_it_test.py b/sdks/python/apache_beam/examples/dataframe/flight_delays_it_test.py
index d1e7a42c5a3..71e09716487 100644
--- a/sdks/python/apache_beam/examples/dataframe/flight_delays_it_test.py
+++ b/sdks/python/apache_beam/examples/dataframe/flight_delays_it_test.py
@@ -35,59 +35,58 @@ from apache_beam.io.filesystems import FileSystems
from apache_beam.testing.test_pipeline import TestPipeline
-@unittest.skip('BEAM-14336')
class FlightDelaysTest(unittest.TestCase):
EXPECTED = {
'2012-12-23': [
('AA', 20.082559339525282, 12.825593395252838),
- ('EV', 10.01901901901902, 4.431431431431432),
+ ('AS', 5.0456273764258555, 1.0722433460076046),
+ ('B6', 20.646569646569645, 16.405405405405407),
+ ('DL', 5.241148325358852, -3.2401913875598085),
+ ('EV', 9.982053838484546, 4.40777666999003),
+ ('F9', 23.67883211678832, 25.27007299270073),
+ ('FL', 4.4602272727272725, -0.8352272727272727),
('HA', -1.0829015544041452, 0.010362694300518135),
+ ('MQ', 8.912912912912914, 3.6936936936936937),
+ ('OO', 30.526699029126213, 31.17961165048544),
('UA', 19.142555438225976, 11.07180570221753),
- ('MQ', 8.902255639097744, 3.6676691729323307),
- ('OO', 31.148883374689827, 31.90818858560794),
('US', 3.092541436464088, -2.350828729281768),
- ('WN', 12.074298711144806, 6.717968157695224),
- ('AS', 5.0456273764258555, 1.0722433460076046),
- ('B6', 20.646569646569645, 16.405405405405407),
- ('DL', 5.2559923298178335, -3.214765100671141),
- ('F9', 23.823529411764707, 25.455882352941178),
- ('FL', 4.492877492877493, -0.8005698005698005),
('VX', 62.755102040816325, 62.61224489795919),
+ ('WN', 12.05824508320726, 6.713313161875946),
('YV', 16.155844155844157, 13.376623376623376),
],
'2012-12-24': [
+ ('AA', 7.049086757990867, -1.5970319634703196),
('AS', 0.5917602996254682, -2.2659176029962547),
('B6', 8.070993914807302, 2.73630831643002),
- ('DL', 3.7171824973319105, -2.2358591248665953),
- ('F9', 14.111940298507463, 15.888059701492537),
- ('FL', 2.4210526315789473, 2.242690058479532),
- ('VX', 3.841666666666667, -2.4166666666666665),
- ('YV', 0.32, 0.78),
- ('MQ', 15.869642857142857, 9.992857142857142),
- ('OO', 11.048517520215633, 10.138814016172507),
- ('US', 1.369281045751634, -1.4101307189542485),
- ('WN', 7.515952597994531, 0.7028258887876025),
- ('AA', 7.049086757990867, -1.5970319634703196),
- ('EV', 7.297101449275362, 2.2693236714975846),
+ ('DL', 3.700745473908413, -2.2396166134185305),
+ ('EV', 7.322115384615385, 2.3653846153846154),
+ ('F9', 13.786764705882351, 15.5),
+ ('FL', 2.416909620991253, 2.224489795918368),
('HA', -2.6785714285714284, -2.4744897959183674),
+ ('MQ', 15.818181818181818, 9.935828877005347),
+ ('OO', 10.902374670184695, 10.08575197889182),
('UA', 10.935406698564593, -1.3337320574162679),
+ ('US', 1.369281045751634, -1.4101307189542485),
+ ('VX', 3.841666666666667, -2.4166666666666665),
+ ('WN', 7.3715753424657535, 0.348458904109589),
+ ('YV', 0.32, 0.78),
],
'2012-12-25': [
+ ('AA', 23.551581843191197, 35.62585969738652),
('AS', 3.4816326530612245, 0.27346938775510204),
('B6', 9.10590631364562, 3.989816700610998),
- ('DL', 2.3022170361726952, -3.6709451575262544),
- ('F9', 19.38255033557047, 21.845637583892618),
- ('FL', 1.3982300884955752, 0.9380530973451328),
- ('VX', 23.62878787878788, 23.636363636363637),
- ('YV', 11.256302521008404, 11.659663865546218),
- ('MQ', 32.6, 44.28666666666667),
- ('OO', 16.2275960170697, 17.11948790896159),
- ('US', 2.7953216374269005, 0.2236842105263158),
- ('WN', 14.405783582089553, 10.111940298507463),
- ('AA', 23.551581843191197, 35.62585969738652),
- ('EV', 17.368638239339752, 16.43191196698762),
+ ('DL', 2.2863795110593714, -3.668218859138533),
+ ('EV', 17.35576923076923, 16.414835164835164),
+ ('F9', 19.38, 21.786666666666665),
+ ('FL', 1.3823529411764706, 0.9205882352941176),
('HA', -4.725806451612903, -3.9946236559139785),
+ ('MQ', 32.527716186252775, 44.148558758314856),
+ ('OO', 15.788595271210012, 16.617524339360223),
('UA', 16.663145539906104, 10.772300469483568),
+ ('US', 2.7953216374269005, 0.2236842105263158),
+ ('VX', 23.62878787878788, 23.636363636363637),
+ ('WN', 14.423791821561338, 10.142193308550183),
+ ('YV', 11.256302521008404, 11.659663865546218),
],
}