You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by bo...@apache.org on 2017/01/13 20:15:21 UTC
incubator-airflow git commit: [AIRFLOW-737] Fix HDFS Sensor directory.
Repository: incubator-airflow
Updated Branches:
refs/heads/master 4a5e17b37 -> 8fd65f8ac
[AIRFLOW-737] Fix HDFS Sensor directory.
Due to a bad ordering in the fake snakebite
client, one test was wrongly True.
Closes #1989 from
vfoucault/fixbug/hdfssensor_folder
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/8fd65f8a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/8fd65f8a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/8fd65f8a
Branch: refs/heads/master
Commit: 8fd65f8ac4a305da01324c5120fbfab8b51f5d46
Parents: 4a5e17b
Author: Vianney Foucault <vi...@gmail.com>
Authored: Fri Jan 13 21:15:14 2017 +0100
Committer: Bolke de Bruin <bo...@xs4all.nl>
Committed: Fri Jan 13 21:15:14 2017 +0100
----------------------------------------------------------------------
airflow/contrib/sensors/hdfs_sensors.py | 1 +
tests/core.py | 8 ++++----
2 files changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/8fd65f8a/airflow/contrib/sensors/hdfs_sensors.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/sensors/hdfs_sensors.py b/airflow/contrib/sensors/hdfs_sensors.py
index a12e295..4e9bb9b 100644
--- a/airflow/contrib/sensors/hdfs_sensors.py
+++ b/airflow/contrib/sensors/hdfs_sensors.py
@@ -62,6 +62,7 @@ class HdfsSensorFolder(HdfsSensor):
return len(result) == 1 and result[0]['path'] == self.filepath
else:
logging.info('Poking for filepath {self.filepath} to a non empty directory'.format(**locals()))
+ result.pop(0)
return bool(result) and result[0]['file_type'] == 'f'
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/8fd65f8a/tests/core.py
----------------------------------------------------------------------
diff --git a/tests/core.py b/tests/core.py
index 6fb735f..0f7e41d 100644
--- a/tests/core.py
+++ b/tests/core.py
@@ -1840,10 +1840,6 @@ class FakeSnakeBiteClient(object):
{'group': u'supergroup', 'permission': 493, 'file_type': 'd', 'access_time': 0, 'block_replication': 0,
'modification_time': 1481132141540, 'length': 0, 'blocksize': 0, 'owner': u'hdfs',
'path': '/datadirectory/empty_directory'}]
- elif path[0] == '/datadirectory/not_empty_directory':
- return [{'group': u'supergroup', 'permission': 420, 'file_type': 'f', 'access_time': 1481122343796,
- 'block_replication': 3, 'modification_time': 1481122343862, 'length': 0, 'blocksize': 134217728,
- 'owner': u'hdfs', 'path': '/datadirectory/not_empty_directory/test_file'}]
elif path[0] == '/datadirectory/not_empty_directory' and include_toplevel:
return [
{'group': u'supergroup', 'permission': 493, 'file_type': 'd', 'access_time': 0, 'block_replication': 0,
@@ -1852,6 +1848,10 @@ class FakeSnakeBiteClient(object):
{'group': u'supergroup', 'permission': 420, 'file_type': 'f', 'access_time': 1481122343796,
'block_replication': 3, 'modification_time': 1481122343862, 'length': 0, 'blocksize': 134217728,
'owner': u'hdfs', 'path': '/datadirectory/not_empty_directory/test_file'}]
+ elif path[0] == '/datadirectory/not_empty_directory':
+ return [{'group': u'supergroup', 'permission': 420, 'file_type': 'f', 'access_time': 1481122343796,
+ 'block_replication': 3, 'modification_time': 1481122343862, 'length': 0, 'blocksize': 134217728,
+ 'owner': u'hdfs', 'path': '/datadirectory/not_empty_directory/test_file'}]
elif path[0] == '/datadirectory/not_existing_file_or_directory':
raise FakeSnakeBiteClientException
elif path[0] == '/datadirectory/regex_dir':