You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by fo...@apache.org on 2018/06/03 13:03:12 UTC
incubator-airflow git commit: [AIRFLOW-2525] Fix a bug introduced by
commit dabf1b9
Repository: incubator-airflow
Updated Branches:
refs/heads/master 2ed618883 -> e5fb9c799
[AIRFLOW-2525] Fix a bug introduced by commit dabf1b9
The previous commit on this issue (#3421)
introduced a new bug on COPY FROM operation.
This PR fixes it by opening a file with 'r+'
mode instead of 'w+' to avoid truncating it.
Closes #3423 from sekikn/AIRFLOW-2525-2
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/e5fb9c79
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/e5fb9c79
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/e5fb9c79
Branch: refs/heads/master
Commit: e5fb9c7998e67be40cb285971fb941ed4e81273a
Parents: 2ed6188
Author: Kengo Seki <se...@apache.org>
Authored: Sun Jun 3 15:03:05 2018 +0200
Committer: Fokko Driesprong <fo...@godatadriven.com>
Committed: Sun Jun 3 15:03:05 2018 +0200
----------------------------------------------------------------------
airflow/hooks/postgres_hook.py | 18 +++++++++++++++---
tests/hooks/test_postgres_hook.py | 2 +-
2 files changed, 16 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/e5fb9c79/airflow/hooks/postgres_hook.py
----------------------------------------------------------------------
diff --git a/airflow/hooks/postgres_hook.py b/airflow/hooks/postgres_hook.py
index bbf125b..0395e70 100644
--- a/airflow/hooks/postgres_hook.py
+++ b/airflow/hooks/postgres_hook.py
@@ -17,6 +17,7 @@
# specific language governing permissions and limitations
# under the License.
+import os
import psycopg2
import psycopg2.extensions
from contextlib import closing
@@ -61,13 +62,24 @@ class PostgresHook(DbApiHook):
def copy_expert(self, sql, filename, open=open):
"""
- Executes SQL using psycopg2 copy_expert method
- Necessary to execute COPY command without access to a superuser
+ Executes SQL using psycopg2 copy_expert method.
+ Necessary to execute COPY command without access to a superuser.
+
+ Note: if this method is called with a "COPY FROM" statement and
+ the specified input file does not exist, it creates an empty
+ file and no data is loaded, but the operation succeeds.
+ So if users want to be aware when the input file does not exist,
+ they have to check its existence by themselves.
"""
- with open(filename, 'w+') as f:
+ if not os.path.isfile(filename):
+ with open(filename, 'w'):
+ pass
+
+ with open(filename, 'r+') as f:
with closing(self.get_conn()) as conn:
with closing(conn.cursor()) as cur:
cur.copy_expert(sql, f)
+ f.truncate(f.tell())
conn.commit()
@staticmethod
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/e5fb9c79/tests/hooks/test_postgres_hook.py
----------------------------------------------------------------------
diff --git a/tests/hooks/test_postgres_hook.py b/tests/hooks/test_postgres_hook.py
index f636b5a..2828f8b 100644
--- a/tests/hooks/test_postgres_hook.py
+++ b/tests/hooks/test_postgres_hook.py
@@ -55,4 +55,4 @@ class TestPostgresHook(unittest.TestCase):
self.cur.close.assert_called_once()
self.conn.commit.assert_called_once()
self.cur.copy_expert.assert_called_once_with(statement, m.return_value)
- m.assert_called_once_with(filename, "w+")
+ self.assertEqual(m.call_args[0], (filename, "r+"))