[buildstream] 07/26: Add utils for file timestamp support

[tv...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

tvb pushed a commit to branch traveltissues/mr4
in repository https://gitbox.apache.org/repos/asf/buildstream.git

commit f735652bb7a3612d0b63b53bb14b62ee6bec4b0e
Author: Darius Makovsky <tr...@protonmail.com>
AuthorDate: Wed Jan 8 16:30:47 2020 +0000

    Add utils for file timestamp support
---
 requirements/cov-requirements.txt    | 14 +++----
 requirements/dev-requirements.txt    | 18 ++++-----
 requirements/requirements.in         |  1 +
 requirements/requirements.txt        |  7 ++--
 src/buildstream/utils.py             | 77 ++++++++++++++++++++++++++++++++++++
 tests/internals/utils_move_atomic.py | 23 ++++++++++-
 6 files changed, 120 insertions(+), 20 deletions(-)

diff --git a/requirements/cov-requirements.txt b/requirements/cov-requirements.txt
index c700aba..abf4936 100644
--- a/requirements/cov-requirements.txt
+++ b/requirements/cov-requirements.txt
@@ -3,13 +3,13 @@ pytest-cov==2.8.1
 Cython==0.29.14
 ## The following requirements were added by pip freeze:
 attrs==19.3.0
-importlib-metadata==1.1.0
-more-itertools==8.0.0
-packaging==19.2
+importlib-metadata==1.3.0
+more-itertools==8.0.2
+packaging==20.0
 pluggy==0.13.1
-py==1.8.0
-pyparsing==2.4.5
-pytest==5.3.1
+py==1.8.1
+pyparsing==2.4.6
+pytest==5.3.2
 six==1.13.0
-wcwidth==0.1.7
+wcwidth==0.1.8
 zipp==0.6.0
diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt
index 15e9b30..f188280 100644
--- a/requirements/dev-requirements.txt
+++ b/requirements/dev-requirements.txt
@@ -1,29 +1,29 @@
 pexpect==4.7.0
 pylint==2.4.4
-pytest==5.3.1
+pytest==5.3.2
 pytest-datafiles==2.0
 pytest-env==0.6.2
-pytest-xdist==1.30.0
-pytest-timeout==1.3.3
+pytest-xdist==1.31.0
+pytest-timeout==1.3.4
 pyftpdlib==1.5.5
 ## The following requirements were added by pip freeze:
 apipkg==1.5
 astroid==2.3.3
 attrs==19.3.0
 execnet==1.7.1
-importlib-metadata==1.1.0
+importlib-metadata==1.3.0
 isort==4.3.21
 lazy-object-proxy==1.4.3
 mccabe==0.6.1
-more-itertools==8.0.0
-packaging==19.2
+more-itertools==8.0.2
+packaging==20.0
 pluggy==0.13.1
 ptyprocess==0.6.0
-py==1.8.0
-pyparsing==2.4.5
+py==1.8.1
+pyparsing==2.4.6
 pytest-forked==1.1.3
 six==1.13.0
 typed-ast==1.4.0
-wcwidth==0.1.7
+wcwidth==0.1.8
 wrapt==1.11.2
 zipp==0.6.0
diff --git a/requirements/requirements.in b/requirements/requirements.in
index 50bb523..a595f69 100644
--- a/requirements/requirements.in
+++ b/requirements/requirements.in
@@ -9,3 +9,4 @@ ruamel.yaml.clib >= 0.1.2
 setuptools
 pyroaring
 ujson
+python-dateutil>= 2.7.0
diff --git a/requirements/requirements.txt b/requirements/requirements.txt
index 9620908..1bc75f2 100644
--- a/requirements/requirements.txt
+++ b/requirements/requirements.txt
@@ -1,14 +1,15 @@
 Click==7.0
-grpcio==1.25.0
+grpcio==1.26.0
 Jinja2==2.10.3
 pluginbase==1.0.0
-protobuf==3.11.0
+protobuf==3.11.2
 psutil==5.6.7
 ruamel.yaml==0.16.5
+ruamel.yaml.clib==0.2.0
 setuptools==39.0.1
 pyroaring==0.2.9
 ujson==1.35
+python-dateutil==2.8.1
 ## The following requirements were added by pip freeze:
 MarkupSafe==1.1.1
-ruamel.yaml.clib==0.2.0
 six==1.13.0
diff --git a/src/buildstream/utils.py b/src/buildstream/utils.py
index 545816e..c2a43d9 100644
--- a/src/buildstream/utils.py
+++ b/src/buildstream/utils.py
@@ -33,10 +33,12 @@ from stat import S_ISDIR
 import subprocess
 import tempfile
 import time
+import datetime
 import itertools
 from contextlib import contextmanager
 from pathlib import Path
 from typing import Callable, IO, Iterable, Iterator, Optional, Tuple, Union
+from dateutil import parser as dateutil_parser
 
 import psutil
 
@@ -132,6 +134,81 @@ class FileListResult:
         return ret
 
 
+def _make_timestamp(timepoint: float) -> str:
+    """Obtain the ISO 8601 timestamp represented by the time given in seconds.
+
+    Args:
+        timepoint (float): the time since the epoch in seconds
+
+    Returns:
+        (str): the timestamp specified by https://www.ietf.org/rfc/rfc3339.txt
+               with a UTC timezone code 'Z'.
+
+    """
+    assert isinstance(timepoint, float), "Time to render as timestamp must be a float: {}".format(str(timepoint))
+    try:
+        return datetime.datetime.utcfromtimestamp(timepoint).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+    except (OverflowError, TypeError):
+        raise UtilError("Failed to make UTC timestamp from {}".format(timepoint))
+
+
+def _get_file_mtimestamp(fullpath: str) -> str:
+    """Obtain the ISO 8601 timestamp represented by the mtime of the
+    file at the given path."""
+    assert isinstance(fullpath, str), "Path to file must be a string: {}".format(str(fullpath))
+    try:
+        mtime = os.path.getmtime(fullpath)
+    except OSError:
+        raise UtilError("Failed to get mtime of file at {}".format(fullpath))
+    return _make_timestamp(mtime)
+
+
+def _parse_timestamp(timestamp: str) -> float:
+    """Parse an ISO 8601 timestamp as specified in
+    https://www.ietf.org/rfc/rfc3339.txt. Only timestamps with the UTC code
+    'Z' or an offset are valid. For example: '2019-12-12T10:23:01.54Z' or
+    '2019-12-12T10:23:01.54+00:00'.
+
+    Args:
+        timestamp (str): the timestamp
+
+    Returns:
+        (float): The time in seconds since epoch represented by the
+            timestamp.
+
+    Raises:
+        UtilError: if extraction of seconds fails
+    """
+    assert isinstance(timestamp, str), "Timestamp to parse must be a string: {}".format(str(timestamp))
+    try:
+        errmsg = "Failed to parse given timestamp: " + timestamp
+        parsed_time = dateutil_parser.isoparse(timestamp)
+        if parsed_time.tzinfo:
+            return parsed_time.timestamp()
+        raise UtilError(errmsg)
+    except (ValueError, OverflowError, TypeError):
+        raise UtilError(errmsg)
+
+
+def _set_file_mtime(fullpath: str, seconds: Union[int, float]) -> None:
+    """Set the access and modification times of the file at the given path
+    to the given time. The time of the file will be set with nanosecond
+    resolution if supported.
+
+    Args:
+        fullpath (str): the string representing the path to the file
+        timestamp (int, float): the time in seconds since the UNIX epoch
+    """
+    assert isinstance(fullpath, str), "Path to file must be a string: {}".format(str(fullpath))
+    assert isinstance(seconds, (int, float)), "Mtime to set must be a float or integer: {}".format(str(seconds))
+    set_mtime = seconds * 10 ** 9
+    try:
+        os.utime(fullpath, times=None, ns=(int(set_mtime), int(set_mtime)))
+    except OSError:
+        errmsg = "Failed to set the times of the file at {} to {}".format(fullpath, str(seconds))
+        raise UtilError(errmsg)
+
+
 def list_relative_paths(directory: str) -> Iterator[str]:
     """A generator for walking directory relative paths
 
diff --git a/tests/internals/utils_move_atomic.py b/tests/internals/utils_move_atomic.py
index cda0208..dd417cb 100644
--- a/tests/internals/utils_move_atomic.py
+++ b/tests/internals/utils_move_atomic.py
@@ -3,7 +3,13 @@
 
 import pytest
 
-from buildstream.utils import move_atomic, DirectoryExistsError
+from buildstream.utils import (
+    move_atomic,
+    DirectoryExistsError,
+    _get_file_mtimestamp,
+    _set_file_mtime,
+    _parse_timestamp,
+)
 
 
 @pytest.fixture
@@ -89,3 +95,18 @@ def test_move_to_existing_non_empty_dir(src, tmp_path):
 
     with pytest.raises(DirectoryExistsError):
         move_atomic(src, dst)
+
+
+def test_move_to_empty_dir_set_mtime(src, tmp_path):
+    dst = tmp_path.joinpath("dst")
+    move_atomic(src, dst)
+    assert dst.joinpath("test").exists()
+    _dst = str(dst)
+    # set the mtime via stamp
+    timestamp1 = "2020-01-08T11:05:50.832123Z"
+    _set_file_mtime(_dst, _parse_timestamp(timestamp1))
+    assert timestamp1 == _get_file_mtimestamp(_dst)
+    # reset the mtime using an offset stamp
+    timestamp2 = "2010-02-12T12:05:50.832123+01:00"
+    _set_file_mtime(_dst, _parse_timestamp(timestamp2))
+    assert _get_file_mtimestamp(_dst) == "2010-02-12T11:05:50.832123Z"