From f698761c877b831c339491f88eb67121462869dd Mon Sep 17 00:00:00 2001 From: Peter Bull Date: Sun, 18 Feb 2024 10:44:19 -0800 Subject: [PATCH 1/5] Implement file scheme --- cloudpathlib/anypath.py | 4 ++++ docs/docs/anypath-polymorphism.md | 25 +++++++++++++++++++++++++ tests/test_anypath.py | 3 +++ 3 files changed, 32 insertions(+) diff --git a/cloudpathlib/anypath.py b/cloudpathlib/anypath.py index ac1980a4..7a0fd946 100644 --- a/cloudpathlib/anypath.py +++ b/cloudpathlib/anypath.py @@ -2,6 +2,7 @@ from abc import ABC from pathlib import Path from typing import Any, Union +from urllib.request import url2pathname from .cloudpath import InvalidPrefixError, CloudPath from .exceptions import AnyPathTypeError @@ -22,6 +23,9 @@ def __new__(cls, *args, **kwargs) -> Union[CloudPath, Path]: # type: ignore return CloudPath(*args, **kwargs) # type: ignore except InvalidPrefixError as cloudpath_exception: try: + if isinstance(args[0], str) and args[0].startswith("file:"): + return Path(url2pathname(args[0].removeprefix("file:")), *args[1:], **kwargs) + return Path(*args, **kwargs) except TypeError as path_exception: raise AnyPathTypeError( diff --git a/docs/docs/anypath-polymorphism.md b/docs/docs/anypath-polymorphism.md index ceab1eb6..0c4f602a 100644 --- a/docs/docs/anypath-polymorphism.md +++ b/docs/docs/anypath-polymorphism.md @@ -23,6 +23,31 @@ isinstance(cloud_path, AnyPath) #> True ``` +## `file:` URI Scheme + +`AnyPath` also supports the [`file:` URI scheme](https://en.wikipedia.org/wiki/File_URI_scheme) _for paths that can be referenced with pathlib_ and returns a `Path` instance for those paths. If you need to roundtrip back to a `file:` URI, you can use the `Path.as_uri` method after any path manipulations that you do. + +For example: + +```python +from cloudpathlib import AnyPath + +# hostname omitted variant +path = AnyPath("file:/root/mydir/myfile.txt") +path +#> PosixPath('/root/mydir/myfile.txt') + +# explicit local path variant +path = AnyPath("file:///root/mydir/myfile.txt") +path +#> PosixPath('/root/mydir/myfile.txt') + +# manipulate the path and return the file:// URI +parent_uri = path.parent.as_uri() +parent_uri +#> 'file:///root/mydir' +``` + ## How It Works The constructor for `AnyPath` will first attempt to run the input through the `CloudPath` base class' constructor, which will validate the input against registered concrete `CloudPath` implementations. This will accept inputs that are already a cloud path class or a string with the appropriate URI scheme prefix (e.g., `s3://`). If no implementation validates successfully, it will then try to run the input through the `Path` constructor. If the `Path` constructor fails and raises a `TypeError`, then the `AnyPath` constructor will raise an `AnyPathTypeError` exception. diff --git a/tests/test_anypath.py b/tests/test_anypath.py index 47cac832..46c5586c 100644 --- a/tests/test_anypath.py +++ b/tests/test_anypath.py @@ -20,6 +20,9 @@ def test_anypath_path(): assert issubclass(WindowsPath, AnyPath) assert not issubclass(str, AnyPath) + # test `file:` scheme + assert AnyPath(path.resolve().as_uri()) == path.resolve() + def test_anypath_cloudpath(rig): cloudpath = rig.create_cloud_path("a/b/c") From 1f4bbf4612ba2bb73f562fa958efd1fec8bed665 Mon Sep 17 00:00:00 2001 From: Peter Bull Date: Sun, 18 Feb 2024 10:47:18 -0800 Subject: [PATCH 2/5] changelog --- HISTORY.md | 1 + 1 file changed, 1 insertion(+) diff --git a/HISTORY.md b/HISTORY.md index 52878ca2..1d4b96a6 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -4,6 +4,7 @@ - Implement sliced downloads in GSClient. (Issue [#387](https://github.com/drivendataorg/cloudpathlib/issues/387), PR [#389](https://github.com/drivendataorg/cloudpathlib/pull/389)) - Implement `as_url` with presigned parameter for all backends. (Issue [#235](https://github.com/drivendataorg/cloudpathlib/issues/235), PR [#236](https://github.com/drivendataorg/cloudpathlib/pull/236)) - Stream to and from Azure Blob Storage. (PR [#403](https://github.com/drivendataorg/cloudpathlib/pull/403)) +- Implement `file:` URI scheme support for `AnyPath`. (Issue [#401](https://github.com/drivendataorg/cloudpathlib/issues/401), PR [#404](https://github.com/drivendataorg/cloudpathlib/pull/404)) ## 0.17.0 (2023-12-21) From 5a31b77f4d6e34e298cbcef81dfea20431b60a01 Mon Sep 17 00:00:00 2001 From: Peter Bull Date: Sun, 18 Feb 2024 10:56:31 -0800 Subject: [PATCH 3/5] living in the past --- cloudpathlib/anypath.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cloudpathlib/anypath.py b/cloudpathlib/anypath.py index 7a0fd946..85729d3c 100644 --- a/cloudpathlib/anypath.py +++ b/cloudpathlib/anypath.py @@ -24,7 +24,8 @@ def __new__(cls, *args, **kwargs) -> Union[CloudPath, Path]: # type: ignore except InvalidPrefixError as cloudpath_exception: try: if isinstance(args[0], str) and args[0].startswith("file:"): - return Path(url2pathname(args[0].removeprefix("file:")), *args[1:], **kwargs) + no_prefix = args[0][5:] # remove file: prefix + return Path(url2pathname(no_prefix), *args[1:], **kwargs) return Path(*args, **kwargs) except TypeError as path_exception: From 747ad3f20641150082ed639b2efd1847138d8adc Mon Sep 17 00:00:00 2001 From: PB Date: Sun, 18 Feb 2024 19:27:13 +0000 Subject: [PATCH 4/5] add absolute --- tests/test_anypath.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_anypath.py b/tests/test_anypath.py index 46c5586c..6650e26b 100644 --- a/tests/test_anypath.py +++ b/tests/test_anypath.py @@ -20,8 +20,8 @@ def test_anypath_path(): assert issubclass(WindowsPath, AnyPath) assert not issubclass(str, AnyPath) - # test `file:` scheme - assert AnyPath(path.resolve().as_uri()) == path.resolve() + # test `file:` scheme (only works with absolute paths; needs .absolute() on Windows) + assert AnyPath(path.absolute().resolve().as_uri()) == path.absolute().resolve() def test_anypath_cloudpath(rig): From 3ebacd1447d69932a772ad01465d117bf3ee167e Mon Sep 17 00:00:00 2001 From: PB Date: Tue, 20 Feb 2024 14:24:46 +0000 Subject: [PATCH 5/5] windows support and tests --- cloudpathlib/anypath.py | 10 ++++++---- cloudpathlib/url_utils.py | 31 +++++++++++++++++++++++++++++++ tests/test_anypath.py | 21 +++++++++++++++++++++ 3 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 cloudpathlib/url_utils.py diff --git a/cloudpathlib/anypath.py b/cloudpathlib/anypath.py index 85729d3c..dbab9db9 100644 --- a/cloudpathlib/anypath.py +++ b/cloudpathlib/anypath.py @@ -2,10 +2,10 @@ from abc import ABC from pathlib import Path from typing import Any, Union -from urllib.request import url2pathname from .cloudpath import InvalidPrefixError, CloudPath from .exceptions import AnyPathTypeError +from .url_utils import path_from_fileurl class AnyPath(ABC): @@ -23,9 +23,11 @@ def __new__(cls, *args, **kwargs) -> Union[CloudPath, Path]: # type: ignore return CloudPath(*args, **kwargs) # type: ignore except InvalidPrefixError as cloudpath_exception: try: - if isinstance(args[0], str) and args[0].startswith("file:"): - no_prefix = args[0][5:] # remove file: prefix - return Path(url2pathname(no_prefix), *args[1:], **kwargs) + if isinstance(args[0], str) and args[0].lower().startswith("file:"): + path = path_from_fileurl(args[0], **kwargs) + for part in args[1:]: + path /= part + return path return Path(*args, **kwargs) except TypeError as path_exception: diff --git a/cloudpathlib/url_utils.py b/cloudpathlib/url_utils.py new file mode 100644 index 00000000..1b3f1959 --- /dev/null +++ b/cloudpathlib/url_utils.py @@ -0,0 +1,31 @@ +from pathlib import PureWindowsPath, Path +from urllib.request import url2pathname +from urllib.parse import urlparse, unquote + + +def path_from_fileurl(urlstr, **kwargs): + """ + Take a file:// url and return a Path. + + Adapted from: + https://github.com/AcademySoftwareFoundation/OpenTimelineIO/blob/4c17494dee2e515aedc8623741556fae3e4afe72/src/py-opentimelineio/opentimelineio/url_utils.py#L43-L72 + """ + # explicitly unquote first in case drive colon is url encoded + unquoted = unquote(urlstr) + + # Parse provided URL + parsed_result = urlparse(unquoted) + + # Convert the parsed URL to a path + filepath = Path(url2pathname(parsed_result.path), **kwargs) + + # If the network location is a window drive, reassemble the path + if PureWindowsPath(parsed_result.netloc).drive: + filepath = Path(parsed_result.netloc + parsed_result.path, **kwargs) + + # Otherwise check if the specified index is a windows drive, then offset the path + elif len(filepath.parts) > 1 and PureWindowsPath(filepath.parts[1]).drive: + # Remove leading "/" if/when `request.url2pathname` yields "/S:/path/file.ext" + filepath = Path(*filepath.parts[1:], **kwargs) + + return filepath diff --git a/tests/test_anypath.py b/tests/test_anypath.py index 6650e26b..04fc5070 100644 --- a/tests/test_anypath.py +++ b/tests/test_anypath.py @@ -1,3 +1,4 @@ +import os from pathlib import Path, PosixPath, WindowsPath import pytest @@ -23,6 +24,26 @@ def test_anypath_path(): # test `file:` scheme (only works with absolute paths; needs .absolute() on Windows) assert AnyPath(path.absolute().resolve().as_uri()) == path.absolute().resolve() + # test file:// + multi arg + assert AnyPath(*path.absolute().resolve().as_uri().rsplit("/", 2)) == path.absolute().resolve() + + # test no hostname + assert Path("/foo/bar") == AnyPath("file:/foo/bar") + assert Path("/foo/bar") == AnyPath("file:///foo/bar") + + # windows tests + if os.name == "nt": + assert Path("c:\\hello\\test.txt") == AnyPath("file:/c:/hello/test.txt") + assert Path("c:\\hello\\test.txt") == AnyPath("file://c:/hello/test.txt") + assert Path("c:\\hello\\test.txt") == AnyPath("file:///c:/hello/test.txt") + assert Path("c:\\hello\\test.txt") == AnyPath("file://c%3A//hello/test.txt") + assert Path("c:\\hello\\test.txt") == AnyPath("file://localhost/c%3a/hello/test.txt") + assert Path("c:\\WINDOWS\\clock.avi") == AnyPath("file://localhost/c|/WINDOWS/clock.avi") + assert Path("c:\\WINDOWS\\clock.avi") == AnyPath("file:///c|/WINDOWS/clock.avi") + assert Path("c:\\hello\\test space.txt") == AnyPath( + "file://localhost/c%3a/hello/test%20space.txt" + ) + def test_anypath_cloudpath(rig): cloudpath = rig.create_cloud_path("a/b/c")