myenv/Lib/site-packages/fsspec/implementations/tests/test_dirfs.py

import pytest

from fsspec.asyn import AsyncFileSystem
from fsspec.implementations.dirfs import DirFileSystem
from fsspec.spec import AbstractFileSystem

PATH = "path/to/dir"
ARGS = ["foo", "bar"]
KWARGS = {"baz": "baz", "qux": "qux"}


@pytest.fixture
def make_fs(mocker):
    def _make_fs(async_impl=False, asynchronous=False):
        attrs = {
            "sep": "/",
            "async_impl": async_impl,
            "_strip_protocol": lambda path: path,
        }

        if async_impl:
            attrs["asynchronous"] = asynchronous
            cls = AsyncFileSystem
        else:
            cls = AbstractFileSystem

        fs = mocker.MagicMock(spec=cls, **attrs)

        return fs

    return _make_fs


@pytest.fixture(
    params=[
        pytest.param(False, id="sync"),
        pytest.param(True, id="async"),
    ]
)
def fs(make_fs, request):
    return make_fs(async_impl=request.param)


@pytest.fixture
def asyncfs(make_fs):
    return make_fs(async_impl=True, asynchronous=True)


@pytest.fixture
def make_dirfs():
    def _make_dirfs(fs, asynchronous=False):
        return DirFileSystem(PATH, fs, asynchronous=asynchronous)

    return _make_dirfs


@pytest.fixture
def dirfs(make_dirfs, fs):
    return make_dirfs(fs)


@pytest.fixture
def adirfs(make_dirfs, asyncfs):
    return make_dirfs(asyncfs, asynchronous=True)


def test_dirfs(fs, asyncfs):
    DirFileSystem("path", fs)
    DirFileSystem("path", asyncfs, asynchronous=True)

    with pytest.raises(ValueError):
        DirFileSystem("path", asyncfs)

    with pytest.raises(ValueError):
        DirFileSystem("path", fs, asynchronous=True)


@pytest.mark.parametrize(
    "root, rel, full",
    [
        ("", "", ""),
        ("", "foo", "foo"),
        ("root", "", "root"),
        ("root", "foo", "root/foo"),
    ],
)
def test_path(fs, root, rel, full):
    dirfs = DirFileSystem(root, fs)
    assert dirfs._join(rel) == full
    assert dirfs._relpath(full) == rel


def test_sep(mocker, dirfs):
    sep = mocker.Mock()
    dirfs.fs.sep = sep
    assert dirfs.sep == sep


@pytest.mark.asyncio
async def test_set_session(mocker, adirfs):
    adirfs.fs.set_session = mocker.AsyncMock()
    assert (
        await adirfs.set_session(*ARGS, **KWARGS) == adirfs.fs.set_session.return_value
    )
    adirfs.fs.set_session.assert_called_once_with(*ARGS, **KWARGS)


@pytest.mark.asyncio
async def test_async_rm_file(adirfs):
    await adirfs._rm_file("file", **KWARGS)
    adirfs.fs._rm_file.assert_called_once_with(f"{PATH}/file", **KWARGS)


def test_rm_file(dirfs):
    dirfs.rm_file("file", **KWARGS)
    dirfs.fs.rm_file.assert_called_once_with("path/to/dir/file", **KWARGS)


@pytest.mark.asyncio
async def test_async_rm(adirfs):
    await adirfs._rm("file", *ARGS, **KWARGS)
    adirfs.fs._rm.assert_called_once_with("path/to/dir/file", *ARGS, **KWARGS)


def test_rm(dirfs):
    dirfs.rm("file", *ARGS, **KWARGS)
    dirfs.fs.rm.assert_called_once_with("path/to/dir/file", *ARGS, **KWARGS)


@pytest.mark.asyncio
async def test_async_cp_file(adirfs):
    await adirfs._cp_file("one", "two", **KWARGS)
    adirfs.fs._cp_file.assert_called_once_with(f"{PATH}/one", f"{PATH}/two", **KWARGS)


def test_cp_file(dirfs):
    dirfs.cp_file("one", "two", **KWARGS)
    dirfs.fs.cp_file.assert_called_once_with(f"{PATH}/one", f"{PATH}/two", **KWARGS)


@pytest.mark.asyncio
async def test_async_copy(adirfs):
    await adirfs._copy("one", "two", *ARGS, **KWARGS)
    adirfs.fs._copy.assert_called_once_with(
        f"{PATH}/one", f"{PATH}/two", *ARGS, **KWARGS
    )


def test_copy(dirfs):
    dirfs.copy("one", "two", *ARGS, **KWARGS)
    dirfs.fs.copy.assert_called_once_with(f"{PATH}/one", f"{PATH}/two", *ARGS, **KWARGS)


@pytest.mark.asyncio
async def test_async_pipe(adirfs):
    await adirfs._pipe("file", *ARGS, **KWARGS)
    adirfs.fs._pipe.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)


def test_pipe(dirfs):
    dirfs.pipe("file", *ARGS, **KWARGS)
    dirfs.fs.pipe.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)


@pytest.mark.asyncio
async def test_async_pipe_file(adirfs):
    await adirfs._pipe_file("file", *ARGS, **KWARGS)
    adirfs.fs._pipe_file.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)


def test_pipe_file(dirfs):
    dirfs.pipe_file("file", *ARGS, **KWARGS)
    dirfs.fs.pipe_file.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)


@pytest.mark.asyncio
async def test_async_cat_file(adirfs):
    assert (
        await adirfs._cat_file("file", *ARGS, **KWARGS)
        == adirfs.fs._cat_file.return_value
    )
    adirfs.fs._cat_file.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)


def test_cat_file(dirfs):
    assert dirfs.cat_file("file", *ARGS, **KWARGS) == dirfs.fs.cat_file.return_value
    dirfs.fs.cat_file.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)


@pytest.mark.asyncio
async def test_async_cat(adirfs):
    assert await adirfs._cat("file", *ARGS, **KWARGS) == adirfs.fs._cat.return_value
    adirfs.fs._cat.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)


def test_cat(dirfs):
    assert dirfs.cat("file", *ARGS, **KWARGS) == dirfs.fs.cat.return_value
    dirfs.fs.cat.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)


@pytest.mark.asyncio
async def test_async_cat_list(adirfs):
    adirfs.fs._cat.return_value = {f"{PATH}/one": "foo", f"{PATH}/two": "bar"}
    assert await adirfs._cat(["one", "two"], *ARGS, **KWARGS) == {
        "one": "foo",
        "two": "bar",
    }
    adirfs.fs._cat.assert_called_once_with(
        [f"{PATH}/one", f"{PATH}/two"], *ARGS, **KWARGS
    )


def test_cat_list(dirfs):
    dirfs.fs.cat.return_value = {f"{PATH}/one": "foo", f"{PATH}/two": "bar"}
    assert dirfs.cat(["one", "two"], *ARGS, **KWARGS) == {"one": "foo", "two": "bar"}
    dirfs.fs.cat.assert_called_once_with(
        [f"{PATH}/one", f"{PATH}/two"], *ARGS, **KWARGS
    )


@pytest.mark.asyncio
async def test_async_put_file(adirfs):
    await adirfs._put_file("local", "file", **KWARGS)
    adirfs.fs._put_file.assert_called_once_with("local", f"{PATH}/file", **KWARGS)


def test_put_file(dirfs):
    dirfs.put_file("local", "file", **KWARGS)
    dirfs.fs.put_file.assert_called_once_with("local", f"{PATH}/file", **KWARGS)


@pytest.mark.asyncio
async def test_async_put(adirfs):
    await adirfs._put("local", "file", **KWARGS)
    adirfs.fs._put.assert_called_once_with("local", f"{PATH}/file", **KWARGS)


def test_put(dirfs):
    dirfs.put("local", "file", **KWARGS)
    dirfs.fs.put.assert_called_once_with("local", f"{PATH}/file", **KWARGS)


@pytest.mark.asyncio
async def test_async_get_file(adirfs):
    await adirfs._get_file("file", "local", **KWARGS)
    adirfs.fs._get_file.assert_called_once_with(f"{PATH}/file", "local", **KWARGS)


def test_get_file(dirfs):
    dirfs.get_file("file", "local", **KWARGS)
    dirfs.fs.get_file.assert_called_once_with(f"{PATH}/file", "local", **KWARGS)


@pytest.mark.asyncio
async def test_async_get(adirfs):
    await adirfs._get("file", "local", **KWARGS)
    adirfs.fs._get.assert_called_once_with(f"{PATH}/file", "local", **KWARGS)


def test_get(dirfs):
    dirfs.get("file", "local", **KWARGS)
    dirfs.fs.get.assert_called_once_with(f"{PATH}/file", "local", **KWARGS)


@pytest.mark.asyncio
async def test_async_isfile(adirfs):
    assert await adirfs._isfile("file") == adirfs.fs._isfile.return_value
    adirfs.fs._isfile.assert_called_once_with(f"{PATH}/file")


def test_isfile(dirfs):
    assert dirfs.isfile("file") == dirfs.fs.isfile.return_value
    dirfs.fs.isfile.assert_called_once_with(f"{PATH}/file")


@pytest.mark.asyncio
async def test_async_isdir(adirfs):
    assert await adirfs._isdir("file") == adirfs.fs._isdir.return_value
    adirfs.fs._isdir.assert_called_once_with(f"{PATH}/file")


def test_isdir(dirfs):
    assert dirfs.isdir("file") == dirfs.fs.isdir.return_value
    dirfs.fs.isdir.assert_called_once_with(f"{PATH}/file")


@pytest.mark.asyncio
async def test_async_size(adirfs):
    assert await adirfs._size("file") == adirfs.fs._size.return_value
    adirfs.fs._size.assert_called_once_with(f"{PATH}/file")


def test_size(dirfs):
    assert dirfs.size("file") == dirfs.fs.size.return_value
    dirfs.fs.size.assert_called_once_with(f"{PATH}/file")


@pytest.mark.asyncio
async def test_async_exists(adirfs):
    assert await adirfs._exists("file") == adirfs.fs._exists.return_value
    adirfs.fs._exists.assert_called_once_with(f"{PATH}/file")


def test_exists(dirfs):
    assert dirfs.exists("file") == dirfs.fs.exists.return_value
    dirfs.fs.exists.assert_called_once_with(f"{PATH}/file")


@pytest.mark.asyncio
async def test_async_info(adirfs):
    assert await adirfs._info("file", **KWARGS) == adirfs.fs._info.return_value
    adirfs.fs._info.assert_called_once_with(f"{PATH}/file", **KWARGS)


def test_info(dirfs):
    assert dirfs.info("file", **KWARGS) == dirfs.fs.info.return_value
    dirfs.fs.info.assert_called_once_with(f"{PATH}/file", **KWARGS)


@pytest.mark.asyncio
async def test_async_ls(adirfs):
    adirfs.fs._ls.return_value = [f"{PATH}/file"]
    assert await adirfs._ls("file", detail=False, **KWARGS) == ["file"]
    adirfs.fs._ls.assert_called_once_with(f"{PATH}/file", detail=False, **KWARGS)


def test_ls(dirfs):
    dirfs.fs.ls.return_value = [f"{PATH}/file"]
    assert dirfs.ls("file", detail=False, **KWARGS) == ["file"]
    dirfs.fs.ls.assert_called_once_with(f"{PATH}/file", detail=False, **KWARGS)


@pytest.mark.asyncio
async def test_async_ls_detail(adirfs):
    adirfs.fs._ls.return_value = [{"name": f"{PATH}/file", "foo": "bar"}]
    assert await adirfs._ls("file", detail=True, **KWARGS) == [
        {"name": "file", "foo": "bar"}
    ]
    adirfs.fs._ls.assert_called_once_with(f"{PATH}/file", detail=True, **KWARGS)


def test_ls_detail(dirfs):
    dirfs.fs.ls.return_value = [{"name": f"{PATH}/file", "foo": "bar"}]
    assert dirfs.ls("file", detail=True, **KWARGS) == [{"name": "file", "foo": "bar"}]
    dirfs.fs.ls.assert_called_once_with(f"{PATH}/file", detail=True, **KWARGS)


@pytest.mark.asyncio
async def test_async_walk(adirfs, mocker):
    async def _walk(path, *args, **kwargs):
        yield (f"{PATH}/root", ["foo", "bar"], ["baz", "qux"])

    adirfs.fs._walk = mocker.MagicMock()
    adirfs.fs._walk.side_effect = _walk

    actual = []
    async for entry in adirfs._walk("root", *ARGS, **KWARGS):
        actual.append(entry)  # noqa: PERF402
    assert actual == [("root", ["foo", "bar"], ["baz", "qux"])]
    adirfs.fs._walk.assert_called_once_with(f"{PATH}/root", *ARGS, **KWARGS)


def test_walk(dirfs):
    dirfs.fs.walk.return_value = iter(
        [(f"{PATH}/root", ["foo", "bar"], ["baz", "qux"])]
    )
    assert list(dirfs.walk("root", *ARGS, **KWARGS)) == [
        ("root", ["foo", "bar"], ["baz", "qux"])
    ]
    dirfs.fs.walk.assert_called_once_with(f"{PATH}/root", *ARGS, **KWARGS)


@pytest.mark.asyncio
async def test_async_glob(adirfs):
    adirfs.fs._glob.return_value = [f"{PATH}/one", f"{PATH}/two"]
    assert await adirfs._glob("*", **KWARGS) == ["one", "two"]
    adirfs.fs._glob.assert_called_once_with(f"{PATH}/*", **KWARGS)


def test_glob(dirfs):
    dirfs.fs.glob.return_value = [f"{PATH}/one", f"{PATH}/two"]
    assert dirfs.glob("*", **KWARGS) == ["one", "two"]
    dirfs.fs.glob.assert_called_once_with(f"{PATH}/*", **KWARGS)


def test_glob_with_protocol(dirfs):
    dirfs.fs.glob.return_value = [f"{PATH}/one", f"{PATH}/two"]
    assert dirfs.glob("dir://*", **KWARGS) == ["one", "two"]
    dirfs.fs.glob.assert_called_once_with(f"{PATH}/*", **KWARGS)


@pytest.mark.asyncio
async def test_async_glob_detail(adirfs):
    adirfs.fs._glob.return_value = {
        f"{PATH}/one": {"foo": "bar"},
        f"{PATH}/two": {"baz": "qux"},
    }
    assert await adirfs._glob("*", detail=True, **KWARGS) == {
        "one": {"foo": "bar"},
        "two": {"baz": "qux"},
    }
    adirfs.fs._glob.assert_called_once_with(f"{PATH}/*", detail=True, **KWARGS)


def test_glob_detail(dirfs):
    dirfs.fs.glob.return_value = {
        f"{PATH}/one": {"foo": "bar"},
        f"{PATH}/two": {"baz": "qux"},
    }
    assert dirfs.glob("*", detail=True, **KWARGS) == {
        "one": {"foo": "bar"},
        "two": {"baz": "qux"},
    }
    dirfs.fs.glob.assert_called_once_with(f"{PATH}/*", detail=True, **KWARGS)


@pytest.mark.asyncio
async def test_async_du(adirfs):
    adirfs.fs._du.return_value = 1234
    assert await adirfs._du("file", *ARGS, **KWARGS) == 1234
    adirfs.fs._du.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)


def test_du(dirfs):
    dirfs.fs.du.return_value = 1234
    assert dirfs.du("file", *ARGS, **KWARGS) == 1234
    dirfs.fs.du.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)


@pytest.mark.asyncio
async def test_async_du_granular(adirfs):
    adirfs.fs._du.return_value = {f"{PATH}/dir/one": 1, f"{PATH}/dir/two": 2}
    assert await adirfs._du("dir", *ARGS, total=False, **KWARGS) == {
        "dir/one": 1,
        "dir/two": 2,
    }
    adirfs.fs._du.assert_called_once_with(f"{PATH}/dir", *ARGS, total=False, **KWARGS)


def test_du_granular(dirfs):
    dirfs.fs.du.return_value = {f"{PATH}/dir/one": 1, f"{PATH}/dir/two": 2}
    assert dirfs.du("dir", *ARGS, total=False, **KWARGS) == {"dir/one": 1, "dir/two": 2}
    dirfs.fs.du.assert_called_once_with(f"{PATH}/dir", *ARGS, total=False, **KWARGS)


@pytest.mark.asyncio
async def test_async_find(adirfs):
    adirfs.fs._find.return_value = [f"{PATH}/dir/one", f"{PATH}/dir/two"]
    assert await adirfs._find("dir", *ARGS, **KWARGS) == ["dir/one", "dir/two"]
    adirfs.fs._find.assert_called_once_with(f"{PATH}/dir", *ARGS, **KWARGS)


def test_find(dirfs):
    dirfs.fs.find.return_value = [f"{PATH}/dir/one", f"{PATH}/dir/two"]
    assert dirfs.find("dir", *ARGS, **KWARGS) == ["dir/one", "dir/two"]
    dirfs.fs.find.assert_called_once_with(f"{PATH}/dir", *ARGS, **KWARGS)


@pytest.mark.asyncio
async def test_async_find_detail(adirfs):
    adirfs.fs._find.return_value = {
        f"{PATH}/dir/one": {"foo": "bar"},
        f"{PATH}/dir/two": {"baz": "qux"},
    }
    assert await adirfs._find("dir", *ARGS, detail=True, **KWARGS) == {
        "dir/one": {"foo": "bar"},
        "dir/two": {"baz": "qux"},
    }
    adirfs.fs._find.assert_called_once_with(f"{PATH}/dir", *ARGS, detail=True, **KWARGS)


def test_find_detail(dirfs):
    dirfs.fs.find.return_value = {
        f"{PATH}/dir/one": {"foo": "bar"},
        f"{PATH}/dir/two": {"baz": "qux"},
    }
    assert dirfs.find("dir", *ARGS, detail=True, **KWARGS) == {
        "dir/one": {"foo": "bar"},
        "dir/two": {"baz": "qux"},
    }
    dirfs.fs.find.assert_called_once_with(f"{PATH}/dir", *ARGS, detail=True, **KWARGS)


@pytest.mark.asyncio
async def test_async_expand_path(adirfs):
    adirfs.fs._expand_path.return_value = [f"{PATH}/file"]
    assert await adirfs._expand_path("*", *ARGS, **KWARGS) == ["file"]
    adirfs.fs._expand_path.assert_called_once_with(f"{PATH}/*", *ARGS, **KWARGS)


def test_expand_path(dirfs):
    dirfs.fs.expand_path.return_value = [f"{PATH}/file"]
    assert dirfs.expand_path("*", *ARGS, **KWARGS) == ["file"]
    dirfs.fs.expand_path.assert_called_once_with(f"{PATH}/*", *ARGS, **KWARGS)


@pytest.mark.asyncio
async def test_async_expand_path_list(adirfs):
    adirfs.fs._expand_path.return_value = [f"{PATH}/1file", f"{PATH}/2file"]
    assert await adirfs._expand_path(["1*", "2*"], *ARGS, **KWARGS) == [
        "1file",
        "2file",
    ]
    adirfs.fs._expand_path.assert_called_once_with(
        [f"{PATH}/1*", f"{PATH}/2*"], *ARGS, **KWARGS
    )


def test_expand_path_list(dirfs):
    dirfs.fs.expand_path.return_value = [f"{PATH}/1file", f"{PATH}/2file"]
    assert dirfs.expand_path(["1*", "2*"], *ARGS, **KWARGS) == ["1file", "2file"]
    dirfs.fs.expand_path.assert_called_once_with(
        [f"{PATH}/1*", f"{PATH}/2*"], *ARGS, **KWARGS
    )


@pytest.mark.asyncio
async def test_async_mkdir(adirfs):
    await adirfs._mkdir("dir", *ARGS, **KWARGS)
    adirfs.fs._mkdir.assert_called_once_with(f"{PATH}/dir", *ARGS, **KWARGS)


def test_mkdir(dirfs):
    dirfs.mkdir("dir", *ARGS, **KWARGS)
    dirfs.fs.mkdir.assert_called_once_with(f"{PATH}/dir", *ARGS, **KWARGS)


@pytest.mark.asyncio
async def test_async_makedirs(adirfs):
    await adirfs._makedirs("dir", *ARGS, **KWARGS)
    adirfs.fs._makedirs.assert_called_once_with(f"{PATH}/dir", *ARGS, **KWARGS)


def test_makedirs(dirfs):
    dirfs.makedirs("dir", *ARGS, **KWARGS)
    dirfs.fs.makedirs.assert_called_once_with(f"{PATH}/dir", *ARGS, **KWARGS)


def test_rmdir(mocker, dirfs):
    dirfs.fs.rmdir = mocker.Mock()
    dirfs.rmdir("dir")
    dirfs.fs.rmdir.assert_called_once_with(f"{PATH}/dir")


def test_mv(mocker, dirfs):
    dirfs.fs.mv = mocker.Mock()
    dirfs.mv("one", "two", **KWARGS)
    dirfs.fs.mv.assert_called_once_with(f"{PATH}/one", f"{PATH}/two", **KWARGS)


def test_touch(mocker, dirfs):
    dirfs.fs.touch = mocker.Mock()
    dirfs.touch("file", **KWARGS)
    dirfs.fs.touch.assert_called_once_with(f"{PATH}/file", **KWARGS)


def test_created(mocker, dirfs):
    dirfs.fs.created = mocker.Mock(return_value="date")
    assert dirfs.created("file") == "date"
    dirfs.fs.created.assert_called_once_with(f"{PATH}/file")


def test_modified(mocker, dirfs):
    dirfs.fs.modified = mocker.Mock(return_value="date")
    assert dirfs.modified("file") == "date"
    dirfs.fs.modified.assert_called_once_with(f"{PATH}/file")


def test_sign(mocker, dirfs):
    dirfs.fs.sign = mocker.Mock(return_value="url")
    assert dirfs.sign("file", *ARGS, **KWARGS) == "url"
    dirfs.fs.sign.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)


def test_open(mocker, dirfs):
    dirfs.fs.open = mocker.Mock()
    assert dirfs.open("file", *ARGS, **KWARGS) == dirfs.fs.open.return_value
    dirfs.fs.open.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)


def test_from_url(m):
    from fsspec.core import url_to_fs

    m.pipe("inner/file", b"data")
    fs, _ = url_to_fs("dir::memory://inner")
    assert fs.ls("", False) == ["file"]
    assert fs.ls("", True)[0]["name"] == "file"
    assert fs.cat("file") == b"data"