myenv/Lib/site-packages/fsspec/implementations/tests/test_cached.py

import json
import os
import pickle
import shutil
import tempfile

import pytest

import fsspec
from fsspec.compression import compr
from fsspec.exceptions import BlocksizeMismatchError
from fsspec.implementations.cache_mapper import (
    BasenameCacheMapper,
    HashCacheMapper,
    create_cache_mapper,
)
from fsspec.implementations.cached import (
    CachingFileSystem,
    LocalTempFile,
    WholeFileCacheFileSystem,
)
from fsspec.implementations.local import make_path_posix
from fsspec.implementations.zip import ZipFileSystem
from fsspec.tests.conftest import win

from .test_ftp import FTPFileSystem


@pytest.fixture
def local_filecache():
    import tempfile

    original_location = tempfile.mkdtemp()
    cache_location = tempfile.mkdtemp()
    original_file = os.path.join(original_location, "afile")
    data = b"test data"
    with open(original_file, "wb") as f:
        f.write(data)

    # we can access the file and read it
    fs = fsspec.filesystem(
        "filecache", target_protocol="file", cache_storage=cache_location
    )

    return data, original_file, cache_location, fs


def test_mapper():
    mapper0 = create_cache_mapper(True)
    assert mapper0("somefile") == "somefile"
    assert mapper0("/somefile") == "somefile"
    assert mapper0("/somedir/somefile") == "somefile"
    assert mapper0("/otherdir/somefile") == "somefile"

    mapper1 = create_cache_mapper(False)
    assert (
        mapper1("somefile")
        == "dd00b9487898b02555b6a2d90a070586d63f93e80c70aaa60c992fa9e81a72fe"
    )
    assert (
        mapper1("/somefile")
        == "884c07bc2efe65c60fb9d280a620e7f180488718fb5d97736521b7f9cf5c8b37"
    )
    assert (
        mapper1("/somedir/somefile")
        == "67a6956e5a5f95231263f03758c1fd9254fdb1c564d311674cec56b0372d2056"
    )
    assert (
        mapper1("/otherdir/somefile")
        == "f043dee01ab9b752c7f2ecaeb1a5e1b2d872018e2d0a1a26c43835ebf34e7d3e"
    )

    assert mapper0 != mapper1
    assert create_cache_mapper(True) == mapper0
    assert create_cache_mapper(False) == mapper1

    assert hash(mapper0) != hash(mapper1)
    assert hash(create_cache_mapper(True)) == hash(mapper0)
    assert hash(create_cache_mapper(False)) == hash(mapper1)

    with pytest.raises(
        ValueError,
        match="BasenameCacheMapper requires zero or positive directory_levels",
    ):
        BasenameCacheMapper(-1)

    mapper2 = BasenameCacheMapper(1)
    assert mapper2("/somefile") == "somefile"
    assert mapper2("/somedir/somefile") == "somedir_@_somefile"
    assert mapper2("/otherdir/somefile") == "otherdir_@_somefile"
    assert mapper2("/dir1/dir2/dir3/somefile") == "dir3_@_somefile"

    assert mapper2 != mapper0
    assert mapper2 != mapper1
    assert BasenameCacheMapper(1) == mapper2

    assert hash(mapper2) != hash(mapper0)
    assert hash(mapper2) != hash(mapper1)
    assert hash(BasenameCacheMapper(1)) == hash(mapper2)

    mapper3 = BasenameCacheMapper(2)
    assert mapper3("/somefile") == "somefile"
    assert mapper3("/somedir/somefile") == "somedir_@_somefile"
    assert mapper3("/otherdir/somefile") == "otherdir_@_somefile"
    assert mapper3("/dir1/dir2/dir3/somefile") == "dir2_@_dir3_@_somefile"

    assert mapper3 != mapper0
    assert mapper3 != mapper1
    assert mapper3 != mapper2
    assert BasenameCacheMapper(2) == mapper3

    assert hash(mapper3) != hash(mapper0)
    assert hash(mapper3) != hash(mapper1)
    assert hash(mapper3) != hash(mapper2)
    assert hash(BasenameCacheMapper(2)) == hash(mapper3)


@pytest.mark.parametrize(
    "cache_mapper", [BasenameCacheMapper(), BasenameCacheMapper(1), HashCacheMapper()]
)
@pytest.mark.parametrize("force_save_pickle", [True, False])
def test_metadata(tmpdir, cache_mapper, force_save_pickle):
    source = os.path.join(tmpdir, "source")
    afile = os.path.join(source, "afile")
    os.mkdir(source)
    open(afile, "w").write("test")

    fs = fsspec.filesystem(
        "filecache",
        target_protocol="file",
        cache_storage=os.path.join(tmpdir, "cache"),
        cache_mapper=cache_mapper,
    )
    fs._metadata._force_save_pickle = force_save_pickle

    with fs.open(afile, "rb") as f:
        assert f.read(5) == b"test"

    afile_posix = make_path_posix(afile)
    detail = fs._metadata.cached_files[0][afile_posix]
    assert sorted(detail.keys()) == ["blocks", "fn", "original", "time", "uid"]
    assert isinstance(detail["blocks"], bool)
    assert isinstance(detail["fn"], str)
    assert isinstance(detail["time"], float)
    assert isinstance(detail["uid"], str)

    assert detail["original"] == afile_posix
    assert detail["fn"] == fs._mapper(afile_posix)

    if isinstance(cache_mapper, BasenameCacheMapper):
        if cache_mapper.directory_levels == 0:
            assert detail["fn"] == "afile"
        else:
            assert detail["fn"] == "source_@_afile"


def test_metadata_replace_pickle_with_json(tmpdir):
    # For backward compatibility will allow reading of old pickled metadata.
    # When the metadata is next saved, it is in json format.
    source = os.path.join(tmpdir, "source")
    afile = os.path.join(source, "afile")
    os.mkdir(source)
    open(afile, "w").write("test")

    # Save metadata in pickle format, to simulate old metadata
    fs = fsspec.filesystem(
        "filecache",
        target_protocol="file",
        cache_storage=os.path.join(tmpdir, "cache"),
    )
    fs._metadata._force_save_pickle = True
    with fs.open(afile, "rb") as f:
        assert f.read(5) == b"test"

    # Confirm metadata is in pickle format
    cache_fn = os.path.join(fs.storage[-1], "cache")
    with open(cache_fn, "rb") as f:
        metadata = pickle.load(f)
    assert list(metadata.keys()) == [make_path_posix(afile)]

    # Force rewrite of metadata, now in json format
    fs._metadata._force_save_pickle = False
    fs.pop_from_cache(afile)
    with fs.open(afile, "rb") as f:
        assert f.read(5) == b"test"

    # Confirm metadata is in json format
    with open(cache_fn, "r") as f:
        metadata = json.load(f)
    assert list(metadata.keys()) == [make_path_posix(afile)]


def test_constructor_kwargs(tmpdir):
    fs = fsspec.filesystem("filecache", target_protocol="file", same_names=True)
    assert isinstance(fs._mapper, BasenameCacheMapper)

    fs = fsspec.filesystem("filecache", target_protocol="file", same_names=False)
    assert isinstance(fs._mapper, HashCacheMapper)

    fs = fsspec.filesystem("filecache", target_protocol="file")
    assert isinstance(fs._mapper, HashCacheMapper)

    with pytest.raises(
        ValueError, match="Cannot specify both same_names and cache_mapper"
    ):
        fs = fsspec.filesystem(
            "filecache",
            target_protocol="file",
            cache_mapper=HashCacheMapper(),
            same_names=True,
        )


def test_idempotent():
    fs = CachingFileSystem("file")
    fs2 = CachingFileSystem("file")
    assert fs2 is fs
    fs3 = pickle.loads(pickle.dumps(fs))
    assert fs3.storage == fs.storage


@pytest.mark.parametrize("force_save_pickle", [True, False])
def test_blockcache_workflow(ftp_writable, tmp_path, force_save_pickle):
    host, port, user, pw = ftp_writable
    fs = FTPFileSystem(host, port, user, pw)
    with fs.open("/out", "wb") as f:
        f.write(b"test\n" * 4096)

    fs_kwargs = {
        "skip_instance_cache": True,
        "cache_storage": str(tmp_path),
        "target_protocol": "ftp",
        "target_options": {
            "host": host,
            "port": port,
            "username": user,
            "password": pw,
        },
    }

    # Open the blockcache and read a little bit of the data
    fs = fsspec.filesystem("blockcache", **fs_kwargs)
    fs._metadata._force_save_pickle = force_save_pickle
    with fs.open("/out", "rb", block_size=5) as f:
        assert f.read(5) == b"test\n"

    # Save the cache/close it
    fs.save_cache()
    del fs

    # Check that cache file only has the first two blocks
    if force_save_pickle:
        with open(tmp_path / "cache", "rb") as f:
            cache = pickle.load(f)
    else:
        with open(tmp_path / "cache", "r") as f:
            cache = json.load(f)
    assert "/out" in cache
    assert cache["/out"]["blocks"] == [0, 1]

    # Reopen the same cache and read some more...
    fs = fsspec.filesystem("blockcache", **fs_kwargs)
    fs._metadata._force_save_pickle = force_save_pickle
    with fs.open("/out", block_size=5) as f:
        assert f.read(5) == b"test\n"
        f.seek(30)
        assert f.read(5) == b"test\n"


@pytest.mark.parametrize("impl", ["filecache", "blockcache"])
def test_workflow(ftp_writable, impl):
    host, port, user, pw = ftp_writable
    fs = FTPFileSystem(host, port, user, pw)
    with fs.open("/out", "wb") as f:
        f.write(b"test")
    fs = fsspec.filesystem(
        impl,
        target_protocol="ftp",
        target_options={"host": host, "port": port, "username": user, "password": pw},
    )
    assert os.listdir(fs.storage[-1]) == []
    with fs.open("/out") as f:
        assert os.listdir(fs.storage[-1])
        assert f.read() == b"test"
        assert fs._metadata.cached_files[-1]["/out"]["blocks"]
    assert fs.cat("/out") == b"test"
    assert fs._metadata.cached_files[-1]["/out"]["blocks"] is True

    with fs.open("/out", "wb") as f:
        f.write(b"changed")

    if impl == "filecache":
        assert (
            fs.cat("/out") == b"changed"
        )  # new value, because we overwrote the cached location


@pytest.mark.parametrize("impl", ["simplecache", "blockcache"])
def test_glob(ftp_writable, impl):
    host, port, user, pw = ftp_writable
    fs = FTPFileSystem(host, port, user, pw)
    with fs.open("/out", "wb") as f:
        f.write(b"test")
    with fs.open("/out2", "wb") as f:
        f.write(b"test2")
    fs = fsspec.filesystem(
        impl,
        target_protocol="ftp",
        target_options={"host": host, "port": port, "username": user, "password": pw},
    )
    assert fs.glob("/wrong*") == []
    assert fs.glob("/ou*") == ["/out", "/out2"]


def test_write():
    tmp = str(tempfile.mkdtemp())
    fn = tmp + "afile"
    url = f"simplecache::file://{fn}"
    with fsspec.open(url, "wb") as f:
        f.write(b"hello")
        assert fn not in f.name
        assert not os.listdir(tmp)

    assert open(fn, "rb").read() == b"hello"


def test_clear():
    import tempfile

    origin = tempfile.mkdtemp()
    cache1 = tempfile.mkdtemp()
    data = b"test data"
    f1 = os.path.join(origin, "afile")
    with open(f1, "wb") as f:
        f.write(data)

    # populates first cache
    fs = fsspec.filesystem("filecache", target_protocol="file", cache_storage=cache1)
    assert fs.cat(f1) == data

    assert "cache" in os.listdir(cache1)
    assert len(os.listdir(cache1)) == 2
    assert fs._check_file(f1)

    fs.clear_cache()
    assert not fs._check_file(f1)
    assert len(os.listdir(cache1)) < 2


@pytest.mark.parametrize("force_save_pickle", [True, False])
def test_clear_expired(tmp_path, force_save_pickle):
    def __ager(cache_fn, fn, del_fn=False):
        """
        Modify the cache file to virtually add time lag to selected files.

        Parameters
        ---------
        cache_fn: str
            cache path
        fn: str
            file name to be modified
        del_fn: bool
            whether or not to delete 'fn' from cache details
        """
        import pathlib
        import time

        if os.path.exists(cache_fn):
            if force_save_pickle:
                with open(cache_fn, "rb") as f:
                    cached_files = pickle.load(f)
            else:
                with open(cache_fn, "r") as f:
                    cached_files = json.load(f)
            fn_posix = pathlib.Path(fn).as_posix()
            cached_files[fn_posix]["time"] = cached_files[fn_posix]["time"] - 691200
            assert os.access(cache_fn, os.W_OK), "Cache is not writable"
            if del_fn:
                del cached_files[fn_posix]["fn"]
            if force_save_pickle:
                with open(cache_fn, "wb") as f:
                    pickle.dump(cached_files, f)
            else:
                with open(cache_fn, "w") as f:
                    json.dump(cached_files, f)
            time.sleep(1)

    origin = tmp_path.joinpath("origin")
    cache1 = tmp_path.joinpath("cache1")
    cache2 = tmp_path.joinpath("cache2")
    cache3 = tmp_path.joinpath("cache3")

    origin.mkdir()
    cache1.mkdir()
    cache2.mkdir()
    cache3.mkdir()

    data = b"test data"
    f1 = origin.joinpath("afile")
    f2 = origin.joinpath("bfile")
    f3 = origin.joinpath("cfile")
    f4 = origin.joinpath("dfile")

    with open(f1, "wb") as f:
        f.write(data)
    with open(f2, "wb") as f:
        f.write(data)
    with open(f3, "wb") as f:
        f.write(data)
    with open(f4, "wb") as f:
        f.write(data)

    # populates first cache
    fs = fsspec.filesystem(
        "filecache", target_protocol="file", cache_storage=str(cache1), cache_check=1
    )
    fs._metadata._force_save_pickle = force_save_pickle
    assert fs.cat(str(f1)) == data

    # populates "last" cache if file not found in first one
    fs = fsspec.filesystem(
        "filecache",
        target_protocol="file",
        cache_storage=[str(cache1), str(cache2)],
        cache_check=1,
    )
    fs._metadata._force_save_pickle = force_save_pickle
    assert fs.cat(str(f2)) == data
    assert fs.cat(str(f3)) == data
    assert len(os.listdir(cache2)) == 3

    # force the expiration
    cache_fn = os.path.join(fs.storage[-1], "cache")
    __ager(cache_fn, f2)

    # remove from cache2 the expired files
    fs.clear_expired_cache()
    assert len(os.listdir(cache2)) == 2

    # check complete cleanup
    __ager(cache_fn, f3)

    fs.clear_expired_cache()
    assert not fs._check_file(f2)
    assert not fs._check_file(f3)
    assert len(os.listdir(cache2)) < 2

    # check cache1 to be untouched after cleaning
    assert len(os.listdir(cache1)) == 2

    # check cleaning with 'same_name' option enabled
    fs = fsspec.filesystem(
        "filecache",
        target_protocol="file",
        cache_storage=[str(cache1), str(cache2), str(cache3)],
        same_names=True,
        cache_check=1,
    )
    fs._metadata._force_save_pickle = force_save_pickle
    assert fs.cat(str(f4)) == data

    cache_fn = os.path.join(fs.storage[-1], "cache")
    __ager(cache_fn, f4)

    fs.clear_expired_cache()
    assert not fs._check_file(str(f4))

    # check cache metadata lacking 'fn' raises RuntimeError.
    fs = fsspec.filesystem(
        "filecache",
        target_protocol="file",
        cache_storage=str(cache1),
        same_names=True,
        cache_check=1,
    )
    fs._metadata._force_save_pickle = force_save_pickle
    assert fs.cat(str(f1)) == data

    cache_fn = os.path.join(fs.storage[-1], "cache")
    __ager(cache_fn, f1, del_fn=True)

    with pytest.raises(RuntimeError, match="Cache metadata does not contain 'fn' for"):
        fs.clear_expired_cache()


def test_pop():
    import tempfile

    origin = tempfile.mkdtemp()
    cache1 = tempfile.mkdtemp()
    cache2 = tempfile.mkdtemp()
    data = b"test data"
    f1 = os.path.join(origin, "afile")
    f2 = os.path.join(origin, "bfile")
    with open(f1, "wb") as f:
        f.write(data)
    with open(f2, "wb") as f:
        f.write(data)

    # populates first cache
    fs = fsspec.filesystem("filecache", target_protocol="file", cache_storage=cache1)
    fs.cat(f1)

    # populates last cache if file not found in first cache
    fs = fsspec.filesystem(
        "filecache", target_protocol="file", cache_storage=[cache1, cache2]
    )
    assert fs.cat(f2) == data
    assert len(os.listdir(cache2)) == 2
    assert fs._check_file(f1)
    with pytest.raises(PermissionError):
        fs.pop_from_cache(f1)
    fs.pop_from_cache(f2)
    fs.pop_from_cache(os.path.join(origin, "uncached-file"))
    assert len(os.listdir(cache2)) == 1
    assert not fs._check_file(f2)
    assert fs._check_file(f1)


def test_write_pickle_context():
    tmp = str(tempfile.mkdtemp())
    fn = tmp + "afile"
    url = f"simplecache::file://{fn}"
    with fsspec.open(url, "wb") as f:
        pickle.loads(pickle.dumps(f))
        f.write(b"hello ")
        pickle.dumps(f)

    with pytest.raises(ValueError):
        pickle.dumps(f)

    assert open(fn, "rb").read() == b"hello "


def test_blocksize(ftp_writable):
    host, port, user, pw = ftp_writable
    fs = FTPFileSystem(host, port, user, pw)
    with fs.open("/out_block", "wb") as f:
        f.write(b"test" * 4000)

    fs = fsspec.filesystem(
        "blockcache",
        target_protocol="ftp",
        target_options={"host": host, "port": port, "username": user, "password": pw},
    )

    with fs.open("/out_block", block_size=20) as f:
        assert f.read(1) == b"t"
    with pytest.raises(BlocksizeMismatchError):
        fs.open("/out_block", block_size=30)


def test_blockcache_multiinstance(ftp_writable):
    host, port, user, pw = ftp_writable
    fs = FTPFileSystem(host, port, user, pw)
    with fs.open("/one", "wb") as f:
        f.write(b"test" * 40)
    with fs.open("/two", "wb") as f:
        f.write(b"test" * 40)
    fs = fsspec.filesystem(
        "blockcache",
        target_protocol="ftp",
        target_options={"host": host, "port": port, "username": user, "password": pw},
    )

    with fs.open("/one", block_size=20) as f:
        assert f.read(1) == b"t"
    fs2 = fsspec.filesystem(
        "blockcache",
        target_protocol="ftp",
        target_options={"host": host, "port": port, "username": user, "password": pw},
        skip_instance_cache=True,
        cache_storage=fs.storage,
    )
    assert fs2._metadata.cached_files  # loaded from metadata for "one"
    with fs2.open("/two", block_size=20) as f:
        assert f.read(1) == b"t"
    assert "/two" in fs2._metadata.cached_files[-1]
    fs.save_cache()
    assert list(fs._metadata.cached_files[-1]) == ["/one", "/two"]
    assert list(fs2._metadata.cached_files[-1]) == ["/one", "/two"]


def test_metadata_save_blocked(ftp_writable, caplog):
    import logging

    host, port, user, pw = ftp_writable
    fs = FTPFileSystem(host, port, user, pw)
    with fs.open("/one", "wb") as f:
        f.write(b"test" * 40)
    fs = fsspec.filesystem(
        "blockcache",
        target_protocol="ftp",
        target_options={"host": host, "port": port, "username": user, "password": pw},
    )

    with fs.open("/one", block_size=20) as f:
        assert f.read(1) == b"t"
    fn = os.path.join(fs.storage[-1], "cache")
    with caplog.at_level(logging.DEBUG):
        with fs.open("/one", block_size=20) as f:
            f.seek(21)
            assert f.read(1)
            os.remove(fn)
            os.mkdir(fn)
    assert "Cache saving failed while closing file" in caplog.text
    os.rmdir(fn)

    def open_raise(*_, **__):
        raise NameError

    try:
        # To simulate an interpreter shutdown we temporarily set an open function in the
        # cache_metadata module which is used on the next attempt to save metadata.
        with caplog.at_level(logging.DEBUG):
            with fs.open("/one", block_size=20) as f:
                fsspec.implementations.cache_metadata.open = open_raise
                f.seek(21)
                assert f.read(1)
    finally:
        fsspec.implementations.cache_metadata.__dict__.pop("open", None)
    assert "Cache save failed due to interpreter shutdown" in caplog.text


@pytest.mark.parametrize("impl", ["filecache", "simplecache", "blockcache"])
def test_local_filecache_creates_dir_if_needed(impl):
    import tempfile

    original_location = tempfile.mkdtemp()
    cache_location = tempfile.mkdtemp()
    os.rmdir(cache_location)
    assert not os.path.exists(cache_location)

    original_file = os.path.join(original_location, "afile")
    data = b"test data"
    with open(original_file, "wb") as f:
        f.write(data)

    # we can access the file and read it
    fs = fsspec.filesystem(impl, target_protocol="file", cache_storage=cache_location)

    with fs.open(original_file, "rb") as f:
        data_in_cache = f.read()

    assert os.path.exists(cache_location)

    assert data_in_cache == data


@pytest.mark.parametrize("toplevel", [True, False])
@pytest.mark.parametrize("impl", ["filecache", "simplecache", "blockcache"])
def test_get_mapper(impl, toplevel):
    import tempfile

    original_location = tempfile.mkdtemp()
    cache_location = tempfile.mkdtemp()
    os.rmdir(cache_location)
    original_file = os.path.join(original_location, "afile")
    data = b"test data"
    with open(original_file, "wb") as f:
        f.write(data)

    if toplevel:
        m = fsspec.get_mapper(
            f"{impl}::file://{original_location}",
            **{impl: {"cache_storage": cache_location}},
        )
    else:
        fs = fsspec.filesystem(
            impl, target_protocol="file", cache_storage=cache_location
        )
        m = fs.get_mapper(original_location)

    assert m["afile"] == data
    assert os.listdir(cache_location)
    assert m["afile"] == data


def test_local_filecache_basic(local_filecache):
    data, original_file, cache_location, fs = local_filecache

    # reading from the file contains the right data
    with fs.open(original_file, "rb") as f:
        assert f.read() == data
    assert "cache" in os.listdir(cache_location)

    # the file in the location contains the right data
    fn = list(fs._metadata.cached_files[-1].values())[0]["fn"]  # this is a hash value
    assert fn in os.listdir(cache_location)
    with open(os.path.join(cache_location, fn), "rb") as f:
        assert f.read() == data

    # still there when original file is removed (check=False)
    os.remove(original_file)
    with fs.open(original_file, "rb") as f:
        assert f.read() == data


def test_local_filecache_does_not_change_when_original_data_changed(local_filecache):
    old_data, original_file, cache_location, fs = local_filecache
    new_data = b"abc"

    with fs.open(original_file, "rb") as f:
        assert f.read() == old_data

    with open(original_file, "wb") as f:
        f.write(new_data)

    with fs.open(original_file, "rb") as f:
        assert f.read() == old_data


def test_local_filecache_gets_from_original_if_cache_deleted(local_filecache):
    old_data, original_file, cache_location, fs = local_filecache
    new_data = b"abc"

    with fs.open(original_file, "rb") as f:
        assert f.read() == old_data

    with open(original_file, "wb") as f:
        f.write(new_data)

    shutil.rmtree(cache_location)
    assert os.path.exists(original_file)

    with open(original_file, "rb") as f:
        assert f.read() == new_data

    with fs.open(original_file, "rb") as f:
        assert f.read() == new_data

    # the file in the location contains the right data
    fn = list(fs._metadata.cached_files[-1].values())[0]["fn"]  # this is a hash value
    assert fn in os.listdir(cache_location)
    with open(os.path.join(cache_location, fn), "rb") as f:
        assert f.read() == new_data


def test_local_filecache_with_new_cache_location_makes_a_new_copy(local_filecache):
    import tempfile

    data, original_file, old_cache_location, old_fs = local_filecache
    new_cache_location = tempfile.mkdtemp()

    with old_fs.open(original_file, "rb") as f:
        assert f.read() == data

    new_fs = fsspec.filesystem(
        "filecache", target_protocol="file", cache_storage=new_cache_location
    )

    with new_fs.open(original_file, "rb") as f:
        assert f.read() == data

    # the file in the location contains the right data
    fn = list(new_fs._metadata.cached_files[-1].values())[0][
        "fn"
    ]  # this is a hash value
    assert fn in os.listdir(old_cache_location)
    assert fn in os.listdir(new_cache_location)

    with open(os.path.join(new_cache_location, fn), "rb") as f:
        assert f.read() == data


def test_filecache_multicache():
    import tempfile

    origin = tempfile.mkdtemp()
    cache1 = tempfile.mkdtemp()
    cache2 = tempfile.mkdtemp()
    data = b"test data"
    f1 = os.path.join(origin, "afile")
    f2 = os.path.join(origin, "bfile")
    with open(f1, "wb") as f:
        f.write(data)
    with open(f2, "wb") as f:
        f.write(data * 2)

    # populates first cache
    fs = fsspec.filesystem("filecache", target_protocol="file", cache_storage=cache1)
    assert fs.cat(f1) == data

    assert len(os.listdir(cache1)) == 2  # cache and hashed afile
    assert len(os.listdir(cache2)) == 0  # hasn't been initialized yet

    # populates last cache if file not found in first cache
    fs = fsspec.filesystem(
        "filecache", target_protocol="file", cache_storage=[cache1, cache2]
    )

    assert fs.cat(f1) == data
    assert fs.cat(f2) == data * 2

    assert "cache" in os.listdir(cache1)
    assert "cache" in os.listdir(cache2)

    cache1_contents = [f for f in os.listdir(cache1) if f != "cache"]
    assert len(cache1_contents) == 1

    with open(os.path.join(cache1, cache1_contents[0]), "rb") as f:
        assert f.read() == data

    cache2_contents = [f for f in os.listdir(cache2) if f != "cache"]
    assert len(cache2_contents) == 1

    with open(os.path.join(cache2, cache2_contents[0]), "rb") as f:
        assert f.read() == data * 2


@pytest.mark.parametrize("impl", ["filecache", "simplecache"])
def test_filecache_multicache_with_same_file_different_data_reads_from_first(impl):
    import tempfile

    origin = tempfile.mkdtemp()
    cache1 = tempfile.mkdtemp()
    cache2 = tempfile.mkdtemp()
    data = b"test data"
    f1 = os.path.join(origin, "afile")
    with open(f1, "wb") as f:
        f.write(data)

    # populate first cache
    fs1 = fsspec.filesystem(impl, target_protocol="file", cache_storage=cache1)
    assert fs1.cat(f1) == data

    with open(f1, "wb") as f:
        f.write(data * 2)

    # populate second cache
    fs2 = fsspec.filesystem(impl, target_protocol="file", cache_storage=cache2)

    assert fs2.cat(f1) == data * 2

    # the filenames in each cache are the same, but the data is different
    assert sorted(os.listdir(cache1)) == sorted(os.listdir(cache2))

    fs = fsspec.filesystem(impl, target_protocol="file", cache_storage=[cache1, cache2])

    assert fs.cat(f1) == data


def test_filecache_with_checks():
    import time

    origin = tempfile.mkdtemp()
    cache1 = tempfile.mkdtemp()
    data = b"test data"
    f1 = os.path.join(origin, "afile")
    with open(f1, "wb") as f:
        f.write(data)

    # populate first cache
    fs = fsspec.filesystem(
        "filecache", target_protocol="file", cache_storage=cache1, expiry_time=0.1
    )
    fs2 = fsspec.filesystem(
        "filecache", target_protocol="file", cache_storage=cache1, check_files=True
    )
    assert fs.cat(f1) == data
    assert fs2.cat(f1) == data

    with open(f1, "wb") as f:
        f.write(data * 2)

    assert fs.cat(f1) == data  # does not change
    assert fs2.cat(f1) == data * 2  # changed, since origin changed
    with fs2.open(f1) as f:
        assert f.read() == data * 2  # read also sees new data
    time.sleep(0.11)  # allow cache details to expire
    assert fs.cat(f1) == data * 2  # changed, since origin changed


@pytest.mark.parametrize("impl", ["filecache", "simplecache", "blockcache"])
@pytest.mark.parametrize("fs", ["local", "multi"], indirect=["fs"])
def test_takes_fs_instance(impl, fs):
    origin = tempfile.mkdtemp()
    data = b"test data"
    f1 = os.path.join(origin, "afile")
    with open(f1, "wb") as f:
        f.write(data)

    fs2 = fsspec.filesystem(impl, fs=fs)

    assert fs2.cat(f1) == data


def test_add_file_to_cache_after_save(local_filecache):
    (data, original_file, cache_location, fs) = local_filecache

    fs.save_cache()

    fs.cat(original_file)
    assert len(fs._metadata.cached_files[-1]) == 1

    fs.save_cache()

    fs2 = fsspec.filesystem(
        "filecache",
        target_protocol="file",
        cache_storage=cache_location,
        do_not_use_cache_for_this_instance=True,  # cache is masking the issue
    )
    assert len(fs2._metadata.cached_files[-1]) == 1


def test_cached_open_close_read(ftp_writable):
    # Regression test for <https://github.com/fsspec/filesystem_spec/issues/799>
    host, port, user, pw = ftp_writable
    fs = FTPFileSystem(host, port, user, pw)
    with fs.open("/out_block", "wb") as f:
        f.write(b"test" * 4000)
    fs = fsspec.filesystem(
        "cached",
        target_protocol="ftp",
        target_options={"host": host, "port": port, "username": user, "password": pw},
    )
    with fs.open("/out_block", block_size=1024) as f:
        pass
    with fs.open("/out_block", block_size=1024) as f:
        assert f.read(1) == b"t"
    # Regression test for <https://github.com/fsspec/filesystem_spec/issues/845>
    assert fs._metadata.cached_files[-1]["/out_block"]["blocks"] == {0}


@pytest.mark.parametrize("impl", ["filecache", "simplecache"])
@pytest.mark.parametrize("compression", ["gzip", "bz2"])
def test_with_compression(impl, compression):
    data = b"123456789"
    tempdir = tempfile.mkdtemp()
    cachedir = tempfile.mkdtemp()
    fn = os.path.join(tempdir, "data")
    f = compr[compression](open(fn, mode="wb"), mode="w")
    f.write(data)
    f.close()

    with fsspec.open(
        f"{impl}::{fn}",
        "rb",
        compression=compression,
        **{impl: {"same_names": True, "cache_storage": cachedir}},
    ) as f:
        # stores original compressed file, uncompress on read
        assert f.read() == data
        assert "data" in os.listdir(cachedir)
        assert open(os.path.join(cachedir, "data"), "rb").read() != data

    cachedir = tempfile.mkdtemp()

    with fsspec.open(
        f"{impl}::{fn}",
        "rb",
        **{
            impl: {
                "same_names": True,
                "compression": compression,
                "cache_storage": cachedir,
            }
        },
    ) as f:
        # stores uncompressed data
        assert f.read() == data
        assert "data" in os.listdir(cachedir)
        assert open(os.path.join(cachedir, "data"), "rb").read() == data


@pytest.mark.parametrize("protocol", ["simplecache", "filecache"])
def test_again(protocol):
    fn = "memory://afile"
    with fsspec.open(fn, "wb") as f:
        f.write(b"hello")
    d2 = tempfile.mkdtemp()
    lurl = fsspec.open_local(f"{protocol}::{fn}", **{protocol: {"cache_storage": d2}})
    assert os.path.exists(lurl)
    assert d2 in lurl
    assert open(lurl, "rb").read() == b"hello"

    # remove cache dir
    shutil.rmtree(d2)
    assert not os.path.exists(lurl)

    # gets recreated
    lurl = fsspec.open_local(f"{protocol}::{fn}", **{protocol: {"cache_storage": d2}})
    assert open(lurl, "rb").read() == b"hello"


@pytest.mark.parametrize("protocol", ["simplecache", "filecache"])
def test_multi_cache(protocol):
    with fsspec.open_files("memory://file*", "wb", num=2) as files:
        for f in files:
            f.write(b"hello")

    d2 = tempfile.mkdtemp()
    lurl = fsspec.open_local(
        f"{protocol}::memory://file*",
        mode="rb",
        **{protocol: {"cache_storage": d2, "same_names": True}},
    )
    assert all(d2 in u for u in lurl)
    assert all(os.path.basename(f) in ["file0", "file1"] for f in lurl)
    assert all(open(u, "rb").read() == b"hello" for u in lurl)

    d2 = tempfile.mkdtemp()
    lurl = fsspec.open_files(
        f"{protocol}::memory://file*",
        mode="rb",
        **{protocol: {"cache_storage": d2, "same_names": True}},
    )
    with lurl as files:
        for f in files:
            assert os.path.basename(f.name) in ["file0", "file1"]
            assert f.read() == b"hello"
    fs = fsspec.filesystem("memory")
    fs.store.clear()
    with lurl as files:
        for f in files:
            assert os.path.basename(f.name) in ["file0", "file1"]
            assert f.read() == b"hello"


@pytest.mark.parametrize("protocol", ["simplecache", "filecache", "blockcache"])
def test_multi_cat(protocol, ftp_writable):
    host, port, user, pw = ftp_writable
    fs = FTPFileSystem(host, port, user, pw)
    for fn in ("/file0", "/file1"):
        with fs.open(fn, "wb") as f:
            f.write(b"hello")

    d2 = tempfile.mkdtemp()
    fs = fsspec.filesystem(protocol, storage=d2, fs=fs)
    assert fs.cat("file*") == {"/file0": b"hello", "/file1": b"hello"}


@pytest.mark.parametrize("protocol", ["simplecache", "filecache"])
def test_multi_cache_chain(protocol):
    import zipfile

    d = tempfile.mkdtemp()
    fn = os.path.join(d, "test.zip")
    zipfile.ZipFile(fn, mode="w").open("test", "w").write(b"hello")

    with fsspec.open_files(f"zip://test::{protocol}::file://{fn}") as files:
        assert d not in files[0]._fileobj._file.name
        assert files[0].read() == b"hello"

    # special test contains "file:" string
    fn = os.path.join(d, "file.zip")
    zipfile.ZipFile(fn, mode="w").open("file", "w").write(b"hello")
    with fsspec.open_files(f"zip://file::{protocol}::file://{fn}") as files:
        assert d not in files[0]._fileobj._file.name
        assert files[0].read() == b"hello"


@pytest.mark.parametrize("protocol", ["blockcache", "simplecache", "filecache"])
def test_strip(protocol):
    fs = fsspec.filesystem(protocol, target_protocol="memory")
    url1 = "memory://afile"
    assert fs._strip_protocol(url1) == "/afile"
    assert fs._strip_protocol(protocol + "://afile") == "/afile"
    assert fs._strip_protocol(protocol + "::memory://afile") == "/afile"


@pytest.mark.parametrize("protocol", ["simplecache", "filecache"])
def test_cached_write(protocol):
    d = tempfile.mkdtemp()
    ofs = fsspec.open_files(f"{protocol}::file://{d}/*.out", mode="wb", num=2)
    with ofs as files:
        for f in files:
            assert isinstance(f, LocalTempFile)
            f.write(b"data")
            fn = f.name

    assert sorted(os.listdir(d)) == ["0.out", "1.out"]
    assert not os.path.exists(fn)


def test_expiry():
    import time

    d = tempfile.mkdtemp()
    fs = fsspec.filesystem("memory")
    fn = "/afile"
    fn0 = "memory://afile"
    data = b"hello"
    with fs.open(fn0, "wb") as f:
        f.write(data)

    fs = fsspec.filesystem(
        "filecache",
        fs=fs,
        cache_storage=d,
        check_files=False,
        expiry_time=0.1,
        same_names=True,
    )

    # get file
    assert fs._check_file(fn0) is False
    assert fs.open(fn0, mode="rb").read() == data
    start_time = fs._metadata.cached_files[-1][fn]["time"]

    # cache time..
    assert fs.last_cache - start_time < 0.19

    # cache should have refreshed
    time.sleep(0.01)

    # file should still be valid... re-read
    assert fs.open(fn0, mode="rb").read() == data
    detail, _ = fs._check_file(fn0)
    assert detail["time"] == start_time

    time.sleep(0.11)
    # file should still be invalid... re-read
    assert fs._check_file(fn0) is False
    assert fs.open(fn0, mode="rb").read() == data
    detail, _ = fs._check_file(fn0)
    assert detail["time"] - start_time > 0.09


def test_equality(tmpdir):
    """Test sane behaviour for equality and hashing.

    Make sure that different CachingFileSystem only test equal to each other
    when they should, and do not test equal to the filesystem they rely upon.
    Similarly, make sure their hashes differ when they should and are equal
    when they should not.

    Related: GitHub#577, GitHub#578
    """
    from fsspec.implementations.local import LocalFileSystem

    lfs = LocalFileSystem()
    dir1 = f"{tmpdir}/raspberry"
    dir2 = f"{tmpdir}/banana"
    cfs1 = CachingFileSystem(fs=lfs, cache_storage=dir1)
    cfs2 = CachingFileSystem(fs=lfs, cache_storage=dir2)
    cfs3 = CachingFileSystem(fs=lfs, cache_storage=dir2)
    assert cfs1 == cfs1
    assert cfs1 != cfs2
    assert cfs1 != cfs3
    assert cfs2 == cfs3
    assert cfs1 != lfs
    assert cfs2 != lfs
    assert cfs3 != lfs
    assert hash(lfs) != hash(cfs1)
    assert hash(lfs) != hash(cfs2)
    assert hash(lfs) != hash(cfs3)
    assert hash(cfs1) != hash(cfs2)
    assert hash(cfs1) != hash(cfs2)
    assert hash(cfs2) == hash(cfs3)


def test_str():
    """Test that the str representation refers to correct class."""
    from fsspec.implementations.local import LocalFileSystem

    lfs = LocalFileSystem()
    cfs = CachingFileSystem(fs=lfs)
    assert "CachingFileSystem" in str(cfs)


def test_getitems_errors(tmpdir):
    tmpdir = str(tmpdir)
    os.makedirs(os.path.join(tmpdir, "afolder"))
    open(os.path.join(tmpdir, "afile"), "w").write("test")
    open(os.path.join(tmpdir, "afolder", "anotherfile"), "w").write("test2")
    m = fsspec.get_mapper(f"file://{tmpdir}")
    assert m.getitems(["afile", "bfile"], on_error="omit") == {"afile": b"test"}

    # my code
    m2 = fsspec.get_mapper(f"simplecache::file://{tmpdir}")
    assert m2.getitems(["afile"], on_error="omit") == {"afile": b"test"}  # works
    assert m2.getitems(["afile", "bfile"], on_error="omit") == {
        "afile": b"test"
    }  # throws KeyError

    with pytest.raises(KeyError):
        m.getitems(["afile", "bfile"])
    out = m.getitems(["afile", "bfile"], on_error="return")
    assert isinstance(out["bfile"], KeyError)
    m = fsspec.get_mapper(f"file://{tmpdir}", missing_exceptions=())
    assert m.getitems(["afile", "bfile"], on_error="omit") == {"afile": b"test"}
    with pytest.raises(FileNotFoundError):
        m.getitems(["afile", "bfile"])


@pytest.mark.parametrize("temp_cache", [False, True])
def test_cache_dir_auto_deleted(temp_cache, tmpdir):
    import gc

    source = os.path.join(tmpdir, "source")
    afile = os.path.join(source, "afile")
    os.mkdir(source)
    open(afile, "w").write("test")

    fs = fsspec.filesystem(
        "filecache",
        target_protocol="file",
        cache_storage="TMP" if temp_cache else os.path.join(tmpdir, "cache"),
        skip_instance_cache=True,  # Important to avoid fs itself being cached
    )

    cache_dir = fs.storage[-1]

    # Force cache to be created
    with fs.open(afile, "rb") as f:
        assert f.read(5) == b"test"

    # Confirm cache exists
    local = fsspec.filesystem("file")
    assert local.exists(cache_dir)

    # Delete file system
    del fs
    gc.collect()

    # Ensure cache has been deleted, if it is temporary
    if temp_cache:
        assert not local.exists(cache_dir)
    else:
        assert local.exists(cache_dir)


@pytest.mark.parametrize("protocol", ["filecache", "blockcache", "simplecache"])
def test_cache_size(tmpdir, protocol):
    if win and protocol == "blockcache":
        pytest.skip("Windows file locking affects blockcache size tests")

    source = os.path.join(tmpdir, "source")
    afile = os.path.join(source, "afile")
    os.mkdir(source)
    open(afile, "w").write("test")

    fs = fsspec.filesystem(protocol, target_protocol="file")
    empty_cache_size = fs.cache_size()

    # Create cache
    with fs.open(afile, "rb") as f:
        assert f.read(5) == b"test"
    single_file_cache_size = fs.cache_size()
    assert single_file_cache_size > empty_cache_size

    # Remove cached file but leave cache metadata file
    fs.pop_from_cache(afile)
    if win and protocol == "filecache":
        empty_cache_size < fs.cache_size()
    elif protocol != "simplecache":
        assert empty_cache_size < fs.cache_size() < single_file_cache_size
    else:
        # simplecache never stores metadata
        assert fs.cache_size() == single_file_cache_size

    # Completely remove cache
    fs.clear_cache()
    if protocol != "simplecache":
        assert fs.cache_size() == empty_cache_size
    else:
        # Whole cache directory has been deleted
        assert fs.cache_size() == 0


def test_spurious_directory_issue1410(tmpdir):
    import zipfile

    os.chdir(tmpdir)
    zipfile.ZipFile("dir.zip", mode="w").open("file.txt", "w").write(b"hello")
    fs = WholeFileCacheFileSystem(fs=ZipFileSystem("dir.zip"))

    assert len(os.listdir()) == 1
    with fs.open("/file.txt", "rb"):
        pass

    # There was a bug reported in issue #1410 in which a directory
    # would be created and the next assertion would fail.
    assert len(os.listdir()) == 1
    assert fs._parent("/any/path") == "any"  # correct for ZIP, which has no leading /


def test_write_transaction(tmpdir, m, monkeypatch):
    called = [0]
    orig = m.put

    def patched_put(*args, **kwargs):
        called[0] += 1
        orig(*args, **kwargs)

    monkeypatch.setattr(m, "put", patched_put)
    tmpdir = str(tmpdir)
    fs, _ = fsspec.core.url_to_fs("simplecache::memory://", cache_storage=tmpdir)
    with fs.transaction:
        fs.pipe("myfile", b"1")
        fs.pipe("otherfile", b"2")
        fs.pipe("deep/dir/otherfile", b"3")
        with fs.open("blarh", "wb") as f:
            f.write(b"ff")
        assert not m.find("")

        assert fs.info("otherfile")["size"] == 1
        assert fs.info("deep")["type"] == "directory"
        assert fs.isdir("deep")
        assert fs.ls("deep", detail=False) == ["/deep/dir"]

    assert m.cat("myfile") == b"1"
    assert m.cat("otherfile") == b"2"
    assert called[0] == 1  # copy was done in one go