myenv/Lib/site-packages/fsspec/implementations/tests/test_ftp.py

import os
import subprocess
import sys
import time

import pytest

import fsspec
from fsspec import open_files
from fsspec.implementations.ftp import FTPFileSystem

ftplib = pytest.importorskip("ftplib")
here = os.path.dirname(os.path.abspath(__file__))


@pytest.fixture()
def ftp():
    pytest.importorskip("pyftpdlib")
    P = subprocess.Popen(
        [sys.executable, "-m", "pyftpdlib", "-d", here],
        stderr=subprocess.STDOUT,
        stdout=subprocess.PIPE,
    )
    try:
        time.sleep(1)
        yield "localhost", 2121
    finally:
        P.terminate()
        P.wait()


def test_basic(ftp):
    host, port = ftp
    fs = FTPFileSystem(host, port)
    assert fs.ls("/", detail=False) == sorted(os.listdir(here))
    out = fs.cat(f"/{os.path.basename(__file__)}")
    assert out == open(__file__, "rb").read()


def test_not_cached(ftp):
    host, port = ftp
    fs = FTPFileSystem(host, port)
    fs2 = FTPFileSystem(host, port)
    assert fs is not fs2


@pytest.mark.parametrize("cache_type", ["bytes", "mmap"])
def test_complex(ftp_writable, cache_type):
    from fsspec.core import BytesCache

    host, port, user, pw = ftp_writable
    files = open_files(
        "ftp:///ou*",
        host=host,
        port=port,
        username=user,
        password=pw,
        block_size=10000,
        cache_type=cache_type,
    )
    assert len(files) == 1
    with files[0] as fo:
        assert fo.read(10) == b"hellohello"
        if isinstance(fo.cache, BytesCache):
            assert len(fo.cache.cache) == 10010
        assert fo.read(2) == b"he"
        assert fo.tell() == 12


def test_write_small(ftp_writable):
    host, port, user, pw = ftp_writable
    fs = FTPFileSystem(host, port, user, pw)
    with fs.open("/out2", "wb") as f:
        f.write(b"oi")
    assert fs.cat("/out2") == b"oi"


def test_with_url(ftp_writable):
    host, port, user, pw = ftp_writable
    fo = fsspec.open(f"ftp://{user}:{pw}@{host}:{port}/out", "wb")
    with fo as f:
        f.write(b"hello")
    fo = fsspec.open(f"ftp://{user}:{pw}@{host}:{port}/out", "rb")
    with fo as f:
        assert f.read() == b"hello"


@pytest.mark.parametrize("cache_type", ["bytes", "mmap"])
def test_write_big(ftp_writable, cache_type):
    host, port, user, pw = ftp_writable
    fs = FTPFileSystem(host, port, user, pw, block_size=1000, cache_type=cache_type)
    fn = "/bigger"
    with fs.open(fn, "wb") as f:
        f.write(b"o" * 500)
        assert not fs.exists(fn)
        f.write(b"o" * 1000)
        fs.invalidate_cache()
        assert fs.exists(fn)
        f.write(b"o" * 200)
        f.flush()

    assert fs.info(fn)["size"] == 1700
    assert fs.cat(fn) == b"o" * 1700


def test_transaction(ftp_writable):
    host, port, user, pw = ftp_writable
    fs = FTPFileSystem(host, port, user, pw)
    fs.mkdir("/tmp")
    fn = "/tr"
    with fs.transaction:
        with fs.open(fn, "wb") as f:
            f.write(b"not")
        assert not fs.exists(fn)
    assert fs.exists(fn)
    assert fs.cat(fn) == b"not"

    fs.rm(fn)
    assert not fs.exists(fn)


def test_transaction_with_cache(ftp_writable, tmpdir):
    host, port, user, pw = ftp_writable
    fs = FTPFileSystem(host, port, user, pw)
    fs.mkdir("/tmp")
    fs.mkdir("/tmp/dir")
    assert "dir" in fs.ls("/tmp", detail=False)

    with fs.transaction:
        fs.rmdir("/tmp/dir")

    assert "dir" not in fs.ls("/tmp", detail=False)
    assert not fs.exists("/tmp/dir")


def test_cat_get(ftp_writable, tmpdir):
    host, port, user, pw = ftp_writable
    fs = FTPFileSystem(host, port, user, pw, block_size=500)
    fs.mkdir("/tmp")
    data = b"hello" * 500
    fs.pipe("/tmp/myfile", data)
    assert fs.cat_file("/tmp/myfile") == data

    fn = os.path.join(tmpdir, "lfile")
    fs.get_file("/tmp/myfile", fn)
    assert open(fn, "rb").read() == data


def test_mkdir(ftp_writable):
    host, port, user, pw = ftp_writable
    fs = FTPFileSystem(host, port, user, pw)
    with pytest.raises(ftplib.error_perm):
        fs.mkdir("/tmp/not/exist", create_parents=False)
    fs.mkdir("/tmp/not/exist")
    assert fs.exists("/tmp/not/exist")
    fs.makedirs("/tmp/not/exist", exist_ok=True)
    with pytest.raises(FileExistsError):
        fs.makedirs("/tmp/not/exist", exist_ok=False)
    fs.makedirs("/tmp/not/exist/inner/inner")
    assert fs.isdir("/tmp/not/exist/inner/inner")


def test_rm_get_recursive(ftp_writable, tmpdir):
    tmpdir = str(tmpdir)
    host, port, user, pw = ftp_writable
    fs = FTPFileSystem(host, port, user, pw)
    fs.mkdir("/tmp/topdir")
    fs.mkdir("/tmp/topdir/underdir")
    fs.touch("/tmp/topdir/afile")
    fs.touch("/tmp/topdir/underdir/afile")

    fs.get("/tmp/topdir", tmpdir, recursive=True)

    with pytest.raises(ftplib.error_perm):
        fs.rmdir("/tmp/topdir")

    fs.rm("/tmp/topdir", recursive=True)
    assert not fs.exists("/tmp/topdir")