Spaces:
Runtime error
Runtime error
File size: 4,877 Bytes
63deadc |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 |
import os
import subprocess
import sys
import time
import pytest
import fsspec
from fsspec import open_files
from fsspec.implementations.ftp import FTPFileSystem
ftplib = pytest.importorskip("ftplib")
here = os.path.dirname(os.path.abspath(__file__))
@pytest.fixture()
def ftp():
pytest.importorskip("pyftpdlib")
P = subprocess.Popen(
[sys.executable, "-m", "pyftpdlib", "-d", here],
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
)
try:
time.sleep(1)
yield "localhost", 2121
finally:
P.terminate()
P.wait()
def test_basic(ftp):
host, port = ftp
fs = FTPFileSystem(host, port)
assert fs.ls("/", detail=False) == sorted(os.listdir(here))
out = fs.cat(f"/{os.path.basename(__file__)}")
assert out == open(__file__, "rb").read()
def test_not_cached(ftp):
host, port = ftp
fs = FTPFileSystem(host, port)
fs2 = FTPFileSystem(host, port)
assert fs is not fs2
@pytest.mark.parametrize("cache_type", ["bytes", "mmap"])
def test_complex(ftp_writable, cache_type):
from fsspec.core import BytesCache
host, port, user, pw = ftp_writable
files = open_files(
"ftp:///ou*",
host=host,
port=port,
username=user,
password=pw,
block_size=10000,
cache_type=cache_type,
)
assert len(files) == 1
with files[0] as fo:
assert fo.read(10) == b"hellohello"
if isinstance(fo.cache, BytesCache):
assert len(fo.cache.cache) == 10010
assert fo.read(2) == b"he"
assert fo.tell() == 12
def test_write_small(ftp_writable):
host, port, user, pw = ftp_writable
fs = FTPFileSystem(host, port, user, pw)
with fs.open("/out2", "wb") as f:
f.write(b"oi")
assert fs.cat("/out2") == b"oi"
def test_with_url(ftp_writable):
host, port, user, pw = ftp_writable
fo = fsspec.open(f"ftp://{user}:{pw}@{host}:{port}/out", "wb")
with fo as f:
f.write(b"hello")
fo = fsspec.open(f"ftp://{user}:{pw}@{host}:{port}/out", "rb")
with fo as f:
assert f.read() == b"hello"
@pytest.mark.parametrize("cache_type", ["bytes", "mmap"])
def test_write_big(ftp_writable, cache_type):
host, port, user, pw = ftp_writable
fs = FTPFileSystem(host, port, user, pw, block_size=1000, cache_type=cache_type)
fn = "/bigger"
with fs.open(fn, "wb") as f:
f.write(b"o" * 500)
assert not fs.exists(fn)
f.write(b"o" * 1000)
fs.invalidate_cache()
assert fs.exists(fn)
f.write(b"o" * 200)
f.flush()
assert fs.info(fn)["size"] == 1700
assert fs.cat(fn) == b"o" * 1700
def test_transaction(ftp_writable):
host, port, user, pw = ftp_writable
fs = FTPFileSystem(host, port, user, pw)
fs.mkdir("/tmp")
fn = "/tr"
with fs.transaction:
with fs.open(fn, "wb") as f:
f.write(b"not")
assert not fs.exists(fn)
assert fs.exists(fn)
assert fs.cat(fn) == b"not"
fs.rm(fn)
assert not fs.exists(fn)
def test_transaction_with_cache(ftp_writable, tmpdir):
host, port, user, pw = ftp_writable
fs = FTPFileSystem(host, port, user, pw)
fs.mkdir("/tmp")
fs.mkdir("/tmp/dir")
assert "dir" in fs.ls("/tmp", detail=False)
with fs.transaction:
fs.rmdir("/tmp/dir")
assert "dir" not in fs.ls("/tmp", detail=False)
assert not fs.exists("/tmp/dir")
def test_cat_get(ftp_writable, tmpdir):
host, port, user, pw = ftp_writable
fs = FTPFileSystem(host, port, user, pw, block_size=500)
fs.mkdir("/tmp")
data = b"hello" * 500
fs.pipe("/tmp/myfile", data)
assert fs.cat_file("/tmp/myfile") == data
fn = os.path.join(tmpdir, "lfile")
fs.get_file("/tmp/myfile", fn)
assert open(fn, "rb").read() == data
def test_mkdir(ftp_writable):
host, port, user, pw = ftp_writable
fs = FTPFileSystem(host, port, user, pw)
with pytest.raises(ftplib.error_perm):
fs.mkdir("/tmp/not/exist", create_parents=False)
fs.mkdir("/tmp/not/exist")
assert fs.exists("/tmp/not/exist")
fs.makedirs("/tmp/not/exist", exist_ok=True)
with pytest.raises(FileExistsError):
fs.makedirs("/tmp/not/exist", exist_ok=False)
fs.makedirs("/tmp/not/exist/inner/inner")
assert fs.isdir("/tmp/not/exist/inner/inner")
def test_rm_get_recursive(ftp_writable, tmpdir):
tmpdir = str(tmpdir)
host, port, user, pw = ftp_writable
fs = FTPFileSystem(host, port, user, pw)
fs.mkdir("/tmp/topdir")
fs.mkdir("/tmp/topdir/underdir")
fs.touch("/tmp/topdir/afile")
fs.touch("/tmp/topdir/underdir/afile")
fs.get("/tmp/topdir", tmpdir, recursive=True)
with pytest.raises(ftplib.error_perm):
fs.rmdir("/tmp/topdir")
fs.rm("/tmp/topdir", recursive=True)
assert not fs.exists("/tmp/topdir")
|