Source code for fsspec.implementations.ftp

import os
import sys
import uuid
import warnings
from ftplib import FTP, FTP_TLS, Error, error_perm
from typing import Any

from ..spec import AbstractBufferedFile, AbstractFileSystem
from ..utils import infer_storage_options, isfilelike


[docs] class FTPFileSystem(AbstractFileSystem): """A filesystem over classic FTP""" root_marker = "/" cachable = False protocol = "ftp"
[docs] def __init__( self, host, port=21, username=None, password=None, acct=None, block_size=None, tempdir=None, timeout=30, encoding="utf-8", tls=False, **kwargs, ): """ You can use _get_kwargs_from_urls to get some kwargs from a reasonable FTP url. Authentication will be anonymous if username/password are not given. Parameters ---------- host: str The remote server name/ip to connect to port: int Port to connect with username: str or None If authenticating, the user's identifier password: str of None User's password on the server, if using acct: str or None Some servers also need an "account" string for auth block_size: int or None If given, the read-ahead or write buffer size. tempdir: str Directory on remote to put temporary files when in a transaction timeout: int Timeout of the ftp connection in seconds encoding: str Encoding to use for directories and filenames in FTP connection tls: bool Use FTP-TLS, by default False """ super().__init__(**kwargs) self.host = host self.port = port self.tempdir = tempdir or "/tmp" self.cred = username or "", password or "", acct or "" self.timeout = timeout self.encoding = encoding if block_size is not None: self.blocksize = block_size else: self.blocksize = 2**16 self.tls = tls self._connect() if self.tls: self.ftp.prot_p()
def _connect(self): if self.tls: ftp_cls = FTP_TLS else: ftp_cls = FTP if sys.version_info >= (3, 9): self.ftp = ftp_cls(timeout=self.timeout, encoding=self.encoding) elif self.encoding: warnings.warn("`encoding` not supported for python<3.9, ignoring") self.ftp = ftp_cls(timeout=self.timeout) else: self.ftp = ftp_cls(timeout=self.timeout) self.ftp.connect(self.host, self.port) self.ftp.login(*self.cred) @classmethod def _strip_protocol(cls, path): return "/" + infer_storage_options(path)["path"].lstrip("/").rstrip("/") @staticmethod def _get_kwargs_from_urls(urlpath): out = infer_storage_options(urlpath) out.pop("path", None) out.pop("protocol", None) return out def ls(self, path, detail=True, **kwargs): path = self._strip_protocol(path) out = [] if path not in self.dircache: try: try: out = [ (fn, details) for (fn, details) in self.ftp.mlsd(path) if fn not in [".", ".."] and details["type"] not in ["pdir", "cdir"] ] except error_perm: out = _mlsd2(self.ftp, path) # Not platform independent for fn, details in out: details["name"] = "/".join( ["" if path == "/" else path, fn.lstrip("/")] ) if details["type"] == "file": details["size"] = int(details["size"]) else: details["size"] = 0 if details["type"] == "dir": details["type"] = "directory" self.dircache[path] = out except Error: try: info = self.info(path) if info["type"] == "file": out = [(path, info)] except (Error, IndexError) as exc: raise FileNotFoundError(path) from exc files = self.dircache.get(path, out) if not detail: return sorted([fn for fn, details in files]) return [details for fn, details in files] def info(self, path, **kwargs): # implement with direct method path = self._strip_protocol(path) if path == "/": # special case, since this dir has no real entry return {"name": "/", "size": 0, "type": "directory"} files = self.ls(self._parent(path).lstrip("/"), True) try: out = next(f for f in files if f["name"] == path) except StopIteration as exc: raise FileNotFoundError(path) from exc return out def get_file(self, rpath, lpath, **kwargs): if self.isdir(rpath): if not os.path.exists(lpath): os.mkdir(lpath) return if isfilelike(lpath): outfile = lpath else: outfile = open(lpath, "wb") def cb(x): outfile.write(x) self.ftp.retrbinary( f"RETR {rpath}", blocksize=self.blocksize, callback=cb, ) if not isfilelike(lpath): outfile.close() def cat_file(self, path, start=None, end=None, **kwargs): if end is not None: return super().cat_file(path, start, end, **kwargs) out = [] def cb(x): out.append(x) try: self.ftp.retrbinary( f"RETR {path}", blocksize=self.blocksize, rest=start, callback=cb, ) except (Error, error_perm) as orig_exc: raise FileNotFoundError(path) from orig_exc return b"".join(out) def _open( self, path, mode="rb", block_size=None, cache_options=None, autocommit=True, **kwargs, ): path = self._strip_protocol(path) block_size = block_size or self.blocksize return FTPFile( self, path, mode=mode, block_size=block_size, tempdir=self.tempdir, autocommit=autocommit, cache_options=cache_options, ) def _rm(self, path): path = self._strip_protocol(path) self.ftp.delete(path) self.invalidate_cache(self._parent(path)) def rm(self, path, recursive=False, maxdepth=None): paths = self.expand_path(path, recursive=recursive, maxdepth=maxdepth) for p in reversed(paths): if self.isfile(p): self.rm_file(p) else: self.rmdir(p) def mkdir(self, path: str, create_parents: bool = True, **kwargs: Any) -> None: path = self._strip_protocol(path) parent = self._parent(path) if parent != self.root_marker and not self.exists(parent) and create_parents: self.mkdir(parent, create_parents=create_parents) self.ftp.mkd(path) self.invalidate_cache(self._parent(path)) def makedirs(self, path: str, exist_ok: bool = False) -> None: path = self._strip_protocol(path) if self.exists(path): # NB: "/" does not "exist" as it has no directory entry if not exist_ok: raise FileExistsError(f"{path} exists without `exist_ok`") # exists_ok=True -> no-op else: self.mkdir(path, create_parents=True) def rmdir(self, path): path = self._strip_protocol(path) self.ftp.rmd(path) self.invalidate_cache(self._parent(path)) def mv(self, path1, path2, **kwargs): path1 = self._strip_protocol(path1) path2 = self._strip_protocol(path2) self.ftp.rename(path1, path2) self.invalidate_cache(self._parent(path1)) self.invalidate_cache(self._parent(path2)) def __del__(self): self.ftp.close() def invalidate_cache(self, path=None): if path is None: self.dircache.clear() else: self.dircache.pop(path, None) super().invalidate_cache(path)
class TransferDone(Exception): """Internal exception to break out of transfer""" pass class FTPFile(AbstractBufferedFile): """Interact with a remote FTP file with read/write buffering""" def __init__( self, fs, path, mode="rb", block_size="default", autocommit=True, cache_type="readahead", cache_options=None, **kwargs, ): super().__init__( fs, path, mode=mode, block_size=block_size, autocommit=autocommit, cache_type=cache_type, cache_options=cache_options, **kwargs, ) if not autocommit: self.target = self.path self.path = "/".join([kwargs["tempdir"], str(uuid.uuid4())]) def commit(self): self.fs.mv(self.path, self.target) def discard(self): self.fs.rm(self.path) def _fetch_range(self, start, end): """Get bytes between given byte limits Implemented by raising an exception in the fetch callback when the number of bytes received reaches the requested amount. Will fail if the server does not respect the REST command on retrieve requests. """ out = [] total = [0] def callback(x): total[0] += len(x) if total[0] > end - start: out.append(x[: (end - start) - total[0]]) if end < self.size: raise TransferDone else: out.append(x) if total[0] == end - start and end < self.size: raise TransferDone try: self.fs.ftp.retrbinary( f"RETR {self.path}", blocksize=self.blocksize, rest=start, callback=callback, ) except TransferDone: try: # stop transfer, we got enough bytes for this block self.fs.ftp.abort() self.fs.ftp.getmultiline() except Error: self.fs._connect() return b"".join(out) def _upload_chunk(self, final=False): self.buffer.seek(0) self.fs.ftp.storbinary( f"STOR {self.path}", self.buffer, blocksize=self.blocksize, rest=self.offset ) return True def _mlsd2(ftp, path="."): """ Fall back to using `dir` instead of `mlsd` if not supported. This parses a Linux style `ls -l` response to `dir`, but the response may be platform dependent. Parameters ---------- ftp: ftplib.FTP path: str Expects to be given path, but defaults to ".". """ lines = [] minfo = [] ftp.dir(path, lines.append) for line in lines: split_line = line.split() if len(split_line) < 9: continue this = ( split_line[-1], { "modify": " ".join(split_line[5:8]), "unix.owner": split_line[2], "unix.group": split_line[3], "unix.mode": split_line[0], "size": split_line[4], }, ) if "d" == this[1]["unix.mode"][0]: this[1]["type"] = "dir" else: this[1]["type"] = "file" minfo.append(this) return minfo