From cc8efbaf13d57021bee8d4b5bd8a1e46ebe54730 Mon Sep 17 00:00:00 2001 From: Rose Hogenson Date: Fri, 11 Apr 2025 21:26:04 -0700 Subject: Add documentation --- internal/cp/cp.go | 102 ++++++++++++++++++++++++++++++------------ internal/wfs/osfs/osfs.go | 2 + internal/wfs/sftpfs/sftpfs.go | 21 ++++++++- internal/wfs/wfs.go | 84 +++++++++++++++++++++++++--------- 4 files changed, 160 insertions(+), 49 deletions(-) (limited to 'internal') diff --git a/internal/cp/cp.go b/internal/cp/cp.go index 7350343..66313c7 100644 --- a/internal/cp/cp.go +++ b/internal/cp/cp.go @@ -1,3 +1,5 @@ +// Package cp implements a concurrent file copy over the abstract [wfs.FS] +// interface. It reports progress and errors using the [Progress] interface. package cp import ( @@ -13,14 +15,26 @@ import ( "gitlab.com/rhogenson/ccp/internal/wfs/sftpfs" ) +// Progress is used to asynchronously report status updates and errors to the +// main program. type Progress interface { + // Max sets the total number of bytes to be copied. It's expected that + // this will only be called once in the program lifetime. Max(int64) - Progress(int64) - FileStart(string, string) - FileDone(string, error) + // Progress reports that n additional bytes have been copied. + Progress(n int64) + // FileStart reports that src is currently being copied to dst. Only + // called for regular files, not directories or symlinks. + FileStart(src, dst string) + // FileDone is called when a regular file has finished copying + // successfully, or when there was an error copying a file. + FileDone(src string, err error) } +// An FSPath is an abstraction over a file path that can point to multiple +// different backing filesystems. type FSPath struct { + // FS is the backing file system where Path is valid. FS wfs.FS Path string } @@ -32,54 +46,59 @@ func (p FSPath) String() string { return p.Path } -func (p FSPath) WalkDir(fn fs.WalkDirFunc) error { +// These helper functions are useful to prevent mismatches between filesystem +// and path. For example it's too easy to write +// +// src.FS.Open(dst.Path) + +func (p FSPath) walkDir(fn fs.WalkDirFunc) error { return fs.WalkDir(p.FS, p.Path, fn) } -func (p FSPath) Stat() (fs.FileInfo, error) { +func (p FSPath) stat() (fs.FileInfo, error) { return fs.Stat(p.FS, p.Path) } -func (p FSPath) Lstat() (fs.FileInfo, error) { +func (p FSPath) lstat() (fs.FileInfo, error) { return wfs.Lstat(p.FS, p.Path) } -func (p FSPath) RemoveAll() error { +func (p FSPath) removeAll() error { return wfs.RemoveAll(p.FS, p.Path) } -func (p FSPath) Open() (fs.File, error) { +func (p FSPath) open() (fs.File, error) { return p.FS.Open(p.Path) } -func (p FSPath) Create(mode fs.FileMode) (io.WriteCloser, error) { +func (p FSPath) create(mode fs.FileMode) (io.WriteCloser, error) { return p.FS.Create(p.Path, mode) } -func (p FSPath) ReadLink() (string, error) { +func (p FSPath) readLink() (string, error) { return wfs.ReadLink(p.FS, p.Path) } -func (p FSPath) SymlinkFrom(target string) error { +func (p FSPath) symlinkFrom(target string) error { return p.FS.Symlink(target, p.Path) } -func (p FSPath) Mkdir() error { +func (p FSPath) mkdir() error { return p.FS.Mkdir(p.Path) } -func (p FSPath) MkdirMode(mode fs.FileMode) error { +func (p FSPath) mkdirMode(mode fs.FileMode) error { return wfs.MkdirMode(p.FS, p.Path, mode) } -func (p FSPath) Chmod(mode fs.FileMode) error { +func (p FSPath) chmod(mode fs.FileMode) error { return p.FS.Chmod(p.Path, mode) } func size(srcs []FSPath) int64 { var n int64 = 0 for _, src := range srcs { - src.WalkDir(func(_ string, d fs.DirEntry, err error) error { + src.walkDir(func(_ string, d fs.DirEntry, err error) error { if err != nil { return nil } @@ -89,7 +108,9 @@ func size(srcs []FSPath) int64 { if err != nil { return nil } - n += 1 + stat.Size() + // The "+ 1" is a fudge factor to make sure that + // the total number of bytes won't be zero. + n += stat.Size() + 1 case fs.ModeSymlink, fs.ModeDir: n++ } @@ -100,7 +121,7 @@ func size(srcs []FSPath) int64 { } func (p FSPath) exists() bool { - _, err := p.Lstat() + _, err := p.lstat() return !errors.Is(err, fs.ErrNotExist) } @@ -113,7 +134,7 @@ func (c *copier) openWithRetry(path FSPath, fn func() error) error { if err := fn(); err == nil || !c.force || !path.exists() { return err } - if err := path.RemoveAll(); err != nil { + if err := path.removeAll(); err != nil { return err } return fn() @@ -122,7 +143,7 @@ func (c *copier) openWithRetry(path FSPath, fn func() error) error { func (c *copier) copyRegularFile(src, dst FSPath) error { c.p.FileStart(src.String(), dst.String()) - in, err := src.Open() + in, err := src.open() if err != nil { return err } @@ -134,12 +155,14 @@ func (c *copier) copyRegularFile(src, dst FSPath) error { var out io.WriteCloser if err := c.openWithRetry(dst, func() error { var err error - out, err = dst.Create(stat.Mode().Perm()) + out, err = dst.create(stat.Mode().Perm()) return err }); err != nil { return err } for { + // io.CopyN will use cool stuff like copy_file_range as long as + // the underlying types are *os.File n, err := io.CopyN(out, in, 1024*1024) if n > 0 { c.p.Progress(n) @@ -161,12 +184,12 @@ func (c *copier) copyRegularFile(src, dst FSPath) error { } func (c *copier) copySymlink(src FSPath, dst FSPath) error { - target, err := src.ReadLink() + target, err := src.readLink() if err != nil { return err } if err := c.openWithRetry(dst, func() error { - return dst.SymlinkFrom(target) + return dst.symlinkFrom(target) }); err != nil { return err } @@ -174,6 +197,9 @@ func (c *copier) copySymlink(src FSPath, dst FSPath) error { return nil } +// Copy copies srcs into dstRoot, reporting progress using the [Progress] +// interface. If force is specified and an existing destination file cannot be +// opened, Copy will remove it and try again. func Copy(progress Progress, srcs []FSPath, dstRoot FSPath, force bool) { done := make(chan struct{}) go func() { @@ -184,7 +210,7 @@ func Copy(progress Progress, srcs []FSPath, dstRoot FSPath, force bool) { dstIsDir := true if len(srcs) == 1 { - stat, err := dstRoot.Stat() + stat, err := dstRoot.stat() dstIsDir = err == nil && stat.IsDir() } @@ -200,13 +226,20 @@ func Copy(progress Progress, srcs []FSPath, dstRoot FSPath, force bool) { mode fs.FileMode } var roDirs []roDir + dstRoot.Path = path.Clean(dstRoot.Path) for _, srcRoot := range srcs { dstRoot := dstRoot if dstIsDir { + // If the destination is a directory, copy into the + // existing directory. dstRoot.Path = path.Join(dstRoot.Path, path.Base(srcRoot.Path)) } srcRoot.Path = path.Clean(srcRoot.Path) - srcRoot.WalkDir(func(srcPath string, d fs.DirEntry, err error) error { + if srcRoot == dstRoot { + progress.FileDone(srcRoot.String(), fmt.Errorf("%q and %q are the same file", srcRoot, dstRoot)) + continue + } + srcRoot.walkDir(func(srcPath string, d fs.DirEntry, err error) error { src := FSPath{srcRoot.FS, srcPath} dst := FSPath{dstRoot.FS, path.Join(dstRoot.Path, strings.TrimPrefix(srcPath, srcRoot.Path))} if err != nil { @@ -214,7 +247,7 @@ func Copy(progress Progress, srcs []FSPath, dstRoot FSPath, force bool) { return nil } switch d.Type() { - case 0: + case 0: // regular file sem <- struct{}{} go func() { defer func() { <-sem }() @@ -222,6 +255,7 @@ func Copy(progress Progress, srcs []FSPath, dstRoot FSPath, force bool) { progress.FileDone(src.String(), err) } }() + case fs.ModeDir: stat, err := d.Info() if err != nil { @@ -231,9 +265,18 @@ func Copy(progress Progress, srcs []FSPath, dstRoot FSPath, force bool) { hasWritePerm := stat.Mode()&0300 == 0300 if err := c.openWithRetry(dst, func() error { if hasWritePerm { - return dst.MkdirMode(stat.Mode().Perm()) + return dst.mkdirMode(stat.Mode().Perm()) } else { - return dst.Mkdir() + // If a directory doesn't have + // write permissions, we won't + // be able to create any files + // inside of it if we create it + // with the correct permissions + // now. So instead create it + // with some default + // permissions, and append it to + // roDirs to be processed later. + return dst.mkdir() } }); err != nil { progress.FileDone(src.String(), err) @@ -254,11 +297,14 @@ func Copy(progress Progress, srcs []FSPath, dstRoot FSPath, force bool) { return nil }) } + // Wait for all jobs to complete. for range maxConcurrency { sem <- struct{}{} } + // Iterate backwards so that directory contents are processed before the + // parent directory itself. for _, d := range slices.Backward(roDirs) { - if err := d.path.Chmod(d.mode); err != nil { + if err := d.path.chmod(d.mode); err != nil { progress.FileDone(d.path.String(), err) continue } diff --git a/internal/wfs/osfs/osfs.go b/internal/wfs/osfs/osfs.go index bf6951c..c879f85 100644 --- a/internal/wfs/osfs/osfs.go +++ b/internal/wfs/osfs/osfs.go @@ -1,3 +1,4 @@ +// Package osfs implements [wfs.FS] backed by the local filesystem. package osfs import ( @@ -15,6 +16,7 @@ var ( _ fs.StatFS = FS{} ) +// An FS is a [wfs.FS] backed by the local filesystem. type FS struct{} func (FS) Open(name string) (fs.File, error) { diff --git a/internal/wfs/sftpfs/sftpfs.go b/internal/wfs/sftpfs/sftpfs.go index d7f0cdc..2ec9635 100644 --- a/internal/wfs/sftpfs/sftpfs.go +++ b/internal/wfs/sftpfs/sftpfs.go @@ -1,3 +1,4 @@ +// Package sftpfs implements [wfs.FS] using [github.com/pkg/sftp]. package sftpfs import ( @@ -24,10 +25,11 @@ var ( _ wfs.FS = (*FS)(nil) _ wfs.ReadLinkFS = (*FS)(nil) _ fs.StatFS = (*FS)(nil) - _ fs.StatFS = (*FS)(nil) _ fs.ReadDirFS = (*FS)(nil) ) +// An FS holds an SFTP connection and wraps its operations into the +// [wfs.FS] interface. type FS struct { User, Host string conn *sftp.Client @@ -46,6 +48,14 @@ var sshAgent = sync.OnceValue(func() agent.ExtendedAgent { return agent.NewClient(conn) }) +// sshKeys returns the available ssh public keys. If an ssh agent can be +// contacted with $SSH_AUTH_SOCK, sshKeys uses the keys from the agent if +// possible. Otherwise sshKeys loads keys from ~/.ssh. If there are any password +// protected keys, sshKeys may prompt the user for the password (although it +// will do so at most once). +// +// If a password-protected key is loaded from ~/.ssh, it will be added to the +// ssh agent if possible. func sshKeys() ([]ssh.Signer, error) { sshAgent := sshAgent() if sshAgent != nil { @@ -121,6 +131,7 @@ func appendToKnownHosts(hostname string, key ssh.PublicKey) error { return f.Close() } +// Dial establishes a new SFTP connection to the given host. func Dial(target string) (*FS, error) { knownHostChecker, err := knownhosts.New(path.Join(os.Getenv("HOME"), ".ssh/known_hosts")) if err != nil { @@ -152,6 +163,9 @@ func Dial(target string) (*FS, error) { if !errors.As(err, &keyErr) || len(keyErr.Want) > 0 { return err } + // scp prompts the user if the host is not found in + // known_hosts, but when is that ever useful? We'll just + // add it to known_hosts without bothering the user. appendToKnownHosts(hostname, key) return nil }, @@ -172,6 +186,7 @@ func Dial(target string) (*FS, error) { }, nil } +// Close closes the underlying SFTP connection. func (f *FS) Close() error { sftpErr := f.conn.Close() if err := f.sshConn.Close(); err != nil { @@ -181,9 +196,13 @@ func (f *FS) Close() error { } func (f *FS) err(op, path string, err error) error { + // github.com/pkg/sftp's errors are pretty terrible. + // We'll wrap them to be more similar to the amazing package os errors. return fmt.Errorf("%s %q: %w", op, f.User+"@"+f.Host+":"+path, err) } +// wfs.FS implementation: + func (f *FS) Open(name string) (fs.File, error) { file, err := f.conn.Open(name) if err != nil { diff --git a/internal/wfs/wfs.go b/internal/wfs/wfs.go index bcdca00..4168b58 100644 --- a/internal/wfs/wfs.go +++ b/internal/wfs/wfs.go @@ -1,11 +1,14 @@ +// Package wfs implements a "writable file system" in the spirit of [fs.FS]. package wfs import ( + "errors" "io" "io/fs" "path" ) +// ReadLinkFS is backported from the latest go master. type ReadLinkFS interface { fs.FS @@ -13,6 +16,9 @@ type ReadLinkFS interface { Lstat(string) (fs.FileInfo, error) } +// ReadLink returns the destination of the named symbolic link. +// +// If fsys does not implement [ReadLinkFS], then ReadLink returns an error. func ReadLink(fsys fs.FS, name string) (string, error) { sym, ok := fsys.(ReadLinkFS) if !ok { @@ -21,6 +27,12 @@ func ReadLink(fsys fs.FS, name string) (string, error) { return sym.ReadLink(name) } +// Lstat returns an [fs.FileInfo] describing the named file. +// If the file is a symbolic link, the returned [fs.FileInfo] describes the +// symbolic link. Lstat makes no attempt to follow the link. +// +// If fsys does not implement [ReadLinkFS], then Lstat is identical +// to [fs.Stat]. func Lstat(fsys fs.FS, name string) (fs.FileInfo, error) { sym, ok := fsys.(ReadLinkFS) if !ok { @@ -29,6 +41,7 @@ func Lstat(fsys fs.FS, name string) (fs.FileInfo, error) { return sym.Lstat(name) } +// An FS provides access to a writable hierarchical file system. type FS interface { fs.FS @@ -39,12 +52,16 @@ type FS interface { Chmod(string, fs.FileMode) error } +// A MkdirModeFS is a file system with a mkdir method that accepts a file mode. type MkdirModeFS interface { FS MkdirMode(string, fs.FileMode) error } +// MkdirMode creates a directory with the given file permission. If fsys +// implements [MkdirModeFS], MkdirMode calls fsys.MkdirMode. Otherwise, +// MkdirMode calls Mkdir and then Chmod to set the mode. func MkdirMode(fsys FS, name string, mode fs.FileMode) error { if fsys, ok := fsys.(MkdirModeFS); ok { return fsys.MkdirMode(name, mode) @@ -56,31 +73,58 @@ func MkdirMode(fsys FS, name string, mode fs.FileMode) error { } func removeDir(fsys FS, dir string) error { - entries, err := fs.ReadDir(fsys, dir) - if err != nil { - return err - } - for _, f := range entries { - name := path.Join(dir, f.Name()) - if f.IsDir() { - err = removeDir(fsys, name) - } else { - err = fsys.Remove(name) - } - if err != nil { - return err + entries, readErr := fs.ReadDir(fsys, dir) + var err error + for _, d := range entries { + if err1 := removeAll(fsys, path.Join(dir, d.Name()), d); err == nil { + err = err1 } } - return fsys.Remove(dir) + if err == nil { + err = readErr + } + err1 := fsys.Remove(dir) + if err1 == nil || errors.Is(err, fs.ErrNotExist) { + return nil + } + if err == nil { + err = err1 + } + return err } -func RemoveAll(fsys FS, path string) error { - stat, err := Lstat(fsys, path) - if err != nil { +func removeAll(fsys FS, path string, d fs.DirEntry) error { + err := fsys.Remove(path) + if err == nil || errors.Is(err, fs.ErrNotExist) { + return nil + } + if !d.IsDir() { return err } - if stat.IsDir() { - return removeDir(fsys, path) + return removeDir(fsys, path) +} + +// RemoveAll removes path and any children it contains. It removes everything it +// can but returns the first error it encounters. If the path does not exist, +// RemoveAll returns nil (no error). +func RemoveAll(fsys FS, path string) error { + // Simple case: if Remove works, we're done. + err := fsys.Remove(path) + if err == nil || errors.Is(err, fs.ErrNotExist) { + return nil + } + + // Otherwise, is this a directory we need to recurse into? + dir, serr := Lstat(fsys, path) + if serr != nil { + if errors.Is(serr, fs.ErrNotExist) { + return nil + } + return serr + } + if !dir.IsDir() { + // Not a directory; return the error from Remove. + return err } - return fsys.Remove(path) + return removeDir(fsys, path) } -- cgit v1.3.1