Google cmd exec.go

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package exec runs external commands. It wraps os.StartProcess to make it
// easier to remap stdin and stdout, connect I/O with pipes, and do other
// adjustments.
//
// Unlike the "system" library call from C and other languages, the
// os/exec package intentionally does not invoke the system shell and
// does not expand any glob patterns or handle other expansions,
// pipelines, or redirections typically done by shells. The package
// behaves more like C's "exec" family of functions. To expand glob
// patterns, either call the shell directly, taking care to escape any
// dangerous input, or use the path/filepath package's Glob function.
// To expand environment variables, use package os's ExpandEnv.
//
// Note that the examples in this package assume a Unix system.
// They may not run on Windows, and they do not run in the Go Playground
// used by golang.org and godoc.org.
package exec

import (
    "bytes"
    "context"
    "errors"
    "internal/syscall/execenv"
    "io"
    "os"
    "path/filepath"
    "runtime"
    "strconv"
    "strings"
    "sync"
    "syscall"
)

// Error is returned by LookPath when it fails to classify a file as an
// executable.
type Error struct {
    // Name is the file name for which the error occurred.
    Name string
    // Err is the underlying error.
    Err error
}

func (e *Error) Error() string {
    return "exec: " + strconv.Quote(e.Name) + ": " + e.Err.Error()
}

func (e *Error) Unwrap() error { return e.Err }

// Cmd represents an external command being prepared or run.
//
// A Cmd cannot be reused after calling its Run, Output or CombinedOutput
// methods.
type Cmd struct {
    // Path is the path of the command to run.
    //
    // This is the only field that must be set to a non-zero
    // value. If Path is relative, it is evaluated relative
    // to Dir.
    Path string

    // Args holds command line arguments, including the command as Args[0].
    // If the Args field is empty or nil, Run uses {Path}.
    //
    // In typical use, both Path and Args are set by calling Command.
    Args []string

    // Env specifies the environment of the process.
    // Each entry is of the form "key=value".
    // If Env is nil, the new process uses the current process's
    // environment.
    // If Env contains duplicate environment keys, only the last
    // value in the slice for each duplicate key is used.
    // As a special case on Windows, SYSTEMROOT is always added if
    // missing and not explicitly set to the empty string.
    Env []string

    // Dir specifies the working directory of the command.
    // If Dir is the empty string, Run runs the command in the
    // calling process's current directory.
    Dir string

    // Stdin specifies the process's standard input.
    //
    // If Stdin is nil, the process reads from the null device (os.DevNull).
    //
    // If Stdin is an *os.File, the process's standard input is connected
    // directly to that file.
    //
    // Otherwise, during the execution of the command a separate
    // goroutine reads from Stdin and delivers that data to the command
    // over a pipe. In this case, Wait does not complete until the goroutine
    // stops copying, either because it has reached the end of Stdin
    // (EOF or a read error) or because writing to the pipe returned an error.
    Stdin io.Reader

    // Stdout and Stderr specify the process's standard output and error.
    //
    // If either is nil, Run connects the corresponding file descriptor
    // to the null device (os.DevNull).
    //
    // If either is an *os.File, the corresponding output from the process
    // is connected directly to that file.
    //
    // Otherwise, during the execution of the command a separate goroutine
    // reads from the process over a pipe and delivers that data to the
    // corresponding Writer. In this case, Wait does not complete until the
    // goroutine reaches EOF or encounters an error.
    //
    // If Stdout and Stderr are the same writer, and have a type that can
    // be compared with ==, at most one goroutine at a time will call Write.
    Stdout io.Writer
    Stderr io.Writer

    // ExtraFiles specifies additional open files to be inherited by the
    // new process. It does not include standard input, standard output, or
    // standard error. If non-nil, entry i becomes file descriptor 3+i.
    //
    // ExtraFiles is not supported on Windows.
    ExtraFiles []*os.File

    // SysProcAttr holds optional, operating system-specific attributes.
    // Run passes it to os.StartProcess as the os.ProcAttr's Sys field.
    SysProcAttr *syscall.SysProcAttr

    // Process is the underlying process, once started.
    Process *os.Process

    // ProcessState contains information about an exited process,
    // available after a call to Wait or Run.
    ProcessState *os.ProcessState

    ctx             context.Context // nil means none
    lookPathErr     error           // LookPath error, if any.
    finished        bool            // when Wait was called
    childFiles      []*os.File
    closeAfterStart []io.Closer
    closeAfterWait  []io.Closer
    goroutine       []func() error
    errch           chan error // one send per goroutine
    waitDone        chan struct{}
}

// Command returns the Cmd struct to execute the named program with
// the given arguments.
//
// It sets only the Path and Args in the returned structure.
//
// If name contains no path separators, Command uses LookPath to
// resolve name to a complete path if possible. Otherwise it uses name
// directly as Path.
//
// The returned Cmd's Args field is constructed from the command name
// followed by the elements of arg, so arg should not include the
// command name itself. For example, Command("echo", "hello").
// Args[0] is always name, not the possibly resolved Path.
//
// On Windows, processes receive the whole command line as a single string
// and do their own parsing. Command combines and quotes Args into a command
// line string with an algorithm compatible with applications using
// CommandLineToArgvW (which is the most common way). Notable exceptions are
// msiexec.exe and cmd.exe (and thus, all batch files), which have a different
// unquoting algorithm. In these or other similar cases, you can do the
// quoting yourself and provide the full command line in SysProcAttr.CmdLine,
// leaving Args empty.
func Command(name string, arg ...string) *Cmd {
    cmd := &Cmd{
        Path: name,
        Args: append([]string{name}, arg...),
    }
    if filepath.Base(name) == name {
        if lp, err := LookPath(name); err != nil {
            cmd.lookPathErr = err
        } else {
            cmd.Path = lp
        }
    }
    return cmd
}

// CommandContext is like Command but includes a context.
//
// The provided context is used to kill the process (by calling
// os.Process.Kill) if the context becomes done before the command
// completes on its own.
func CommandContext(ctx context.Context, name string, arg ...string) *Cmd {
    if ctx == nil {
        panic("nil Context")
    }
    cmd := Command(name, arg...)
    cmd.ctx = ctx
    return cmd
}

// String returns a human-readable description of c.
// It is intended only for debugging.
// In particular, it is not suitable for use as input to a shell.
// The output of String may vary across Go releases.
func (c *Cmd) String() string {
    if c.lookPathErr != nil {
        // failed to resolve path; report the original requested path (plus args)
        return strings.Join(c.Args, " ")
    }
    // report the exact executable path (plus args)
    b := new(strings.Builder)
    b.WriteString(c.Path)
    for _, a := range c.Args[1:] {
        b.WriteByte(' ')
        b.WriteString(a)
    }
    return b.String()
}

// interfaceEqual protects against panics from doing equality tests on
// two interfaces with non-comparable underlying types.
func interfaceEqual(a, b interface{}) bool {
    defer func() {
        recover()
    }()
    return a == b
}

func (c *Cmd) envv() ([]string, error) {
    if c.Env != nil {
        return c.Env, nil
    }
    return execenv.Default(c.SysProcAttr)
}

func (c *Cmd) argv() []string {
    if len(c.Args) > 0 {
        return c.Args
    }
    return []string{c.Path}
}

// skipStdinCopyError optionally specifies a function which reports
// whether the provided stdin copy error should be ignored.
var skipStdinCopyError func(error) bool

func (c *Cmd) stdin() (f *os.File, err error) {
    if c.Stdin == nil {
        f, err = os.Open(os.DevNull)
        if err != nil {
            return
        }
        c.closeAfterStart = append(c.closeAfterStart, f)
        return
    }

    if f, ok := c.Stdin.(*os.File); ok {
        return f, nil
    }

    pr, pw, err := os.Pipe()
    if err != nil {
        return
    }

    c.closeAfterStart = append(c.closeAfterStart, pr)
    c.closeAfterWait = append(c.closeAfterWait, pw)
    c.goroutine = append(c.goroutine, func() error {
        _, err := io.Copy(pw, c.Stdin)
        if skip := skipStdinCopyError; skip != nil && skip(err) {
            err = nil
        }
        if err1 := pw.Close(); err == nil {
            err = err1
        }
        return err
    })
    return pr, nil
}

func (c *Cmd) stdout() (f *os.File, err error) {
    return c.writerDescriptor(c.Stdout)
}

func (c *Cmd) stderr() (f *os.File, err error) {
    if c.Stderr != nil && interfaceEqual(c.Stderr, c.Stdout) {
        return c.childFiles[1], nil
    }
    return c.writerDescriptor(c.Stderr)
}

func (c *Cmd) writerDescriptor(w io.Writer) (f *os.File, err error) {
    if w == nil {
        f, err = os.OpenFile(os.DevNull, os.O_WRONLY, 0)
        if err != nil {
            return
        }
        c.closeAfterStart = append(c.closeAfterStart, f)
        return
    }

    if f, ok := w.(*os.File); ok {
        return f, nil
    }

    pr, pw, err := os.Pipe()
    if err != nil {
        return
    }

    c.closeAfterStart = append(c.closeAfterStart, pw)
    c.closeAfterWait = append(c.closeAfterWait, pr)
    c.goroutine = append(c.goroutine, func() error {
        _, err := io.Copy(w, pr)
        pr.Close() // in case io.Copy stopped due to write error
        return err
    })
    return pw, nil
}

func (c *Cmd) closeDescriptors(closers []io.Closer) {
    for _, fd := range closers {
        fd.Close()
    }
}

// Run starts the specified command and waits for it to complete.
//
// The returned error is nil if the command runs, has no problems
// copying stdin, stdout, and stderr, and exits with a zero exit
// status.
//
// If the command starts but does not complete successfully, the error is of
// type *ExitError. Other error types may be returned for other situations.
//
// If the calling goroutine has locked the operating system thread
// with runtime.LockOSThread and modified any inheritable OS-level
// thread state (for example, Linux or Plan 9 name spaces), the new
// process will inherit the caller's thread state.
func (c *Cmd) Run() error {
    if err := c.Start(); err != nil {
        return err
    }
    return c.Wait()
}

// lookExtensions finds windows executable by its dir and path.
// It uses LookPath to try appropriate extensions.
// lookExtensions does not search PATH, instead it converts `prog` into `.\prog`.
func lookExtensions(path, dir string) (string, error) {
    if filepath.Base(path) == path {
        path = filepath.Join(".", path)
    }
    if dir == "" {
        return LookPath(path)
    }
    if filepath.VolumeName(path) != "" {
        return LookPath(path)
    }
    if len(path) > 1 && os.IsPathSeparator(path[0]) {
        return LookPath(path)
    }
    dirandpath := filepath.Join(dir, path)
    // We assume that LookPath will only add file extension.
    lp, err := LookPath(dirandpath)
    if err != nil {
        return "", err
    }
    ext := strings.TrimPrefix(lp, dirandpath)
    return path + ext, nil
}

// Start starts the specified command but does not wait for it to complete.
//
// If Start returns successfully, the c.Process field will be set.
//
// The Wait method will return the exit code and release associated resources
// once the command exits.
func (c *Cmd) Start() error {
    if c.lookPathErr != nil {
        c.closeDescriptors(c.closeAfterStart)
        c.closeDescriptors(c.closeAfterWait)
        return c.lookPathErr
    }
    if runtime.GOOS == "windows" {
        lp, err := lookExtensions(c.Path, c.Dir)
        if err != nil {
            c.closeDescriptors(c.closeAfterStart)
            c.closeDescriptors(c.closeAfterWait)
            return err
        }
        c.Path = lp
    }
    if c.Process != nil {
        return errors.New("exec: already started")
    }
    if c.ctx != nil {
        select {
        case <-c.ctx.Done():
            c.closeDescriptors(c.closeAfterStart)
            c.closeDescriptors(c.closeAfterWait)
            return c.ctx.Err()
        default:
        }
    }

    c.childFiles = make([]*os.File, 0, 3+len(c.ExtraFiles))
    type F func(*Cmd) (*os.File, error)
    for _, setupFd := range []F{(*Cmd).stdin, (*Cmd).stdout, (*Cmd).stderr} {
        fd, err := setupFd(c)
        if err != nil {
            c.closeDescriptors(c.closeAfterStart)
            c.closeDescriptors(c.closeAfterWait)
            return err
        }
        c.childFiles = append(c.childFiles, fd)
    }
    c.childFiles = append(c.childFiles, c.ExtraFiles...)

    envv, err := c.envv()
    if err != nil {
        return err
    }

    c.Process, err = os.StartProcess(c.Path, c.argv(), &os.ProcAttr{
        Dir:   c.Dir,
        Files: c.childFiles,
        Env:   addCriticalEnv(dedupEnv(envv)),
        Sys:   c.SysProcAttr,
    })
    if err != nil {
        c.closeDescriptors(c.closeAfterStart)
        c.closeDescriptors(c.closeAfterWait)
        return err
    }

    c.closeDescriptors(c.closeAfterStart)

    // Don't allocate the channel unless there are goroutines to fire.
    if len(c.goroutine) > 0 {
        c.errch = make(chan error, len(c.goroutine))
        for _, fn := range c.goroutine {
            go func(fn func() error) {
                c.errch <- fn()
            }(fn)
        }
    }

    if c.ctx != nil {
        c.waitDone = make(chan struct{})
        go func() {
            select {
            case <-c.ctx.Done():
                c.Process.Kill()
            case <-c.waitDone:
            }
        }()
    }

    return nil
}

// An ExitError reports an unsuccessful exit by a command.
type ExitError struct {
    *os.ProcessState

    // Stderr holds a subset of the standard error output from the
    // Cmd.Output method if standard error was not otherwise being
    // collected.
    //
    // If the error output is long, Stderr may contain only a prefix
    // and suffix of the output, with the middle replaced with
    // text about the number of omitted bytes.
    //
    // Stderr is provided for debugging, for inclusion in error messages.
    // Users with other needs should redirect Cmd.Stderr as needed.
    Stderr []byte
}

func (e *ExitError) Error() string {
    return e.ProcessState.String()
}

// Wait waits for the command to exit and waits for any copying to
// stdin or copying from stdout or stderr to complete.
//
// The command must have been started by Start.
//
// The returned error is nil if the command runs, has no problems
// copying stdin, stdout, and stderr, and exits with a zero exit
// status.
//
// If the command fails to run or doesn't complete successfully, the
// error is of type *ExitError. Other error types may be
// returned for I/O problems.
//
// If any of c.Stdin, c.Stdout or c.Stderr are not an *os.File, Wait also waits
// for the respective I/O loop copying to or from the process to complete.
//
// Wait releases any resources associated with the Cmd.
func (c *Cmd) Wait() error {
    if c.Process == nil {
        return errors.New("exec: not started")
    }
    if c.finished {
        return errors.New("exec: Wait was already called")
    }
    c.finished = true

    state, err := c.Process.Wait()
    if c.waitDone != nil {
        close(c.waitDone)
    }
    c.ProcessState = state

    var copyError error
    for range c.goroutine {
        if err := <-c.errch; err != nil && copyError == nil {
            copyError = err
        }
    }

    c.closeDescriptors(c.closeAfterWait)

    if err != nil {
        return err
    } else if !state.Success() {
        return &ExitError{ProcessState: state}
    }

    return copyError
}

// Output runs the command and returns its standard output.
// Any returned error will usually be of type *ExitError.
// If c.Stderr was nil, Output populates ExitError.Stderr.
func (c *Cmd) Output() ([]byte, error) {
    if c.Stdout != nil {
        return nil, errors.New("exec: Stdout already set")
    }
    var stdout bytes.Buffer
    c.Stdout = &stdout

    captureErr := c.Stderr == nil
    if captureErr {
        c.Stderr = &prefixSuffixSaver{N: 32 << 10}
    }

    err := c.Run()
    if err != nil && captureErr {
        if ee, ok := err.(*ExitError); ok {
            ee.Stderr = c.Stderr.(*prefixSuffixSaver).Bytes()
        }
    }
    return stdout.Bytes(), err
}

// CombinedOutput runs the command and returns its combined standard
// output and standard error.
func (c *Cmd) CombinedOutput() ([]byte, error) {
    if c.Stdout != nil {
        return nil, errors.New("exec: Stdout already set")
    }
    if c.Stderr != nil {
        return nil, errors.New("exec: Stderr already set")
    }
    var b bytes.Buffer
    c.Stdout = &b
    c.Stderr = &b
    err := c.Run()
    return b.Bytes(), err
}

// StdinPipe returns a pipe that will be connected to the command's
// standard input when the command starts.
// The pipe will be closed automatically after Wait sees the command exit.
// A caller need only call Close to force the pipe to close sooner.
// For example, if the command being run will not exit until standard input
// is closed, the caller must close the pipe.
func (c *Cmd) StdinPipe() (io.WriteCloser, error) {
    if c.Stdin != nil {
        return nil, errors.New("exec: Stdin already set")
    }
    if c.Process != nil {
        return nil, errors.New("exec: StdinPipe after process started")
    }
    pr, pw, err := os.Pipe()
    if err != nil {
        return nil, err
    }
    c.Stdin = pr
    c.closeAfterStart = append(c.closeAfterStart, pr)
    wc := &closeOnce{File: pw}
    c.closeAfterWait = append(c.closeAfterWait, wc)
    return wc, nil
}

type closeOnce struct {
    *os.File

    once sync.Once
    err  error
}

func (c *closeOnce) Close() error {
    c.once.Do(c.close)
    return c.err
}

func (c *closeOnce) close() {
    c.err = c.File.Close()
}

// StdoutPipe returns a pipe that will be connected to the command's
// standard output when the command starts.
//
// Wait will close the pipe after seeing the command exit, so most callers
// need not close the pipe themselves. It is thus incorrect to call Wait
// before all reads from the pipe have completed.
// For the same reason, it is incorrect to call Run when using StdoutPipe.
// See the example for idiomatic usage.
func (c *Cmd) StdoutPipe() (io.ReadCloser, error) {
    if c.Stdout != nil {
        return nil, errors.New("exec: Stdout already set")
    }
    if c.Process != nil {
        return nil, errors.New("exec: StdoutPipe after process started")
    }
    pr, pw, err := os.Pipe()
    if err != nil {
        return nil, err
    }
    c.Stdout = pw
    c.closeAfterStart = append(c.closeAfterStart, pw)
    c.closeAfterWait = append(c.closeAfterWait, pr)
    return pr, nil
}

// StderrPipe returns a pipe that will be connected to the command's
// standard error when the command starts.
//
// Wait will close the pipe after seeing the command exit, so most callers
// need not close the pipe themselves. It is thus incorrect to call Wait
// before all reads from the pipe have completed.
// For the same reason, it is incorrect to use Run when using StderrPipe.
// See the StdoutPipe example for idiomatic usage.
func (c *Cmd) StderrPipe() (io.ReadCloser, error) {
    if c.Stderr != nil {
        return nil, errors.New("exec: Stderr already set")
    }
    if c.Process != nil {
        return nil, errors.New("exec: StderrPipe after process started")
    }
    pr, pw, err := os.Pipe()
    if err != nil {
        return nil, err
    }
    c.Stderr = pw
    c.closeAfterStart = append(c.closeAfterStart, pw)
    c.closeAfterWait = append(c.closeAfterWait, pr)
    return pr, nil
}

// prefixSuffixSaver is an io.Writer which retains the first N bytes
// and the last N bytes written to it. The Bytes() methods reconstructs
// it with a pretty error message.
type prefixSuffixSaver struct {
    N         int // max size of prefix or suffix
    prefix    []byte
    suffix    []byte // ring buffer once len(suffix) == N
    suffixOff int    // offset to write into suffix
    skipped   int64

    // TODO(bradfitz): we could keep one large []byte and use part of it for
    // the prefix, reserve space for the '... Omitting N bytes ...' message,
    // then the ring buffer suffix, and just rearrange the ring buffer
    // suffix when Bytes() is called, but it doesn't seem worth it for
    // now just for error messages. It's only ~64KB anyway.
}

func (w *prefixSuffixSaver) Write(p []byte) (n int, err error) {
    lenp := len(p)
    p = w.fill(&w.prefix, p)

    // Only keep the last w.N bytes of suffix data.
    if overage := len(p) - w.N; overage > 0 {
        p = p[overage:]
        w.skipped += int64(overage)
    }
    p = w.fill(&w.suffix, p)

    // w.suffix is full now if p is non-empty. Overwrite it in a circle.
    for len(p) > 0 { // 0, 1, or 2 iterations.
        n := copy(w.suffix[w.suffixOff:], p)
        p = p[n:]
        w.skipped += int64(n)
        w.suffixOff += n
        if w.suffixOff == w.N {
            w.suffixOff = 0
        }
    }
    return lenp, nil
}

// fill appends up to len(p) bytes of p to *dst, such that *dst does not
// grow larger than w.N. It returns the un-appended suffix of p.
func (w *prefixSuffixSaver) fill(dst *[]byte, p []byte) (pRemain []byte) {
    if remain := w.N - len(*dst); remain > 0 {
        add := minInt(len(p), remain)
        *dst = append(*dst, p[:add]...)
        p = p[add:]
    }
    return p
}

func (w *prefixSuffixSaver) Bytes() []byte {
    if w.suffix == nil {
        return w.prefix
    }
    if w.skipped == 0 {
        return append(w.prefix, w.suffix...)
    }
    var buf bytes.Buffer
    buf.Grow(len(w.prefix) + len(w.suffix) + 50)
    buf.Write(w.prefix)
    buf.WriteString("\n... omitting ")
    buf.WriteString(strconv.FormatInt(w.skipped, 10))
    buf.WriteString(" bytes ...\n")
    buf.Write(w.suffix[w.suffixOff:])
    buf.Write(w.suffix[:w.suffixOff])
    return buf.Bytes()
}

func minInt(a, b int) int {
    if a < b {
        return a
    }
    return b
}

// dedupEnv returns a copy of env with any duplicates removed, in favor of
// later values.
// Items not of the normal environment "key=value" form are preserved unchanged.
func dedupEnv(env []string) []string {
    return dedupEnvCase(runtime.GOOS == "windows", env)
}

// dedupEnvCase is dedupEnv with a case option for testing.
// If caseInsensitive is true, the case of keys is ignored.
func dedupEnvCase(caseInsensitive bool, env []string) []string {
    out := make([]string, 0, len(env))
    saw := make(map[string]int, len(env)) // key => index into out
    for _, kv := range env {
        eq := strings.Index(kv, "=")
        if eq < 0 {
            out = append(out, kv)
            continue
        }
        k := kv[:eq]
        if caseInsensitive {
            k = strings.ToLower(k)
        }
        if dupIdx, isDup := saw[k]; isDup {
            out[dupIdx] = kv
            continue
        }
        saw[k] = len(out)
        out = append(out, kv)
    }
    return out
}

// addCriticalEnv adds any critical environment variables that are required
// (or at least almost always required) on the operating system.
// Currently this is only used for Windows.
func addCriticalEnv(env []string) []string {
    if runtime.GOOS != "windows" {
        return env
    }
    for _, kv := range env {
        eq := strings.Index(kv, "=")
        if eq < 0 {
            continue
        }
        k := kv[:eq]
        if strings.EqualFold(k, "SYSTEMROOT") {
            // We already have it.
            return env
        }
    }
    return append(env, "SYSTEMROOT="+os.Getenv("SYSTEMROOT"))
}