| // Copyright 2023 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| package work |
| |
| import ( |
| "bytes" |
| "cmd/go/internal/base" |
| "cmd/go/internal/cache" |
| "cmd/go/internal/cfg" |
| "cmd/go/internal/load" |
| "cmd/go/internal/par" |
| "cmd/go/internal/str" |
| "errors" |
| "fmt" |
| "internal/lazyregexp" |
| "io" |
| "io/fs" |
| "os" |
| "os/exec" |
| "path/filepath" |
| "runtime" |
| "strconv" |
| "strings" |
| "sync" |
| "time" |
| ) |
| |
| // A Shell runs shell commands and performs shell-like file system operations. |
| // |
| // Shell tracks context related to running commands, and form a tree much like |
| // context.Context. |
| type Shell struct { |
| action *Action // nil for the root shell |
| *shellShared // per-Builder state shared across Shells |
| } |
| |
| // shellShared is Shell state shared across all Shells derived from a single |
| // root shell (generally a single Builder). |
| type shellShared struct { |
| workDir string // $WORK, immutable |
| |
| printLock sync.Mutex |
| printFunc func(args ...any) (int, error) |
| scriptDir string // current directory in printed script |
| |
| mkdirCache par.Cache[string, error] // a cache of created directories |
| } |
| |
| // NewShell returns a new Shell. |
| // |
| // Shell will internally serialize calls to the print function. |
| // If print is nil, it defaults to printing to stderr. |
| func NewShell(workDir string, print func(a ...any) (int, error)) *Shell { |
| if print == nil { |
| print = func(a ...any) (int, error) { |
| return fmt.Fprint(os.Stderr, a...) |
| } |
| } |
| shared := &shellShared{ |
| workDir: workDir, |
| printFunc: print, |
| } |
| return &Shell{shellShared: shared} |
| } |
| |
| // Print emits a to this Shell's output stream, formatting it like fmt.Print. |
| // It is safe to call concurrently. |
| func (sh *Shell) Print(a ...any) { |
| sh.printLock.Lock() |
| defer sh.printLock.Unlock() |
| sh.printFunc(a...) |
| } |
| |
| func (sh *Shell) printLocked(a ...any) { |
| sh.printFunc(a...) |
| } |
| |
| // WithAction returns a Shell identical to sh, but bound to Action a. |
| func (sh *Shell) WithAction(a *Action) *Shell { |
| sh2 := *sh |
| sh2.action = a |
| return &sh2 |
| } |
| |
| // Shell returns a shell for running commands on behalf of Action a. |
| func (b *Builder) Shell(a *Action) *Shell { |
| if a == nil { |
| // The root shell has a nil Action. The point of this method is to |
| // create a Shell bound to an Action, so disallow nil Actions here. |
| panic("nil Action") |
| } |
| if a.sh == nil { |
| a.sh = b.backgroundSh.WithAction(a) |
| } |
| return a.sh |
| } |
| |
| // BackgroundShell returns a Builder-wide Shell that's not bound to any Action. |
| // Try not to use this unless there's really no sensible Action available. |
| func (b *Builder) BackgroundShell() *Shell { |
| return b.backgroundSh |
| } |
| |
| // moveOrCopyFile is like 'mv src dst' or 'cp src dst'. |
| func (sh *Shell) moveOrCopyFile(dst, src string, perm fs.FileMode, force bool) error { |
| if cfg.BuildN { |
| sh.ShowCmd("", "mv %s %s", src, dst) |
| return nil |
| } |
| |
| // If we can update the mode and rename to the dst, do it. |
| // Otherwise fall back to standard copy. |
| |
| // If the source is in the build cache, we need to copy it. |
| if strings.HasPrefix(src, cache.DefaultDir()) { |
| return sh.CopyFile(dst, src, perm, force) |
| } |
| |
| // On Windows, always copy the file, so that we respect the NTFS |
| // permissions of the parent folder. https://golang.org/issue/22343. |
| // What matters here is not cfg.Goos (the system we are building |
| // for) but runtime.GOOS (the system we are building on). |
| if runtime.GOOS == "windows" { |
| return sh.CopyFile(dst, src, perm, force) |
| } |
| |
| // If the destination directory has the group sticky bit set, |
| // we have to copy the file to retain the correct permissions. |
| // https://golang.org/issue/18878 |
| if fi, err := os.Stat(filepath.Dir(dst)); err == nil { |
| if fi.IsDir() && (fi.Mode()&fs.ModeSetgid) != 0 { |
| return sh.CopyFile(dst, src, perm, force) |
| } |
| } |
| |
| // The perm argument is meant to be adjusted according to umask, |
| // but we don't know what the umask is. |
| // Create a dummy file to find out. |
| // This avoids build tags and works even on systems like Plan 9 |
| // where the file mask computation incorporates other information. |
| mode := perm |
| f, err := os.OpenFile(filepath.Clean(dst)+"-go-tmp-umask", os.O_WRONLY|os.O_CREATE|os.O_EXCL, perm) |
| if err == nil { |
| fi, err := f.Stat() |
| if err == nil { |
| mode = fi.Mode() & 0777 |
| } |
| name := f.Name() |
| f.Close() |
| os.Remove(name) |
| } |
| |
| if err := os.Chmod(src, mode); err == nil { |
| if err := os.Rename(src, dst); err == nil { |
| if cfg.BuildX { |
| sh.ShowCmd("", "mv %s %s", src, dst) |
| } |
| return nil |
| } |
| } |
| |
| return sh.CopyFile(dst, src, perm, force) |
| } |
| |
| // copyFile is like 'cp src dst'. |
| func (sh *Shell) CopyFile(dst, src string, perm fs.FileMode, force bool) error { |
| if cfg.BuildN || cfg.BuildX { |
| sh.ShowCmd("", "cp %s %s", src, dst) |
| if cfg.BuildN { |
| return nil |
| } |
| } |
| |
| sf, err := os.Open(src) |
| if err != nil { |
| return err |
| } |
| defer sf.Close() |
| |
| // Be careful about removing/overwriting dst. |
| // Do not remove/overwrite if dst exists and is a directory |
| // or a non-empty non-object file. |
| if fi, err := os.Stat(dst); err == nil { |
| if fi.IsDir() { |
| return fmt.Errorf("build output %q already exists and is a directory", dst) |
| } |
| if !force && fi.Mode().IsRegular() && fi.Size() != 0 && !isObject(dst) { |
| return fmt.Errorf("build output %q already exists and is not an object file", dst) |
| } |
| } |
| |
| // On Windows, remove lingering ~ file from last attempt. |
| if runtime.GOOS == "windows" { |
| if _, err := os.Stat(dst + "~"); err == nil { |
| os.Remove(dst + "~") |
| } |
| } |
| |
| mayberemovefile(dst) |
| df, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) |
| if err != nil && runtime.GOOS == "windows" { |
| // Windows does not allow deletion of a binary file |
| // while it is executing. Try to move it out of the way. |
| // If the move fails, which is likely, we'll try again the |
| // next time we do an install of this binary. |
| if err := os.Rename(dst, dst+"~"); err == nil { |
| os.Remove(dst + "~") |
| } |
| df, err = os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) |
| } |
| if err != nil { |
| return fmt.Errorf("copying %s: %w", src, err) // err should already refer to dst |
| } |
| |
| _, err = io.Copy(df, sf) |
| df.Close() |
| if err != nil { |
| mayberemovefile(dst) |
| return fmt.Errorf("copying %s to %s: %v", src, dst, err) |
| } |
| return nil |
| } |
| |
| // mayberemovefile removes a file only if it is a regular file |
| // When running as a user with sufficient privileges, we may delete |
| // even device files, for example, which is not intended. |
| func mayberemovefile(s string) { |
| if fi, err := os.Lstat(s); err == nil && !fi.Mode().IsRegular() { |
| return |
| } |
| os.Remove(s) |
| } |
| |
| // writeFile writes the text to file. |
| func (sh *Shell) writeFile(file string, text []byte) error { |
| if cfg.BuildN || cfg.BuildX { |
| switch { |
| case len(text) == 0: |
| sh.ShowCmd("", "echo -n > %s # internal", file) |
| case bytes.IndexByte(text, '\n') == len(text)-1: |
| // One line. Use a simpler "echo" command. |
| sh.ShowCmd("", "echo '%s' > %s # internal", bytes.TrimSuffix(text, []byte("\n")), file) |
| default: |
| // Use the most general form. |
| sh.ShowCmd("", "cat >%s << 'EOF' # internal\n%sEOF", file, text) |
| } |
| } |
| if cfg.BuildN { |
| return nil |
| } |
| return os.WriteFile(file, text, 0666) |
| } |
| |
| // Mkdir makes the named directory. |
| func (sh *Shell) Mkdir(dir string) error { |
| // Make Mkdir(a.Objdir) a no-op instead of an error when a.Objdir == "". |
| if dir == "" { |
| return nil |
| } |
| |
| // We can be a little aggressive about being |
| // sure directories exist. Skip repeated calls. |
| return sh.mkdirCache.Do(dir, func() error { |
| if cfg.BuildN || cfg.BuildX { |
| sh.ShowCmd("", "mkdir -p %s", dir) |
| if cfg.BuildN { |
| return nil |
| } |
| } |
| |
| return os.MkdirAll(dir, 0777) |
| }) |
| } |
| |
| // RemoveAll is like 'rm -rf'. It attempts to remove all paths even if there's |
| // an error, and returns the first error. |
| func (sh *Shell) RemoveAll(paths ...string) error { |
| if cfg.BuildN || cfg.BuildX { |
| // Don't say we are removing the directory if we never created it. |
| show := func() bool { |
| for _, path := range paths { |
| if _, ok := sh.mkdirCache.Get(path); ok { |
| return true |
| } |
| if _, err := os.Stat(path); !os.IsNotExist(err) { |
| return true |
| } |
| } |
| return false |
| } |
| if show() { |
| sh.ShowCmd("", "rm -rf %s", strings.Join(paths, " ")) |
| } |
| } |
| if cfg.BuildN { |
| return nil |
| } |
| |
| var err error |
| for _, path := range paths { |
| if err2 := os.RemoveAll(path); err2 != nil && err == nil { |
| err = err2 |
| } |
| } |
| return err |
| } |
| |
| // Symlink creates a symlink newname -> oldname. |
| func (sh *Shell) Symlink(oldname, newname string) error { |
| // It's not an error to try to recreate an existing symlink. |
| if link, err := os.Readlink(newname); err == nil && link == oldname { |
| return nil |
| } |
| |
| if cfg.BuildN || cfg.BuildX { |
| sh.ShowCmd("", "ln -s %s %s", oldname, newname) |
| if cfg.BuildN { |
| return nil |
| } |
| } |
| return os.Symlink(oldname, newname) |
| } |
| |
| // fmtCmd formats a command in the manner of fmt.Sprintf but also: |
| // |
| // fmtCmd replaces the value of b.WorkDir with $WORK. |
| func (sh *Shell) fmtCmd(dir string, format string, args ...any) string { |
| cmd := fmt.Sprintf(format, args...) |
| if sh.workDir != "" && !strings.HasPrefix(cmd, "cat ") { |
| cmd = strings.ReplaceAll(cmd, sh.workDir, "$WORK") |
| escaped := strconv.Quote(sh.workDir) |
| escaped = escaped[1 : len(escaped)-1] // strip quote characters |
| if escaped != sh.workDir { |
| cmd = strings.ReplaceAll(cmd, escaped, "$WORK") |
| } |
| } |
| return cmd |
| } |
| |
| // ShowCmd prints the given command to standard output |
| // for the implementation of -n or -x. |
| // |
| // ShowCmd also replaces the name of the current script directory with dot (.) |
| // but only when it is at the beginning of a space-separated token. |
| // |
| // If dir is not "" or "/" and not the current script directory, ShowCmd first |
| // prints a "cd" command to switch to dir and updates the script directory. |
| func (sh *Shell) ShowCmd(dir string, format string, args ...any) { |
| // Use the output lock directly so we can manage scriptDir. |
| sh.printLock.Lock() |
| defer sh.printLock.Unlock() |
| |
| cmd := sh.fmtCmd(dir, format, args...) |
| |
| if dir != "" && dir != "/" { |
| if dir != sh.scriptDir { |
| // Show changing to dir and update the current directory. |
| sh.printLocked(sh.fmtCmd("", "cd %s\n", dir)) |
| sh.scriptDir = dir |
| } |
| // Replace scriptDir is our working directory. Replace it |
| // with "." in the command. |
| dot := " ." |
| if dir[len(dir)-1] == filepath.Separator { |
| dot += string(filepath.Separator) |
| } |
| cmd = strings.ReplaceAll(" "+cmd, " "+dir, dot)[1:] |
| } |
| |
| sh.printLocked(cmd + "\n") |
| } |
| |
| // reportCmd reports the output and exit status of a command. The cmdOut and |
| // cmdErr arguments are the output and exit error of the command, respectively. |
| // |
| // The exact reporting behavior is as follows: |
| // |
| // cmdOut cmdErr Result |
| // "" nil print nothing, return nil |
| // !="" nil print output, return nil |
| // "" !=nil print nothing, return cmdErr (later printed) |
| // !="" !=nil print nothing, ignore err, return output as error (later printed) |
| // |
| // reportCmd returns a non-nil error if and only if cmdErr != nil. It assumes |
| // that the command output, if non-empty, is more detailed than the command |
| // error (which is usually just an exit status), so prefers using the output as |
| // the ultimate error. Typically, the caller should return this error from an |
| // Action, which it will be printed by the Builder. |
| // |
| // reportCmd formats the output as "# desc" followed by the given output. The |
| // output is expected to contain references to 'dir', usually the source |
| // directory for the package that has failed to build. reportCmd rewrites |
| // mentions of dir with a relative path to dir when the relative path is |
| // shorter. This is usually more pleasant. For example, if fmt doesn't compile |
| // and we are in src/html, the output is |
| // |
| // $ go build |
| // # fmt |
| // ../fmt/print.go:1090: undefined: asdf |
| // $ |
| // |
| // instead of |
| // |
| // $ go build |
| // # fmt |
| // /usr/gopher/go/src/fmt/print.go:1090: undefined: asdf |
| // $ |
| // |
| // reportCmd also replaces references to the work directory with $WORK, replaces |
| // cgo file paths with the original file path, and replaces cgo-mangled names |
| // with "C.name". |
| // |
| // desc is optional. If "", a.Package.Desc() is used. |
| // |
| // dir is optional. If "", a.Package.Dir is used. |
| func (sh *Shell) reportCmd(desc, dir string, cmdOut []byte, cmdErr error) error { |
| if len(cmdOut) == 0 && cmdErr == nil { |
| // Common case |
| return nil |
| } |
| if len(cmdOut) == 0 && cmdErr != nil { |
| // Just return the error. |
| // |
| // TODO: This is what we've done for a long time, but it may be a |
| // mistake because it loses all of the extra context and results in |
| // ultimately less descriptive output. We should probably just take the |
| // text of cmdErr as the output in this case and do everything we |
| // otherwise would. We could chain the errors if we feel like it. |
| return cmdErr |
| } |
| |
| // Fetch defaults from the package. |
| var p *load.Package |
| a := sh.action |
| if a != nil { |
| p = a.Package |
| } |
| var importPath string |
| if p != nil { |
| importPath = p.ImportPath |
| if desc == "" { |
| desc = p.Desc() |
| } |
| if dir == "" { |
| dir = p.Dir |
| } |
| } |
| |
| out := string(cmdOut) |
| |
| if !strings.HasSuffix(out, "\n") { |
| out = out + "\n" |
| } |
| |
| // Replace workDir with $WORK |
| out = replacePrefix(out, sh.workDir, "$WORK") |
| |
| // Rewrite mentions of dir with a relative path to dir |
| // when the relative path is shorter. |
| for { |
| // Note that dir starts out long, something like |
| // /foo/bar/baz/root/a |
| // The target string to be reduced is something like |
| // (blah-blah-blah) /foo/bar/baz/root/sibling/whatever.go:blah:blah |
| // /foo/bar/baz/root/a doesn't match /foo/bar/baz/root/sibling, but the prefix |
| // /foo/bar/baz/root does. And there may be other niblings sharing shorter |
| // prefixes, the only way to find them is to look. |
| // This doesn't always produce a relative path -- |
| // /foo is shorter than ../../.., for example. |
| if reldir := base.ShortPath(dir); reldir != dir { |
| out = replacePrefix(out, dir, reldir) |
| if filepath.Separator == '\\' { |
| // Don't know why, sometimes this comes out with slashes, not backslashes. |
| wdir := strings.ReplaceAll(dir, "\\", "/") |
| out = replacePrefix(out, wdir, reldir) |
| } |
| } |
| dirP := filepath.Dir(dir) |
| if dir == dirP { |
| break |
| } |
| dir = dirP |
| } |
| |
| // Fix up output referring to cgo-generated code to be more readable. |
| // Replace x.go:19[/tmp/.../x.cgo1.go:18] with x.go:19. |
| // Replace *[100]_Ctype_foo with *[100]C.foo. |
| // If we're using -x, assume we're debugging and want the full dump, so disable the rewrite. |
| if !cfg.BuildX && cgoLine.MatchString(out) { |
| out = cgoLine.ReplaceAllString(out, "") |
| out = cgoTypeSigRe.ReplaceAllString(out, "C.") |
| } |
| |
| // Usually desc is already p.Desc(), but if not, signal cmdError.Error to |
| // add a line explicitly metioning the import path. |
| needsPath := importPath != "" && p != nil && desc != p.Desc() |
| |
| err := &cmdError{desc, out, importPath, needsPath} |
| if cmdErr != nil { |
| // The command failed. Report the output up as an error. |
| return err |
| } |
| // The command didn't fail, so just print the output as appropriate. |
| if a != nil && a.output != nil { |
| // The Action is capturing output. |
| a.output = append(a.output, err.Error()...) |
| } else { |
| // Write directly to the Builder output. |
| sh.Print(err.Error()) |
| } |
| return nil |
| } |
| |
| // replacePrefix is like strings.ReplaceAll, but only replaces instances of old |
| // that are preceded by ' ', '\t', or appear at the beginning of a line. |
| func replacePrefix(s, old, new string) string { |
| n := strings.Count(s, old) |
| if n == 0 { |
| return s |
| } |
| |
| s = strings.ReplaceAll(s, " "+old, " "+new) |
| s = strings.ReplaceAll(s, "\n"+old, "\n"+new) |
| s = strings.ReplaceAll(s, "\n\t"+old, "\n\t"+new) |
| if strings.HasPrefix(s, old) { |
| s = new + s[len(old):] |
| } |
| return s |
| } |
| |
| type cmdError struct { |
| desc string |
| text string |
| importPath string |
| needsPath bool // Set if desc does not already include the import path |
| } |
| |
| func (e *cmdError) Error() string { |
| var msg string |
| if e.needsPath { |
| // Ensure the import path is part of the message. |
| // Clearly distinguish the description from the import path. |
| msg = fmt.Sprintf("# %s\n# [%s]\n", e.importPath, e.desc) |
| } else { |
| msg = "# " + e.desc + "\n" |
| } |
| return msg + e.text |
| } |
| |
| func (e *cmdError) ImportPath() string { |
| return e.importPath |
| } |
| |
| var cgoLine = lazyregexp.New(`\[[^\[\]]+\.(cgo1|cover)\.go:[0-9]+(:[0-9]+)?\]`) |
| var cgoTypeSigRe = lazyregexp.New(`\b_C2?(type|func|var|macro)_\B`) |
| |
| // run runs the command given by cmdline in the directory dir. |
| // If the command fails, run prints information about the failure |
| // and returns a non-nil error. |
| func (sh *Shell) run(dir string, desc string, env []string, cmdargs ...any) error { |
| out, err := sh.runOut(dir, env, cmdargs...) |
| if desc == "" { |
| desc = sh.fmtCmd(dir, "%s", strings.Join(str.StringList(cmdargs...), " ")) |
| } |
| return sh.reportCmd(desc, dir, out, err) |
| } |
| |
| // runOut runs the command given by cmdline in the directory dir. |
| // It returns the command output and any errors that occurred. |
| // It accumulates execution time in a. |
| func (sh *Shell) runOut(dir string, env []string, cmdargs ...any) ([]byte, error) { |
| a := sh.action |
| |
| cmdline := str.StringList(cmdargs...) |
| |
| for _, arg := range cmdline { |
| // GNU binutils commands, including gcc and gccgo, interpret an argument |
| // @foo anywhere in the command line (even following --) as meaning |
| // "read and insert arguments from the file named foo." |
| // Don't say anything that might be misinterpreted that way. |
| if strings.HasPrefix(arg, "@") { |
| return nil, fmt.Errorf("invalid command-line argument %s in command: %s", arg, joinUnambiguously(cmdline)) |
| } |
| } |
| |
| if cfg.BuildN || cfg.BuildX { |
| var envcmdline string |
| for _, e := range env { |
| if j := strings.IndexByte(e, '='); j != -1 { |
| if strings.ContainsRune(e[j+1:], '\'') { |
| envcmdline += fmt.Sprintf("%s=%q", e[:j], e[j+1:]) |
| } else { |
| envcmdline += fmt.Sprintf("%s='%s'", e[:j], e[j+1:]) |
| } |
| envcmdline += " " |
| } |
| } |
| envcmdline += joinUnambiguously(cmdline) |
| sh.ShowCmd(dir, "%s", envcmdline) |
| if cfg.BuildN { |
| return nil, nil |
| } |
| } |
| |
| var buf bytes.Buffer |
| path, err := cfg.LookPath(cmdline[0]) |
| if err != nil { |
| return nil, err |
| } |
| cmd := exec.Command(path, cmdline[1:]...) |
| if cmd.Path != "" { |
| cmd.Args[0] = cmd.Path |
| } |
| cmd.Stdout = &buf |
| cmd.Stderr = &buf |
| cleanup := passLongArgsInResponseFiles(cmd) |
| defer cleanup() |
| if dir != "." { |
| cmd.Dir = dir |
| } |
| cmd.Env = cmd.Environ() // Pre-allocate with correct PWD. |
| |
| // Add the TOOLEXEC_IMPORTPATH environment variable for -toolexec tools. |
| // It doesn't really matter if -toolexec isn't being used. |
| // Note that a.Package.Desc is not really an import path, |
| // but this is consistent with 'go list -f {{.ImportPath}}'. |
| // Plus, it is useful to uniquely identify packages in 'go list -json'. |
| if a != nil && a.Package != nil { |
| cmd.Env = append(cmd.Env, "TOOLEXEC_IMPORTPATH="+a.Package.Desc()) |
| } |
| |
| cmd.Env = append(cmd.Env, env...) |
| start := time.Now() |
| err = cmd.Run() |
| if a != nil && a.json != nil { |
| aj := a.json |
| aj.Cmd = append(aj.Cmd, joinUnambiguously(cmdline)) |
| aj.CmdReal += time.Since(start) |
| if ps := cmd.ProcessState; ps != nil { |
| aj.CmdUser += ps.UserTime() |
| aj.CmdSys += ps.SystemTime() |
| } |
| } |
| |
| // err can be something like 'exit status 1'. |
| // Add information about what program was running. |
| // Note that if buf.Bytes() is non-empty, the caller usually |
| // shows buf.Bytes() and does not print err at all, so the |
| // prefix here does not make most output any more verbose. |
| if err != nil { |
| err = errors.New(cmdline[0] + ": " + err.Error()) |
| } |
| return buf.Bytes(), err |
| } |
| |
| // joinUnambiguously prints the slice, quoting where necessary to make the |
| // output unambiguous. |
| // TODO: See issue 5279. The printing of commands needs a complete redo. |
| func joinUnambiguously(a []string) string { |
| var buf strings.Builder |
| for i, s := range a { |
| if i > 0 { |
| buf.WriteByte(' ') |
| } |
| q := strconv.Quote(s) |
| // A gccgo command line can contain -( and -). |
| // Make sure we quote them since they are special to the shell. |
| // The trimpath argument can also contain > (part of =>) and ;. Quote those too. |
| if s == "" || strings.ContainsAny(s, " ()>;") || len(q) > len(s)+2 { |
| buf.WriteString(q) |
| } else { |
| buf.WriteString(s) |
| } |
| } |
| return buf.String() |
| } |