mirror of https://go.googlesource.com/go
440 lines
11 KiB
Go
440 lines
11 KiB
Go
// Copyright 2023 The Go Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
// Distpack creates the tgz and zip files for a Go distribution.
|
|
// It writes into GOROOT/pkg/distpack:
|
|
//
|
|
// - a binary distribution (tgz or zip) for the current GOOS and GOARCH
|
|
// - a source distribution that is independent of GOOS/GOARCH
|
|
// - the module mod, info, and zip files for a distribution in module form
|
|
// (as used by GOTOOLCHAIN support in the go command).
|
|
//
|
|
// Distpack is typically invoked by the -distpack flag to make.bash.
|
|
// A cross-compiled distribution for goos/goarch can be built using:
|
|
//
|
|
// GOOS=goos GOARCH=goarch ./make.bash -distpack
|
|
//
|
|
// To test that the module downloads are usable with the go command:
|
|
//
|
|
// ./make.bash -distpack
|
|
// mkdir -p /tmp/goproxy/golang.org/toolchain/
|
|
// ln -sf $(pwd)/../pkg/distpack /tmp/goproxy/golang.org/toolchain/@v
|
|
// GOPROXY=file:///tmp/goproxy GOTOOLCHAIN=$(sed 1q ../VERSION) gotip version
|
|
//
|
|
// gotip can be replaced with an older released Go version once there is one.
|
|
// It just can't be the one make.bash built, because it knows it is already that
|
|
// version and will skip the download.
|
|
package main
|
|
|
|
import (
|
|
"archive/tar"
|
|
"archive/zip"
|
|
"compress/flate"
|
|
"compress/gzip"
|
|
"crypto/sha256"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"io/fs"
|
|
"log"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strings"
|
|
"time"
|
|
|
|
"cmd/internal/telemetry"
|
|
)
|
|
|
|
func usage() {
|
|
fmt.Fprintf(os.Stderr, "usage: distpack\n")
|
|
os.Exit(2)
|
|
}
|
|
|
|
const (
|
|
modPath = "golang.org/toolchain"
|
|
modVersionPrefix = "v0.0.1"
|
|
)
|
|
|
|
var (
|
|
goroot string
|
|
gohostos string
|
|
gohostarch string
|
|
goos string
|
|
goarch string
|
|
)
|
|
|
|
func main() {
|
|
log.SetPrefix("distpack: ")
|
|
log.SetFlags(0)
|
|
telemetry.Start()
|
|
flag.Usage = usage
|
|
flag.Parse()
|
|
telemetry.Inc("distpack/invocations")
|
|
telemetry.CountFlags("distpack/flag:", *flag.CommandLine)
|
|
if flag.NArg() != 0 {
|
|
usage()
|
|
}
|
|
|
|
// Load context.
|
|
goroot = runtime.GOROOT()
|
|
if goroot == "" {
|
|
log.Fatalf("missing $GOROOT")
|
|
}
|
|
gohostos = runtime.GOOS
|
|
gohostarch = runtime.GOARCH
|
|
goos = os.Getenv("GOOS")
|
|
if goos == "" {
|
|
goos = gohostos
|
|
}
|
|
goarch = os.Getenv("GOARCH")
|
|
if goarch == "" {
|
|
goarch = gohostarch
|
|
}
|
|
goosUnderGoarch := goos + "_" + goarch
|
|
goosDashGoarch := goos + "-" + goarch
|
|
exe := ""
|
|
if goos == "windows" {
|
|
exe = ".exe"
|
|
}
|
|
version, versionTime := readVERSION(goroot)
|
|
|
|
// Start with files from GOROOT, filtering out non-distribution files.
|
|
base, err := NewArchive(goroot)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
base.SetTime(versionTime)
|
|
base.SetMode(mode)
|
|
base.Remove(
|
|
".git/**",
|
|
".gitattributes",
|
|
".github/**",
|
|
".gitignore",
|
|
"VERSION.cache",
|
|
"misc/cgo/*/_obj/**",
|
|
"**/.DS_Store",
|
|
"**/*.exe~", // go.dev/issue/23894
|
|
// Generated during make.bat/make.bash.
|
|
"src/cmd/dist/dist",
|
|
"src/cmd/dist/dist.exe",
|
|
)
|
|
|
|
// The source distribution removes files generated during the release build.
|
|
// See ../dist/build.go's deptab.
|
|
srcArch := base.Clone()
|
|
srcArch.Remove(
|
|
"bin/**",
|
|
"pkg/**",
|
|
|
|
// Generated during cmd/dist. See ../dist/build.go:/gentab.
|
|
"src/cmd/go/internal/cfg/zdefaultcc.go",
|
|
"src/go/build/zcgo.go",
|
|
"src/runtime/internal/sys/zversion.go",
|
|
"src/time/tzdata/zzipdata.go",
|
|
|
|
// Generated during cmd/dist by bootstrapBuildTools.
|
|
"src/cmd/cgo/zdefaultcc.go",
|
|
"src/cmd/internal/objabi/zbootstrap.go",
|
|
"src/internal/buildcfg/zbootstrap.go",
|
|
|
|
// Generated by earlier versions of cmd/dist .
|
|
"src/cmd/go/internal/cfg/zosarch.go",
|
|
)
|
|
srcArch.AddPrefix("go")
|
|
testSrc(srcArch)
|
|
|
|
// The binary distribution includes only a subset of bin and pkg.
|
|
binArch := base.Clone()
|
|
binArch.Filter(func(name string) bool {
|
|
// Discard bin/ for now, will add back later.
|
|
if strings.HasPrefix(name, "bin/") {
|
|
return false
|
|
}
|
|
// Discard most of pkg.
|
|
if strings.HasPrefix(name, "pkg/") {
|
|
// Keep pkg/include.
|
|
if strings.HasPrefix(name, "pkg/include/") {
|
|
return true
|
|
}
|
|
// Discard other pkg except pkg/tool.
|
|
if !strings.HasPrefix(name, "pkg/tool/") {
|
|
return false
|
|
}
|
|
// Inside pkg/tool, keep only $GOOS_$GOARCH.
|
|
if !strings.HasPrefix(name, "pkg/tool/"+goosUnderGoarch+"/") {
|
|
return false
|
|
}
|
|
// Inside pkg/tool/$GOOS_$GOARCH, discard helper tools.
|
|
switch strings.TrimSuffix(path.Base(name), ".exe") {
|
|
case "api", "dist", "distpack", "metadata":
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
})
|
|
|
|
// Add go and gofmt to bin, using cross-compiled binaries
|
|
// if this is a cross-compiled distribution.
|
|
binExes := []string{
|
|
"go",
|
|
"gofmt",
|
|
}
|
|
crossBin := "bin"
|
|
if goos != gohostos || goarch != gohostarch {
|
|
crossBin = "bin/" + goosUnderGoarch
|
|
}
|
|
for _, b := range binExes {
|
|
name := "bin/" + b + exe
|
|
src := filepath.Join(goroot, crossBin, b+exe)
|
|
info, err := os.Stat(src)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
binArch.Add(name, src, info)
|
|
}
|
|
binArch.Sort()
|
|
binArch.SetTime(versionTime) // fix added files
|
|
binArch.SetMode(mode) // fix added files
|
|
|
|
zipArch := binArch.Clone()
|
|
zipArch.AddPrefix("go")
|
|
testZip(zipArch)
|
|
|
|
// The module distribution is the binary distribution with unnecessary files removed
|
|
// and file names using the necessary prefix for the module.
|
|
modArch := binArch.Clone()
|
|
modArch.Remove(
|
|
"api/**",
|
|
"doc/**",
|
|
"misc/**",
|
|
"test/**",
|
|
)
|
|
modVers := modVersionPrefix + "-" + version + "." + goosDashGoarch
|
|
modArch.AddPrefix(modPath + "@" + modVers)
|
|
modArch.RenameGoMod()
|
|
modArch.Sort()
|
|
testMod(modArch)
|
|
|
|
// distpack returns the full path to name in the distpack directory.
|
|
distpack := func(name string) string {
|
|
return filepath.Join(goroot, "pkg/distpack", name)
|
|
}
|
|
if err := os.MkdirAll(filepath.Join(goroot, "pkg/distpack"), 0777); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
writeTgz(distpack(version+".src.tar.gz"), srcArch)
|
|
|
|
if goos == "windows" {
|
|
writeZip(distpack(version+"."+goos+"-"+goarch+".zip"), zipArch)
|
|
} else {
|
|
writeTgz(distpack(version+"."+goos+"-"+goarch+".tar.gz"), zipArch)
|
|
}
|
|
|
|
writeZip(distpack(modVers+".zip"), modArch)
|
|
writeFile(distpack(modVers+".mod"),
|
|
[]byte(fmt.Sprintf("module %s\n", modPath)))
|
|
writeFile(distpack(modVers+".info"),
|
|
[]byte(fmt.Sprintf("{%q:%q, %q:%q}\n",
|
|
"Version", modVers,
|
|
"Time", versionTime.Format(time.RFC3339))))
|
|
}
|
|
|
|
// mode computes the mode for the given file name.
|
|
func mode(name string, _ fs.FileMode) fs.FileMode {
|
|
if strings.HasPrefix(name, "bin/") ||
|
|
strings.HasPrefix(name, "pkg/tool/") ||
|
|
strings.HasSuffix(name, ".bash") ||
|
|
strings.HasSuffix(name, ".sh") ||
|
|
strings.HasSuffix(name, ".pl") ||
|
|
strings.HasSuffix(name, ".rc") {
|
|
return 0o755
|
|
} else if ok, _ := amatch("**/go_?*_?*_exec", name); ok {
|
|
return 0o755
|
|
}
|
|
return 0o644
|
|
}
|
|
|
|
// readVERSION reads the VERSION file.
|
|
// The first line of the file is the Go version.
|
|
// Additional lines are 'key value' pairs setting other data.
|
|
// The only valid key at the moment is 'time', which sets the modification time for file archives.
|
|
func readVERSION(goroot string) (version string, t time.Time) {
|
|
data, err := os.ReadFile(filepath.Join(goroot, "VERSION"))
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
version, rest, _ := strings.Cut(string(data), "\n")
|
|
for _, line := range strings.Split(rest, "\n") {
|
|
f := strings.Fields(line)
|
|
if len(f) == 0 {
|
|
continue
|
|
}
|
|
switch f[0] {
|
|
default:
|
|
log.Fatalf("VERSION: unexpected line: %s", line)
|
|
case "time":
|
|
if len(f) != 2 {
|
|
log.Fatalf("VERSION: unexpected time line: %s", line)
|
|
}
|
|
t, err = time.ParseInLocation(time.RFC3339, f[1], time.UTC)
|
|
if err != nil {
|
|
log.Fatalf("VERSION: bad time: %s", err)
|
|
}
|
|
}
|
|
}
|
|
return version, t
|
|
}
|
|
|
|
// writeFile writes a file with the given name and data or fatals.
|
|
func writeFile(name string, data []byte) {
|
|
if err := os.WriteFile(name, data, 0666); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
reportHash(name)
|
|
}
|
|
|
|
// check panics if err is not nil. Otherwise it returns x.
|
|
// It is only meant to be used in a function that has deferred
|
|
// a function to recover appropriately from the panic.
|
|
func check[T any](x T, err error) T {
|
|
check1(err)
|
|
return x
|
|
}
|
|
|
|
// check1 panics if err is not nil.
|
|
// It is only meant to be used in a function that has deferred
|
|
// a function to recover appropriately from the panic.
|
|
func check1(err error) {
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
|
|
// writeTgz writes the archive in tgz form to the file named name.
|
|
func writeTgz(name string, a *Archive) {
|
|
out, err := os.Create(name)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
var f File
|
|
defer func() {
|
|
if err := recover(); err != nil {
|
|
extra := ""
|
|
if f.Name != "" {
|
|
extra = " " + f.Name
|
|
}
|
|
log.Fatalf("writing %s%s: %v", name, extra, err)
|
|
}
|
|
}()
|
|
|
|
zw := check(gzip.NewWriterLevel(out, gzip.BestCompression))
|
|
tw := tar.NewWriter(zw)
|
|
|
|
// Find the mode and mtime to use for directory entries,
|
|
// based on the mode and mtime of the first file we see.
|
|
// We know that modes and mtimes are uniform across the archive.
|
|
var dirMode fs.FileMode
|
|
var mtime time.Time
|
|
for _, f := range a.Files {
|
|
dirMode = fs.ModeDir | f.Mode | (f.Mode&0444)>>2 // copy r bits down to x bits
|
|
mtime = f.Time
|
|
break
|
|
}
|
|
|
|
// mkdirAll ensures that the tar file contains directory
|
|
// entries for dir and all its parents. Some programs reading
|
|
// these tar files expect that. See go.dev/issue/61862.
|
|
haveDir := map[string]bool{".": true}
|
|
var mkdirAll func(string)
|
|
mkdirAll = func(dir string) {
|
|
if dir == "/" {
|
|
panic("mkdirAll /")
|
|
}
|
|
if haveDir[dir] {
|
|
return
|
|
}
|
|
haveDir[dir] = true
|
|
mkdirAll(path.Dir(dir))
|
|
df := &File{
|
|
Name: dir + "/",
|
|
Time: mtime,
|
|
Mode: dirMode,
|
|
}
|
|
h := check(tar.FileInfoHeader(df.Info(), ""))
|
|
h.Name = dir + "/"
|
|
if err := tw.WriteHeader(h); err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
|
|
for _, f = range a.Files {
|
|
h := check(tar.FileInfoHeader(f.Info(), ""))
|
|
mkdirAll(path.Dir(f.Name))
|
|
h.Name = f.Name
|
|
if err := tw.WriteHeader(h); err != nil {
|
|
panic(err)
|
|
}
|
|
r := check(os.Open(f.Src))
|
|
check(io.Copy(tw, r))
|
|
check1(r.Close())
|
|
}
|
|
f.Name = ""
|
|
check1(tw.Close())
|
|
check1(zw.Close())
|
|
check1(out.Close())
|
|
reportHash(name)
|
|
}
|
|
|
|
// writeZip writes the archive in zip form to the file named name.
|
|
func writeZip(name string, a *Archive) {
|
|
out, err := os.Create(name)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
var f File
|
|
defer func() {
|
|
if err := recover(); err != nil {
|
|
extra := ""
|
|
if f.Name != "" {
|
|
extra = " " + f.Name
|
|
}
|
|
log.Fatalf("writing %s%s: %v", name, extra, err)
|
|
}
|
|
}()
|
|
|
|
zw := zip.NewWriter(out)
|
|
zw.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) {
|
|
return flate.NewWriter(out, flate.BestCompression)
|
|
})
|
|
for _, f = range a.Files {
|
|
h := check(zip.FileInfoHeader(f.Info()))
|
|
h.Name = f.Name
|
|
h.Method = zip.Deflate
|
|
w := check(zw.CreateHeader(h))
|
|
r := check(os.Open(f.Src))
|
|
check(io.Copy(w, r))
|
|
check1(r.Close())
|
|
}
|
|
f.Name = ""
|
|
check1(zw.Close())
|
|
check1(out.Close())
|
|
reportHash(name)
|
|
}
|
|
|
|
func reportHash(name string) {
|
|
f, err := os.Open(name)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
h := sha256.New()
|
|
io.Copy(h, f)
|
|
f.Close()
|
|
fmt.Printf("distpack: %x %s\n", h.Sum(nil)[:8], filepath.Base(name))
|
|
}
|