Fix dh-make-golang estimate

estimate.go

@@ -3,41 +3,43 @@ package main
 import (
 	"flag"
 	"fmt"
-	"go/build"
 	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
-	"sort"
+	"regexp"
+	"strconv"
 	"strings"
 
 	"golang.org/x/tools/go/vcs"
-	"golang.org/x/tools/refactor/importgraph"
 )
 
-func get(gopath, repo string) error {
+// majorVersionRegexp checks if an import path contains a major version suffix.
+var majorVersionRegexp = regexp.MustCompile(`([/.])v([0-9]+)$`)
+
+func get(gopath, repodir, repo string) error {
 	done := make(chan struct{})
 	defer close(done)
-	go progressSize("go get", filepath.Join(gopath, "src"), done)
+	go progressSize("go get", repodir, done)
 
 	// As per https://groups.google.com/forum/#!topic/golang-nuts/N5apfenE4m4,
 	// the arguments to “go get” are packages, not repositories. Hence, we
 	// specify “gopkg/...” in order to cover all packages.
 	// As a concrete example, github.com/jacobsa/util is a repository we want
-	// to package into a single Debian package, and using “go get -d
+	// to package into a single Debian package, and using “go get -t
 	// github.com/jacobsa/util” fails because there are no buildable go files
 	// in the top level of that repository.
-	cmd := exec.Command("go", "get", "-d", "-t", repo+"/...")
+	cmd := exec.Command("go", "get", "-t", repo+"/...")
+	cmd.Dir = repodir
 	cmd.Stderr = os.Stderr
 	cmd.Env = append([]string{
-		"GO111MODULE=off",
 		"GOPATH=" + gopath,
 	}, passthroughEnv()...)
 	return cmd.Run()
 }
 
 func removeVendor(gopath string) (found bool, _ error) {
-	err := filepath.Walk(filepath.Join(gopath, "src"), func(path string, info os.FileInfo, err error) error {
+	err := filepath.Walk(gopath, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -56,117 +58,179 @@ func removeVendor(gopath string) (found bool, _ error) {
 	return found, err
 }
 
+// otherVersions guesses the import paths of potential other major version
+// of the given module import path, based on [majorVersionRegex].
+func otherVersions(mod string) (mods []string) {
+	matches := majorVersionRegexp.FindStringSubmatch(mod)
+	if matches == nil {
+		return
+	}
+	matchFull, matchSep, matchVer := matches[0], matches[1], matches[2]
+	matchIndex := len(mod) - len(matchFull)
+	prefix := mod[:matchIndex]
+	version, _ := strconv.Atoi(matchVer)
+	for v := version - 1; v > 1; v-- {
+		mods = append(mods, prefix+matchSep+"v"+strconv.Itoa(v))
+	}
+	mods = append(mods, prefix)
+	return
+}
+
 func estimate(importpath string) error {
+	removeTemp := func(path string) {
+		if err := forceRemoveAll(path); err != nil {
+			log.Printf("could not remove all %s: %v", path, err)
+		}
+	}
+
 	// construct a separate GOPATH in a temporary directory
 	gopath, err := os.MkdirTemp("", "dh-make-golang")
 	if err != nil {
 		return fmt.Errorf("create temp dir: %w", err)
 	}
-	defer os.RemoveAll(gopath)
+	defer removeTemp(gopath)
+	// second temporary directosy for the repo sources
+	repodir, err := os.MkdirTemp("", "dh-make-golang")
+	if err != nil {
+		return fmt.Errorf("create temp dir: %w", err)
+	}
+	defer removeTemp(repodir)
+
+	// Create a dummy go module in repodir to be able to use go get.
+	err = os.WriteFile(filepath.Join(repodir, "go.mod"), []byte("module dummymod\n"), 0644)
+	if err != nil {
+		return fmt.Errorf("create dummymod: %w", err)
+	}
 
-	if err := get(gopath, importpath); err != nil {
+	if err := get(gopath, repodir, importpath); err != nil {
 		return fmt.Errorf("go get: %w", err)
 	}
 
-	found, err := removeVendor(gopath)
+	found, err := removeVendor(repodir)
 	if err != nil {
 		return fmt.Errorf("remove vendor: %w", err)
 	}
 
 	if found {
 		// Fetch un-vendored dependencies
-		if err := get(gopath, importpath); err != nil {
+		if err := get(gopath, repodir, importpath); err != nil {
 			return fmt.Errorf("fetch un-vendored: go get: %w", err)
 		}
 	}
 
-	// Remove standard lib packages
-	cmd := exec.Command("go", "list", "std")
+	// Get dependency graph from go mod graph
+	cmd := exec.Command("go", "mod", "graph")
+	cmd.Dir = repodir
 	cmd.Stderr = os.Stderr
 	cmd.Env = append([]string{
-		"GO111MODULE=off",
 		"GOPATH=" + gopath,
 	}, passthroughEnv()...)
-
 	out, err := cmd.Output()
 	if err != nil {
-		return fmt.Errorf("go list std: args: %v; error: %w", cmd.Args, err)
-	}
-	stdlib := make(map[string]bool)
-	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
-		stdlib[line] = true
+		return fmt.Errorf("go mod graph: args: %v; error: %w", cmd.Args, err)
 	}
 
-	stdlib["C"] = true // would fail resolving anyway
-
-	// Filter out all already-packaged ones:
+	// Retrieve already-packaged ones
 	golangBinaries, err := getGolangBinaries()
 	if err != nil {
 		return nil
 	}
 
-	build.Default.GOPATH = gopath
-	forward, _, errors := importgraph.Build(&build.Default)
-	if len(errors) > 0 {
-		lines := make([]string, 0, len(errors))
-		for importPath, err := range errors {
-			lines = append(lines, fmt.Sprintf("%s: %v", importPath, err))
+	// Build a graph in memory from the output of go mod graph
+	type Node struct {
+		name     string
+		children []*Node
+	}
+	root := &Node{name: importpath}
+	nodes := make(map[string]*Node)
+	nodes[importpath] = root
+	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
+		// go mod graph outputs one line for each dependency. Each line
+		// consists of the dependency preceded by the module that
+		// imported it, separated by a single space. The module names
+		// can have a version information delimited by the @ character
+		src, dep, _ := strings.Cut(line, " ")
+		// The root module is the only one that does not have a version
+		// indication with @ in the output of go mod graph. We use this
+		// to filter out the depencencies of the "dummymod" module.
+		if mod, _, found := strings.Cut(src, "@"); !found {
+			continue
+		} else if mod == importpath || strings.HasPrefix(mod, importpath+"/") {
+			src = importpath
+		}
+		depNode, ok := nodes[dep]
+		if !ok {
+			depNode = &Node{name: dep}
+			nodes[dep] = depNode
 		}
-		return fmt.Errorf("could not load packages: %v", strings.Join(lines, "\n"))
+		srcNode, ok := nodes[src]
+		if !ok {
+			srcNode = &Node{name: src}
+			nodes[src] = srcNode
+		}
+		srcNode.children = append(srcNode.children, depNode)
 	}
 
+	// Analyse the dependency graph
 	var lines []string
 	seen := make(map[string]bool)
-	rrseen := make(map[string]bool)
-	node := func(importPath string, indent int) {
-		rr, err := vcs.RepoRootForImportPath(importPath, false)
-		if err != nil {
-			log.Printf("Could not determine repo path for import path %q: %v\n", importPath, err)
+	var visit func(n *Node, indent int)
+	visit = func(n *Node, indent int) {
+		// Get the module name without its version, as go mod graph
+		// can return multiple times the same module with different
+		// versions.
+		mod, _, _ := strings.Cut(n.name, "@")
+		if seen[mod] {
 			return
 		}
-		if rrseen[rr.Root] {
+		seen[mod] = true
+		// Go version dependency is indicated as a dependency to "go" and
+		// "toolchain", we do not use this information for now.
+		if mod == "go" || mod == "toolchain" {
 			return
 		}
-		rrseen[rr.Root] = true
-		if _, ok := golangBinaries[rr.Root]; ok {
+		if _, ok := golangBinaries[mod]; ok {
 			return // already packaged in Debian
 		}
-		lines = append(lines, fmt.Sprintf("%s%s", strings.Repeat("  ", indent), rr.Root))
-	}
-	var visit func(x string, indent int)
-	visit = func(x string, indent int) {
-		if seen[x] {
-			return
+		var debianVersion string
+		// Check for potential other major versions already in Debian.
+		for _, otherVersion := range otherVersions(mod) {
+			if _, ok := golangBinaries[otherVersion]; ok {
+				debianVersion = otherVersion
+				break
+			}
 		}
-		seen[x] = true
-		if !stdlib[x] {
-			node(x, indent)
+		if debianVersion == "" {
+			// When multiple modules are developped in the same repo,
+			// the repo root is often used as the import path metadata
+			// in Debian, so we do a last try with that.
+			rr, err := vcs.RepoRootForImportPath(mod, false)
+			if err != nil {
+				log.Printf("Could not determine repo path for import path %q: %v\n", mod, err)
+			} else if _, ok := golangBinaries[rr.Root]; ok {
+				// Log info to indicate that it is an approximate match
+				// but consider that it is packaged and skip the children.
+				log.Printf("%s is packaged as %s in Debian", mod, rr.Root)
+				return
+			}
 		}
-		for y := range forward[x] {
-			visit(y, indent+1)
-		}
-	}
-
-	keys := make([]string, 0, len(forward))
-	for key := range forward {
-		keys = append(keys, key)
-	}
-	sort.Strings(keys)
-	for _, key := range keys {
-		if !strings.HasPrefix(key, importpath) {
-			continue
+		if debianVersion != "" {
+			lines = append(lines, fmt.Sprintf("%s%s\t(%s in Debian)", strings.Repeat("  ", indent), mod, debianVersion))
+		} else {
+			lines = append(lines, fmt.Sprintf("%s%s", strings.Repeat("  ", indent), mod))
 		}
-		if seen[key] {
-			continue // already covered in a previous visit call
+		for _, n := range n.children {
+			visit(n, indent+1)
 		}
-		visit(key, 0)
 	}
 
+	visit(root, 0)
+
 	if len(lines) == 0 {
 		log.Printf("%s is already fully packaged in Debian", importpath)
 		return nil
 	}
-	log.Printf("Bringing %s to Debian requires packaging the following Go packages:", importpath)
+	log.Printf("Bringing %s to Debian requires packaging the following Go modules:", importpath)
 	for _, line := range lines {
 		fmt.Println(line)
 	}
@@ -178,8 +242,8 @@ func execEstimate(args []string) {
 	fs := flag.NewFlagSet("estimate", flag.ExitOnError)
 
 	fs.Usage = func() {
-		fmt.Fprintf(os.Stderr, "Usage: %s estimate <go-package-importpath>\n", os.Args[0])
-		fmt.Fprintf(os.Stderr, "Estimates the work necessary to bring <go-package-importpath> into Debian\n"+
+		fmt.Fprintf(os.Stderr, "Usage: %s estimate <go-module-importpath>\n", os.Args[0])
+		fmt.Fprintf(os.Stderr, "Estimates the work necessary to bring <go-module-importpath> into Debian\n"+
 			"by printing all currently unpacked repositories.\n")
 		fmt.Fprintf(os.Stderr, "Example: %s estimate github.com/Debian/dh-make-golang\n", os.Args[0])
 	}

fs_utils.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"io/fs"
+	"os"
+	"path/filepath"
+)
+
+// forceRemoveAll is a more robust alternative to [os.RemoveAll] that tries
+// harder to remove all the files and directories.
+func forceRemoveAll(path string) error {
+	// first pass to make sure all the directories are writable
+	err := filepath.Walk(path, func(path string, info fs.FileInfo, err error) error {
+		if info.IsDir() {
+			return os.Chmod(path, 0777)
+		} else {
+			// remove files by the way
+			return os.Remove(path)
+		}
+	})
+	if err != nil {
+		return err
+	}
+	// remove the remaining directories
+	return os.RemoveAll(path)
+}