[RFC] tools: refactor all binary packages as library packages

toolkit/docs/formats/imageconfig.md

@@ -1,6 +1,6 @@
 # Image configuration
 
-Image configuration consists of two sections - Disks and SystemConfigs - that describe the produced artifact(image). Image configuration code can be found in (configuration.go)[../../tools/imagegen/configuration/configuration.go] and validity of the configuration file can be verified by the [imageconfigvalidator](../../tools/imageconfigvalidator/imageconfigvalidator.go)
+Image configuration consists of two sections - Disks and SystemConfigs - that describe the produced artifact(image). Image configuration code can be found in (configuration.go)[../../tools/pkg/imagegen/configuration/configuration.go] and validity of the configuration file can be verified by the [imageconfigvalidator](../../tools/imageconfigvalidator/imageconfigvalidator.go)
 
 
 ## Disks

toolkit/docs/security/read-only-roots.md

@@ -49,7 +49,7 @@ Files in `/etc` such as the `passwd` and `machine-id` files are also part of the
 Ideally as much of `/etc` as possible should be left read-only to avoid miss-configuration.
 
 ### Forward Error Correction (FEC)
-Verity supports error correction which will return the original data even if the underlying blocks have been modified (it does not restore the underlying data stored on disk however). FEC incurs some overhead but the hash tree, which is a sunk cost, makes it much more effective than normal Reed-Solomon codes. For a 2GiB disk, FEC with 2 roots (i.e. 2 bytes of error correction codes per 255 bytes of real data) can correct ~16MiB of errors with ~16MiB of overhead. See [veritydisk.go](../../tools/imagegen/configuration/veritydisk.go) for calculation details.
+Verity supports error correction which will return the original data even if the underlying blocks have been modified (it does not restore the underlying data stored on disk however). FEC incurs some overhead but the hash tree, which is a sunk cost, makes it much more effective than normal Reed-Solomon codes. For a 2GiB disk, FEC with 2 roots (i.e. 2 bytes of error correction codes per 255 bytes of real data) can correct ~16MiB of errors with ~16MiB of overhead. See [veritydisk.go](../../tools/pkg/imagegen/configuration/veritydisk.go) for calculation details.
 
 ### Hash Tree and FEC Overhead
 The extra data required for verity needs to be stored outside the measured partition. In the case of Mariner it is stored in the initramfs. Assuming the Merkle tree is a full m-ary tree with m=128 (128 branches per node, from `4k/sizeof(sha256)`), the size of the Merkle tree is:

toolkit/scripts/tools.mk

@@ -14,7 +14,6 @@ $(call create_folder,$(BUILD_DIR)/tools)
 
 # List of go utilities in tools/ directory
 go_tool_list = \
-	boilerplate \
 	depsearch \
 	grapher \
 	graphpkgfetcher \
@@ -37,7 +36,7 @@ go_tool_targets = $(foreach target,$(go_tool_list),$(TOOL_BINS_DIR)/$(target))
 # Common files to monitor for all go targets
 go_module_files = $(TOOLS_DIR)/go.mod $(TOOLS_DIR)/go.sum
 go_internal_files = $(shell find $(TOOLS_DIR)/internal/ -type f -name '*.go')
-go_imagegen_files = $(shell find $(TOOLS_DIR)/imagegen/ -type f -name '*.go')
+go_imagegen_files = $(shell find $(TOOLS_DIR)/pkg/imagegen/ -type f -name '*.go')
 go_common_files = $(go_module_files) $(go_internal_files) $(go_imagegen_files) $(BUILD_DIR)/tools/internal.test_coverage
 # A report on test coverage for all the go tools
 test_coverage_report=$(TOOL_BINS_DIR)/test_coverage_report.html

toolkit/tools/depsearch/depsearch.go

@@ -9,14 +9,14 @@ import (
 	"path/filepath"
 	"sort"
 
-	"github.com/microsoft/CBL-Mariner/toolkit/tools/internal/exe"
-	"github.com/microsoft/CBL-Mariner/toolkit/tools/internal/file"
-	"github.com/microsoft/CBL-Mariner/toolkit/tools/internal/logger"
-	"github.com/microsoft/CBL-Mariner/toolkit/tools/internal/pkggraph"
-	"github.com/microsoft/CBL-Mariner/toolkit/tools/scheduler/schedulerutils"
-
 	"gonum.org/v1/gonum/graph"
 	"gopkg.in/alecthomas/kingpin.v2"
+
+	"github.com/microsoft/CBL-Mariner/toolkit/tools/internal/exe"
+	"github.com/microsoft/CBL-Mariner/toolkit/tools/internal/file"
+	"github.com/microsoft/CBL-Mariner/toolkit/tools/pkg/graph/pkggraph"
+	"github.com/microsoft/CBL-Mariner/toolkit/tools/pkg/logger"
+	"github.com/microsoft/CBL-Mariner/toolkit/tools/pkg/scheduler/schedulerutils"
 )
 
 const (

toolkit/tools/graphPreprocessor/graphPreprocessor.go

@@ -7,8 +7,9 @@ import (
 	"os"
 
 	"github.com/microsoft/CBL-Mariner/toolkit/tools/internal/exe"
-	"github.com/microsoft/CBL-Mariner/toolkit/tools/internal/logger"
-	"github.com/microsoft/CBL-Mariner/toolkit/tools/internal/pkggraph"
+	"github.com/microsoft/CBL-Mariner/toolkit/tools/pkg/graph/pkggraph"
+	"github.com/microsoft/CBL-Mariner/toolkit/tools/pkg/graph/preprocessor"
+	"github.com/microsoft/CBL-Mariner/toolkit/tools/pkg/logger"
 
 	"gopkg.in/alecthomas/kingpin.v2"
 )
@@ -23,69 +24,28 @@ var (
 	logLevel = exe.LogLevelFlag(app)
 )
 
-func replaceRunNodesWithPrebuiltNodes(pkgGraph *pkggraph.PkgGraph) (err error) {
-	for _, node := range pkgGraph.AllNodes() {
-
-		if node.Type != pkggraph.TypeRun {
-			continue
-		}
-
-		isPrebuilt, _, missing := pkggraph.IsSRPMPrebuilt(node.SrpmPath, pkgGraph, nil)
-
-		if isPrebuilt == false {
-			logger.Log.Tracef("Can't mark %s as prebuilt, missing: %v", node.SrpmPath, missing)
-			continue
-		}
-
-		preBuiltNode := pkgGraph.CloneNode(node)
-		preBuiltNode.State = pkggraph.StateUpToDate
-		preBuiltNode.Type = pkggraph.TypePreBuilt
-
-		parentNodes := pkgGraph.To(node.ID())
-		for parentNodes.Next() {
-			parentNode := parentNodes.Node().(*pkggraph.PkgNode)
-
-			if parentNode.Type != pkggraph.TypeGoal {
-				pkgGraph.RemoveEdge(parentNode.ID(), node.ID())
-
-				logger.Log.Debugf("Adding a 'PreBuilt' node '%s' with id %d. For '%s'", preBuiltNode.FriendlyName(), preBuiltNode.ID(), parentNode.FriendlyName())
-				err = pkgGraph.AddEdge(parentNode, preBuiltNode)
-
-				if err != nil {
-					logger.Log.Errorf("Adding edge failed for %v -> %v", parentNode, preBuiltNode)
-					return
-				}
-			}
-		}
+func populatePreprocessorConfig() *preprocessor.Config {
+	return &preprocessor.Config{
+		InputGraphFile:  *inputGraphFile,
+		OutputGraphFile: *outputGraphFile,
+		HydratedBuild:   *hydratedBuild,
 	}
 
-	return
 }
 
 func main() {
 	app.Version(exe.ToolkitVersion)
 	kingpin.MustParse(app.Parse(os.Args[1:]))
 	logger.InitBestEffort(*logFile, *logLevel)
 
-	scrubbedGraph := pkggraph.NewPkgGraph()
-
-	err := pkggraph.ReadDOTGraphFile(scrubbedGraph, *inputGraphFile)
+	cfg := populatePreprocessorConfig()
+	scrubbedGraph, err := cfg.ReadAndPreprocessGraph()
 	if err != nil {
-		logger.Log.Panicf("Failed to read graph to file, %s. Error: %s", *inputGraphFile, err)
+		logger.Log.Panic(err)
 	}
-
-	if *hydratedBuild {
-		logger.Log.Debugf("Nodes before replacing prebuilt nodes: %d", len(scrubbedGraph.AllNodes()))
-		err = replaceRunNodesWithPrebuiltNodes(scrubbedGraph)
-		logger.Log.Debugf("Nodes after replacing prebuilt nodes: %d", len(scrubbedGraph.AllNodes()))
-		if err != nil {
-			logger.Log.Panicf("Failed to replace run nodes with preBuilt nodes. Error: %s", err)
-		}
-	}
-
-	err = pkggraph.WriteDOTGraphFile(scrubbedGraph, *outputGraphFile)
+	err = pkggraph.WriteDOTGraphFile(scrubbedGraph, cfg.OutputGraphFile)
 	if err != nil {
-		logger.Log.Panicf("Failed to write cache graph to file, %s. Error: %s", *outputGraphFile, err)
+		logger.Log.Panicf("Failed to write cache graph to file, %s. Error: %s", cfg.OutputGraphFile, err)
 	}
 	return
 }

toolkit/tools/graphanalytics/graphanalytics.go

@@ -11,9 +11,9 @@ import (
 	"strings"
 
 	"github.com/microsoft/CBL-Mariner/toolkit/tools/internal/exe"
-	"github.com/microsoft/CBL-Mariner/toolkit/tools/internal/logger"
-	"github.com/microsoft/CBL-Mariner/toolkit/tools/internal/pkggraph"
 	"github.com/microsoft/CBL-Mariner/toolkit/tools/internal/sliceutils"
+	"github.com/microsoft/CBL-Mariner/toolkit/tools/pkg/graph/pkggraph"
+	"github.com/microsoft/CBL-Mariner/toolkit/tools/pkg/logger"
 
 	"gonum.org/v1/gonum/graph"
 	graphpath "gonum.org/v1/gonum/graph/path"