diff --git a/src/go/api/vm/util.go b/src/go/api/vm/util.go index e2cf623d..deea968d 100644 --- a/src/go/api/vm/util.go +++ b/src/go/api/vm/util.go @@ -2,6 +2,7 @@ package vm import ( "context" + "encoding/json" "fmt" "io" "os" @@ -12,12 +13,19 @@ import ( "time" "phenix/api/experiment" + "phenix/util/common" + "phenix/util/mm" ) +type qemuBackingChain struct { + Filename string `json:"filename"` + BackingFile string `json:"backing-filename"` +} + var diskNameWithTstampRegex = regexp.MustCompile(`(.*)_\d{14}`) func GetNewDiskName(expName, vmName string) (string, error) { - base, err := getBaseImage(expName, vmName) + base, err := getVMImage(expName, vmName) if err != nil { return "", fmt.Errorf("getting base disk image: %w", err) } @@ -39,7 +47,7 @@ func GetNewDiskName(expName, vmName string) (string, error) { return name, nil } -func getBaseImage(expName, vmName string) (string, error) { +func getVMImage(expName, vmName string) (string, error) { exp, err := experiment.Get(expName) if err != nil { return "", fmt.Errorf("getting experiment %s: %w", expName, err) @@ -50,9 +58,79 @@ func getBaseImage(expName, vmName string) (string, error) { return "", fmt.Errorf("getting vm %s for experiment %s", vmName, expName) } + // base image from topology return vm.Hardware().Drives()[0].Image(), nil } +func getBackingImage(path string) (string, error) { + if !filepath.IsAbs(path) { + path = common.PhenixBase + "/images/" + path + } + + chain, err := getImageBackingChain(path) + if err != nil { + return "", fmt.Errorf("getting image backing chain for %s: %w", path, err) + } + + // backing image should always be last image in chain + path = chain[len(chain)-1].Filename + + stats, err := os.Lstat(path) + if err != nil { + return "", fmt.Errorf("getting file stats for %s: %w", path, err) + } + + // base image path is a symlink + if stats.Mode()&os.ModeSymlink != 0 { + origPath := path + + path, err = os.Readlink(path) + if err != nil { + return "", fmt.Errorf("getting symlink target for %s: %w", origPath, err) + } + } + + return path, nil +} + +func getImageSnapshots(path string) ([]string, error) { + chain, err := getImageBackingChain(path) + if err != nil { + return nil, fmt.Errorf("getting image backing chain for %s: %w", path, err) + } + + if len(chain) <= 1 { + return nil, nil + } + + var snapshots []string + + // range chain in reverse to get snapshots in correct order for rebasing + // skip last entry since it will be the base image (not a snapshot) + for i := len(chain) - 2; i >= 0; i-- { + snapshots = append(snapshots, chain[i].Filename) + } + + return snapshots, nil +} + +func getImageBackingChain(path string) ([]qemuBackingChain, error) { + cmd := fmt.Sprintf("qemu-img info --backing-chain %s --output json", path) + + resp, err := mm.MeshShellResponse("", cmd) + if err != nil { + return nil, fmt.Errorf("getting image info for %s: %w", path, err) + } + + var chain []qemuBackingChain + + if err := json.Unmarshal([]byte(resp), &chain); err != nil { + return nil, fmt.Errorf("parsing image info for %s: %w", path, err) + } + + return chain, nil +} + type copier struct { subs []chan float64 } diff --git a/src/go/api/vm/vm.go b/src/go/api/vm/vm.go index 72d10797..457cb177 100644 --- a/src/go/api/vm/vm.go +++ b/src/go/api/vm/vm.go @@ -940,9 +940,27 @@ func CommitToDisk(expName, vmName, out string, cb func(float64)) (string, error) } } - base, err := getBaseImage(expName, vmName) + disk, err := getVMImage(expName, vmName) if err != nil { - return "", fmt.Errorf("getting base image for VM %s in experiment %s: %w", vmName, expName, err) + return "", fmt.Errorf("getting disk image for VM %s in experiment %s: %w", vmName, expName, err) + } + + if !filepath.IsAbs(disk) { + disk = common.PhenixBase + "/images/" + disk + } + + if !filepath.IsAbs(out) { + out = common.PhenixBase + "/images/" + out + } + + base, err := getBackingImage(disk) + if err != nil { + return "", fmt.Errorf("getting backing image for VM %s in experiment %s: %w", vmName, expName, err) + } + + snapshots, err := getImageSnapshots(disk) + if err != nil { + return "", fmt.Errorf("getting disk snapshots for VM %s in experiment %s: %w", vmName, expName, err) } // Get status of VM (scheduled host, VM state). @@ -964,14 +982,6 @@ func CommitToDisk(expName, vmName, out string, cb func(float64)) (string, error) node = status[0]["host"] ) - if !filepath.IsAbs(base) { - base = common.PhenixBase + "/images/" + base - } - - if !filepath.IsAbs(out) { - out = common.PhenixBase + "/images/" + out - } - wait, ctx := errgroup.WithContext(context.Background()) // Make copy of base image locally on headnode. Using a context here will help @@ -1007,6 +1017,37 @@ func CommitToDisk(expName, vmName, out string, cb func(float64)) (string, error) } } + // Make a copy of any snapshots pointed at by the VM disk image in the + // experiment topology so they can be rebased onto the base backing image for + // the VM. + + for idx, snapshot := range snapshots { + var ( + id = idx + file = snapshot + ) + + wait.Go(func() error { + copier := newCopier() + + tmp := fmt.Sprintf("%s/images/%s/tmp/snapshot-%d.qc2", common.PhenixBase, expName, id) + + if err := os.MkdirAll(filepath.Dir(tmp), 0755); err != nil { + return fmt.Errorf("creating experiment tmp directory: %w", err) + } + + if err := copier.copy(ctx, file, tmp); err != nil { + os.Remove(tmp) + return fmt.Errorf("making copy of snapshot %s: %w", file, err) + } + + // update snapshots entry to point at new copy of snapshot + snapshots[id] = tmp + + return nil + }) + } + // Copy minimega snapshot disk on remote machine to a location (still on // remote machine) that can be seen by minimega files. Then use minimega `file // get` to copy it to the headnode. @@ -1048,13 +1089,27 @@ func CommitToDisk(expName, vmName, out string, cb func(float64)) (string, error) return "", fmt.Errorf("preparing images for rebase/commit: %w", err) } - snap = fmt.Sprintf("%s/images/%s/tmp/%s.qc2", common.PhenixBase, expName, vmName) + // Add the final boss (the minimega snapshot disk) to the end of the list of + // snapshots to rebase onto the copy of the original backing image. If the + // image pointed at by the VM config in the topology was not a snapshot, then + // this will be the only snapshot in the list. + snapshots = append(snapshots, fmt.Sprintf("%s/images/%s/tmp/%s.qc2", common.PhenixBase, expName, vmName)) - shell := exec.Command("qemu-img", "rebase", "-f", "qcow2", "-b", out, "-F", "qcow2", snap) + for idx, snapshot := range snapshots { + // first parent should be copy of original backing image + parent := out + + if idx != 0 { + parent = snapshots[idx-1] + } + + shell := exec.Command("qemu-img", "rebase", "-f", "qcow2", "-b", parent, "-F", "qcow2", snapshot) + + res, err := shell.CombinedOutput() + if err != nil { + return "", fmt.Errorf("rebasing snapshot (%s): %w", string(res), err) + } - res, err := shell.CombinedOutput() - if err != nil { - return "", fmt.Errorf("rebasing snapshot (%s): %w", string(res), err) } done := make(chan struct{}) @@ -1091,9 +1146,9 @@ func CommitToDisk(expName, vmName, out string, cb func(float64)) (string, error) }() } - shell = exec.Command("qemu-img", "commit", snap) + shell := exec.Command("qemu-img", "commit", "-b", out, snapshots[len(snapshots)-1]) - res, err = shell.CombinedOutput() + res, err := shell.CombinedOutput() if err != nil { return "", fmt.Errorf("committing snapshot (%s): %w", string(res), err) } @@ -1477,16 +1532,13 @@ func ChangeOpticalDisc(expName, vmName, isoPath string) error { return fmt.Errorf("no optical disc path provided") } - - cmd := mmcli.NewNamespacedCommand(expName) - cmd.Command = fmt.Sprintf("vm cdrom change %s %s",vmName,isoPath) + cmd := mmcli.NewNamespacedCommand(expName) + cmd.Command = fmt.Sprintf("vm cdrom change %s %s", vmName, isoPath) if err := mmcli.ErrorResponse(mmcli.Run(cmd)); err != nil { return fmt.Errorf("changing optical disc for VM %s: %w", vmName, err) } - - return nil } @@ -1501,15 +1553,12 @@ func EjectOpticalDisc(expName, vmName string) error { return fmt.Errorf("no VM name provided") } - - cmd := mmcli.NewNamespacedCommand(expName) - cmd.Command = fmt.Sprintf("vm cdrom eject %s",vmName) + cmd := mmcli.NewNamespacedCommand(expName) + cmd.Command = fmt.Sprintf("vm cdrom eject %s", vmName) if err := mmcli.ErrorResponse(mmcli.Run(cmd)); err != nil { return fmt.Errorf("ejecting optical disc for VM %s: %w", vmName, err) } - return nil } - diff --git a/src/go/util/mm/mm.go b/src/go/util/mm/mm.go index 83ecb091..7f5fa3bd 100644 --- a/src/go/util/mm/mm.go +++ b/src/go/util/mm/mm.go @@ -47,5 +47,6 @@ type MM interface { TapVLAN(...TapOption) error MeshShell(string, string) error + MeshShellResponse(string, string) (string, error) MeshSend(string, string, string) error } diff --git a/src/go/util/mm/package.go b/src/go/util/mm/package.go index 67c515e5..e5d84cb5 100644 --- a/src/go/util/mm/package.go +++ b/src/go/util/mm/package.go @@ -140,6 +140,10 @@ func MeshShell(host, cmd string) error { return DefaultMM.MeshShell(host, cmd) } +func MeshShellResponse(host, cmd string) (string, error) { + return DefaultMM.MeshShellResponse(host, cmd) +} + func MeshSend(ns, host, command string) error { return DefaultMM.MeshSend(ns, host, command) }