diff --git a/configs/disk-image-configs/gem5init b/configs/disk-image-configs/gem5init index cf40cc6..dda9800 100644 --- a/configs/disk-image-configs/gem5init +++ b/configs/disk-image-configs/gem5init @@ -1,5 +1,17 @@ #!/bin/bash - + +if [ $(uname -i) == "aarch64" ]; +then + echo "On Arm machine" + M5_ADDR="--addr=0x10010000" +else + echo "On x86 machine" + M5_ADDR=0xffff0000 + M5_ADDR="" +fi + + function gem5_run() { # ==== # gem5 init service @@ -8,19 +20,11 @@ function gem5_run() { # Try to read in a run script from the host system. # For gem5 use the special magic instruction `m5 readfile` # - /sbin/m5 readfile > /tmp/script chmod 755 /tmp/script - if [ -s /tmp/script ] - then - # If there is a script, execute the script and then exit the simulation - su root -c '/tmp/script' # gives script full privileges as root user in multi-user mode - sync - sleep 10 - /sbin/m5 exit - else - echo "No script found" - fi - echo "Gem5 init done" + /tmp/script || true + echo "Done running script, exiting." + rm -f /tmp/script + /sbin/m5 $M5_ADDR exit } @@ -36,35 +40,35 @@ function qemu_run() { # curl "http://10.0.2.2:3003/run.sh" -f -o /tmp/script - chmod 755 /tmp/script if [ 0 -eq $? ]; then + echo "Found file server in qemu." echo "Run script found... run it." + chmod 755 /tmp/script # If there is a script, execute the script and then shutdown the machine - su root -c '/tmp/script' # gives script full privileges as root user in multi-user mode - sync - sleep 10 + /tmp/script || true + echo "Done running script, exiting." + rm -f /tmp/script else - echo "No script found" + echo "No file server found" fi } -## -## Check if we are in gem5 or qemu -## -CPU=`cat /proc/cpuinfo | grep vendor_id | head -n 1 | cut -d ' ' -f2-` -echo "Got CPU type: $CPU" -if [ "$CPU" == "M5 Simulator" ]; -then +printf "Starting gem5 init... trying to read run script file via readfile.\n" + +if ! m5 $M5_ADDR readfile > /tmp/script; then + + printf "Failed to run m5 readfile. Try QEMU read.\n" + rm -f /tmp/script + + # Read was not successful + # Try to read script from file server + qemu_run +else echo "In gem5. Try loading script" gem5_run -else - echo "Not in gem5. Try to load script from http server" - qemu_run fi -exit 0 - diff --git a/docs/simulation/simple_component.md b/docs/simulation/simple_component.md new file mode 100644 index 0000000..ce21ddc --- /dev/null +++ b/docs/simulation/simple_component.md @@ -0,0 +1,48 @@ +--- +layout: default +title: Simple Simulation with gem5 Component Library +parent: Simulation +nav_order: 1 +--- + +# Simulation Methodology +{: .no_toc } + +
+ + Table of contents + + {: .text-delta } +1. TOC +{:toc} +
+ + + + +--- + +Gem5 offers a wide range of prebuild configurations, aka. standard library, for cpus, boards, caches, etc. To simplify the configuration process we provide a configuration script that is mostly constructed out the component models provided by gem5. This script is called `vswarm_simple.py` and will be copied into your working directory. + +After installing the functions on the disk you can perform the *setup* step with the following command: + +```bash +/build//gem5.opt vswarm_simple.py --kernel kernel --disk disk.img --mode setup --atomic-warming=50 --num-invocations=20 +``` + +This will boot the kernel, function, invoke the function for 50 times and then create a checkpoint. The simulation will exit or can be killed otherwise + +Once the checkpoint is created the simulation can be performed with: + +```bash +/build//gem5.opt vswarm_simple.py --kernel kernel --disk disk.img --mode evaluation --atomic-warming=50 --num-invocations=20 +``` + +The script was tested with ATOMIC,TIMING and O3 core which can be configured commenting the corresponding lines in the script +``` +eval_core = CPUTypes. +``` \ No newline at end of file diff --git a/gem5utils/systems/simple/system.py b/gem5utils/systems/simple/system.py index b19994d..24bc10a 100644 --- a/gem5utils/systems/simple/system.py +++ b/gem5utils/systems/simple/system.py @@ -36,7 +36,7 @@ class SimpleSystem(System): - def __init__(self, kernel, disk, num_cpus=2, CPUModel=AtomicSimpleCPU, kvm=True): + def __init__(self, kernel, disk, num_cpus=2, CPUModel=X86AtomicSimpleCPU, kvm=True): super(SimpleSystem, self).__init__() self._host_parallel = True if kvm and num_cpus > 1 else False @@ -115,7 +115,7 @@ def __init__(self, kernel, disk, num_cpus=2, CPUModel=AtomicSimpleCPU, kvm=True) def getHostParallel(self): return self._host_parallel - def createCPU(self, num_cpus=2, CPUModel=AtomicSimpleCPU): + def createCPU(self, num_cpus=2, CPUModel=X86AtomicSimpleCPU): """ Create the CPUs for the system """ # Beside the CPU we use for simulation we will use @@ -123,7 +123,7 @@ def createCPU(self, num_cpus=2, CPUModel=AtomicSimpleCPU): # Note KVM needs a VM and atomic_noncaching print("Create CPU: ", CPUModel) self.cpu = [CPUModel(cpu_id = i) for i in range(num_cpus)] - self.atomic_cpu = [AtomicSimpleCPU( + self.atomic_cpu = [X86AtomicSimpleCPU( cpu_id = i, switched_out=True) for i in range(num_cpus)] diff --git a/gem5utils/systems/skylake/core.py b/gem5utils/systems/skylake/core.py index ba09740..d9344b1 100644 --- a/gem5utils/systems/skylake/core.py +++ b/gem5utils/systems/skylake/core.py @@ -174,16 +174,13 @@ class LTAGE_BP(LTAGE_TAGE): logUResetPeriod = 19 class BranchPred(LTAGE): - BTBEntries = 512 - BTBTagSize = 19 - RASSize = 32 indirectBranchPred = IndirectPred() # use NULL to disable tage = LTAGE_BP() depth = 3 width = 4 -class SklVerbatimCPU(DerivO3CPU): +class SklVerbatimCPU(X86O3CPU): """ Uncalibrated: Configured based on micro-architecture documentation """ branchPred = BranchPred() @@ -222,7 +219,7 @@ class SklVerbatimCPU(DerivO3CPU): depth = 3 width = 4 -class SklTunedCPU(DerivO3CPU): +class SklTunedCPU(X86O3CPU): """ Calibrated Skylake: configured to match the performance of hardware """ branchPred = BranchPred() @@ -269,7 +266,7 @@ class SklTunedCPU(DerivO3CPU): depth = 3 width = 32 -class UnconstrainedCPU(DerivO3CPU): +class UnconstrainedCPU(X86O3CPU): """ Configuration with maximum pipeline widths and mininum delays """ branchPred = BranchPred() diff --git a/setup/disk.Makefile b/setup/disk.Makefile index 505d960..e89925d 100644 --- a/setup/disk.Makefile +++ b/setup/disk.Makefile @@ -42,14 +42,14 @@ CPUS := 4 UBUNTU_VERSION ?= focal ifeq ($(UBUNTU_VERSION), focal) - CLOUD_IMAGE_FILE := ubuntu-20.04.5-live-server-amd64.iso + CLOUD_IMAGE_FILE := ubuntu-20.04.6-live-server-amd64.iso CLOUD_IMAGE_BASE_URL := https://releases.ubuntu.com/20.04/ - CLOUD_IMAGE_HASH := 5035be37a7e9abbdc09f0d257f3e33416c1a0fb322ba860d42d74aa75c3468d4 + CLOUD_IMAGE_HASH := b8f31413336b9393ad5d8ef0282717b2ab19f007df2e9ed5196c13d8f9153c8b KERNEL_CUSTOM ?= $(RESOURCES)/vmlinux-focal-amd64 else ifeq ($(UBUNTU_VERSION), jammy) - CLOUD_IMAGE_FILE := ubuntu-22.04.1-live-server-amd64.iso + CLOUD_IMAGE_FILE := ubuntu-22.04.4-live-server-amd64.iso CLOUD_IMAGE_BASE_URL := https://releases.ubuntu.com/22.04/ - CLOUD_IMAGE_HASH := 10f19c5b2b8d6db711582e0e27f5116296c34fe4b313ba45f9b201a5007056cb + CLOUD_IMAGE_HASH := 45f873de9f8cb637345d6e66a583762730bbea30277ef7b32c9c3bd6700a32b2 KERNEL_CUSTOM ?= $(RESOURCES)/vmlinux-jammy-amd64 else @echo "Unsupported ubuntu version $(UBUNTU_VERSION)" diff --git a/setup/disk_arm.Makefile b/setup/disk_arm.Makefile index deb9154..042e1a4 100644 --- a/setup/disk_arm.Makefile +++ b/setup/disk_arm.Makefile @@ -47,9 +47,9 @@ ifeq ($(UBUNTU_VERSION), focal) CLOUD_IMAGE_HASH := e42d6373dd39173094af5c26cbf2497770426f42049f8b9ea3e60ce35bebdedf KERNEL_CUSTOM ?= $(RESOURCES)/vmlinux-focal-arm64 else ifeq ($(UBUNTU_VERSION), jammy) - CLOUD_IMAGE_FILE := ubuntu-22.04.1-live-server-arm64.iso + CLOUD_IMAGE_FILE := ubuntu-22.04.4-live-server-arm64.iso CLOUD_IMAGE_BASE_URL := https://cdimage.ubuntu.com/releases/22.04/release/ - CLOUD_IMAGE_HASH := bc5a8015651c6f8699ab262d333375d3930b824f03d14ae51e551d89d9bb571c + CLOUD_IMAGE_HASH := 74b8a9f71288ae0ac79075c2793a0284ef9b9729a3dcf41b693d95d724622b65 KERNEL_CUSTOM ?= $(RESOURCES)/vmlinux-jammy-arm64 else @echo "Unsupported ubuntu version $(UBUNTU_VERSION)" diff --git a/setup/gem5.Makefile b/setup/gem5.Makefile index e983990..d033499 100644 --- a/setup/gem5.Makefile +++ b/setup/gem5.Makefile @@ -28,8 +28,8 @@ ROOT := $(abspath $(dir $(mkfile_path))/../) ## User specific inputs RESOURCES ?=$(ROOT)/resources/ -ARCH := X86 -VERSION := v22.0.0.1 +ARCH := ALL +VERSION := v24.0.0.0 GEM5_DIR ?= $(RESOURCES)/gem5/ GEM5 := $(GEM5_DIR)/build/$(ARCH)/gem5.opt @@ -53,7 +53,7 @@ dep_install: ## Clone repo -- $(GEM5_DIR): - git clone https://github.com/ease-lab/gem5.git $@ + git clone https://github.com/gem5/gem5.git $@ cd $@; git checkout $(VERSION) diff --git a/simulation/Makefile b/simulation/Makefile index 7cc8669..022f234 100644 --- a/simulation/Makefile +++ b/simulation/Makefile @@ -91,12 +91,13 @@ SERVE := $(WORKING_DIR)/server.pid FUNCTIONS_YAML := $(WORKING_DIR)/functions.yaml FUNCTIONS_LIST := $(WORKING_DIR)/functions.list GEM5_CONFIG := $(WORKING_DIR)/run_sim.py +GEM5_SIMPLE_CONFIG := $(WORKING_DIR)/vswarm_simple.py SETUP_ALL_SCRIPT := $(WORKING_DIR)/setup_all_functions.sh SETUP_FN_SCRIPT := $(WORKING_DIR)/setup_function.sh SIM_ALL_SCRIPT := $(WORKING_DIR)/sim_all_functions.sh SIM_FN_SCRIPT := $(WORKING_DIR)/sim_function.sh -templates: $(SETUP_ALL_SCRIPT) $(SETUP_FN_SCRIPT) $(SIM_ALL_SCRIPT) $(SIM_FN_SCRIPT) $(GEM5_CONFIG) $(FUNCTIONS_YAML) $(FUNCTIONS_LIST) +templates: $(SETUP_ALL_SCRIPT) $(SETUP_FN_SCRIPT) $(SIM_ALL_SCRIPT) $(SIM_FN_SCRIPT) $(GEM5_CONFIG) $(GEM5_SIMPLE_CONFIG) $(FUNCTIONS_YAML) $(FUNCTIONS_LIST) $(WORKING_DIR)/functions.%: $(ROOT)/simulation/functions/functions.% diff --git a/simulation/wkdir-tmpl/vswarm_simple.tmpl.py b/simulation/wkdir-tmpl/vswarm_simple.tmpl.py new file mode 100644 index 0000000..feef106 --- /dev/null +++ b/simulation/wkdir-tmpl/vswarm_simple.tmpl.py @@ -0,0 +1,278 @@ +# +# Copyright (c) 2024 David Schall and EASE lab +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# Install dependencies + +""" +This script show a simple example to run serverless functions on a x86 based full system +simulation. The script boots a full system Ubuntu image and starts the function container. +The function is invoked using a test client. + +The workflow has two steps +1. Use the "setup" mode to boot the full system from scratch using the KVM core. The + script will perform functional warming and then take a checkpoint of the system. +2. Use the "evaluation" mode to start from the previously taken checkpoint and perform + the actual measurements using a detailed core model. + +Usage +----- + +``` +scons build//gem5.opt -j +./build//gem5.opt run_vswarm.py + --mode --function + --kernel --disk + --atomic-warming --num-invocations +``` + +""" +import m5 + +from gem5.coherence_protocol import CoherenceProtocol +from gem5.components.boards.x86_board import X86Board +from gem5.components.memory import DualChannelDDR4_2400 +from gem5.components.memory.simple import SingleChannelSimpleMemory +from gem5.components.processors.cpu_types import CPUTypes +from gem5.components.processors.simple_processor import SimpleProcessor +from gem5.isas import ISA +from gem5.resources.resource import obtain_resource,KernelResource,DiskImageResource +from gem5.simulate.exit_event import ExitEvent +from gem5.simulate.simulator import Simulator +from gem5.utils.requires import requires + +# This runs a check to ensure the gem5 binary is compiled for X86. +requires(isa_required=ISA.X86) + +from gem5.components.cachehierarchies.classic.private_l1_private_l2_cache_hierarchy import ( + PrivateL1PrivateL2CacheHierarchy, +) + +import os +from pathlib import Path + + +import argparse +def parse_arguments(): + parser = argparse.ArgumentParser(description= + "gem5 config file to run vSwarm benchmarks") + parser.add_argument("--kernel", type = str, help = "Path to vmlinux") + parser.add_argument("--disk", type = str, + help = "Path to the disk image containing your function image") + parser.add_argument("-f", "--function", action="store", type=str, default="fibonacci-go", + help="""Specify a function that should run in the simulator.""") + parser.add_argument("--atomic-warming", type=int, default=0, + help="""Perform warming of the cache hierarchy using the atomic core.""") + parser.add_argument("--num-invocations", type=int, default=5, + help="""Number of invocation to be measured.""") + parser.add_argument("--mode", type=str, default="setup",choices=["setup", "evaluation",], + help="""Setup mode: Will boot linux using the kvm core, perform functional + warming and then take a snapshot. + Evaluation mode: Will start from a previously taken checkpoint + do some """) + parser.add_argument("--checkpoint-dir", type = str, default="checkpoints/", + help = "Directory of") + return parser.parse_args() + + +args = parse_arguments() + +if args.mode == "setup": + Path("{}/{}".format(args.checkpoint_dir, args.function)).mkdir(parents=True, exist_ok=True) + + + +# Here we setup the parameters of the l1 and l2 caches. +cache_hierarchy = PrivateL1PrivateL2CacheHierarchy( + l1d_size="16kB", l1i_size="16kB", l2_size="256kB" +) + +# Memory: Dual Channel DDR4 2400 DRAM device. +memory = DualChannelDDR4_2400(size="2GB") + +# Here we setup the processor. For booting we take the KVM core and +# for the evaluation we can take ATOMIC, TIMING or O3 +# eval_core = CPUTypes.ATOMIC +eval_core = CPUTypes.TIMING +# eval_core = CPUTypes.O3 + +processor = SimpleProcessor( + cpu_type=CPUTypes.KVM if args.mode=="setup" else eval_core, + isa=ISA.X86, + num_cores=2, +) + + +# Here we setup the board. The ArmBoard allows for Full-System ARM simulations. +board = X86Board( + clk_freq="3GHz", + processor=processor, + memory=memory, + cache_hierarchy=cache_hierarchy, +) + + + +def writeRunScript(function_name): + n_invocations=args.num_invocations + n_warming=args.atomic_warming + return f""" +#!/bin/bash + +## Define the image name of your function. + +# We use the 'm5 exit' magic instruction to indicate the +# python script where in workflow the system currently is. + +m5 exit ## 1: BOOTING complete + +## Spin up Container +echo "Start the container..." +docker-compose -f /root/functions.yaml up -d {function_name} +m5 exit ## 2: Started container + +echo "Pin function container to core 1" +docker update function --cpuset-cpus 1 + +sleep 5 +m5 exit ## 3: Pinned container + + +# # The client will perform some functional warming +# and then send a fail code before invoking the +# function again for the actual measurement. +# /root/test-client -function-name aes -url localhost -port 50000 -n 10 -w 100 -m5ops -input 10 +# /root/test-client -function-name fibonacci -url localhost -port 50000 -n 2 -w 2 -m5ops -v -input 10 +/root/test-client \ + -function-name {function_name} \ + -url localhost \ + -port 50000 \ + -n {n_invocations} \ + -w {n_warming} \ + -m5ops \ + -input 10 + + +m5 exit ## 4: Stop client +# ------------------------------------------- + + +## Stop container +#docker-compose -f /root/functions.yaml down +m5 exit ## 5: Container stop + + +## exit the simulations +m5 exit ## 6: Test done + +""" + +def workitems(start) -> bool: + cnt = 1 + while True: + if start: + print("Begin Invocation ", cnt) + # m5.stats.reset() + else: + print("End Invocation ", cnt) + # m5.stats.dump() + cnt += 1 + yield False + + +def executeExit() -> bool: + + if args.mode == "setup": + + print("1: BOOTING complete") + yield False + + print("2: Started container") + yield False + + print("3: Pinned container") + yield False + + print("4: Stop client") + yield False + + print("5: Stop container") + yield False + + print("6: Stop simulation") + yield False + yield False + yield False + yield False + yield False + + else: + print("Simulation done") + m5.stats.dump() + m5.exit() + + + +def executeFail() -> bool: + print("1: Client started") + yield False + print("1: Function warming starts") + yield False + print("1: Function warming done") + # processor.switch() + if args.mode == "setup": + m5.checkpoint("{}/{}".format(args.checkpoint_dir, args.function)) + + yield False + + while True: + print("1: Function warming done") + yield False + + + +# Here we set a full system workload. +board.set_kernel_disk_workload( + kernel=KernelResource(args.kernel), + disk_image=DiskImageResource(args.disk), + readfile_contents=writeRunScript(args.function), + kernel_args=['earlyprintk=ttyS0', 'console=ttyS0', 'lpj=7999923', + 'root=/dev/hda2', + 'isolcpus=1', + 'cloud-init=disabled' + ], + checkpoint=Path("{}/{}".format(args.checkpoint_dir, args.function)) if args.mode=="evaluation" else None, +) +# We define the system with the aforementioned system defined. +simulator = Simulator( + board=board, + on_exit_event={ + # ExitEvent.EXIT: (func() for func in [processor.switch]), + ExitEvent.WORKBEGIN: workitems(True), + ExitEvent.WORKEND: workitems(False), + ExitEvent.EXIT: executeExit(), + ExitEvent.FAIL: executeFail(), + }, +) + +# Once the system successfully boots, it encounters an +# `m5_exit instruction encountered`. We stop the simulation then. When the +# simulation has ended you may inspect `m5out/board.terminal` to see +# the stdout. +simulator.run() diff --git a/tools/client/main.go b/tools/client/main.go index 2fc74e8..101d457 100644 --- a/tools/client/main.go +++ b/tools/client/main.go @@ -95,8 +95,6 @@ func main() { m5.Fail(0, 20) // 20: Connection established } - // ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) - // defer cancel() ctx := context.Background() // Set up a connection to the function server. @@ -180,33 +178,3 @@ func invokeFunction(ctx context.Context, n int, instrument bool) { } } } - -func invokeFunctionInstrumented(ctx context.Context, n int) { - // Print 5 times the progress - mod := 1 - if n > 2*5 { - mod = n / 5 - } - for i := 0; i < n; i++ { - - pkt := generator.Next() - - m5.WorkBegin(100+i, 0) // 21: Send Request - - rep, err := client.Request(ctx, pkt) - - m5.WorkEnd(100+i, 0) // 21: Response received - - if err != nil { - log.Warnf("Fail to invoke: %s\n", err) - } - - log.Debugf("Invocation %d: %s", i, rep) - if i%mod == 0 { - log.Printf("Invoked for %d times\n", i) - } - if *delay > 0 { - time.Sleep(time.Duration(*delay) * time.Microsecond) - } - } -}