require actor cpu and memory limits (#2285)

We can still consider more flexible overcommit in the future, but I
think leaving actors uncapped is unsafe, and it's hard to pick a
reasonable universal default, it's too workload dependent.

Actor authors should really be aware of how much resources their actor
requires, and tuning them can have significant performance impacts.

In the future we could do something like determining how much memory was
in use at snapshot time and consider that for scheduling resumes.

#### Breaking change

ActorTemplates *must* declare CPU/memory limits.

Previously `microvm` actors had defaulting, and gvisor actors were
simply uncapped.

This made the scheduler unaware of placement for gvisor.
Requiring it also ensures actor owners are aware of how much resources
they cost.

Setting actor size will be important for performance particularly with
microvms where higher memory limits have a real cost for e.g.
snapshot/resume.

Author: BenTheElder

benchmarking/nighthawk-egress/traffic-driver-template.yaml.tmpl

@@ -58,6 +58,13 @@ volumes:
         - egress-mitm.ate.dev
         - system-roots.ate.dev
         path: egress-trust-bundle/trust-bundle.pem
+# Sandbox size: headroom so the load generator is not the bottleneck.
+resources:
+  limits:
+  - name: cpu
+    quantity: "2"
+  - name: memory
+    quantity: 1Gi
 sandboxConfig:
   sandboxClass: gvisor
   configName: gvisor-default

benchmarking/workloads/deploy.sh

@@ -48,8 +48,8 @@ WORKER_COUNT=1
 SANDBOX_CLASS="gvisor"
 # Actor memory limit (ActorTemplate resources.limits.memory). The default
 # is the smallest size microvm admits (128Mi VMM reserve + 128Mi guest floor),
-# so benchmark actors do not inherit the 2 GiB kata default and drag its page
-# cache into every memory snapshot. Raise it for RAM-consuming suites.
+# since a bigger guest drags more page cache into every memory snapshot. Raise
+# it for RAM-consuming suites.
 ACTOR_MEMORY="256Mi"
 # Worker pod memory (WorkerPool spec.template.resources), set as both request
 # and limit. Empty leaves the pod unsized. The limit is also the memory the

benchmarking/workloads/manifests/glutton-durdir-data-template.yaml.tmpl

@@ -43,6 +43,13 @@ containers:
 volumes:
 - name: data
   durableDir: {}
+# Sandbox size: see the note on the sleep template.
+resources:
+  limits:
+  - name: cpu
+    quantity: "1"
+  - name: memory
+    quantity: ${ACTOR_MEMORY}
 snapshotConfig:
   preferredFidelity: SNAPSHOT_FIDELITY_VOLUMES
   storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/glutton-durdir-data/

benchmarking/workloads/manifests/glutton-durdir-full-template.yaml.tmpl

@@ -40,6 +40,13 @@ containers:
 volumes:
 - name: data
   durableDir: {}
+# Sandbox size: see the note on the sleep template.
+resources:
+  limits:
+  - name: cpu
+    quantity: "1"
+  - name: memory
+    quantity: ${ACTOR_MEMORY}
 snapshotConfig:
   preferredFidelity: SNAPSHOT_FIDELITY_MEMORY
   storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/glutton-durdir-full/

benchmarking/workloads/manifests/glutton-storage-template.yaml.tmpl

@@ -52,6 +52,8 @@ containers:
       port: 80
 resources:
   limits:
+  - name: cpu
+    quantity: "1"
   - name: memory
     quantity: ${ACTOR_MEMORY}
 snapshotConfig:

benchmarking/workloads/manifests/glutton-template.yaml.tmpl

@@ -53,11 +53,12 @@ containers:
     httpGet:
       path: /readyz
       port: 80
-# See the note on the sleep template: unset means the 2 GiB microvm default,
-# which the benchmarks should not silently inherit. Raise ${ACTOR_MEMORY}
-# for suites that exercise glutton's WriteRAM path.
+# See the note on the sleep template. Raise ${ACTOR_MEMORY} for suites that
+# exercise glutton's WriteRAM path.
 resources:
   limits:
+  - name: cpu
+    quantity: "1"
   - name: memory
     quantity: ${ACTOR_MEMORY}
 snapshotConfig:

benchmarking/workloads/manifests/kernelmem-template.yaml.tmpl

@@ -16,10 +16,10 @@
 # benchmark workload (locust/tests/kernelmem.py): fills kernel memory with
 # dentry cache and sleeping processes to size kernel-state snapshots. Not
 # part of deploy.sh's default template set; deploy with
-# WORKLOAD_TEMPLATES="kernelmem ...". No memory limit: the 50000 sleeping
-# processes need more than the 256Mi default --actor-memory the other
-# templates take. The sandbox class must match the WorkerPool's so snapshots
-# stay within a class.
+# WORKLOAD_TEMPLATES="kernelmem ...". Its memory limit is fixed rather than
+# --actor-memory: the 50000 sleeping processes need far more than the 256Mi
+# default. The sandbox class must match the WorkerPool's so snapshots stay
+# within a class.
 metadata:
   atespace: benchmark-workloads
   name: kernelmem
@@ -30,6 +30,14 @@ containers:
 - name: kernelmem
   image: containerstack/alpine-stress:latest
   command: ["sh", "-c", "find /; for i in $(seq 50000); do sleep 10000 & done; sleep 10000"]
+# Sandbox size: the 50000 sleeping processes take ~6.3GiB on Linux (~130KB
+# each, mostly page tables and kernel state).
+resources:
+  limits:
+  - name: cpu
+    quantity: "1"
+  - name: memory
+    quantity: 8Gi
 snapshotConfig:
   preferredFidelity: SNAPSHOT_FIDELITY_MEMORY
   storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/kernelmem/

benchmarking/workloads/manifests/sleep-template.yaml.tmpl

@@ -26,12 +26,14 @@ containers:
 - name: sleep
   image: busybox@sha256:1487d0af5f52b4ba31c7e465126ee2123fe3f2305d638e7827681e7cf6c83d5e
   command: ["/bin/sleep", "1000000"]
-# Without a limit a microvm actor gets the kata default 2 GiB guest, and
-# anything the guest kernel caches rides along in the memory snapshot —
-# bigger guest, bigger snapshot, slower suspend/resume. microvm requires at
-# least 256Mi (128Mi VMM reserve + 128Mi guest floor).
+# Keep the memory small: anything the micro-VM guest kernel caches rides
+# along in the memory snapshot, so a bigger guest means slower
+# suspend/resume. microvm requires at least 256Mi (128Mi VMM reserve +
+# 128Mi guest floor).
 resources:
   limits:
+  - name: cpu
+    quantity: "1"
   - name: memory
     quantity: ${ACTOR_MEMORY}
 snapshotConfig:

benchmarking/workloads/manifests/swebench-astropy-7336-template.yaml.tmpl

@@ -46,6 +46,14 @@ containers:
   env:
   - name: SWEPERF_DISABLE_INTERNAL_SLEEP
     value: "1"
+# Sandbox size: enough for the astropy test runs the trace replays. The cpu
+# limit caps the measured execution time, so keep it fixed across runs.
+resources:
+  limits:
+  - name: cpu
+    quantity: "2"
+  - name: memory
+    quantity: 4Gi
 sandboxConfig:
   sandboxClass: ${SANDBOX_CLASS}
   configName: ${SANDBOX_CONFIG_NAME}

benchmarking/workloads/manifests/usermem-template.yaml.tmpl

@@ -15,9 +15,9 @@
 # ActorTemplate (protojson-shaped ateapipb.ActorTemplate) for the usermem
 # benchmark workload (locust/tests/usermem.py): stress holds 1 GiB of user
 # memory to size memory snapshots. Not part of deploy.sh's default template
-# set; deploy with WORKLOAD_TEMPLATES="usermem ...". No memory limit: the
-# workload allocates 1 GiB, well above the 256Mi default --actor-memory the
-# other templates take. The sandbox class must match the WorkerPool's so
+# set; deploy with WORKLOAD_TEMPLATES="usermem ...". Its memory limit is
+# fixed rather than --actor-memory: the workload allocates 1 GiB, well above
+# the 256Mi default. The sandbox class must match the WorkerPool's so
 # snapshots stay within a class.
 metadata:
   atespace: benchmark-workloads
@@ -29,6 +29,13 @@ containers:
 - name: usermem
   image: containerstack/alpine-stress:latest
   command: ["stress", "--vm", "1", "--vm-bytes", "1G", "--vm-keep", "--vm-hang", "0"]
+# Sandbox size: the 1 GiB stress allocation plus headroom.
+resources:
+  limits:
+  - name: cpu
+    quantity: "1"
+  - name: memory
+    quantity: 1536Mi
 snapshotConfig:
   preferredFidelity: SNAPSHOT_FIDELITY_MEMORY
   storageLocation: gs://${BUCKET_NAME}/benchmark-workloads/usermem/