From e85a79fb5fa4e48215aec995a3e483e767f2ee89 Mon Sep 17 00:00:00 2001
From: Tyler Caraza-Harter <tylerharter@users.noreply.github.com>
Date: Wed, 17 Dec 2025 21:59:34 -0600
Subject: [PATCH 01/55] better errors for admin install and pip install (#373)

---
 go/admin/commands.go                         | 4 ++++
 go/worker/embedded/packagePullerInstaller.py | 7 ++++---
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/go/admin/commands.go b/go/admin/commands.go
index c4c87189a..ce04ad662 100644
--- a/go/admin/commands.go
+++ b/go/admin/commands.go
@@ -136,6 +136,10 @@ func createTarGz(funcDir string) ([]byte, error) {
 			return nil
 		}
 
+		if !info.Mode().IsRegular() {
+			return fmt.Errorf("cannot archive non-regular file %q (mode: %s)", path, info.Mode().String())
+		}
+
 		relPath, err := filepath.Rel(funcDir, path)
 		if err != nil {
 			return fmt.Errorf("unable to compute relative path: %v", err)
diff --git a/go/worker/embedded/packagePullerInstaller.py b/go/worker/embedded/packagePullerInstaller.py
index cead4c0c3..cc1307bac 100644
--- a/go/worker/embedded/packagePullerInstaller.py
+++ b/go/worker/embedded/packagePullerInstaller.py
@@ -69,10 +69,11 @@ def f(event):
     if not alreadyInstalled:
         try:
             subprocess.check_output(
-                ['pip3', 'install', '--no-deps', pkg, '--cache-dir', '/tmp/.cache', '-t', '/host/files'])
+                ['pip3', 'install', '--no-deps', pkg, '--cache-dir', '/tmp/.cache', '-t', '/host/files'],
+                stderr=subprocess.STDOUT)
         except subprocess.CalledProcessError as e:
-            print(f'pip install failed with error code {e.returncode}')
-            print(f'Output: {e.output}')
+            output = e.output.decode('utf-8') if e.output else ''
+            raise Exception(f'pip install failed for {pkg} (exit code {e.returncode}): {output}') from None
 
     name = pkg.split("==")[0]
     d = deps("/host/files")

From 13027d774f61bc69d1e4ae302e980b5ace30f7f0 Mon Sep 17 00:00:00 2001
From: Tyler Caraza-Harter <tylerharter@users.noreply.github.com>
Date: Mon, 29 Dec 2025 14:22:37 -0600
Subject: [PATCH 02/55] check admin failure cases; fix boss test (#375)

---
 go/admin/commands.go         | 10 ++++++++--
 go/boss/lambdastore/store.go | 13 ++++++++-----
 scripts/boss_test.py         |  5 +----
 3 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/go/admin/commands.go b/go/admin/commands.go
index ce04ad662..7962136c3 100644
--- a/go/admin/commands.go
+++ b/go/admin/commands.go
@@ -31,7 +31,10 @@ func checkStatus(port string) error {
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusOK {
-		body, _ := io.ReadAll(resp.Body)
+		body, readErr := io.ReadAll(resp.Body)
+		if readErr != nil {
+			return fmt.Errorf("boss/worker returned status %d (failed to read response body: %v)", resp.StatusCode, readErr)
+		}
 		return fmt.Errorf("boss/worker returned status %d: %s", resp.StatusCode, string(body))
 	}
 
@@ -205,7 +208,10 @@ func uploadToLambdaStore(funcName string, tarData []byte, port string) error {
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusCreated {
-		body, _ := io.ReadAll(resp.Body)
+		body, readErr := io.ReadAll(resp.Body)
+		if readErr != nil {
+			return fmt.Errorf("upload failed with status %d (failed to read response body: %v)", resp.StatusCode, readErr)
+		}
 		return fmt.Errorf("upload failed with status %d: %s", resp.StatusCode, string(body))
 	}
 
diff --git a/go/boss/lambdastore/store.go b/go/boss/lambdastore/store.go
index a43f04e95..8c97564fe 100644
--- a/go/boss/lambdastore/store.go
+++ b/go/boss/lambdastore/store.go
@@ -262,16 +262,19 @@ func (s *LambdaStore) addToRegistry(funcName string, body io.Reader) error {
 	if err != nil {
 		return fmt.Errorf("failed to create blob writer: %w", err)
 	}
-	defer func() {
-		if err := writer.Close(); err != nil {
-			slog.Error(fmt.Sprintf("warning: failed to close blob writer: %v", err))
-		}
-	}()
 
 	if _, err := io.Copy(writer, tempFile); err != nil {
+		// Close writer to release resources (ignore close error since we already have an error)
+		writer.Close()
 		return fmt.Errorf("failed to upload to blob storage: %w", err)
 	}
 
+	// Close the writer to finalize the upload - this is where the actual commit happens
+	// for many blob storage implementations, so we must check the error
+	if err := writer.Close(); err != nil {
+		return fmt.Errorf("failed to finalize blob upload: %w", err)
+	}
+
 	lambdaEntry.Config = cfg
 
 	if s.eventManager != nil {
diff --git a/scripts/boss_test.py b/scripts/boss_test.py
index 0494efa8c..7ba250a2c 100644
--- a/scripts/boss_test.py
+++ b/scripts/boss_test.py
@@ -200,11 +200,8 @@ def tester(platform):
     clear_config()
     launch_boss(platform)
 
-    # Step 1: scale up worker
-    status = json.loads(boss_get("status"))
-    assert status["state"]["running"] == 0
+    # Step 1: scale to 1 worker (boss may auto-launch 1 on some platforms)
     scale_workers(1)
-    assert json.loads(boss_get("status"))["state"]["starting"] == 1
     wait_for_workers(1)
 
     # Step 2: upload and verify lambda

From c230f41e77865c35b1c4ebb5194ee8e29749859b Mon Sep 17 00:00:00 2001
From: Tyler Caraza-Harter <tylerharter@users.noreply.github.com>
Date: Mon, 29 Dec 2025 14:34:30 -0600
Subject: [PATCH 03/55] allow specification of env variables for ol functions
 (#374)

* allow specification of env variables for ol functions

* docs, cleanup parsing

* fix dockerfile installs and stale test examples

* fix docker sandbox too

* fix boss test for new config

* linter issues
---
 docs/worker/lambda-config.md               | 24 ++++++++-
 examples/env-test/f.py                     | 30 ++++++++++++
 examples/env-test/ol.yaml                  | 11 +++++
 go/common/lambdaConfig.go                  | 18 ++++++-
 go/worker/lambda/lambdaFunction.go         | 48 ++++++++++++------
 min-image/Dockerfile                       | 10 ++--
 min-image/runtimes/python/server.py        |  7 +++
 min-image/runtimes/python/server_legacy.py |  7 +++
 scripts/boss_test.py                       |  3 +-
 scripts/helper/test.py                     | 32 ++++++------
 scripts/test.py                            | 57 ++++++++++++++++++++--
 wasm-image/Dockerfile                      |  2 +-
 12 files changed, 203 insertions(+), 46 deletions(-)
 create mode 100644 examples/env-test/f.py
 create mode 100644 examples/env-test/ol.yaml

diff --git a/docs/worker/lambda-config.md b/docs/worker/lambda-config.md
index 2ce9aa11b..fe2e014ad 100644
--- a/docs/worker/lambda-config.md
+++ b/docs/worker/lambda-config.md
@@ -13,12 +13,18 @@ triggers:
   http:
     - method: PUT
     - method: PATCH
+
+environment:
+  MY_ENV_VAR1: "value1"
+  MY_ENV_VAR2: "value2"
 ```
 
-## 3. Trigger Types
+## 3. Configuration Options
+
+### a. Triggers
 OpenLambda only supports HTTP trigger for now, but future development plans include supporting other trigger types.
 
-### a. HTTP Triggers
+#### HTTP Triggers
 Defines which HTTP methods can be used to invoke the lambda.
 
 Example:
@@ -30,6 +36,20 @@ triggers:
 ```
 In this case, the lambda accepts GET and POST requests.
 
+### b. Environment Variables
+Defines environment variables that will be available to the lambda function at runtime.
+
+Example:
+```yaml
+environment:
+  MY_ENV_VAR1: "production"
+  MY_ENV_VAR2: "enabled"
+```
+
+These variables can be accessed in your lambda code using standard environment variable methods (e.g., `os.environ` in Python).
+
+**Note:** Environment variables defined in `ol.yaml` are written to a `.env` file in the lambda's directory during execution. If your lambda already has a `.env` file, it will be overwritten with the values from `ol.yaml`.
+
 ## 4. How to Use
 ### a. Define Configuration
 Create an `ol.yaml` file inside the lambda function directory with the desired configuration.
diff --git a/examples/env-test/f.py b/examples/env-test/f.py
new file mode 100644
index 000000000..7fe1f5f4a
--- /dev/null
+++ b/examples/env-test/f.py
@@ -0,0 +1,30 @@
+import os
+import json
+
+def f(event):
+    """
+    Lambda function that demonstrates environment variable usage.
+    Returns all environment variables that were configured in ol.yaml
+    """
+    
+    # Get environment variables from config
+    env_vars = {
+        "MY_ENV_VAR": os.environ.get("MY_ENV_VAR", "not set"),
+        "DATABASE_URL": os.environ.get("DATABASE_URL", "not set"),
+        "DEBUG_MODE": os.environ.get("DEBUG_MODE", "not set"),
+        "API_KEY": os.environ.get("API_KEY", "not set"),
+        "CUSTOM_PATH": os.environ.get("CUSTOM_PATH", "not set"),
+    }
+    
+    response = {
+        "message": "Environment variables test",
+        "event": event,
+        "configured_env_vars": env_vars,
+        "all_env_vars_count": len(os.environ),
+    }
+    
+    # If debug mode is enabled, show all environment variables
+    if os.environ.get("DEBUG_MODE") == "true":
+        response["all_env_vars"] = dict(os.environ)
+    
+    return response
\ No newline at end of file
diff --git a/examples/env-test/ol.yaml b/examples/env-test/ol.yaml
new file mode 100644
index 000000000..387a52a3a
--- /dev/null
+++ b/examples/env-test/ol.yaml
@@ -0,0 +1,11 @@
+triggers:
+  http:
+    - method: GET
+    - method: POST
+
+environment:
+  MY_ENV_VAR: "Hello from environment"
+  DATABASE_URL: "postgresql://user:pass@localhost/db"
+  DEBUG_MODE: "true"
+  API_KEY: "secret-key-789"
+  CUSTOM_PATH: "/usr/local/bin"
\ No newline at end of file
diff --git a/go/common/lambdaConfig.go b/go/common/lambdaConfig.go
index 59c0138d1..c3029960b 100644
--- a/go/common/lambdaConfig.go
+++ b/go/common/lambdaConfig.go
@@ -10,6 +10,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"strings"
 
 	"gopkg.in/yaml.v3"
 )
@@ -42,7 +43,8 @@ type KafkaTrigger struct {
 
 // LambdaConfig defines the overall configuration for the lambda function.
 type LambdaConfig struct {
-	Triggers Triggers `yaml:"triggers"` // List of HTTP triggers
+	Triggers    Triggers          `yaml:"triggers"`    // List of HTTP triggers
+	Environment map[string]string `yaml:"environment"` // Environment variables for the lambda
 	// Additional configurations can be added here.
 }
 
@@ -54,6 +56,7 @@ func LoadDefaultLambdaConfig() *LambdaConfig {
 				{Method: "*"}, // Default to allow all methods
 			},
 		},
+		Environment: make(map[string]string),
 	}
 }
 
@@ -87,6 +90,19 @@ func checkLambdaConfig(config *LambdaConfig) error {
 		}
 	}
 
+	// Validate environment variables
+	for key, value := range config.Environment {
+		if key == "" {
+			return fmt.Errorf("Environment variable key cannot be empty")
+		}
+		// Optionally validate that keys are valid environment variable names
+		if strings.Contains(key, "=") {
+			return fmt.Errorf("Environment variable key '%s' cannot contain '='", key)
+		}
+		// Value can be empty (that's valid)
+		_ = value
+	}
+
 	return nil
 }
 
diff --git a/go/worker/lambda/lambdaFunction.go b/go/worker/lambda/lambdaFunction.go
index f625ad43e..0c3eca751 100644
--- a/go/worker/lambda/lambdaFunction.go
+++ b/go/worker/lambda/lambdaFunction.go
@@ -18,8 +18,8 @@ import (
 )
 
 type FunctionMeta struct {
-	Sandbox *sandbox.SandboxMeta `json:"sandbox"` // Existing sandbox metadata
 	Config  *common.LambdaConfig `json:"config"`  // New Lambda config (from YAML)
+	Sandbox *sandbox.SandboxMeta `json:"sandbox"` // Existing sandbox metadata
 }
 
 // LambdaFunc represents a single lambda function (the code)
@@ -72,18 +72,19 @@ func (f *LambdaFunc) printf(format string, args ...any) {
 	slog.Info(fmt.Sprintf("%s [FUNC %s]", strings.TrimRight(msg, "\n"), f.name))
 }
 
-// parseMeta reads in a requirements.txt file that was built from pip-compile
+// parseMeta constructs a FunctionMeta based on contents of a code
+// directory, such as an ol.yaml and requirements.txt (generated by
+// pip-compile)
 func parseMeta(codeDir string) (*FunctionMeta, error) {
 	sandboxMeta := &sandbox.SandboxMeta{
 		Installs: []string{},
 		Imports:  []string{},
 	}
 
+	// having a requirements.txt is optional
 	path := filepath.Join(codeDir, "requirements.txt")
 	file, err := os.Open(path)
-	if errors.Is(err, os.ErrNotExist) {
-		// having a requirements.txt is optional
-	} else if err != nil {
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
 		return nil, err
 	}
 	defer file.Close()
@@ -152,14 +153,13 @@ func (f *LambdaFunc) pullHandlerIfStale() (err error) {
 		}
 	}()
 
-	if rtType == common.RT_PYTHON {
-		// inspect new code for dependencies; if we can install
-		// everything necessary, start using new code
-		meta, err := parseMeta(codeDir)
-		if err != nil {
-			return err
-		}
+	// Parse meta for native functions to get config
+	meta, err := parseMeta(codeDir)
+	if err != nil {
+		return err
+	}
 
+	if rtType == common.RT_PYTHON {
 		// make sure all specified dependencies are installed
 		// (but don't recursively find others)
 		for _, pkg := range meta.Sandbox.Installs {
@@ -172,14 +172,30 @@ func (f *LambdaFunc) pullHandlerIfStale() (err error) {
 		f.Meta = meta
 	} else if rtType == common.RT_NATIVE {
 		slog.Info("Got native function")
+	}
 
-		// Initialize f.Meta for native functions for consistensy.
-		f.Meta = &FunctionMeta{
-			Sandbox: nil,                              // Sandbox is nil for native functions
-			Config:  common.LoadDefaultLambdaConfig(), // Load default configuration
+	// Write environment variables to .env file if any are specified
+	if meta.Config.Environment != nil && len(meta.Config.Environment) > 0 {
+		slog.Info("creating .env for lambda", "entries", len(meta.Config.Environment))
+		envPath := filepath.Join(codeDir, ".env")
+		envFile, err := os.Create(envPath)
+		if err != nil {
+			return fmt.Errorf("failed to create .env file: %w", err)
+		}
+		defer envFile.Close()
+
+		for key, value := range meta.Config.Environment {
+			// Quote the value if it contains spaces or special characters
+			if strings.ContainsAny(value, " \t\n#=") {
+				escapedValue := strings.ReplaceAll(value, `"`, `\"`)
+				fmt.Fprintf(envFile, "%s=\"%s\"\n", key, escapedValue)
+			} else {
+				fmt.Fprintf(envFile, "%s=%s\n", key, value)
+			}
 		}
 	}
 
+	f.Meta = meta
 	f.codeDir = codeDir
 	f.lastPull = &now
 	return nil
diff --git a/min-image/Dockerfile b/min-image/Dockerfile
index a8c7ec0eb..a475453d3 100644
--- a/min-image/Dockerfile
+++ b/min-image/Dockerfile
@@ -1,11 +1,11 @@
 FROM ubuntu:22.04
 
-RUN apt-get -y --fix-missing update
-RUN apt-get -y install wget apt-transport-https curl
-RUN apt-get -y install python3 python3-dev python3-pip python-is-python3
-RUN apt-get -y install build-essential libseccomp-dev
+RUN apt-get update && apt-get -y install \
+    wget apt-transport-https curl \
+    python3 python3-dev python3-pip python-is-python3 \
+    build-essential libseccomp-dev
 RUN pip3 install --upgrade pip
-RUN pip3 install virtualenv requests tornado==6.1.0
+RUN pip3 install virtualenv requests tornado==6.1.0 python-dotenv
 
 RUN mkdir /runtimes
 
diff --git a/min-image/runtimes/python/server.py b/min-image/runtimes/python/server.py
index b6b1a4e96..abd57bbe8 100644
--- a/min-image/runtimes/python/server.py
+++ b/min-image/runtimes/python/server.py
@@ -6,6 +6,7 @@
 
 sys.path.append("/usr/local/lib/python3.10/dist-packages")
 
+from dotenv import load_dotenv
 import tornado.ioloop
 import tornado.web
 import tornado.httpserver
@@ -21,6 +22,12 @@
 def web_server():
     print(f"server.py: start web server on fd: {file_sock.fileno()}")
     sys.path.append('/handler')
+    
+    # Load environment variables from .env file if it exists
+    env_path = '/handler/.env'
+    if os.path.exists(env_path):
+        load_dotenv(env_path)
+        print(f"server.py: loaded environment variables from {env_path}")
 
     # TODO: as a safeguard, we should add a mechanism so that the
     # import doesn't happen until the cgroup move completes, so that a
diff --git a/min-image/runtimes/python/server_legacy.py b/min-image/runtimes/python/server_legacy.py
index 4b19156a4..057465683 100644
--- a/min-image/runtimes/python/server_legacy.py
+++ b/min-image/runtimes/python/server_legacy.py
@@ -14,6 +14,7 @@
 import importlib
 import traceback
 
+from dotenv import load_dotenv
 import tornado.ioloop
 import tornado.web
 import tornado.httpserver
@@ -24,6 +25,12 @@
 PKGS_DIR = '/packages'
 HANDLER_DIR = '/handler'
 
+# Load environment variables from .env file if it exists
+env_path = f'{HANDLER_DIR}/.env'
+if os.path.exists(env_path):
+    load_dotenv(env_path)
+    print(f"server_legacy.py: loaded environment variables from {env_path}")
+
 sys.path.append(PKGS_DIR)
 sys.path.append(HANDLER_DIR)
 
diff --git a/scripts/boss_test.py b/scripts/boss_test.py
index 7ba250a2c..60366666e 100644
--- a/scripts/boss_test.py
+++ b/scripts/boss_test.py
@@ -134,7 +134,8 @@ def verify_lambda_config(lambda_name):
             "HTTP": [{"Method": "POST"}],
             "Cron": None,
             "Kafka": None,
-        }
+        },
+        "Environment": None,
     }
     assert actual_config == expected_config, (
         f"Lambda config mismatch!\nExpected: {expected_config}\nActual: {actual_config}"
diff --git a/scripts/helper/test.py b/scripts/helper/test.py
index 36e3c90a4..1245f8f36 100644
--- a/scripts/helper/test.py
+++ b/scripts/helper/test.py
@@ -14,7 +14,7 @@
 
 TEST_FILTER = []
 TEST_BLOCKLIST = []
-WORKER_TYPE = []
+WORKER_TYPE = None
 RESULTS = OrderedDict({"runs": []})
 START_TIME = None
 
@@ -23,6 +23,9 @@ def set_worker_type(new_val):
     global WORKER_TYPE
     WORKER_TYPE = new_val
 
+def get_worker_type():
+    return WORKER_TYPE
+
 def set_test_filter(new_val):
     ''' Sets up the filter for all following tests '''
 
@@ -114,21 +117,18 @@ def _wrapper(*args, **kwargs):
         worker = WORKER_TYPE()
         assert worker
         print("Worker started")
-
-        if worker:
-            try:
-                # run test/benchmark
-                test_t0 = time()
-                return_val = func(**kwargs)
-                test_t1 = time()
-                result["test_seconds"] = test_t1 - test_t0
-                result["pass"] = True
-            except Exception as err:
-                print(f"Failed to run test: {err}")
-                result["pass"] = False
-                result["errors"].append(traceback.format_exc().split("\n"))
-
-            worker.stop()
+        try:
+            # run test/benchmark
+            test_t0 = time()
+            return_val = func(**kwargs)
+            test_t1 = time()
+            result["test_seconds"] = test_t1 - test_t0
+            result["pass"] = True
+        except Exception as err:
+            print(f"Failed to run test: {err}")
+            result["pass"] = False
+            result["errors"].append(traceback.format_exc().split("\n"))
+        worker.stop()
 
         mounts1 = mounts()
         if len(mounts0) != len(mounts1):
diff --git a/scripts/test.py b/scripts/test.py
index 58afd5384..559d0b86f 100755
--- a/scripts/test.py
+++ b/scripts/test.py
@@ -28,7 +28,8 @@
     start_tests,
     check_test_results,
     set_worker_type,
-    test
+    get_worker_type,
+    test,
 )
 
 # You can either install the OpenLambda Python bindings
@@ -39,7 +40,6 @@
 # These will be set by argparse in main()
 OL_DIR = None
 
-@test
 def install_examples_to_worker_registry():
     """Install all lambda functions from examples directory to
     worker registry using admin install"""
@@ -76,8 +76,10 @@ def install_examples_to_worker_registry():
                 print(f"✓ Successfully installed {func_name}")
             else:
                 print(f"✗ Failed to install {func_name}: {result.stderr}")
+                raise Exception(f"install failed for {func_name}")
         except Exception as e:
             print(f"✗ Error installing {func_name}: {e}")
+            raise e
     print("Finished installing example functions")
 
 
@@ -336,8 +338,54 @@ def test_http_method_restrictions():
             f"for PUT, not {repr(r.text)}"
         )
 
+@test
+def env_test():
+    """Test that environment variables from ol.yaml are properly loaded"""
+    open_lambda = OpenLambda()
+
+    # Call the env-test function
+    result = open_lambda.run("env-test", {})
+
+    # Verify that all configured environment variables are present
+    expected_vars = {
+        "MY_ENV_VAR": "Hello from environment",
+        "DATABASE_URL": "postgresql://user:pass@localhost/db", 
+        "DEBUG_MODE": "true",
+        "API_KEY": "secret-key-789",
+        "CUSTOM_PATH": "/usr/local/bin"
+    }
+
+    # Check that the configured_env_vars match what we expect
+    if "configured_env_vars" not in result:
+        raise ValueError(f"configured_env_vars not found in response: {result}")
+
+    configured = result["configured_env_vars"]
+
+    for key, expected_value in expected_vars.items():
+        if key not in configured:
+            raise ValueError(f"Environment variable {key} not found in response")
+        if configured[key] != expected_value:
+            raise ValueError(
+                f"Environment variable {key}={configured[key]} but expected {expected_value}")
+
+    print(f"✓ All {len(expected_vars)} environment variables loaded correctly")
+
+    # Verify DEBUG_MODE enabled all env vars to be returned
+    if "all_env_vars" not in result:
+        raise ValueError("DEBUG_MODE=true but all_env_vars not returned")
+
+    return {"env_vars_tested": len(expected_vars)}
+
 
 def run_tests():
+    worker_type = get_worker_type()
+    worker = worker_type()
+    assert worker
+    print("Worker started")
+    install_examples_to_worker_registry()
+    print("Examples installed")
+    worker.stop()
+
     ping_test()
 
     # do smoke tests under various configs
@@ -360,6 +408,9 @@ def run_tests():
     flask_test()
     test_http_method_restrictions()
 
+    # test environment variables from ol.yaml
+    env_test()
+
     # make sure code updates get pulled within the cache time
     with tempfile.TemporaryDirectory() as reg_dir:
         with TestConfContext(registry=reg_dir, registry_cache_ms=3000):
@@ -424,8 +475,6 @@ def main():
         else:
             raise RuntimeError(f"Invalid worker type {args.worker_type}")
 
-        install_examples_to_worker_registry()
-
         start_tests()
         run_tests()
 
diff --git a/wasm-image/Dockerfile b/wasm-image/Dockerfile
index b5d071510..a95167662 100644
--- a/wasm-image/Dockerfile
+++ b/wasm-image/Dockerfile
@@ -1,6 +1,6 @@
 FROM ol-min
 
-RUN apt-get -y install clang
+RUN apt-get update && apt-get -y install clang
 
 # Setup rust environment (prereq for native runtime)
 RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain nightly-2025-02-15

From be696ac8bed7daf09137d63a42a3b827b615534d Mon Sep 17 00:00:00 2001
From: Tyler Caraza-Harter <tylerharter@users.noreply.github.com>
Date: Mon, 29 Dec 2025 20:33:25 -0600
Subject: [PATCH 04/55] install lambda from git (#376)

---
 docs/worker/getting-started.md | 16 ++++++++++
 go/admin/commands.go           | 55 ++++++++++++++++++++++++++++------
 2 files changed, 62 insertions(+), 9 deletions(-)

diff --git a/docs/worker/getting-started.md b/docs/worker/getting-started.md
index 117822218..b89b1fcfd 100644
--- a/docs/worker/getting-started.md
+++ b/docs/worker/getting-started.md
@@ -197,6 +197,22 @@ If you initialized a worker with a specific path (e.g., `./ol worker init -p myw
 
 If no `-p` flag is specified, the command will default to the worker running on port 5000 using the default config.
 
+### Installing from a Git Repository
+
+You can also install lambdas directly from a Git repository (GitHub, GitLab, etc.):
+
+```bash
+./ol admin install https://github.com/open-lambda/hello-lambda-example.git
+```
+
+This works with both HTTPS and SSH URLs:
+
+```bash
+./ol admin install git@github.com:open-lambda/hello-lambda-example.git
+```
+
+The function name is derived from the repository name (e.g., `hello-lambda-example`).
+
 ## Invoke Lambda
 
 Invoke your lambda with `curl` (the result should be the same as the POST body):
diff --git a/go/admin/commands.go b/go/admin/commands.go
index 7962136c3..de2dd5380 100644
--- a/go/admin/commands.go
+++ b/go/admin/commands.go
@@ -8,6 +8,7 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 	"time"
@@ -41,6 +42,28 @@ func checkStatus(port string) error {
 	return nil
 }
 
+// isGitURL returns true if the path looks like a git repository URL
+func isGitURL(path string) bool {
+	return strings.HasSuffix(path, ".git")
+}
+
+// cloneGitRepo clones a git repository to a temporary directory
+func cloneGitRepo(gitURL string) (string, error) {
+	tmpDir, err := os.MkdirTemp("", "ol-install-*")
+	if err != nil {
+		return "", fmt.Errorf("failed to create temp directory: %v", err)
+	}
+
+	cmd := exec.Command("git", "clone", "--depth", "1", gitURL, tmpDir)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		os.RemoveAll(tmpDir)
+		return "", fmt.Errorf("git clone failed: %v\n%s", err, string(output))
+	}
+
+	return tmpDir, nil
+}
+
 func adminInstall(ctx *cli.Context) error {
 	args := ctx.Args().Slice()
 	var installTarget string
@@ -49,7 +72,7 @@ func adminInstall(ctx *cli.Context) error {
 	workerPath := ctx.String("path")
 
 	if len(args) == 0 {
-		return fmt.Errorf("usage: ol admin install [boss | -p <worker_path>] <function_directory>")
+		return fmt.Errorf("usage: ol admin install [boss | -p <worker_path>] <directory_or_git_url>")
 	}
 	if len(args) == 1 {
 		funcDir = args[0]
@@ -62,7 +85,7 @@ func adminInstall(ctx *cli.Context) error {
 			return fmt.Errorf("cannot use both 'boss' and '-p' flags together")
 		}
 	} else {
-		return fmt.Errorf("usage: ol admin install [boss | -p <worker_path>] <function_directory>")
+		return fmt.Errorf("usage: ol admin install [boss | -p <worker_path>] <directory_or_git_url>")
 	}
 
 	var portToUploadLambda string
@@ -99,15 +122,29 @@ func adminInstall(ctx *cli.Context) error {
 		portToUploadLambda = common.Conf.Worker_port
 	}
 
-	funcDir = strings.TrimSuffix(funcDir, "/")
+	var funcName string
+	var tmpDir string
 
-	funcName := filepath.Base(funcDir)
-
-	if _, err := os.Stat(funcDir); os.IsNotExist(err) {
-		return fmt.Errorf("directory %s does not exist", funcDir)
+	if isGitURL(funcDir) {
+		funcName = strings.TrimSuffix(filepath.Base(funcDir), ".git")
+		clonedDir, err := cloneGitRepo(funcDir)
+		if err != nil {
+			return err
+		}
+		tmpDir = clonedDir
+		funcDir = clonedDir
+	} else {
+		funcDir = strings.TrimSuffix(funcDir, "/")
+		funcName = filepath.Base(funcDir)
+		if _, err := os.Stat(funcDir); os.IsNotExist(err) {
+			return fmt.Errorf("directory %s does not exist", funcDir)
+		}
 	}
 
 	tarData, err := createTarGz(funcDir)
+	if tmpDir != "" {
+		os.RemoveAll(tmpDir)
+	}
 	if err != nil {
 		return fmt.Errorf("failed to create tar.gz: %v", err)
 	}
@@ -222,8 +259,8 @@ func AdminCommands() []*cli.Command {
 	return []*cli.Command{
 		{
 			Name:      "install",
-			Usage:     "Install a lambda function from directory",
-			UsageText: "ol admin install [boss | -p <worker_path>] <function_directory>",
+			Usage:     "Install a lambda function from directory or git repo",
+			UsageText: "ol admin install [boss | -p <worker_path>] <function_directory_or_git_url>",
 			Action:    adminInstall,
 			Flags: []cli.Flag{
 				&cli.StringFlag{

From ab3d0ba3acc8e7da501a2ef55b94a1989bfe2fc9 Mon Sep 17 00:00:00 2001
From: Tyler Caraza-Harter <tylerharter@users.noreply.github.com>
Date: Tue, 30 Dec 2025 13:11:52 -0600
Subject: [PATCH 05/55] more "admin install" flags (#377)

---
 go/admin/commands.go | 84 ++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 78 insertions(+), 6 deletions(-)

diff --git a/go/admin/commands.go b/go/admin/commands.go
index de2dd5380..aa78d6827 100644
--- a/go/admin/commands.go
+++ b/go/admin/commands.go
@@ -42,9 +42,14 @@ func checkStatus(port string) error {
 	return nil
 }
 
+const installUsage = "ol admin install [-c <config>] [-n <name>] [boss | -p <worker_path>] <directory_or_git_url>"
+
 // isGitURL returns true if the path looks like a git repository URL
 func isGitURL(path string) bool {
-	return strings.HasSuffix(path, ".git")
+	if !strings.HasSuffix(path, ".git") {
+		return false
+	}
+	return strings.HasPrefix(path, "https://") || strings.HasPrefix(path, "git@")
 }
 
 // cloneGitRepo clones a git repository to a temporary directory
@@ -72,7 +77,7 @@ func adminInstall(ctx *cli.Context) error {
 	workerPath := ctx.String("path")
 
 	if len(args) == 0 {
-		return fmt.Errorf("usage: ol admin install [boss | -p <worker_path>] <directory_or_git_url>")
+		return fmt.Errorf("usage: %s", installUsage)
 	}
 	if len(args) == 1 {
 		funcDir = args[0]
@@ -85,7 +90,7 @@ func adminInstall(ctx *cli.Context) error {
 			return fmt.Errorf("cannot use both 'boss' and '-p' flags together")
 		}
 	} else {
-		return fmt.Errorf("usage: ol admin install [boss | -p <worker_path>] <directory_or_git_url>")
+		return fmt.Errorf("usage: %s", installUsage)
 	}
 
 	var portToUploadLambda string
@@ -141,7 +146,27 @@ func adminInstall(ctx *cli.Context) error {
 		}
 	}
 
-	tarData, err := createTarGz(funcDir)
+	// Override function name if specified
+	if name := ctx.String("name"); name != "" {
+		funcName = name
+	}
+
+	// Build overrides map
+	overrides := make(map[string]string)
+	configPath := ctx.String("config")
+	if configPath != "" {
+		if _, err := os.Stat(configPath); os.IsNotExist(err) {
+			return fmt.Errorf("config file %s does not exist", configPath)
+		}
+		// Warn if ol.yaml already exists in the source
+		existingConfig := filepath.Join(funcDir, "ol.yaml")
+		if _, err := os.Stat(existingConfig); err == nil {
+			fmt.Printf("Warning: overriding existing ol.yaml in source with %s\n", configPath)
+		}
+		overrides["ol.yaml"] = configPath
+	}
+
+	tarData, err := createTarGz(funcDir, overrides)
 	if tmpDir != "" {
 		os.RemoveAll(tmpDir)
 	}
@@ -157,7 +182,7 @@ func adminInstall(ctx *cli.Context) error {
 	return nil
 }
 
-func createTarGz(funcDir string) ([]byte, error) {
+func createTarGz(funcDir string, overrides map[string]string) ([]byte, error) {
 	var buf bytes.Buffer
 	gzWriter := gzip.NewWriter(&buf)
 	tarWriter := tar.NewWriter(gzWriter)
@@ -185,6 +210,11 @@ func createTarGz(funcDir string) ([]byte, error) {
 			return fmt.Errorf("unable to compute relative path: %v", err)
 		}
 
+		// Skip files that will be overridden
+		if _, ok := overrides[relPath]; ok {
+			return nil
+		}
+
 		header, err := tar.FileInfoHeader(info, "")
 		if err != nil {
 			return fmt.Errorf("unable to create header: %v", err)
@@ -215,6 +245,38 @@ func createTarGz(funcDir string) ([]byte, error) {
 		return nil, err
 	}
 
+	// Add override files
+	for relPath, localPath := range overrides {
+		info, err := os.Stat(localPath)
+		if err != nil {
+			return nil, fmt.Errorf("unable to stat override file %s: %v", localPath, err)
+		}
+
+		header, err := tar.FileInfoHeader(info, "")
+		if err != nil {
+			return nil, fmt.Errorf("unable to create header for override %s: %v", relPath, err)
+		}
+		header.Name = relPath
+
+		if err := tarWriter.WriteHeader(header); err != nil {
+			return nil, fmt.Errorf("failed to write header for override %s: %v", relPath, err)
+		}
+
+		file, err := os.Open(localPath)
+		if err != nil {
+			return nil, fmt.Errorf("unable to open override file %s: %v", localPath, err)
+		}
+
+		if _, err := io.Copy(tarWriter, file); err != nil {
+			file.Close()
+			return nil, fmt.Errorf("error copying override file %s: %v", localPath, err)
+		}
+
+		if err := file.Close(); err != nil {
+			return nil, fmt.Errorf("error closing override file %s: %v", localPath, err)
+		}
+	}
+
 	if err := tarWriter.Close(); err != nil {
 		return nil, fmt.Errorf("failed to close tar writer: %v", err)
 	}
@@ -260,7 +322,7 @@ func AdminCommands() []*cli.Command {
 		{
 			Name:      "install",
 			Usage:     "Install a lambda function from directory or git repo",
-			UsageText: "ol admin install [boss | -p <worker_path>] <function_directory_or_git_url>",
+			UsageText: installUsage,
 			Action:    adminInstall,
 			Flags: []cli.Flag{
 				&cli.StringFlag{
@@ -268,6 +330,16 @@ func AdminCommands() []*cli.Command {
 					Aliases: []string{"p"},
 					Usage:   "Worker directory path (e.g., -p myworker)",
 				},
+				&cli.StringFlag{
+					Name:    "config",
+					Aliases: []string{"c"},
+					Usage:   "Path to ol.yaml config file to include (overrides existing ol.yaml in source)",
+				},
+				&cli.StringFlag{
+					Name:    "name",
+					Aliases: []string{"n"},
+					Usage:   "Lambda function name (defaults to directory or repo name)",
+				},
 			},
 		},
 	}

From d47664ba40f347c8f7faea0d5c9f7d6c8725d148 Mon Sep 17 00:00:00 2001
From: Tyler Caraza-Harter <tylerharter@users.noreply.github.com>
Date: Wed, 31 Dec 2025 14:12:07 -0600
Subject: [PATCH 06/55] refactor to include runtime in sandbox meta (#378)

---
 go/worker/event/sockServer.go              |  5 +-
 go/worker/lambda/handlerPuller.go          | 49 ++++++---------
 go/worker/lambda/lambdaFunction.go         | 70 ++++++++++++----------
 go/worker/lambda/lambdaInstance.go         |  6 +-
 go/worker/lambda/packages/packagePuller.go |  3 +-
 go/worker/lambda/zygote/api.go             |  4 +-
 go/worker/lambda/zygote/importCache.go     | 22 +++----
 go/worker/lambda/zygote/multiTree.go       |  4 +-
 go/worker/sandbox/api.go                   | 13 ++--
 go/worker/sandbox/docker.go                |  8 +--
 go/worker/sandbox/dockerPool.go            |  2 +-
 go/worker/sandbox/sock.go                  |  9 +--
 go/worker/sandbox/sockPool.go              |  7 +--
 13 files changed, 95 insertions(+), 107 deletions(-)

diff --git a/go/worker/event/sockServer.go b/go/worker/event/sockServer.go
index 51ceed985..1bb2cdbb0 100644
--- a/go/worker/event/sockServer.go
+++ b/go/worker/event/sockServer.go
@@ -104,15 +104,16 @@ func (server *SOCKServer) Create(w http.ResponseWriter, _ []string, args map[str
 		}
 	}
 
-	if parent != nil && parent.GetRuntimeType() != rtType {
+	if parent != nil && parent.Meta().Runtime != rtType {
 		return fmt.Errorf("Parent and child have different runtimes")
 	}
 
 	meta := &sandbox.SandboxMeta{
+		Runtime:  rtType,
 		Installs: packages,
 	}
 
-	c, err := server.sbPool.Create(parent, leaf, codeDir, scratchDir, meta, rtType)
+	c, err := server.sbPool.Create(parent, leaf, codeDir, scratchDir, meta)
 	if err != nil {
 		return err
 	}
diff --git a/go/worker/lambda/handlerPuller.go b/go/worker/lambda/handlerPuller.go
index 2c9cb73c6..8c4b75d83 100644
--- a/go/worker/lambda/handlerPuller.go
+++ b/go/worker/lambda/handlerPuller.go
@@ -7,7 +7,6 @@ import (
 	"io"
 	"os"
 	"os/exec"
-	"path/filepath"
 	"strings"
 	"sync"
 	"time"
@@ -23,8 +22,6 @@ import (
 
 var errNotFound404 = errors.New("lambda not found in blob store")
 
-var RT_UNKNOWN common.RuntimeType
-
 type HandlerPuller struct {
 	bucket   *blob.Bucket
 	dirCache sync.Map // key=lambda name, value=*CacheEntry
@@ -34,7 +31,6 @@ type HandlerPuller struct {
 type CacheEntry struct {
 	version time.Time // blob modification time
 	path    string
-	runtime common.RuntimeType
 }
 
 func NewHandlerPuller(dirMaker *common.DirMaker) (*HandlerPuller, error) {
@@ -67,12 +63,12 @@ func NewHandlerPuller(dirMaker *common.DirMaker) (*HandlerPuller, error) {
 	}, nil
 }
 
-func (cp *HandlerPuller) Pull(name string) (common.RuntimeType, string, error) {
+func (cp *HandlerPuller) Pull(name string) (string, error) {
 	t := common.T0("pull-lambda")
 	defer t.T1()
 
 	if err := common.ValidateFunctionName(name); err != nil {
-		return RT_UNKNOWN, "", err
+		return "", err
 	}
 
 	key := name + common.LambdaFileExtension
@@ -81,71 +77,62 @@ func (cp *HandlerPuller) Pull(name string) (common.RuntimeType, string, error) {
 	if err == nil {
 		version := attrs.ModTime
 		if cached := cp.getCache(name); cached != nil && cached.version.Equal(version) {
-			return cached.runtime, cached.path, nil
+			return cached.path, nil
 		}
 	}
 
-	rt, dir, err := cp.pullFromBlob(key, name)
+	dir, err := cp.pullFromBlob(key, name)
 	if err == nil {
 		var version time.Time
 		if attrs != nil {
 			version = attrs.ModTime
 		}
-		cp.putCache(name, version, dir, rt)
-		return rt, dir, nil
+		cp.putCache(name, version, dir)
+		return dir, nil
 	} else if err != errNotFound404 {
-		return RT_UNKNOWN, "", err
+		return "", err
 	}
-	return RT_UNKNOWN, "", fmt.Errorf(
+	return "", fmt.Errorf(
 		"lambda %q not found in blob store (bucket=%q, key=%q)",
 		name, common.Conf.Registry, key,
 	)
 }
 
-func (cp *HandlerPuller) pullFromBlob(key, lambdaName string) (common.RuntimeType, string, error) {
+func (cp *HandlerPuller) pullFromBlob(key, lambdaName string) (string, error) {
 	ctx := context.Background()
 	reader, err := cp.bucket.NewReader(ctx, key, nil)
 	if err != nil {
 		if gcerrors.Code(err) == gcerrors.NotFound {
-			return RT_UNKNOWN, "", errNotFound404
+			return "", errNotFound404
 		}
-		return RT_UNKNOWN, "", err
+		return "", err
 	}
 	defer reader.Close()
 
 	tmpFile, err := os.CreateTemp("", lambdaName+"_blob")
-
 	if err != nil {
-		return RT_UNKNOWN, "", err
+		return "", err
 	}
 
 	tmpPath := tmpFile.Name()
 	if _, err := io.Copy(tmpFile, reader); err != nil {
 		tmpFile.Close()
-		return RT_UNKNOWN, "", err
+		return "", err
 	}
 	tmpFile.Close()
 	defer os.Remove(tmpPath)
 
 	targetDir := cp.dirMaker.Get(lambdaName)
 	if err := os.MkdirAll(targetDir, 0755); err != nil {
-		return RT_UNKNOWN, "", err
+		return "", err
 	}
 
 	cmd := exec.Command("tar", "-xzf", tmpPath, "--directory", targetDir)
 	if output, err := cmd.CombinedOutput(); err != nil {
-		return RT_UNKNOWN, "", fmt.Errorf("tar extract failed: %v :: %s", err, output)
+		return "", fmt.Errorf("tar extract failed: %v :: %s", err, output)
 	}
 
-	var rt common.RuntimeType
-	if _, err := os.Stat(filepath.Join(targetDir, "f.py")); err == nil {
-		rt = common.RT_PYTHON
-	} else if _, err := os.Stat(filepath.Join(targetDir, "f.bin")); err == nil {
-		rt = common.RT_NATIVE
-	} else {
-		return RT_UNKNOWN, "", fmt.Errorf("runtime type not found in extracted archive")
-	}
-	return rt, targetDir, nil
+	return targetDir, nil
 }
 
 func (cp *HandlerPuller) Reset(name string) {
@@ -159,10 +146,10 @@ func (cp *HandlerPuller) getCache(name string) *CacheEntry {
 	}
 	return entry.(*CacheEntry)
 }
-func (cp *HandlerPuller) putCache(name string, version time.Time, path string, runtime common.RuntimeType) {
+func (cp *HandlerPuller) putCache(name string, version time.Time, path string) {
 	// Clean up old cache entry if it exists
 	if old := cp.getCache(name); old != nil && old.path != path {
 		os.RemoveAll(old.path)
 	}
-	cp.dirCache.Store(name, &CacheEntry{version, path, runtime})
+	cp.dirCache.Store(name, &CacheEntry{version, path})
 }
diff --git a/go/worker/lambda/lambdaFunction.go b/go/worker/lambda/lambdaFunction.go
index 0c3eca751..233307198 100644
--- a/go/worker/lambda/lambdaFunction.go
+++ b/go/worker/lambda/lambdaFunction.go
@@ -18,8 +18,11 @@ import (
 )
 
 type FunctionMeta struct {
-	Config  *common.LambdaConfig `json:"config"`  // New Lambda config (from YAML)
-	Sandbox *sandbox.SandboxMeta `json:"sandbox"` // Existing sandbox metadata
+	// user-specified config (via ol.yaml)
+	Config  *common.LambdaConfig `json:"config"`
+	// container-specific settings, inferred by file contents
+	// (e.g., do we have Python or native code?  what is in requirements.txt?)
+	Sandbox *sandbox.SandboxMeta `json:"sandbox"`
 }
 
 // LambdaFunc represents a single lambda function (the code)
@@ -27,8 +30,6 @@ type LambdaFunc struct {
 	lmgr *LambdaMgr
 	name string
 
-	rtType common.RuntimeType
-
 	// lambda code
 	lastPull *time.Time
 	codeDir  string
@@ -81,21 +82,33 @@ func parseMeta(codeDir string) (*FunctionMeta, error) {
 		Imports:  []string{},
 	}
 
-	// having a requirements.txt is optional
-	path := filepath.Join(codeDir, "requirements.txt")
-	file, err := os.Open(path)
-	if err != nil && !errors.Is(err, os.ErrNotExist) {
-		return nil, err
+	// Determine runtime type by checking for f.py or f.bin
+	if _, err := os.Stat(filepath.Join(codeDir, "f.py")); err == nil {
+		sandboxMeta.Runtime = common.RT_PYTHON
+	} else if _, err := os.Stat(filepath.Join(codeDir, "f.bin")); err == nil {
+		sandboxMeta.Runtime = common.RT_NATIVE
+	} else {
+		return nil, fmt.Errorf("cannot determine runtime: no f.py or f.bin found in %s", codeDir)
 	}
-	defer file.Close()
-
-	scnr := bufio.NewScanner(file)
-	for scnr.Scan() {
-		line := strings.ReplaceAll(scnr.Text(), " ", "")
-		pkg := strings.Split(line, "#")[0]
-		if pkg != "" {
-			pkg = packages.NormalizePkg(pkg)
-			sandboxMeta.Installs = append(sandboxMeta.Installs, pkg)
+
+	// Parse requirements.txt for Python functions (optional)
+	if sandboxMeta.Runtime == common.RT_PYTHON {
+		path := filepath.Join(codeDir, "requirements.txt")
+		file, err := os.Open(path)
+		if err != nil && !errors.Is(err, os.ErrNotExist) {
+			return nil, err
+		}
+		if err == nil {
+			defer file.Close()
+			scnr := bufio.NewScanner(file)
+			for scnr.Scan() {
+				line := strings.ReplaceAll(scnr.Text(), " ", "")
+				pkg := strings.Split(line, "#")[0]
+				if pkg != "" {
+					pkg = packages.NormalizePkg(pkg)
+					sandboxMeta.Installs = append(sandboxMeta.Installs, pkg)
+				}
+			}
 		}
 	}
 
@@ -126,7 +139,7 @@ func (f *LambdaFunc) pullHandlerIfStale() (err error) {
 	}
 
 	// is there new code?
-	rtType, codeDir, err := f.lmgr.HandlerPuller.Pull(f.name)
+	codeDir, err := f.lmgr.HandlerPuller.Pull(f.name)
 	if err != nil {
 		return err
 	}
@@ -135,7 +148,11 @@ func (f *LambdaFunc) pullHandlerIfStale() (err error) {
 		return nil
 	}
 
-	f.rtType = rtType
+	// Parse meta to get runtime type and config
+	meta, err := parseMeta(codeDir)
+	if err != nil {
+		return err
+	}
 
 	defer func() {
 		if err != nil {
@@ -143,7 +160,7 @@ func (f *LambdaFunc) pullHandlerIfStale() (err error) {
 				slog.Error(fmt.Sprintf("could not cleanup %s after failed pull", codeDir))
 			}
 
-			if rtType == common.RT_PYTHON {
+			if meta.Sandbox.Runtime == common.RT_PYTHON {
 				// we dirty this dir (e.g., by setting up
 				// symlinks to packages, so we want the
 				// HandlerPuller to give us a new one next
@@ -153,13 +170,7 @@ func (f *LambdaFunc) pullHandlerIfStale() (err error) {
 		}
 	}()
 
-	// Parse meta for native functions to get config
-	meta, err := parseMeta(codeDir)
-	if err != nil {
-		return err
-	}
-
-	if rtType == common.RT_PYTHON {
+	if meta.Sandbox.Runtime == common.RT_PYTHON {
 		// make sure all specified dependencies are installed
 		// (but don't recursively find others)
 		for _, pkg := range meta.Sandbox.Installs {
@@ -169,8 +180,7 @@ func (f *LambdaFunc) pullHandlerIfStale() (err error) {
 		}
 
 		f.lmgr.DepTracer.TraceFunction(codeDir, meta.Sandbox.Installs)
-		f.Meta = meta
-	} else if rtType == common.RT_NATIVE {
+	} else if meta.Sandbox.Runtime == common.RT_NATIVE {
 		slog.Info("Got native function")
 	}
 
diff --git a/go/worker/lambda/lambdaInstance.go b/go/worker/lambda/lambdaInstance.go
index af4300ea9..d8b004afb 100644
--- a/go/worker/lambda/lambdaInstance.go
+++ b/go/worker/lambda/lambdaInstance.go
@@ -99,11 +99,11 @@ func (linst *LambdaInstance) Task() {
 		if sb == nil {
 			sb = nil
 
-			if f.lmgr.ZygoteProvider != nil && f.rtType == common.RT_PYTHON {
+			if f.lmgr.ZygoteProvider != nil && linst.meta.Sandbox.Runtime == common.RT_PYTHON {
 				scratchDir := f.lmgr.scratchDirs.Make(f.name)
 
 				// we don't specify parent SB, because ImportCache.Create chooses it for us
-				sb, err = f.lmgr.ZygoteProvider.Create(f.lmgr.sbPool, true, linst.codeDir, scratchDir, linst.meta.Sandbox, f.rtType)
+				sb, err = f.lmgr.ZygoteProvider.Create(f.lmgr.sbPool, true, linst.codeDir, scratchDir, linst.meta.Sandbox)
 				if err != nil {
 					f.printf("failed to get Sandbox from import cache")
 					sb = nil
@@ -116,7 +116,7 @@ func (linst *LambdaInstance) Task() {
 			if sb == nil {
 				t2 := common.T0("LambdaInstance-WaitSandbox-NoImportCache")
 				scratchDir := f.lmgr.scratchDirs.Make(f.name)
-				sb, err = f.lmgr.sbPool.Create(nil, true, linst.codeDir, scratchDir, linst.meta.Sandbox, f.rtType)
+				sb, err = f.lmgr.sbPool.Create(nil, true, linst.codeDir, scratchDir, linst.meta.Sandbox)
 				t2.T1()
 			}
 
diff --git a/go/worker/lambda/packages/packagePuller.go b/go/worker/lambda/packages/packagePuller.go
index 1546e4cad..d971ee77b 100644
--- a/go/worker/lambda/packages/packagePuller.go
+++ b/go/worker/lambda/packages/packagePuller.go
@@ -180,9 +180,10 @@ func (pp *PackagePuller) sandboxInstall(p *Package) (err error) {
 	inst := common.Conf.InstallerLimits.WithDefaults(&common.Conf.Limits)
 
 	meta := &sandbox.SandboxMeta{
+		Runtime:    common.RT_PYTHON,
 		MemLimitMB: inst.Mem_mb,
 	}
-	sb, err := pp.sbPool.Create(nil, true, pp.pipLambda, scratchDir, meta, common.RT_PYTHON)
+	sb, err := pp.sbPool.Create(nil, true, pp.pipLambda, scratchDir, meta)
 	if err != nil {
 		return err
 	}
diff --git a/go/worker/lambda/zygote/api.go b/go/worker/lambda/zygote/api.go
index 36b721878..52d666d4f 100644
--- a/go/worker/lambda/zygote/api.go
+++ b/go/worker/lambda/zygote/api.go
@@ -1,13 +1,11 @@
 package zygote
 
 import (
-	"github.com/open-lambda/open-lambda/go/common"
 	"github.com/open-lambda/open-lambda/go/worker/sandbox"
 )
 
 type ZygoteProvider interface {
 	Create(childSandboxPool sandbox.SandboxPool, isLeaf bool,
-		codeDir, scratchDir string, meta *sandbox.SandboxMeta,
-		rt_type common.RuntimeType) (sandbox.Sandbox, error)
+		codeDir, scratchDir string, meta *sandbox.SandboxMeta) (sandbox.Sandbox, error)
 	Cleanup()
 }
diff --git a/go/worker/lambda/zygote/importCache.go b/go/worker/lambda/zygote/importCache.go
index a0145be98..743daf981 100644
--- a/go/worker/lambda/zygote/importCache.go
+++ b/go/worker/lambda/zygote/importCache.go
@@ -146,7 +146,7 @@ func (cache *ImportCache) recursiveKill(node *ImportCacheNode) {
 }
 
 // Create creates a new sandbox using the import cache.
-func (cache *ImportCache) Create(childSandboxPool sandbox.SandboxPool, isLeaf bool, codeDir, scratchDir string, meta *sandbox.SandboxMeta, rt_type common.RuntimeType) (sandbox.Sandbox, error) {
+func (cache *ImportCache) Create(childSandboxPool sandbox.SandboxPool, isLeaf bool, codeDir, scratchDir string, meta *sandbox.SandboxMeta) (sandbox.Sandbox, error) {
 	t := common.T0("ImportCache.Create")
 	defer t.T1()
 
@@ -158,7 +158,7 @@ func (cache *ImportCache) Create(childSandboxPool sandbox.SandboxPool, isLeaf bo
 		panic(fmt.Errorf("did not find Zygote; at least expected to find the root"))
 	}
 	slog.Info(fmt.Sprintf("Try using Zygote from <%v>", node))
-	return cache.createChildSandboxFromNode(childSandboxPool, node, isLeaf, codeDir, scratchDir, meta, rt_type)
+	return cache.createChildSandboxFromNode(childSandboxPool, node, isLeaf, codeDir, scratchDir, meta)
 }
 
 // use getSandboxInNode to get a Zygote Sandbox for the node (creating one
@@ -167,20 +167,20 @@ func (cache *ImportCache) Create(childSandboxPool sandbox.SandboxPool, isLeaf bo
 // the new Sandbox may either be for a Zygote, or a leaf Sandbox
 func (cache *ImportCache) createChildSandboxFromNode(
 	childSandboxPool sandbox.SandboxPool, node *ImportCacheNode, isLeaf bool,
-	codeDir, scratchDir string, meta *sandbox.SandboxMeta, rt_type common.RuntimeType) (sandbox.Sandbox, error) {
+	codeDir, scratchDir string, meta *sandbox.SandboxMeta) (sandbox.Sandbox, error) {
 	t := common.T0("ImportCache.createChildSandboxFromNode")
 	defer t.T1()
 
 	// try twice, restarting parent Sandbox if it fails the first time
 	forceNew := false
 	for i := 0; i < 2; i++ {
-		zygoteSB, isNew, err := cache.getSandboxInNode(node, forceNew, rt_type)
+		zygoteSB, isNew, err := cache.getSandboxInNode(node, forceNew)
 		if err != nil {
 			return nil, err
 		}
 
 		t2 := common.T0("ImportCache.createChildSandboxFromNode:childSandboxPool.Create")
-		sb, err := childSandboxPool.Create(zygoteSB, isLeaf, codeDir, scratchDir, meta, rt_type)
+		sb, err := childSandboxPool.Create(zygoteSB, isLeaf, codeDir, scratchDir, meta)
 
 		if err == nil {
 			if isLeaf {
@@ -212,7 +212,7 @@ func (cache *ImportCache) createChildSandboxFromNode(
 //
 // the Sandbox returned is guaranteed to be in Unpaused state.  After
 // use, caller must also call putSandboxInNode to release ref count
-func (cache *ImportCache) getSandboxInNode(node *ImportCacheNode, forceNew bool, rt_type common.RuntimeType) (sb sandbox.Sandbox, isNew bool, err error) {
+func (cache *ImportCache) getSandboxInNode(node *ImportCacheNode, forceNew bool) (sb sandbox.Sandbox, isNew bool, err error) {
 	t := common.T0("ImportCache.getSandboxInNode")
 	defer t.T1()
 
@@ -239,7 +239,7 @@ func (cache *ImportCache) getSandboxInNode(node *ImportCacheNode, forceNew bool,
 	}
 
 	// SLOW PATH
-	if err := cache.createSandboxInNode(node, rt_type); err != nil {
+	if err := cache.createSandboxInNode(node); err != nil {
 		return nil, false, err
 	}
 	node.sbRefCount = 1
@@ -281,7 +281,7 @@ func (*ImportCache) putSandboxInNode(node *ImportCacheNode, sb sandbox.Sandbox)
 	}
 }
 
-func (cache *ImportCache) createSandboxInNode(node *ImportCacheNode, rt_type common.RuntimeType) (err error) {
+func (cache *ImportCache) createSandboxInNode(node *ImportCacheNode) (err error) {
 	// populate codeDir/packages with deps, and record top-level mods)
 	if node.codeDir == "" {
 		codeDir := cache.codeDirs.Make("import-cache")
@@ -305,7 +305,9 @@ func (cache *ImportCache) createSandboxInNode(node *ImportCacheNode, rt_type com
 
 		// policy: what modules should we pre-import?  Top-level of
 		// pre-initialized packages is just one possibility...
+		// Import cache is Python-specific, so always use RT_PYTHON
 		node.meta = &sandbox.SandboxMeta{
+			Runtime:  common.RT_PYTHON,
 			Installs: installs,
 			Imports:  topLevelMods,
 		}
@@ -314,9 +316,9 @@ func (cache *ImportCache) createSandboxInNode(node *ImportCacheNode, rt_type com
 	scratchDir := cache.scratchDirs.Make("import-cache")
 	var sb sandbox.Sandbox
 	if node.parent != nil {
-		sb, err = cache.createChildSandboxFromNode(cache.sbPool, node.parent, false, node.codeDir, scratchDir, node.meta, rt_type)
+		sb, err = cache.createChildSandboxFromNode(cache.sbPool, node.parent, false, node.codeDir, scratchDir, node.meta)
 	} else {
-		sb, err = cache.sbPool.Create(nil, false, node.codeDir, scratchDir, node.meta, common.RT_PYTHON)
+		sb, err = cache.sbPool.Create(nil, false, node.codeDir, scratchDir, node.meta)
 	}
 
 	if err != nil {
diff --git a/go/worker/lambda/zygote/multiTree.go b/go/worker/lambda/zygote/multiTree.go
index e0a29637a..aa607bef5 100644
--- a/go/worker/lambda/zygote/multiTree.go
+++ b/go/worker/lambda/zygote/multiTree.go
@@ -44,9 +44,9 @@ func NewMultiTree(codeDirs *common.DirMaker, scratchDirs *common.DirMaker, sbPoo
 }
 
 // Create creates a new sandbox using a randomly selected ImportCache tree.
-func (mt *MultiTree) Create(childSandboxPool sandbox.SandboxPool, isLeaf bool, codeDir, scratchDir string, meta *sandbox.SandboxMeta, rt_type common.RuntimeType) (sandbox.Sandbox, error) {
+func (mt *MultiTree) Create(childSandboxPool sandbox.SandboxPool, isLeaf bool, codeDir, scratchDir string, meta *sandbox.SandboxMeta) (sandbox.Sandbox, error) {
 	idx := rand.Intn(len(mt.trees))
-	return mt.trees[idx].Create(childSandboxPool, isLeaf, codeDir, scratchDir, meta, rt_type)
+	return mt.trees[idx].Create(childSandboxPool, isLeaf, codeDir, scratchDir, meta)
 }
 
 // Cleanup performs cleanup operations for all ImportCache trees in the MultiTree.
diff --git a/go/worker/sandbox/api.go b/go/worker/sandbox/api.go
index bbe215cf3..b109e9496 100644
--- a/go/worker/sandbox/api.go
+++ b/go/worker/sandbox/api.go
@@ -12,8 +12,8 @@ type SandboxPool interface {
 	// isLeaf: true iff this is not being created as a sandbox we can fork later
 	// codeDir: directory where lambda code exists
 	// scratchDir: directory where handler code can write (caller is responsible for creating and deleting)
-	// meta: details about installs, imports, etc.  Will be populated with defaults if not specified
-	Create(parent Sandbox, isLeaf bool, codeDir, scratchDir string, meta *SandboxMeta, rtType common.RuntimeType) (sb Sandbox, err error)
+	// meta: details about runtime, installs, imports, etc.  Will be populated with defaults if not specified
+	Create(parent Sandbox, isLeaf bool, codeDir, scratchDir string, meta *SandboxMeta) (sb Sandbox, err error)
 
 	// blocks until all Sandboxes are deleted, so caller must
 	// either delete them before this call, or from another asyncronously
@@ -80,15 +80,16 @@ type Sandbox interface {
 
 	// Child calls this on parent to notify of child Destroy
 	childExit(child Sandbox)
-
-	GetRuntimeType() common.RuntimeType // TODO: make it part of SandboxMeta?
 }
 
 type SandboxMeta struct {
-	Installs   []string
-	Imports    []string
+	Runtime    common.RuntimeType
 	MemLimitMB int
 	CPUPercent int
+
+	// Python specific fields:
+	Installs []string
+	Imports  []string
 }
 
 type SandboxError string
diff --git a/go/worker/sandbox/docker.go b/go/worker/sandbox/docker.go
index 5087f5d81..80ea35ef6 100644
--- a/go/worker/sandbox/docker.go
+++ b/go/worker/sandbox/docker.go
@@ -33,7 +33,6 @@ type DockerContainer struct {
 	client     *docker.Client
 	installed  map[string]bool
 	meta       *SandboxMeta
-	rtType     common.RuntimeType
 	httpClient *http.Client
 }
 
@@ -253,11 +252,6 @@ func (container *DockerContainer) ID() string {
 	return container.hostID
 }
 
-// GetRuntimeType returns what runtime is being used by this container?
-func (container *DockerContainer) GetRuntimeType() common.RuntimeType {
-	return container.rtType
-}
-
 // DockerID returns the id assigned by docker itself, not by open lambda
 func (container *DockerContainer) DockerID() string {
 	return container.container.ID
@@ -269,7 +263,7 @@ func (container *DockerContainer) HostDir() string {
 }
 
 func (container *DockerContainer) runServer() error {
-	if container.rtType != common.RT_PYTHON {
+	if container.meta.Runtime != common.RT_PYTHON {
 		return fmt.Errorf("Unsupported runtime")
 	}
 
diff --git a/go/worker/sandbox/dockerPool.go b/go/worker/sandbox/dockerPool.go
index f00bd87a2..28dd30d3f 100644
--- a/go/worker/sandbox/dockerPool.go
+++ b/go/worker/sandbox/dockerPool.go
@@ -60,7 +60,7 @@ func NewDockerPool(pidMode string, caps []string) (*DockerPool, error) {
 }
 
 // Create creates a docker sandbox from the handler and sandbox directory.
-func (pool *DockerPool) Create(parent Sandbox, isLeaf bool, codeDir, scratchDir string, meta *SandboxMeta, _ common.RuntimeType) (sb Sandbox, err error) {
+func (pool *DockerPool) Create(parent Sandbox, isLeaf bool, codeDir, scratchDir string, meta *SandboxMeta) (sb Sandbox, err error) {
 	meta = fillMetaDefaults(meta)
 	t := common.T0("Create()")
 	defer t.T1()
diff --git a/go/worker/sandbox/sock.go b/go/worker/sandbox/sock.go
index 9ee3c9491..12a153ceb 100644
--- a/go/worker/sandbox/sock.go
+++ b/go/worker/sandbox/sock.go
@@ -26,7 +26,6 @@ type SOCKContainer struct {
 	codeDir          string
 	scratchDir       string
 	cg               cgroups.Cgroup
-	rtType           common.RuntimeType
 	client           *http.Client
 
 	// 1 for self, plus 1 for each child (we can't release memory
@@ -53,10 +52,6 @@ func (container *SOCKContainer) ID() string {
 	return container.id
 }
 
-func (container *SOCKContainer) GetRuntimeType() common.RuntimeType {
-	return container.rtType
-}
-
 func (container *SOCKContainer) freshProc() (err error) {
 	// get FD to cgroup
 	cgFiles := make([]*os.File, 1)
@@ -70,13 +65,13 @@ func (container *SOCKContainer) freshProc() (err error) {
 
 	var cmd *exec.Cmd
 
-	if container.rtType == common.RT_PYTHON {
+	if container.meta.Runtime == common.RT_PYTHON {
 		cmd = exec.Command(
 			"chroot", container.containerRootDir, "python3", "-u",
 			"/runtimes/python/server.py", "/host/bootstrap.py", strconv.Itoa(1),
 			strconv.FormatBool(common.Conf.Features.Enable_seccomp),
 		)
-	} else if container.rtType == common.RT_NATIVE {
+	} else if container.meta.Runtime == common.RT_NATIVE {
 		if container.containerProxy == nil {
 			err := container.launchContainerProxy()
 
diff --git a/go/worker/sandbox/sockPool.go b/go/worker/sandbox/sockPool.go
index d8d793cd9..a71f9616b 100644
--- a/go/worker/sandbox/sockPool.go
+++ b/go/worker/sandbox/sockPool.go
@@ -64,7 +64,7 @@ func sbStr(sb Sandbox) string {
 	return fmt.Sprintf("<SB %s>", sb.ID())
 }
 
-func (pool *SOCKPool) Create(parent Sandbox, isLeaf bool, codeDir, scratchDir string, meta *SandboxMeta, rtType common.RuntimeType) (sb Sandbox, err error) {
+func (pool *SOCKPool) Create(parent Sandbox, isLeaf bool, codeDir, scratchDir string, meta *SandboxMeta) (sb Sandbox, err error) {
 	id := fmt.Sprintf("%d", atomic.AddInt64(&nextId, 1))
 	meta = fillMetaDefaults(meta)
 	pool.printf("<%v>.Create(%v, %v, %v, %v, %v)=%s...", pool.name, sbStr(parent), isLeaf, codeDir, scratchDir, meta, id)
@@ -84,7 +84,6 @@ func (pool *SOCKPool) Create(parent Sandbox, isLeaf bool, codeDir, scratchDir st
 		cgRefCount:       1,
 		children:         make(map[string]Sandbox),
 		meta:             meta,
-		rtType:           rtType,
 		containerProxy:   nil,
 	}
 	var c Sandbox = cSock
@@ -122,7 +121,7 @@ func (pool *SOCKPool) Create(parent Sandbox, isLeaf bool, codeDir, scratchDir st
 	}
 	t2.T1()
 
-	if rtType == common.RT_PYTHON {
+	if meta.Runtime == common.RT_PYTHON {
 		// add installed packages to the path, and import the modules we'll need
 		var pyCode []string
 
@@ -153,7 +152,7 @@ func (pool *SOCKPool) Create(parent Sandbox, isLeaf bool, codeDir, scratchDir st
 		if err := ioutil.WriteFile(path, code, 0600); err != nil {
 			return nil, err
 		}
-	} else if rtType == common.RT_NATIVE {
+	} else if meta.Runtime == common.RT_NATIVE {
 		// nothing to do?
 	} else {
 		return nil, fmt.Errorf("Unsupported runtime")

From a1df905abb6af541262449a1d432b9b37a8f94aa Mon Sep 17 00:00:00 2001
From: Tyler Caraza-Harter <tylerharter@users.noreply.github.com>
Date: Fri, 2 Jan 2026 10:23:44 -0600
Subject: [PATCH 07/55] make entry point configurable (#379)

* make entry point configurable

* example cleanup

* fix Docker entry
---
 .github/workflows/ci.yml                   |  6 ++--
 docs/worker/lambda-config.md               | 19 +++++++++++++
 examples/flask-entry-test/app.py           | 14 +++++++++
 examples/flask-entry-test/ol.yaml          |  7 +++++
 examples/flask-entry-test/requirements.in  |  2 ++
 examples/flask-entry-test/requirements.txt | 20 +++++++++++++
 examples/server/echo.py                    |  2 --
 examples/server/hello.py                   |  2 --
 go/admin/commands.go                       | 17 +++++++++--
 go/worker/lambda/lambdaFunction.go         | 27 ++++++++++++------
 min-image/runtimes/python/server.py        | 14 +++++----
 min-image/runtimes/python/server_legacy.py | 15 ++++++----
 scripts/test.py                            | 33 +++++++++++++++++++---
 13 files changed, 144 insertions(+), 34 deletions(-)
 create mode 100644 examples/flask-entry-test/app.py
 create mode 100644 examples/flask-entry-test/ol.yaml
 create mode 100644 examples/flask-entry-test/requirements.in
 create mode 100644 examples/flask-entry-test/requirements.txt
 delete mode 100644 examples/server/echo.py
 delete mode 100644 examples/server/hello.py

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2d3b86577..2fbc6b696 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -62,12 +62,12 @@ jobs:
       working-directory: go/common
       run: go test -v
       timeout-minutes: 5
-    - name: Test Python (Docker)
-      # not all features are supported by docker yet, so we only run some of the tests
-      run: sudo env "PATH=$PATH" ./scripts/test.py --worker_type=docker --test_blocklist=max_mem_alloc
     - name: Test Python (SOCK)
       run: sudo env "PATH=$PATH" ./scripts/test.py --worker_type=sock
       timeout-minutes: 20
+    - name: Test Python (Docker)
+      # not all features are supported by docker yet, so we only run some of the tests
+      run: sudo env "PATH=$PATH" ./scripts/test.py --worker_type=docker --test_blocklist=max_mem_alloc
     - name: Boss/Lambda Store Tests
       run: sudo env "PATH=$PATH" python3 scripts/boss_test.py local
       timeout-minutes: 5
diff --git a/docs/worker/lambda-config.md b/docs/worker/lambda-config.md
index fe2e014ad..2e613da31 100644
--- a/docs/worker/lambda-config.md
+++ b/docs/worker/lambda-config.md
@@ -50,6 +50,25 @@ These variables can be accessed in your lambda code using standard environment v
 
 **Note:** Environment variables defined in `ol.yaml` are written to a `.env` file in the lambda's directory during execution. If your lambda already has a `.env` file, it will be overwritten with the values from `ol.yaml`.
 
+### c. Special Environment Variables
+
+#### OL_ENTRY_FILE
+By default, OpenLambda expects Python lambda functions to be defined in a file named `f.py`. You can override this by setting the `OL_ENTRY_FILE` environment variable to specify a different entry file.
+
+Example:
+```yaml
+environment:
+  OL_ENTRY_FILE: "app.py"
+```
+
+With this configuration:
+- OpenLambda will look for `app.py` instead of `f.py` when detecting the Python runtime
+- The Python runtime will import the `app` module instead of `f`
+- For standard functions, define your handler as `def f(event)` in the specified file
+- For Flask/WSGI applications, define your `app` object in the specified file
+
+This is useful when you want to use conventional naming (e.g., `app.py` for Flask applications) or integrate existing code without renaming files.
+
 ## 4. How to Use
 ### a. Define Configuration
 Create an `ol.yaml` file inside the lambda function directory with the desired configuration.
diff --git a/examples/flask-entry-test/app.py b/examples/flask-entry-test/app.py
new file mode 100644
index 000000000..fa1b6fc60
--- /dev/null
+++ b/examples/flask-entry-test/app.py
@@ -0,0 +1,14 @@
+from flask import Flask, request, Response
+
+app = Flask("flask-entry-test")
+
+@app.route("/")
+def index():
+    return Response("Hello from app.py!\n", status=200)
+
+@app.route("/info")
+def info():
+    return {
+        "entry_file": "app.py",
+        "message": "This function uses OL_ENTRY_FILE to specify app.py as the entry point"
+    }
diff --git a/examples/flask-entry-test/ol.yaml b/examples/flask-entry-test/ol.yaml
new file mode 100644
index 000000000..fa6be1c13
--- /dev/null
+++ b/examples/flask-entry-test/ol.yaml
@@ -0,0 +1,7 @@
+triggers:
+  http:
+    - method: GET
+    - method: POST
+
+environment:
+  OL_ENTRY_FILE: "app.py"
diff --git a/examples/flask-entry-test/requirements.in b/examples/flask-entry-test/requirements.in
new file mode 100644
index 000000000..308c5b964
--- /dev/null
+++ b/examples/flask-entry-test/requirements.in
@@ -0,0 +1,2 @@
+flask
+werkzeug<3.1
diff --git a/examples/flask-entry-test/requirements.txt b/examples/flask-entry-test/requirements.txt
new file mode 100644
index 000000000..d29ce9e0e
--- /dev/null
+++ b/examples/flask-entry-test/requirements.txt
@@ -0,0 +1,20 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile requirements.in -o requirements.txt
+blinker==1.9.0
+    # via flask
+click==8.1.7
+    # via flask
+flask==3.1.0
+    # via -r requirements.in
+itsdangerous==2.2.0
+    # via flask
+jinja2==3.1.4
+    # via flask
+markupsafe==3.0.2
+    # via
+    #   jinja2
+    #   werkzeug
+werkzeug==3.0.6
+    # via
+    #   -r requirements.in
+    #   flask
diff --git a/examples/server/echo.py b/examples/server/echo.py
deleted file mode 100644
index f209c6344..000000000
--- a/examples/server/echo.py
+++ /dev/null
@@ -1,2 +0,0 @@
-def f(event):
-    return event
diff --git a/examples/server/hello.py b/examples/server/hello.py
deleted file mode 100644
index f048f4749..000000000
--- a/examples/server/hello.py
+++ /dev/null
@@ -1,2 +0,0 @@
-def f(event):
-    return 'hello'
diff --git a/go/admin/commands.go b/go/admin/commands.go
index aa78d6827..9a4911ebf 100644
--- a/go/admin/commands.go
+++ b/go/admin/commands.go
@@ -187,9 +187,20 @@ func createTarGz(funcDir string, overrides map[string]string) ([]byte, error) {
 	gzWriter := gzip.NewWriter(&buf)
 	tarWriter := tar.NewWriter(gzWriter)
 
-	fpyPath := filepath.Join(funcDir, "f.py")
-	if _, err := os.Stat(fpyPath); os.IsNotExist(err) {
-		return nil, fmt.Errorf("required file f.py not found in %s", funcDir)
+	// Determine the Python entry file (default to f.py, or use OL_ENTRY_FILE from ol.yaml)
+	// TODO: support OL_ENTRY_FILE for native runtime
+	pythonEntryFile := "f.py"
+	if lambdaConfig, err := common.LoadLambdaConfig(funcDir); err == nil {
+		if lambdaConfig.Environment != nil {
+			if entryFile, ok := lambdaConfig.Environment["OL_ENTRY_FILE"]; ok {
+				pythonEntryFile = entryFile
+			}
+		}
+	}
+
+	entryPath := filepath.Join(funcDir, pythonEntryFile)
+	if _, err := os.Stat(entryPath); os.IsNotExist(err) {
+		return nil, fmt.Errorf("required file %s not found in %s", pythonEntryFile, funcDir)
 	}
 
 	err := filepath.Walk(funcDir, func(path string, info os.FileInfo, err error) error {
diff --git a/go/worker/lambda/lambdaFunction.go b/go/worker/lambda/lambdaFunction.go
index 233307198..21ff11825 100644
--- a/go/worker/lambda/lambdaFunction.go
+++ b/go/worker/lambda/lambdaFunction.go
@@ -82,13 +82,28 @@ func parseMeta(codeDir string) (*FunctionMeta, error) {
 		Imports:  []string{},
 	}
 
-	// Determine runtime type by checking for f.py or f.bin
-	if _, err := os.Stat(filepath.Join(codeDir, "f.py")); err == nil {
+	// Load Lambda configuration from ol.yaml first (needed to check OL_ENTRY_FILE)
+	lambdaConfig, err := common.LoadLambdaConfig(codeDir)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse lambda configuration file: %v", err)
+	}
+
+	// Determine the Python entry file (default to f.py)
+	pythonEntryFile := "f.py"
+	if lambdaConfig.Environment != nil {
+		if entryFile, ok := lambdaConfig.Environment["OL_ENTRY_FILE"]; ok {
+			pythonEntryFile = entryFile
+		}
+	}
+
+	// Determine runtime type by checking for entry file or f.bin
+	// TODO: support OL_ENTRY_FILE for native runtime
+	if _, err := os.Stat(filepath.Join(codeDir, pythonEntryFile)); err == nil {
 		sandboxMeta.Runtime = common.RT_PYTHON
 	} else if _, err := os.Stat(filepath.Join(codeDir, "f.bin")); err == nil {
 		sandboxMeta.Runtime = common.RT_NATIVE
 	} else {
-		return nil, fmt.Errorf("cannot determine runtime: no f.py or f.bin found in %s", codeDir)
+		return nil, fmt.Errorf("cannot determine runtime: no %s or f.bin found in %s", pythonEntryFile, codeDir)
 	}
 
 	// Parse requirements.txt for Python functions (optional)
@@ -112,12 +127,6 @@ func parseMeta(codeDir string) (*FunctionMeta, error) {
 		}
 	}
 
-	// Load Lambda configuration from ol.yaml
-	lambdaConfig, err := common.LoadLambdaConfig(codeDir)
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse lambda configuration file: %v", err)
-	}
-
 	// Return combined FunctionMeta
 	return &FunctionMeta{
 		Sandbox: sandboxMeta,
diff --git a/min-image/runtimes/python/server.py b/min-image/runtimes/python/server.py
index abd57bbe8..ca3eb5ee4 100644
--- a/min-image/runtimes/python/server.py
+++ b/min-image/runtimes/python/server.py
@@ -32,7 +32,11 @@ def web_server():
     # TODO: as a safeguard, we should add a mechanism so that the
     # import doesn't happen until the cgroup move completes, so that a
     # malicious child cannot eat up Zygote resources
-    import f
+    entry_file = os.environ.get('OL_ENTRY_FILE', 'f.py')
+    if not entry_file.endswith('.py'):
+        raise ValueError(f"OL_ENTRY_FILE must end with .py, got: {entry_file}")
+    module_name = entry_file[:-3]
+    handler_module = importlib.import_module(module_name)
 
     class SockFileHandler(tornado.web.RequestHandler):
         # TODO: we should consider how are the different requests used in the context of different applications and functions
@@ -47,7 +51,7 @@ def handle_request(self):
                     self.write(f'bad request data: "{data}"')
                     return
 
-                result = f.f(event) if event is not None else f.f({}) 
+                result = handler_module.f(event) if event is not None else handler_module.f({}) 
                 self.write(json.dumps(result))  # Return the result as JSON
             except Exception:
                 self.set_status(500)  # Internal server error for unhandled exceptions
@@ -74,7 +78,7 @@ def options(self):
             self.handle_request()
     
 
-    if hasattr(f, "app"):
+    if hasattr(handler_module, "app"):
         def path_wrapper(environ, start_response):
             path = environ.get("PATH_INFO", "")
             # split path to get individual components
@@ -87,8 +91,8 @@ def path_wrapper(environ, start_response):
             # set the root of the application
             app_name = parts[2]
             environ["SCRIPT_NAME"] = '/run/' + app_name
-            
-            return f.app(environ, start_response)
+
+            return handler_module.app(environ, start_response)
         
         # use WSGI entry
         # call wrapper to strip /run/<func-name> from path
diff --git a/min-image/runtimes/python/server_legacy.py b/min-image/runtimes/python/server_legacy.py
index 057465683..2c5312ed4 100644
--- a/min-image/runtimes/python/server_legacy.py
+++ b/min-image/runtimes/python/server_legacy.py
@@ -48,12 +48,15 @@
 
 # run after forking into sandbox
 def init():
-    global initialized, f
+    global initialized, handler_module
     if initialized:
         return
 
-    # assume submitted .py file is /handler/f.py
-    import f
+    entry_file = os.environ.get('OL_ENTRY_FILE', 'f.py')
+    if not entry_file.endswith('.py'):
+        raise ValueError(f"OL_ENTRY_FILE must end with .py, got: {entry_file}")
+    module_name = entry_file[:-3]
+    handler_module = importlib.import_module(module_name)
 
     initialized = True
 
@@ -68,7 +71,7 @@ def handle_request(self):
                 self.write(f'bad request data: "{data}"')
                 return
 
-            result = f.f(event) if event is not None else f.f({}) 
+            result = handler_module.f(event) if event is not None else handler_module.f({}) 
             self.write(json.dumps(result))  # Return the result as JSON
         except Exception:
             self.set_status(500)  # Internal server error for unhandled exceptions
@@ -98,7 +101,7 @@ def options(self):
 # listen on sock file with Tornado
 def lambda_server():
     init()
-    if hasattr(f, "app"):
+    if hasattr(handler_module, "app"):
         def path_wrapper(environ, start_response):
             path = environ.get("PATH_INFO", "")
             # split path to get individual components
@@ -112,7 +115,7 @@ def path_wrapper(environ, start_response):
             app_name = parts[2]
             environ["SCRIPT_NAME"] = '/run/' + app_name
 
-            return f.app(environ, start_response)
+            return handler_module.app(environ, start_response)
 
         # use WSGI entry
         # call wrapper to strip /run/<func-name> from path
diff --git a/scripts/test.py b/scripts/test.py
index 559d0b86f..dc10933e3 100755
--- a/scripts/test.py
+++ b/scripts/test.py
@@ -49,14 +49,12 @@ def install_examples_to_worker_registry():
     if not os.path.exists(examples_dir):
         print(f"Examples directory not found at {examples_dir}")
         return
-    # Get all directories in examples
+    # Get all directories in examples (each directory is a lambda function)
     example_functions = []
     for item in os.listdir(examples_dir):
         item_path = os.path.join(examples_dir, item)
         if os.path.isdir(item_path):
-            # Check if it has f.py (required for lambda functions)
-            if os.path.exists(os.path.join(item_path, "f.py")):
-                example_functions.append(item_path)
+            example_functions.append(item_path)
     print(f"Found {len(example_functions)} lambda functions in examples directory")
     # Install each function using admin install command
     # Find the ol binary - it should be in the project root
@@ -309,6 +307,32 @@ def flask_test():
     if r.text != "hi\n":
         raise ValueError(f"r.text should be 'hi\n', not {repr(r.text)}")
 
+@test
+def flask_entry_test():
+    """Test OL_ENTRY_FILE feature with a Flask app using app.py instead of f.py"""
+    # Test the index route
+    url = 'http://localhost:5000/run/flask-entry-test'
+    print("URL", url)
+    r = requests.get(url)
+    print("RESPONSE", r)
+
+    if r.status_code != 200:
+        raise ValueError(f"expected status code 200, but got {r.status_code}")
+    if r.text != "Hello from app.py!\n":
+        raise ValueError(f"r.text should be 'Hello from app.py!\\n', not {repr(r.text)}")
+
+    # Test the info route
+    url_info = 'http://localhost:5000/run/flask-entry-test/info'
+    print("URL", url_info)
+    r = requests.get(url_info)
+    print("RESPONSE", r)
+
+    if r.status_code != 200:
+        raise ValueError(f"expected status code 200, but got {r.status_code}")
+    data = r.json()
+    if data.get("entry_file") != "app.py":
+        raise ValueError(f"expected entry_file='app.py', got {data}")
+
 @test
 def test_http_method_restrictions():
     url = 'http://localhost:5000/run/lambda-config-test'
@@ -406,6 +430,7 @@ def run_tests():
 
     # make sure we can use WSGI apps based on frameworks like Flask
     flask_test()
+    flask_entry_test()
     test_http_method_restrictions()
 
     # test environment variables from ol.yaml

From 4ed16fa161b33a646b15f4497ecb20a907ae29d6 Mon Sep 17 00:00:00 2001
From: Tyler Caraza-Harter <tylerharter@users.noreply.github.com>
Date: Fri, 2 Jan 2026 11:08:46 -0600
Subject: [PATCH 08/55] fix header forwarding and add pip-compile lambda (#383)

* pip compile in a lambda

* fix header forwarding and support pip-compile as a lambda
---
 docs/worker/pypi-packages.md             | 15 +++++++
 examples/pip-compile/f.py                | 51 ++++++++++++++++++++++++
 examples/pip-compile/requirements.in     |  3 ++
 examples/pip-compile/requirements.txt    | 32 +++++++++++++++
 examples/wsgi-post-echo/f.py             | 12 ++++++
 examples/wsgi-post-echo/requirements.in  |  1 +
 examples/wsgi-post-echo/requirements.txt | 24 +++++++++++
 go/worker/lambda/lambdaInstance.go       |  9 +++++
 scripts/test.py                          | 20 ++++++++++
 9 files changed, 167 insertions(+)
 create mode 100644 examples/pip-compile/f.py
 create mode 100644 examples/pip-compile/requirements.in
 create mode 100644 examples/pip-compile/requirements.txt
 create mode 100644 examples/wsgi-post-echo/f.py
 create mode 100644 examples/wsgi-post-echo/requirements.in
 create mode 100644 examples/wsgi-post-echo/requirements.txt

diff --git a/docs/worker/pypi-packages.md b/docs/worker/pypi-packages.md
index 87618e1c0..b6b32dc17 100644
--- a/docs/worker/pypi-packages.md
+++ b/docs/worker/pypi-packages.md
@@ -45,6 +45,21 @@ six==1.16.0
     # via python-dateutil
 ```
 
+### Using the pip-compile Lambda
+
+If you don't have pip-tools installed locally, you can use the
+`pip-compile` lambda included in OpenLambda. First, install it:
+
+```bash
+ol admin install ./examples/pip-compile
+```
+
+Then compile your requirements:
+
+```bash
+curl -X POST -H "Content-Type: text/plain" --data-binary @requirements.in http://localhost:5000/run/pip-compile > requirements.txt
+```
+
 ## Try It
 
 Start an OpenLambda worker (if not already started).  For example, you
diff --git a/examples/pip-compile/f.py b/examples/pip-compile/f.py
new file mode 100644
index 000000000..a3222bb3e
--- /dev/null
+++ b/examples/pip-compile/f.py
@@ -0,0 +1,51 @@
+import os
+
+SCRATCH_DIR = "/host/tmp"
+
+# Set cache directories to writable scratch dir BEFORE importing pip-tools
+os.environ["PIP_TOOLS_CACHE_DIR"] = SCRATCH_DIR
+os.environ["XDG_CACHE_HOME"] = SCRATCH_DIR
+os.environ["HOME"] = SCRATCH_DIR
+
+from flask import Flask, request, Response
+from piptools.scripts.compile import cli
+from click.testing import CliRunner
+
+app = Flask(__name__)
+
+
+@app.route("/", methods=["POST"])
+def compile_requirements():
+    """
+    Compile requirements.in to requirements.txt using pip-compile.
+
+    Input (POST body): requirements.in content
+    Output: compiled requirements.txt content (plain text)
+    """
+    requirements_in = request.get_data(as_text=True)
+
+    if not requirements_in:
+        return Response("No requirements provided", status=400, mimetype="text/plain")
+
+    in_path = os.path.join(SCRATCH_DIR, "requirements.in")
+    out_path = os.path.join(SCRATCH_DIR, "requirements.txt")
+
+    with open(in_path, "w") as f:
+        f.write(requirements_in)
+
+    runner = CliRunner()
+    result = runner.invoke(cli, [
+        "--index-url", "https://pypi.org/simple/",
+        "--output-file", out_path,
+        in_path
+    ])
+
+    if result.exit_code != 0:
+        return Response(
+            result.output or str(result.exception),
+            status=400,
+            mimetype="text/plain"
+        )
+
+    with open(out_path, "r") as f:
+        return Response(f.read(), mimetype="text/plain")
diff --git a/examples/pip-compile/requirements.in b/examples/pip-compile/requirements.in
new file mode 100644
index 000000000..f5f1e8bcf
--- /dev/null
+++ b/examples/pip-compile/requirements.in
@@ -0,0 +1,3 @@
+flask
+pip-tools==5.5.0
+tomli
diff --git a/examples/pip-compile/requirements.txt b/examples/pip-compile/requirements.txt
new file mode 100644
index 000000000..8a012d536
--- /dev/null
+++ b/examples/pip-compile/requirements.txt
@@ -0,0 +1,32 @@
+#
+# This file is autogenerated by pip-compile with Python 3.12
+# by the following command:
+#
+#    pip-compile requirements.in
+#
+blinker==1.9.0
+    # via flask
+click==8.3.1
+    # via
+    #   flask
+    #   pip-tools
+flask==3.1.2
+    # via -r requirements.in
+itsdangerous==2.2.0
+    # via flask
+jinja2==3.1.6
+    # via flask
+markupsafe==3.0.3
+    # via
+    #   flask
+    #   jinja2
+    #   werkzeug
+pip-tools==5.5.0
+    # via -r requirements.in
+tomli==2.3.0
+    # via -r requirements.in
+werkzeug==3.1.4
+    # via flask
+
+# The following packages are considered to be unsafe in a requirements file:
+# pip
diff --git a/examples/wsgi-post-echo/f.py b/examples/wsgi-post-echo/f.py
new file mode 100644
index 000000000..e2bc9a6e1
--- /dev/null
+++ b/examples/wsgi-post-echo/f.py
@@ -0,0 +1,12 @@
+from flask import Flask, request, Response
+
+app = Flask(__name__)
+
+
+@app.route("/", methods=["GET", "POST", "PUT"])
+def echo():
+    """Echo back the POST body."""
+    return Response(
+        request.get_data(as_text=True),
+        mimetype=request.content_type or "text/plain"
+    )
diff --git a/examples/wsgi-post-echo/requirements.in b/examples/wsgi-post-echo/requirements.in
new file mode 100644
index 000000000..7e1060246
--- /dev/null
+++ b/examples/wsgi-post-echo/requirements.in
@@ -0,0 +1 @@
+flask
diff --git a/examples/wsgi-post-echo/requirements.txt b/examples/wsgi-post-echo/requirements.txt
new file mode 100644
index 000000000..96159fc91
--- /dev/null
+++ b/examples/wsgi-post-echo/requirements.txt
@@ -0,0 +1,24 @@
+#
+# This file is autogenerated by pip-compile with Python 3.13
+# by the following command:
+#
+#    pip-compile requirements.in
+#
+blinker==1.6.2
+    # via flask
+click==8.1.7
+    # via flask
+flask==2.3.2
+    # via -r requirements.in
+itsdangerous==2.1.2
+    # via flask
+jinja2==3.1.4
+    # via flask
+markupsafe==2.1.3
+    # via
+    #   jinja2
+    #   werkzeug
+werkzeug==3.0.3
+    # via
+    #   -r requirements.in
+    #   flask
diff --git a/go/worker/lambda/lambdaInstance.go b/go/worker/lambda/lambdaInstance.go
index d8b004afb..af0a8a21b 100644
--- a/go/worker/lambda/lambdaInstance.go
+++ b/go/worker/lambda/lambdaInstance.go
@@ -143,6 +143,15 @@ func (linst *LambdaInstance) Task() {
 			if err != nil {
 				linst.TrySendError(req, http.StatusInternalServerError, "Could not create NewRequest: "+err.Error(), sb)
 			} else {
+				// Copy headers from original request
+				for k, vv := range req.r.Header {
+					for _, v := range vv {
+						httpReq.Header.Add(k, v)
+					}
+				}
+				// Preserve ContentLength (parsed from Content-Length header)
+				httpReq.ContentLength = req.r.ContentLength
+
 				resp, err := sb.Client().Do(httpReq)
 
 				// copy response out
diff --git a/scripts/test.py b/scripts/test.py
index dc10933e3..aab3776f4 100755
--- a/scripts/test.py
+++ b/scripts/test.py
@@ -307,6 +307,25 @@ def flask_test():
     if r.text != "hi\n":
         raise ValueError(f"r.text should be 'hi\n', not {repr(r.text)}")
 
+@test
+def wsgi_post_echo_test():
+    """Test that POST body is properly forwarded to WSGI/Flask apps"""
+    url = 'http://localhost:5000/run/wsgi-post-echo'
+
+    # Test with plain text body
+    test_body = "hello world"
+    r = requests.post(url, data=test_body, headers={"Content-Type": "text/plain"})
+    check_status_code(r)
+    if r.text != test_body:
+        raise ValueError(f"expected '{test_body}', but got '{r.text}'")
+
+    # Test with JSON body
+    test_json = '{"key": "value"}'
+    r = requests.post(url, data=test_json, headers={"Content-Type": "application/json"})
+    check_status_code(r)
+    if r.text != test_json:
+        raise ValueError(f"expected '{test_json}', but got '{r.text}'")
+
 @test
 def flask_entry_test():
     """Test OL_ENTRY_FILE feature with a Flask app using app.py instead of f.py"""
@@ -430,6 +449,7 @@ def run_tests():
 
     # make sure we can use WSGI apps based on frameworks like Flask
     flask_test()
+    wsgi_post_echo_test()
     flask_entry_test()
     test_http_method_restrictions()
 

From daa770d8f72971c0b464b14c3323d386dc1a1e77 Mon Sep 17 00:00:00 2001
From: Tyler Caraza-Harter <tylerharter@users.noreply.github.com>
Date: Fri, 2 Jan 2026 12:41:45 -0600
Subject: [PATCH 09/55] after OOM or other errors, start fresh sandbox (#385)

---
 go/worker/lambda/lambdaInstance.go | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/go/worker/lambda/lambdaInstance.go b/go/worker/lambda/lambdaInstance.go
index af0a8a21b..44824b1e1 100644
--- a/go/worker/lambda/lambdaInstance.go
+++ b/go/worker/lambda/lambdaInstance.go
@@ -157,6 +157,8 @@ func (linst *LambdaInstance) Task() {
 				// copy response out
 				if err != nil {
 					linst.TrySendError(req, http.StatusBadGateway, "RoundTrip failed: "+err.Error()+"\n", sb)
+					sb.Destroy("Sandbox's HTTP client returned an error")
+					sb = nil
 				} else {
 					// copy headers
 					// (adapted from copyHeaders: https://go.dev/src/net/http/httputil/reverseproxy.go)
@@ -193,17 +195,18 @@ func (linst *LambdaInstance) Task() {
 			// check whether we should shutdown (non-blocking)
 			select {
 			case killed := <-linst.killChan:
-				rtLog := sb.GetRuntimeLog()
-				sb.Destroy("Lambda instance kill signal received")
-
-				slog.Info("Stopped sandbox")
-
-				if common.Conf.Log_output {
-					if rtLog != "" {
-						slog.Info("Runtime output is:")
-
-						for _, line := range strings.Split(rtLog, "\n") {
-							slog.Info(fmt.Sprintf("   %s", line))
+				if sb != nil {
+					rtLog := sb.GetRuntimeLog()
+					sb.Destroy("Lambda instance kill signal received")
+					slog.Info("Stopped sandbox")
+
+					if common.Conf.Log_output {
+						if rtLog != "" {
+							slog.Info("Runtime output is:")
+
+							for _, line := range strings.Split(rtLog, "\n") {
+								slog.Info(fmt.Sprintf("   %s", line))
+							}
 						}
 					}
 				}

From 503cd3b8c976f72ee1e9fd7798ad472ae222b255 Mon Sep 17 00:00:00 2001
From: Tyler Caraza-Harter <tylerharter@users.noreply.github.com>
Date: Sun, 11 Jan 2026 13:30:54 -0600
Subject: [PATCH 10/55] better pip-compile and requirements.txt override for
 admin install (#393)

---
 docs/worker/pypi-packages.md          |  8 ++-
 examples/pip-compile/f.py             | 86 +++++++++++++++++++++------
 examples/pip-compile/requirements.in  |  1 -
 examples/pip-compile/requirements.txt | 10 +---
 go/admin/commands.go                  | 42 +++++++++----
 go/worker/lambda/lambdaInstance.go    |  5 ++
 6 files changed, 109 insertions(+), 43 deletions(-)

diff --git a/docs/worker/pypi-packages.md b/docs/worker/pypi-packages.md
index b6b32dc17..dc8fa361d 100644
--- a/docs/worker/pypi-packages.md
+++ b/docs/worker/pypi-packages.md
@@ -54,10 +54,14 @@ If you don't have pip-tools installed locally, you can use the
 ol admin install ./examples/pip-compile
 ```
 
-Then compile your requirements:
+Then compile your requirements (from a file or URL):
 
 ```bash
-curl -X POST -H "Content-Type: text/plain" --data-binary @requirements.in http://localhost:5000/run/pip-compile > requirements.txt
+# From a local file
+curl -X POST --data-binary @requirements.in http://localhost:5000/run/pip-compile/text > requirements.txt
+
+# From a URL
+curl -X POST -d 'https://example.com/requirements.in' http://localhost:5000/run/pip-compile/url > requirements.txt
 ```
 
 ## Try It
diff --git a/examples/pip-compile/f.py b/examples/pip-compile/f.py
index a3222bb3e..fa0ecd3c8 100644
--- a/examples/pip-compile/f.py
+++ b/examples/pip-compile/f.py
@@ -1,11 +1,14 @@
 import os
+import urllib.request
+import urllib.error
 
 SCRATCH_DIR = "/host/tmp"
 
-# Set cache directories to writable scratch dir BEFORE importing pip-tools
-os.environ["PIP_TOOLS_CACHE_DIR"] = SCRATCH_DIR
-os.environ["XDG_CACHE_HOME"] = SCRATCH_DIR
+# Set cache/temp directories to writable location BEFORE importing pip-tools
 os.environ["HOME"] = SCRATCH_DIR
+os.environ["TMPDIR"] = SCRATCH_DIR
+os.environ["XDG_CACHE_HOME"] = SCRATCH_DIR
+os.environ["PIP_CACHE_DIR"] = SCRATCH_DIR
 
 from flask import Flask, request, Response
 from piptools.scripts.compile import cli
@@ -14,16 +17,8 @@
 app = Flask(__name__)
 
 
-@app.route("/", methods=["POST"])
-def compile_requirements():
-    """
-    Compile requirements.in to requirements.txt using pip-compile.
-
-    Input (POST body): requirements.in content
-    Output: compiled requirements.txt content (plain text)
-    """
-    requirements_in = request.get_data(as_text=True)
-
+def do_compile(requirements_in, quiet=True):
+    """Compile requirements.in content to requirements.txt."""
     if not requirements_in:
         return Response("No requirements provided", status=400, mimetype="text/plain")
 
@@ -33,12 +28,16 @@ def compile_requirements():
     with open(in_path, "w") as f:
         f.write(requirements_in)
 
-    runner = CliRunner()
-    result = runner.invoke(cli, [
-        "--index-url", "https://pypi.org/simple/",
+    args = [
         "--output-file", out_path,
-        in_path
-    ])
+        "--pip-args", "--only-binary=:all:",
+    ]
+    if quiet:
+        args.extend(["--no-header", "--no-annotate"])
+    args.append(in_path)
+
+    runner = CliRunner()
+    result = runner.invoke(cli, args)
 
     if result.exit_code != 0:
         return Response(
@@ -48,4 +47,53 @@ def compile_requirements():
         )
 
     with open(out_path, "r") as f:
-        return Response(f.read(), mimetype="text/plain")
+        lines = [l for l in f if not l.startswith("--")]
+        return Response("".join(lines), mimetype="text/plain")
+
+
+@app.route("/", methods=["GET"])
+def docs():
+    """Return documentation with curl examples."""
+    return Response("""pip-compile Lambda Service
+==========================
+
+Compiles requirements.in files to pinned requirements.txt using pip-compile.
+
+Endpoints
+---------
+
+POST /text
+    Pass requirements.in content directly in the request body.
+
+    curl -X POST -d $'flask>=2.0\\nrequests' http://localhost:5000/run/pip-compile/text
+
+POST /url
+    Pass a URL to fetch requirements.in from.
+
+    curl -X POST -d 'https://example.com/requirements.in' http://localhost:5000/run/pip-compile/url
+""", mimetype="text/plain")
+
+
+@app.route("/text", methods=["POST"])
+def compile_from_text():
+    """Compile requirements.in from POST body text."""
+    quiet = request.args.get("quiet", "1") == "1"
+    return do_compile(request.get_data(as_text=True), quiet=quiet)
+
+
+@app.route("/url", methods=["POST"])
+def compile_from_url():
+    """Fetch requirements.in from a URL and compile it."""
+    url = request.get_data(as_text=True).strip()
+    quiet = request.args.get("quiet", "1") == "1"
+
+    if not url:
+        return Response("No URL provided", status=400, mimetype="text/plain")
+
+    try:
+        with urllib.request.urlopen(url, timeout=30) as response:
+            requirements_in = response.read().decode('utf-8')
+    except urllib.error.URLError as e:
+        return Response(f"Failed to fetch URL: {e}", status=400, mimetype="text/plain")
+
+    return do_compile(requirements_in, quiet=quiet)
diff --git a/examples/pip-compile/requirements.in b/examples/pip-compile/requirements.in
index f5f1e8bcf..a7c1f43db 100644
--- a/examples/pip-compile/requirements.in
+++ b/examples/pip-compile/requirements.in
@@ -1,3 +1,2 @@
 flask
 pip-tools==5.5.0
-tomli
diff --git a/examples/pip-compile/requirements.txt b/examples/pip-compile/requirements.txt
index 8a012d536..7b76e82d5 100644
--- a/examples/pip-compile/requirements.txt
+++ b/examples/pip-compile/requirements.txt
@@ -1,8 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.12
-# by the following command:
-#
-#    pip-compile requirements.in
+# This file is autogenerated by pip-compile
 #
 blinker==1.9.0
     # via flask
@@ -23,10 +20,5 @@ markupsafe==3.0.3
     #   werkzeug
 pip-tools==5.5.0
     # via -r requirements.in
-tomli==2.3.0
-    # via -r requirements.in
 werkzeug==3.1.4
     # via flask
-
-# The following packages are considered to be unsafe in a requirements file:
-# pip
diff --git a/go/admin/commands.go b/go/admin/commands.go
index 9a4911ebf..33ac9630a 100644
--- a/go/admin/commands.go
+++ b/go/admin/commands.go
@@ -42,7 +42,7 @@ func checkStatus(port string) error {
 	return nil
 }
 
-const installUsage = "ol admin install [-c <config>] [-n <name>] [boss | -p <worker_path>] <directory_or_git_url>"
+const installUsage = "ol admin install [-c <config>] [-r <requirements>] [-n <name>] [boss | -p <worker_path>] <directory_or_git_url>"
 
 // isGitURL returns true if the path looks like a git repository URL
 func isGitURL(path string) bool {
@@ -153,17 +153,26 @@ func adminInstall(ctx *cli.Context) error {
 
 	// Build overrides map
 	overrides := make(map[string]string)
-	configPath := ctx.String("config")
-	if configPath != "" {
-		if _, err := os.Stat(configPath); os.IsNotExist(err) {
-			return fmt.Errorf("config file %s does not exist", configPath)
+	addOverride := func(flagName, targetFile string) error {
+		path := ctx.String(flagName)
+		if path == "" {
+			return nil
+		}
+		if _, err := os.Stat(path); os.IsNotExist(err) {
+			return fmt.Errorf("%s file %s does not exist", flagName, path)
 		}
-		// Warn if ol.yaml already exists in the source
-		existingConfig := filepath.Join(funcDir, "ol.yaml")
-		if _, err := os.Stat(existingConfig); err == nil {
-			fmt.Printf("Warning: overriding existing ol.yaml in source with %s\n", configPath)
+		if _, err := os.Stat(filepath.Join(funcDir, targetFile)); err == nil {
+			fmt.Printf("Warning: overriding existing %s in source with %s\n", targetFile, path)
 		}
-		overrides["ol.yaml"] = configPath
+		overrides[targetFile] = path
+		return nil
+	}
+
+	if err := addOverride("config", "ol.yaml"); err != nil {
+		return err
+	}
+	if err := addOverride("requirements", "requirements.txt"); err != nil {
+		return err
 	}
 
 	tarData, err := createTarGz(funcDir, overrides)
@@ -188,9 +197,13 @@ func createTarGz(funcDir string, overrides map[string]string) ([]byte, error) {
 	tarWriter := tar.NewWriter(gzWriter)
 
 	// Determine the Python entry file (default to f.py, or use OL_ENTRY_FILE from ol.yaml)
-	// TODO: support OL_ENTRY_FILE for native runtime
+	// Check override config first, then fall back to source config
 	pythonEntryFile := "f.py"
-	if lambdaConfig, err := common.LoadLambdaConfig(funcDir); err == nil {
+	configDir := funcDir
+	if configOverride, ok := overrides["ol.yaml"]; ok {
+		configDir = filepath.Dir(configOverride)
+	}
+	if lambdaConfig, err := common.LoadLambdaConfig(configDir); err == nil {
 		if lambdaConfig.Environment != nil {
 			if entryFile, ok := lambdaConfig.Environment["OL_ENTRY_FILE"]; ok {
 				pythonEntryFile = entryFile
@@ -346,6 +359,11 @@ func AdminCommands() []*cli.Command {
 					Aliases: []string{"c"},
 					Usage:   "Path to ol.yaml config file to include (overrides existing ol.yaml in source)",
 				},
+				&cli.StringFlag{
+					Name:    "requirements",
+					Aliases: []string{"r"},
+					Usage:   "Path to requirements.txt file to include (overrides existing requirements.txt in source)",
+				},
 				&cli.StringFlag{
 					Name:    "name",
 					Aliases: []string{"n"},
diff --git a/go/worker/lambda/lambdaInstance.go b/go/worker/lambda/lambdaInstance.go
index 44824b1e1..36eb0cc3d 100644
--- a/go/worker/lambda/lambdaInstance.go
+++ b/go/worker/lambda/lambdaInstance.go
@@ -222,6 +222,11 @@ func (linst *LambdaInstance) Task() {
 			default:
 				req = nil
 			}
+
+			// if sandbox was destroyed, break out so outer loop can create a new one
+			if sb == nil {
+				break
+			}
 		}
 
 		if sb != nil {

From 16ca3ee3d88271c79b9b1feb479b10ceb60b32c4 Mon Sep 17 00:00:00 2001
From: Tyler Caraza-Harter <tylerharter@users.noreply.github.com>
Date: Sun, 11 Jan 2026 20:51:42 -0600
Subject: [PATCH 11/55] support WSGI entry points other than "app" (#396)

---
 examples/wsgi-entry-test/main.py           | 16 ++++++++++++
 examples/wsgi-entry-test/ol.yaml           |  8 ++++++
 examples/wsgi-entry-test/requirements.in   |  2 ++
 examples/wsgi-entry-test/requirements.txt  | 24 ++++++++++++++++++
 min-image/runtimes/python/server.py        | 23 ++++++++++++++---
 min-image/runtimes/python/server_legacy.py | 23 ++++++++++++++---
 scripts/test.py                            | 29 ++++++++++++++++++++++
 7 files changed, 117 insertions(+), 8 deletions(-)
 create mode 100644 examples/wsgi-entry-test/main.py
 create mode 100644 examples/wsgi-entry-test/ol.yaml
 create mode 100644 examples/wsgi-entry-test/requirements.in
 create mode 100644 examples/wsgi-entry-test/requirements.txt

diff --git a/examples/wsgi-entry-test/main.py b/examples/wsgi-entry-test/main.py
new file mode 100644
index 000000000..e2609ff27
--- /dev/null
+++ b/examples/wsgi-entry-test/main.py
@@ -0,0 +1,16 @@
+from flask import Flask, Response
+
+# Intentionally NOT named "app" to test OL_WSGI_ENTRY
+my_wsgi_app = Flask("wsgi-entry-test")
+
+@my_wsgi_app.route("/")
+def index():
+    return Response("Hello from my_wsgi_app!\n", status=200)
+
+@my_wsgi_app.route("/info")
+def info():
+    return {
+        "entry_file": "main.py",
+        "entry_point": "my_wsgi_app",
+        "message": "This tests OL_WSGI_ENTRY with a non-standard name"
+    }
diff --git a/examples/wsgi-entry-test/ol.yaml b/examples/wsgi-entry-test/ol.yaml
new file mode 100644
index 000000000..f6172d6c0
--- /dev/null
+++ b/examples/wsgi-entry-test/ol.yaml
@@ -0,0 +1,8 @@
+triggers:
+  http:
+    - method: GET
+    - method: POST
+
+environment:
+  OL_ENTRY_FILE: "main.py"
+  OL_WSGI_ENTRY: "my_wsgi_app"
diff --git a/examples/wsgi-entry-test/requirements.in b/examples/wsgi-entry-test/requirements.in
new file mode 100644
index 000000000..b44f14a28
--- /dev/null
+++ b/examples/wsgi-entry-test/requirements.in
@@ -0,0 +1,2 @@
+flask==2.3.2
+werkzeug==3.0.3
diff --git a/examples/wsgi-entry-test/requirements.txt b/examples/wsgi-entry-test/requirements.txt
new file mode 100644
index 000000000..96159fc91
--- /dev/null
+++ b/examples/wsgi-entry-test/requirements.txt
@@ -0,0 +1,24 @@
+#
+# This file is autogenerated by pip-compile with Python 3.13
+# by the following command:
+#
+#    pip-compile requirements.in
+#
+blinker==1.6.2
+    # via flask
+click==8.1.7
+    # via flask
+flask==2.3.2
+    # via -r requirements.in
+itsdangerous==2.1.2
+    # via flask
+jinja2==3.1.4
+    # via flask
+markupsafe==2.1.3
+    # via
+    #   jinja2
+    #   werkzeug
+werkzeug==3.0.3
+    # via
+    #   -r requirements.in
+    #   flask
diff --git a/min-image/runtimes/python/server.py b/min-image/runtimes/python/server.py
index ca3eb5ee4..657fa12ec 100644
--- a/min-image/runtimes/python/server.py
+++ b/min-image/runtimes/python/server.py
@@ -38,6 +38,21 @@ def web_server():
     module_name = entry_file[:-3]
     handler_module = importlib.import_module(module_name)
 
+    # Determine entry point
+    # TODO: add OL_FUNCTION_ENTRY for explicit function entry points
+    wsgi_entry = os.environ.get('OL_WSGI_ENTRY')
+    if wsgi_entry:
+        entry_point = getattr(handler_module, wsgi_entry)
+        is_wsgi = True
+    elif hasattr(handler_module, 'f'):
+        entry_point = handler_module.f
+        is_wsgi = False
+    elif hasattr(handler_module, 'app'):
+        entry_point = handler_module.app
+        is_wsgi = True
+    else:
+        raise ValueError("No entry point found. Set OL_WSGI_ENTRY or define 'f' or 'app' in your module.")
+
     class SockFileHandler(tornado.web.RequestHandler):
         # TODO: we should consider how are the different requests used in the context of different applications and functions
         # and consider what does the validations should look like for example, should we allow POST requests with no payload etc.
@@ -51,7 +66,7 @@ def handle_request(self):
                     self.write(f'bad request data: "{data}"')
                     return
 
-                result = handler_module.f(event) if event is not None else handler_module.f({}) 
+                result = entry_point(event) if event is not None else entry_point({}) 
                 self.write(json.dumps(result))  # Return the result as JSON
             except Exception:
                 self.set_status(500)  # Internal server error for unhandled exceptions
@@ -78,7 +93,7 @@ def options(self):
             self.handle_request()
     
 
-    if hasattr(handler_module, "app"):
+    if is_wsgi:
         def path_wrapper(environ, start_response):
             path = environ.get("PATH_INFO", "")
             # split path to get individual components
@@ -92,8 +107,8 @@ def path_wrapper(environ, start_response):
             app_name = parts[2]
             environ["SCRIPT_NAME"] = '/run/' + app_name
 
-            return handler_module.app(environ, start_response)
-        
+            return entry_point(environ, start_response)
+
         # use WSGI entry
         # call wrapper to strip /run/<func-name> from path
         app = tornado.wsgi.WSGIContainer(path_wrapper)
diff --git a/min-image/runtimes/python/server_legacy.py b/min-image/runtimes/python/server_legacy.py
index 2c5312ed4..b5d9b9380 100644
--- a/min-image/runtimes/python/server_legacy.py
+++ b/min-image/runtimes/python/server_legacy.py
@@ -48,7 +48,7 @@
 
 # run after forking into sandbox
 def init():
-    global initialized, handler_module
+    global initialized, handler_module, entry_point, is_wsgi
     if initialized:
         return
 
@@ -58,6 +58,21 @@ def init():
     module_name = entry_file[:-3]
     handler_module = importlib.import_module(module_name)
 
+    # Determine entry point
+    # TODO: add OL_FUNCTION_ENTRY for explicit function entry points
+    wsgi_entry = os.environ.get('OL_WSGI_ENTRY')
+    if wsgi_entry:
+        entry_point = getattr(handler_module, wsgi_entry)
+        is_wsgi = True
+    elif hasattr(handler_module, 'f'):
+        entry_point = handler_module.f
+        is_wsgi = False
+    elif hasattr(handler_module, 'app'):
+        entry_point = handler_module.app
+        is_wsgi = True
+    else:
+        raise ValueError("No entry point found. Set OL_WSGI_ENTRY or define 'f' or 'app' in your module.")
+
     initialized = True
 
 class SockFileHandler(tornado.web.RequestHandler):
@@ -71,7 +86,7 @@ def handle_request(self):
                 self.write(f'bad request data: "{data}"')
                 return
 
-            result = handler_module.f(event) if event is not None else handler_module.f({}) 
+            result = entry_point(event) if event is not None else entry_point({}) 
             self.write(json.dumps(result))  # Return the result as JSON
         except Exception:
             self.set_status(500)  # Internal server error for unhandled exceptions
@@ -101,7 +116,7 @@ def options(self):
 # listen on sock file with Tornado
 def lambda_server():
     init()
-    if hasattr(handler_module, "app"):
+    if is_wsgi:
         def path_wrapper(environ, start_response):
             path = environ.get("PATH_INFO", "")
             # split path to get individual components
@@ -115,7 +130,7 @@ def path_wrapper(environ, start_response):
             app_name = parts[2]
             environ["SCRIPT_NAME"] = '/run/' + app_name
 
-            return handler_module.app(environ, start_response)
+            return entry_point(environ, start_response)
 
         # use WSGI entry
         # call wrapper to strip /run/<func-name> from path
diff --git a/scripts/test.py b/scripts/test.py
index aab3776f4..08f489732 100755
--- a/scripts/test.py
+++ b/scripts/test.py
@@ -352,6 +352,34 @@ def flask_entry_test():
     if data.get("entry_file") != "app.py":
         raise ValueError(f"expected entry_file='app.py', got {data}")
 
+@test
+def wsgi_entry_test():
+    """Test OL_WSGI_ENTRY feature with a WSGI entry point not named 'app'"""
+    # Test the index route
+    url = 'http://localhost:5000/run/wsgi-entry-test'
+    print("URL", url)
+    r = requests.get(url)
+    print("RESPONSE", r)
+
+    if r.status_code != 200:
+        raise ValueError(f"expected status code 200, but got {r.status_code}")
+    if r.text != "Hello from my_wsgi_app!\n":
+        raise ValueError(f"r.text should be 'Hello from my_wsgi_app!\\n', not {repr(r.text)}")
+
+    # Test the info route
+    url_info = 'http://localhost:5000/run/wsgi-entry-test/info'
+    print("URL", url_info)
+    r = requests.get(url_info)
+    print("RESPONSE", r)
+
+    if r.status_code != 200:
+        raise ValueError(f"expected status code 200, but got {r.status_code}")
+    data = r.json()
+    if data.get("entry_point") != "my_wsgi_app":
+        raise ValueError(f"expected entry_point='my_wsgi_app', got {data}")
+    if data.get("entry_file") != "main.py":
+        raise ValueError(f"expected entry_file='main.py', got {data}")
+
 @test
 def test_http_method_restrictions():
     url = 'http://localhost:5000/run/lambda-config-test'
@@ -451,6 +479,7 @@ def run_tests():
     flask_test()
     wsgi_post_echo_test()
     flask_entry_test()
+    wsgi_entry_test()
     test_http_method_restrictions()
 
     # test environment variables from ol.yaml

From f076c726ffa6209bc6b6204cae0a91a385c2c9a3 Mon Sep 17 00:00:00 2001
From: cblin321 <126987173+cblin321@users.noreply.github.com>
Date: Mon, 12 Jan 2026 09:51:17 -0600
Subject: [PATCH 12/55] update setFreezeState to watch cgroup events file
 (#386)

* watch event files for setFreeze

* fix implementation

* Update ci.yml

* remove goroutine

* restart on EINTR

* remove read

* linter errs

* revert ci.yml

* remove eintr err handling

* improve err msgs; improve err handling

* update setFreezeState

* update warning; update err msgs

* updates
---
 go/worker/sandbox/cgroups/cgroup.go | 53 +++++++++++++++++++++++------
 1 file changed, 42 insertions(+), 11 deletions(-)

diff --git a/go/worker/sandbox/cgroups/cgroup.go b/go/worker/sandbox/cgroups/cgroup.go
index f5d2fddf6..03303e3ef 100644
--- a/go/worker/sandbox/cgroups/cgroup.go
+++ b/go/worker/sandbox/cgroups/cgroup.go
@@ -2,6 +2,7 @@ package cgroups
 
 import (
 	"bufio"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"log/slog"
@@ -12,6 +13,7 @@ import (
 	"time"
 
 	"github.com/open-lambda/open-lambda/go/common"
+	"golang.org/x/sys/unix"
 )
 
 type CgroupImpl struct {
@@ -189,26 +191,55 @@ func (cg *CgroupImpl) AddPid(pid string) error {
 }
 
 func (cg *CgroupImpl) setFreezeState(state int64) error {
-	cg.WriteInt("cgroup.freeze", state)
+	timeout := 20 * time.Second
+
+	resourcePath := cg.ResourcePath("cgroup.events")
 
-	timeout := 5 * time.Second
+	eventFile, err := os.Open(resourcePath)
+	if err != nil {
+		return fmt.Errorf("failed to open %s: %w", resourcePath, err)
+	}
+	defer eventFile.Close()
+
+	// poll(2): POLLPRI indicates "cgroup.events file modified"
+	pollFDs := []unix.PollFd{
+		{
+			Fd:     int32(eventFile.Fd()),
+			Events: unix.POLLPRI,
+		},
+	}
 
 	start := time.Now()
-	for {
-		freezerState, err := cg.TryReadInt("cgroup.freeze")
-		if err != nil {
-			return fmt.Errorf("failed to check self_freezing state :: %v", err)
+
+	defer func(start time.Time) {
+		elapsed := time.Since(start)
+		if elapsed >= 250*time.Millisecond {
+			cg.printf("WARNING!  setFreezeState to state %v took %v to complete", state, elapsed)
 		}
+	}(start)
 
-		if freezerState == state {
-			return nil
+	cg.WriteInt("cgroup.freeze", state)
+
+	for {
+		elapsed := time.Since(start)
+
+		remaining := timeout - elapsed
+		if remaining < 0 {
+			return fmt.Errorf("cgroup freeze timeout after %v (expected state %v)", timeout, state)
 		}
 
-		if time.Since(start) > timeout {
-			return fmt.Errorf("cgroup stuck on %v after %v (should be %v)", freezerState, timeout, state)
+		_, err := unix.Poll(pollFDs, int(remaining.Milliseconds()))
+		if err != nil && !errors.Is(err, unix.EINTR) {
+			return fmt.Errorf("poll syscall failed on %s: %w", resourcePath, err)
 		}
 
-		time.Sleep(1 * time.Millisecond)
+		freezerState, err := cg.TryReadIntKV("cgroup.events", "frozen")
+		if err != nil {
+			return fmt.Errorf("failed to check self_freezing state :: %w", err)
+		}
+		if freezerState == state {
+			return nil
+		}
 	}
 }
 

From 33cf5356842d48f4768e0961ad02a1997ee789e1 Mon Sep 17 00:00:00 2001
From: Tyler Caraza-Harter <tylerharter@users.noreply.github.com>
Date: Tue, 13 Jan 2026 10:40:27 -0600
Subject: [PATCH 13/55] support ASGI applications (#399)

* support ASGI and document ag forecasting as an example

* share common code between server.py and server_legacy.py

* use buffering to fix early pipe closure bug

* indicate no keep alive (with Close header) on Python side because we do not support it

* timeout for stress_one_lambda_task

* cleanup ol.sock properly at destroy time, not lazily when reused
---
 docs/worker/README.md                      |   1 +
 docs/worker/apps.md                        |  64 ++++++
 examples/fastapi-test/f.py                 |   7 +
 examples/fastapi-test/requirements.in      |   1 +
 examples/fastapi-test/requirements.txt     |  12 +
 go/admin/commands.go                       |  14 +-
 go/worker/sandbox/sock.go                  |   6 +
 min-image/Dockerfile                       |   3 +-
 min-image/runtimes/python/server.py        | 124 ++---------
 min-image/runtimes/python/server_common.py | 247 +++++++++++++++++++++
 min-image/runtimes/python/server_legacy.py | 136 ++----------
 python/src/open_lambda.py                  |   8 +-
 scripts/test.py                            |  23 +-
 13 files changed, 410 insertions(+), 236 deletions(-)
 create mode 100644 docs/worker/apps.md
 create mode 100644 examples/fastapi-test/f.py
 create mode 100644 examples/fastapi-test/requirements.in
 create mode 100644 examples/fastapi-test/requirements.txt
 create mode 100644 min-image/runtimes/python/server_common.py

diff --git a/docs/worker/README.md b/docs/worker/README.md
index 4d8ac7f43..0ec6cc657 100644
--- a/docs/worker/README.md
+++ b/docs/worker/README.md
@@ -10,6 +10,7 @@ See how to build the code, deploy a worker, and write/run lambdas [here](getting
 * [dependency management](pypi-packages.md)
 * [manual cleanup](manual-cleanup.md)
 * [lambda configuration](lambda-config.md)
+* [deploying example applications](apps.md)
 * [setup development environment](../boss/setup-dev-env.md)
 * registry config (TODO)
 * Zygote tree (TODO)
diff --git a/docs/worker/apps.md b/docs/worker/apps.md
new file mode 100644
index 000000000..b651da849
--- /dev/null
+++ b/docs/worker/apps.md
@@ -0,0 +1,64 @@
+# Deploying Applications
+
+## Agricultural Forecasting API (FastAPI)
+
+[ag_forecasting_api](https://github.com/UW-Madison-DSI/ag_forecasting_api) is a FastAPI application that provides crop disease forecasting models for corn and soybean in Wisconsin, developed by University of Wisconsin-Madison plant pathology experts.
+
+Initialize a worker with the min image:
+
+```bash
+./ol worker init -i ol-min
+```
+
+Edit `myworker/config.json` to increase memory limit (512MB needed for this app):
+
+```json
+"limits": {
+    "mem_mb": 512,
+    ...
+}
+```
+
+Start the worker:
+
+```bash
+./ol worker up -d
+```
+
+Create `ol.yaml` to configure the app for OpenLambda:
+
+```yaml
+triggers:
+  http:
+    - method: "*"
+environment:
+  OL_ENTRY_FILE: app.py
+  MEASUREMENTS_CACHE_DIR: /host/tmp/cache
+```
+
+Install pip-compile and pin requirements.txt to versions suitable for OpenLambda:
+
+```bash
+./ol admin install examples/pip-compile
+curl -X POST -d 'https://raw.githubusercontent.com/tylerharter/ag_forecasting_api/main/requirements.txt' http://localhost:5000/run/pip-compile/url > requirements.txt
+```
+
+Install and test:
+
+```bash
+./ol admin install -c ol.yaml -r requirements.txt https://github.com/tylerharter/ag_forecasting_api.git
+
+# simple test
+curl http://localhost:5000/run/ag_forecasting_api/
+```
+
+NOTE: the full app doesn't work yet (we need to make sure code and writable directories are as expected).  This fails:
+
+```bash
+# get a forecast for the ALTN station
+curl "http://localhost:5000/run/ag_forecasting_api/ag_models_wrappers/wisconet?forecasting_date=2024-07-01&risk_days=1&station_id=ALTN"
+```
+
+TODO: update ag_forecasting_api URLs from tylerharter fork to UW-Madison-DSI once env option is merged upstream.
+
+## TODO: add more example apps
diff --git a/examples/fastapi-test/f.py b/examples/fastapi-test/f.py
new file mode 100644
index 000000000..44f9163e7
--- /dev/null
+++ b/examples/fastapi-test/f.py
@@ -0,0 +1,7 @@
+from fastapi import FastAPI
+
+app = FastAPI()
+
+@app.get("/")
+def hello():
+    return {"message": "hello world"}
diff --git a/examples/fastapi-test/requirements.in b/examples/fastapi-test/requirements.in
new file mode 100644
index 000000000..6b0b9396e
--- /dev/null
+++ b/examples/fastapi-test/requirements.in
@@ -0,0 +1 @@
+fastapi
diff --git a/examples/fastapi-test/requirements.txt b/examples/fastapi-test/requirements.txt
new file mode 100644
index 000000000..04d1cbda0
--- /dev/null
+++ b/examples/fastapi-test/requirements.txt
@@ -0,0 +1,12 @@
+
+annotated-doc==0.0.4
+annotated-types==0.7.0
+anyio==4.12.1
+exceptiongroup==1.3.1
+fastapi==0.128.0
+idna==3.11
+pydantic-core==2.41.5
+pydantic==2.12.5
+starlette==0.50.0
+typing-extensions==4.15.0
+typing-inspection==0.4.2
diff --git a/go/admin/commands.go b/go/admin/commands.go
index 33ac9630a..e6b6c72f7 100644
--- a/go/admin/commands.go
+++ b/go/admin/commands.go
@@ -203,11 +203,13 @@ func createTarGz(funcDir string, overrides map[string]string) ([]byte, error) {
 	if configOverride, ok := overrides["ol.yaml"]; ok {
 		configDir = filepath.Dir(configOverride)
 	}
-	if lambdaConfig, err := common.LoadLambdaConfig(configDir); err == nil {
-		if lambdaConfig.Environment != nil {
-			if entryFile, ok := lambdaConfig.Environment["OL_ENTRY_FILE"]; ok {
-				pythonEntryFile = entryFile
-			}
+	lambdaConfig, err := common.LoadLambdaConfig(configDir)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse config in %s: %v", configDir, err)
+	}
+	if lambdaConfig.Environment != nil {
+		if entryFile, ok := lambdaConfig.Environment["OL_ENTRY_FILE"]; ok {
+			pythonEntryFile = entryFile
 		}
 	}
 
@@ -216,7 +218,7 @@ func createTarGz(funcDir string, overrides map[string]string) ([]byte, error) {
 		return nil, fmt.Errorf("required file %s not found in %s", pythonEntryFile, funcDir)
 	}
 
-	err := filepath.Walk(funcDir, func(path string, info os.FileInfo, err error) error {
+	err = filepath.Walk(funcDir, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return fmt.Errorf("walk error: %v", err)
 		}
diff --git a/go/worker/sandbox/sock.go b/go/worker/sandbox/sock.go
index 12a153ceb..bb22e0241 100644
--- a/go/worker/sandbox/sock.go
+++ b/go/worker/sandbox/sock.go
@@ -317,6 +317,12 @@ func (container *SOCKContainer) decCgRefCount() {
 		}
 		t.T1()
 
+		// Clean up ol.sock from scratchDir (scratchDir itself may be reused, e.g., for package caching)
+		sockPath := filepath.Join(container.scratchDir, "ol.sock")
+		if err := os.Remove(sockPath); err != nil && !os.IsNotExist(err) {
+			container.printf("remove socket %s failed :: %v\n", sockPath, err)
+		}
+
 		if container.parent != nil {
 			container.parent.childExit(container)
 		}
diff --git a/min-image/Dockerfile b/min-image/Dockerfile
index a475453d3..7a80bdc76 100644
--- a/min-image/Dockerfile
+++ b/min-image/Dockerfile
@@ -5,7 +5,7 @@ RUN apt-get update && apt-get -y install \
     python3 python3-dev python3-pip python-is-python3 \
     build-essential libseccomp-dev
 RUN pip3 install --upgrade pip
-RUN pip3 install virtualenv requests tornado==6.1.0 python-dotenv
+RUN pip3 install virtualenv requests python-dotenv
 
 RUN mkdir /runtimes
 
@@ -16,6 +16,7 @@ RUN cd /tmp/py-runtime && python3 setup.py build_ext --inplace
 RUN mv /tmp/py-runtime/ol.*.so /runtimes/python/ol.so
 RUN mv /tmp/py-runtime/server.py /runtimes/python/server.py
 RUN mv /tmp/py-runtime/server_legacy.py /runtimes/python/server_legacy.py
+RUN mv /tmp/py-runtime/server_common.py /runtimes/python/server_common.py
 RUN rm -rf /tmp/py-runtime
 
 # for the Docker container engine
diff --git a/min-image/runtimes/python/server.py b/min-image/runtimes/python/server.py
index 657fa12ec..ad3072b8d 100644
--- a/min-image/runtimes/python/server.py
+++ b/min-image/runtimes/python/server.py
@@ -2,125 +2,26 @@
 
 ''' Python runtime for sock '''
 
-import os, sys, json, argparse, importlib, traceback, time, fcntl, array, socket, struct
+import os
+import sys
+import socket
+import struct
+import traceback
 
 sys.path.append("/usr/local/lib/python3.10/dist-packages")
-
-from dotenv import load_dotenv
-import tornado.ioloop
-import tornado.web
-import tornado.httpserver
-import tornado.wsgi
-import tornado.netutil
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
 
 import ol
+from server_common import web_server_on_sock
 
 file_sock_path = "/host/ol.sock"
 file_sock = None
 bootstrap_path = None
 
+
 def web_server():
-    print(f"server.py: start web server on fd: {file_sock.fileno()}")
-    sys.path.append('/handler')
-    
-    # Load environment variables from .env file if it exists
-    env_path = '/handler/.env'
-    if os.path.exists(env_path):
-        load_dotenv(env_path)
-        print(f"server.py: loaded environment variables from {env_path}")
-
-    # TODO: as a safeguard, we should add a mechanism so that the
-    # import doesn't happen until the cgroup move completes, so that a
-    # malicious child cannot eat up Zygote resources
-    entry_file = os.environ.get('OL_ENTRY_FILE', 'f.py')
-    if not entry_file.endswith('.py'):
-        raise ValueError(f"OL_ENTRY_FILE must end with .py, got: {entry_file}")
-    module_name = entry_file[:-3]
-    handler_module = importlib.import_module(module_name)
-
-    # Determine entry point
-    # TODO: add OL_FUNCTION_ENTRY for explicit function entry points
-    wsgi_entry = os.environ.get('OL_WSGI_ENTRY')
-    if wsgi_entry:
-        entry_point = getattr(handler_module, wsgi_entry)
-        is_wsgi = True
-    elif hasattr(handler_module, 'f'):
-        entry_point = handler_module.f
-        is_wsgi = False
-    elif hasattr(handler_module, 'app'):
-        entry_point = handler_module.app
-        is_wsgi = True
-    else:
-        raise ValueError("No entry point found. Set OL_WSGI_ENTRY or define 'f' or 'app' in your module.")
-
-    class SockFileHandler(tornado.web.RequestHandler):
-        # TODO: we should consider how are the different requests used in the context of different applications and functions
-        # and consider what does the validations should look like for example, should we allow POST requests with no payload etc.
-        def handle_request(self):
-            try:
-                data = self.request.body
-                try:
-                    event = json.loads(data) if data else None
-                except:
-                    self.set_status(400)  # Bad request if JSON parsing fails
-                    self.write(f'bad request data: "{data}"')
-                    return
-
-                result = entry_point(event) if event is not None else entry_point({}) 
-                self.write(json.dumps(result))  # Return the result as JSON
-            except Exception:
-                self.set_status(500)  # Internal server error for unhandled exceptions
-                self.write(traceback.format_exc())  # Include traceback in response
-        
-        
-        # Define methods for each HTTP method
-        def get(self):
-            self.handle_request()
-
-        def post(self):
-            self.handle_request()
-
-        def put(self):
-            self.handle_request()
-
-        def delete(self):
-            self.handle_request()
-
-        def patch(self):
-            self.handle_request()
-
-        def options(self):
-            self.handle_request()
-    
-
-    if is_wsgi:
-        def path_wrapper(environ, start_response):
-            path = environ.get("PATH_INFO", "")
-            # split path to get individual components
-            parts = path.split("/") # ["", "run", <app-name>]
-
-            # set new environment path
-            # `/run/<func-name>/a/b/c` -> `/a/b/c`
-            environ["PATH_INFO"] = '/' + '/'.join(parts[3:])
-
-            # set the root of the application
-            app_name = parts[2]
-            environ["SCRIPT_NAME"] = '/run/' + app_name
-
-            return entry_point(environ, start_response)
-
-        # use WSGI entry
-        # call wrapper to strip /run/<func-name> from path
-        app = tornado.wsgi.WSGIContainer(path_wrapper)
-    else:
-        # use function entry
-        app = tornado.web.Application([
-            (".*", SockFileHandler),
-        ])
-    server = tornado.httpserver.HTTPServer(app)
-    server.add_socket(file_sock)
-    tornado.ioloop.IOLoop.instance().start()
-    server.start()
+    """Wrapper that calls web_server_on_sock with the global file_sock."""
+    web_server_on_sock(file_sock, server_name="server.py")
 
 
 def fork_server():
@@ -189,7 +90,9 @@ def start_container():
     # child, which will actually use it.  This is so that the parent
     # can know that once the child exits, it is safe to start sending
     # messages to the sock file.
-    file_sock = tornado.netutil.bind_unix_socket(file_sock_path)
+    file_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    file_sock.bind(file_sock_path)
+    file_sock.listen(1)  # backlog=1: we handle one request at a time, no concurrency
 
     pid = os.fork()
     assert pid >= 0
@@ -211,6 +114,7 @@ def start_container():
             print("Exception: " + traceback.format_exc())
             print("Problematic Python Code:\n" + code)
 
+
 def main():
     '''
     caller is expected to do chroot, because we want to use the
diff --git a/min-image/runtimes/python/server_common.py b/min-image/runtimes/python/server_common.py
new file mode 100644
index 000000000..6e1a0c863
--- /dev/null
+++ b/min-image/runtimes/python/server_common.py
@@ -0,0 +1,247 @@
+# pylint: disable=line-too-long,invalid-name,broad-except
+
+'''
+Common code shared between server.py and server_legacy.py
+'''
+
+import os
+import sys
+import json
+import asyncio
+import http.client
+import importlib
+import traceback
+from enum import Enum
+from urllib.parse import urlparse
+
+from dotenv import load_dotenv
+
+
+class EntryType(Enum):
+    FUNC = "func"    # f(event) -> result
+    WSGI = "wsgi"    # app(environ, start_response) -> iterable
+    ASGI = "asgi"    # await app(scope, receive, send)
+
+
+class RequestParser:
+    '''
+    Parses an HTTP/1.x request from a connection.
+
+    Uses iso-8859-1 (Latin-1) for the request line per RFC 7230 Section 3.2.4:
+    "Historically, HTTP has allowed field content with text in the ISO-8859-1
+    charset." This encoding also provides a safe 1-to-1 byte-to-codepoint mapping,
+    ensuring any byte sequence decodes without error.
+    '''
+    def __init__(self, conn):
+        self.rfile = conn.makefile('rb', buffering=65536)
+
+        # Parse request line: "METHOD /path HTTP/1.1\r\n"
+        line = self.rfile.readline().decode('iso-8859-1').rstrip('\r\n')
+        self.command, self.path, self.request_version = line.split(None, 2)
+
+        # Parse headers using stdlib (documented API)
+        self.headers = http.client.parse_headers(self.rfile)
+
+        self.remaining = int(self.headers.get('Content-Length', 0))
+
+    def read(self, size=-1):
+        if size < 0:
+            size = self.remaining
+        size = min(size, self.remaining)
+        data = self.rfile.read(size)
+        self.remaining -= len(data)
+        return data
+
+
+def handle_func(conn, request, entry_point):
+    """Handle direct function calls: f(event) -> result"""
+    try:
+        body = request.read()
+        event = json.loads(body) if body else {}
+        result = entry_point(event)
+        response_body = json.dumps(result).encode()
+        status, status_text = 200, "OK"
+        content_type = "application/json"
+    except Exception:
+        response_body = traceback.format_exc().encode()
+        status, status_text = 500, "Internal Server Error"
+        content_type = "text/plain"
+
+    conn.sendall(f"HTTP/1.1 {status} {status_text}\r\n".encode())
+    conn.sendall(f"Content-Type: {content_type}\r\n".encode())
+    conn.sendall(f"Content-Length: {len(response_body)}\r\n".encode())
+    conn.sendall(b"Connection: close\r\n")
+    conn.sendall(b"\r\n")
+    conn.sendall(response_body)
+
+
+def handle_wsgi(conn, request, entry_point, app_name, path_info, query_string):
+    """Handle WSGI apps: app(environ, start_response) -> iterable"""
+    # Host header is required in HTTP/1.1 (RFC 2616 section 14.23)
+    # Note: we listen on a Unix socket, so port may not be meaningful
+    host = request.headers['Host']
+    if ':' in host:
+        server_name, server_port = host.split(':', 1)
+    else:
+        server_name, server_port = host, ""
+
+    # WSGI 1.0 (PEP 3333): https://peps.python.org/pep-3333/#environ-variables
+    environ = {
+        # CGI variables (required)
+        "REQUEST_METHOD": request.command,
+        "SCRIPT_NAME": "/run/" + app_name,
+        "PATH_INFO": path_info,
+        "QUERY_STRING": query_string,
+        "SERVER_NAME": server_name,
+        "SERVER_PORT": server_port,
+        "SERVER_PROTOCOL": request.request_version,
+        # wsgi.* variables (required)
+        "wsgi.version": (1, 0),       # PEP 3333 specifies tuple (1, 0)
+        "wsgi.url_scheme": "http",
+        "wsgi.input": request,  # request.read() handles Content-Length limiting
+        "wsgi.errors": sys.stderr,
+        "wsgi.multithread": False,
+        "wsgi.multiprocess": False,
+        "wsgi.run_once": False,
+    }
+    # HTTP headers -> environ per CGI spec (RFC 3875 section 4.1.18):
+    # - Convert to uppercase, replace "-" with "_"
+    # - Prefix with "HTTP_" except Content-Type and Content-Length
+    for key, value in request.headers.items():
+        key = key.upper().replace("-", "_")
+        if key in ("CONTENT_TYPE", "CONTENT_LENGTH"):
+            environ[key] = value
+        else:
+            environ["HTTP_" + key] = value
+
+    def start_response(status, response_headers, exc_info=None):
+        conn.sendall(f"HTTP/1.1 {status}\r\n".encode())
+        for name, value in response_headers:
+            conn.sendall(f"{name}: {value}\r\n".encode())
+        conn.sendall(b"Connection: close\r\n")
+        conn.sendall(b"\r\n")
+
+    result = entry_point(environ, start_response)
+    for chunk in result:
+        conn.sendall(chunk)
+    # PEP 3333: if iterable has close(), server must call it for cleanup
+    if hasattr(result, 'close'):
+        result.close()
+
+
+def handle_asgi(conn, request, entry_point, app_name, path_info, query_string):
+    """Handle ASGI apps: await app(scope, receive, send)"""
+    # TODO: stream body using more_body flag instead of reading all upfront
+    body = request.read()
+
+    # ASGI 3.0: https://asgi.readthedocs.io/en/latest/specs/www.html#http-connection-scope
+    scope = {
+        "type": "http",
+        "asgi": {"version": "3.0"},
+        "http_version": request.request_version.split("/")[1],  # "HTTP/1.1" -> "1.1"
+        "method": request.command,
+        "scheme": "http",
+        "path": path_info,
+        "query_string": query_string.encode(),
+        "root_path": "/run/" + app_name,
+        "headers": [(k.lower().encode(), v.encode()) for k, v in request.headers.items()],
+    }
+
+    response_started = False
+
+    async def receive():
+        return {"type": "http.request", "body": body, "more_body": False}
+
+    async def send(message):
+        nonlocal response_started
+        if message["type"] == "http.response.start":
+            response_started = True
+            status = message["status"]
+            conn.sendall(f"HTTP/1.1 {status} OK\r\n".encode())
+            for name, value in message.get("headers", []):
+                conn.sendall(name + b": " + value + b"\r\n")
+            conn.sendall(b"Connection: close\r\n")
+            conn.sendall(b"\r\n")
+        elif message["type"] == "http.response.body":
+            conn.sendall(message.get("body", b""))
+
+    try:
+        asyncio.run(entry_point(scope, receive, send))
+    except Exception:
+        if not response_started:
+            error = traceback.format_exc().encode()
+            conn.sendall(b"HTTP/1.1 500 Internal Server Error\r\n")
+            conn.sendall(b"Content-Type: text/plain\r\n")
+            conn.sendall(f"Content-Length: {len(error)}\r\n".encode())
+            conn.sendall(b"Connection: close\r\n")
+            conn.sendall(b"\r\n")
+            conn.sendall(error)
+
+
+def web_server_on_sock(file_sock, server_name="server"):
+    """
+    Main web server loop. Accepts connections and dispatches to appropriate handler.
+
+    Args:
+        file_sock: The socket to accept connections on
+        server_name: Name for logging (e.g., "server.py" or "server_legacy.py")
+    """
+    print(f"{server_name}: start web server on fd: {file_sock.fileno()}")
+    sys.path.append('/handler')
+
+    # Load environment variables from .env file if it exists
+    env_path = '/handler/.env'
+    if os.path.exists(env_path):
+        load_dotenv(env_path)
+        print(f"{server_name}: loaded environment variables from {env_path}")
+
+    # Import handler module
+    entry_file = os.environ.get('OL_ENTRY_FILE', 'f.py')
+    if not entry_file.endswith('.py'):
+        raise ValueError(f"OL_ENTRY_FILE must end with .py, got: {entry_file}")
+    module_name = entry_file[:-3]
+    handler_module = importlib.import_module(module_name)
+
+    # Determine entry point and type
+    wsgi_entry = os.environ.get('OL_WSGI_ENTRY')
+    asgi_entry = os.environ.get('OL_ASGI_ENTRY')
+    if wsgi_entry:
+        entry_point = getattr(handler_module, wsgi_entry)
+        entry_type = EntryType.WSGI
+    elif asgi_entry:
+        entry_point = getattr(handler_module, asgi_entry)
+        entry_type = EntryType.ASGI
+    elif hasattr(handler_module, 'f'):
+        entry_point = handler_module.f
+        entry_type = EntryType.FUNC
+    elif hasattr(handler_module, 'app'):
+        entry_point = handler_module.app
+        # Detect ASGI vs WSGI: ASGI apps have async __call__
+        if asyncio.iscoroutinefunction(getattr(entry_point, '__call__', None)):
+            entry_type = EntryType.ASGI
+        else:
+            entry_type = EntryType.WSGI
+    else:
+        raise ValueError("No entry point found. Define 'f' or 'app' in your module.")
+
+    print(f"{server_name}: entry_type={entry_type.value}")
+
+    while True:
+        conn, _ = file_sock.accept()
+        request = RequestParser(conn)
+
+        # Parse path: `/run/<app-name>/a/b/c` -> app_name, `/a/b/c`, query
+        parsed = urlparse(request.path)
+        parts = parsed.path.split("/")  # ["", "run", <app-name>, ...]
+        app_name = parts[2]
+        path_info = '/' + '/'.join(parts[3:])
+        query_string = parsed.query
+
+        if entry_type == EntryType.FUNC:
+            handle_func(conn, request, entry_point)
+        elif entry_type == EntryType.WSGI:
+            handle_wsgi(conn, request, entry_point, app_name, path_info, query_string)
+        elif entry_type == EntryType.ASGI:
+            handle_asgi(conn, request, entry_point, app_name, path_info, query_string)
+
+        conn.close()
diff --git a/min-image/runtimes/python/server_legacy.py b/min-image/runtimes/python/server_legacy.py
index b5d9b9380..e98330ed3 100644
--- a/min-image/runtimes/python/server_legacy.py
+++ b/min-image/runtimes/python/server_legacy.py
@@ -9,17 +9,14 @@
 
 import os
 import sys
-import json
 import argparse
 import importlib
-import traceback
+import socket
+
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
 
 from dotenv import load_dotenv
-import tornado.ioloop
-import tornado.web
-import tornado.httpserver
-import tornado.netutil
-import tornado.wsgi
+from server_common import web_server_on_sock
 
 HOST_DIR = '/host'
 PKGS_DIR = '/packages'
@@ -34,129 +31,41 @@
 sys.path.append(PKGS_DIR)
 sys.path.append(HANDLER_DIR)
 
-FS_PATH = os.path.join(HOST_DIR, 'fs.sock')
 SOCK_PATH = os.path.join(HOST_DIR, 'ol.sock')
+FS_PATH = os.path.join(HOST_DIR, 'fs.sock')
 STDOUT_PATH = os.path.join(HOST_DIR, 'stdout')
 STDERR_PATH = os.path.join(HOST_DIR, 'stderr')
 SERVER_PIPE_PATH = os.path.join(HOST_DIR, 'server_pipe')
 
 PROCESSES_DEFAULT = 10
-initialized = False
 
 parser = argparse.ArgumentParser(description='Listen and serve cache requests or lambda invocations.')
 parser.add_argument('--cache', action='store_true', default=False, help='Begin as a cache entry.')
 
-# run after forking into sandbox
-def init():
-    global initialized, handler_module, entry_point, is_wsgi
-    if initialized:
-        return
-
-    entry_file = os.environ.get('OL_ENTRY_FILE', 'f.py')
-    if not entry_file.endswith('.py'):
-        raise ValueError(f"OL_ENTRY_FILE must end with .py, got: {entry_file}")
-    module_name = entry_file[:-3]
-    handler_module = importlib.import_module(module_name)
-
-    # Determine entry point
-    # TODO: add OL_FUNCTION_ENTRY for explicit function entry points
-    wsgi_entry = os.environ.get('OL_WSGI_ENTRY')
-    if wsgi_entry:
-        entry_point = getattr(handler_module, wsgi_entry)
-        is_wsgi = True
-    elif hasattr(handler_module, 'f'):
-        entry_point = handler_module.f
-        is_wsgi = False
-    elif hasattr(handler_module, 'app'):
-        entry_point = handler_module.app
-        is_wsgi = True
-    else:
-        raise ValueError("No entry point found. Set OL_WSGI_ENTRY or define 'f' or 'app' in your module.")
-
-    initialized = True
-
-class SockFileHandler(tornado.web.RequestHandler):
-    def handle_request(self):
-        try:
-            data = self.request.body
-            try:
-                event = json.loads(data) if data else None
-            except:
-                self.set_status(400)  # Bad request if JSON parsing fails
-                self.write(f'bad request data: "{data}"')
-                return
-
-            result = entry_point(event) if event is not None else entry_point({}) 
-            self.write(json.dumps(result))  # Return the result as JSON
-        except Exception:
-            self.set_status(500)  # Internal server error for unhandled exceptions
-            self.write(traceback.format_exc())  # Include traceback in response
-    
-    
-    # Define methods for each HTTP method
-    def get(self):
-        self.handle_request()
-
-    def post(self):
-        self.handle_request()
-
-    def put(self):
-        self.handle_request()
-
-    def delete(self):
-        self.handle_request()
-
-    def patch(self):
-        self.handle_request()
-
-    def options(self):
-        self.handle_request()
-        
-
-# listen on sock file with Tornado
+
 def lambda_server():
-    init()
-    if is_wsgi:
-        def path_wrapper(environ, start_response):
-            path = environ.get("PATH_INFO", "")
-            # split path to get individual components
-            parts = path.split("/") # ["", "run", <app-name>]
-
-            # set new environment path
-            # `/run/<func-name>/a/b/c` -> `/a/b/c`
-            environ["PATH_INFO"] = '/' + '/'.join(parts[3:])
-
-            # set the root of the application
-            app_name = parts[2]
-            environ["SCRIPT_NAME"] = '/run/' + app_name
-
-            return entry_point(environ, start_response)
-
-        # use WSGI entry
-        # call wrapper to strip /run/<func-name> from path
-        tornado_app = tornado.wsgi.WSGIContainer(path_wrapper)
-    else:
-        # use function entry
-        tornado_app = tornado.web.Application([
-            (".*", SockFileHandler),
-        ])
-    server = tornado.httpserver.HTTPServer(tornado_app)
-    socket = tornado.netutil.bind_unix_socket(SOCK_PATH)
-    server.add_socket(socket)
-    # notify worker server that we are ready through stdout
-    # flush is necessary, and don't put it after tornado start; won't work
+    """Start the lambda server on a Unix socket."""
+    # Create and bind the socket
+    if os.path.exists(SOCK_PATH):
+        os.remove(SOCK_PATH)
+    file_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    file_sock.bind(SOCK_PATH)
+    file_sock.listen(1)
+
+    # Notify worker server that we are ready
     with open(SERVER_PIPE_PATH, 'w', encoding='utf-8') as pipe:
         pipe.write('ready')
-    tornado.ioloop.IOLoop.instance().start()
-    server.start(PROCESSES_DEFAULT)
 
-# listen for fds to forkenter
+    # Run the web server
+    web_server_on_sock(file_sock, server_name="server_legacy.py")
+
+
 def cache_loop():
+    """Listen for fds to forkenter (Docker cache mode)."""
     import ns
 
     signal = "cache"
     r = -1
-    count = 0
     # only child meant to serve ever escapes the loop
     while r != 0 or signal == "cache":
         if r == 0:
@@ -190,22 +99,23 @@ def cache_loop():
         print('')
         flush()
 
-        count += 1
-
     print('SERVING HANDLERS')
     flush()
     lambda_server()
 
+
 def flush():
     sys.stdout.flush()
     sys.stderr.flush()
 
+
 def redirect():
     sys.stdout.close()
     sys.stderr.close()
     sys.stdout = open(STDOUT_PATH, 'w')
     sys.stderr = open(STDERR_PATH, 'w')
 
+
 if __name__ == '__main__':
     args = parser.parse_args()
     redirect()
diff --git a/python/src/open_lambda.py b/python/src/open_lambda.py
index 8853582a2..3184cc3f9 100644
--- a/python/src/open_lambda.py
+++ b/python/src/open_lambda.py
@@ -11,14 +11,14 @@ def __init__(self, address="localhost:5000"):
         self._address = address
         self._session = Session()
 
-    def _post(self, path, data=None):
+    def _post(self, path, data=None, timeout=None):
         ''' Issues a _post request to the OL worker '''
-        return self._session.post(f'http://{self._address}/{path}', pyjson.dumps(data))
+        return self._session.post(f'http://{self._address}/{path}', pyjson.dumps(data), timeout=timeout)
 
-    def run(self, fn_name, args, json=True):
+    def run(self, fn_name, args, json=True, timeout=60):
         ''' Execute a serverless function '''
 
-        resp = self._post(f"run/{fn_name}", args)
+        resp = self._post(f"run/{fn_name}", args, timeout=timeout)
         self._check_status_code(resp, "run")
 
         if json:
diff --git a/scripts/test.py b/scripts/test.py
index 08f489732..bc876a329 100755
--- a/scripts/test.py
+++ b/scripts/test.py
@@ -14,6 +14,7 @@
 import subprocess
 
 from time import time
+from datetime import datetime
 from subprocess import call
 from multiprocessing import Pool
 
@@ -154,7 +155,7 @@ def stress_one_lambda_task(args):
     start, seconds = args
     pos = 0
     while time() < start + seconds:
-        result = open_lambda.run("echo", pos, json=False)
+        result = open_lambda.run("echo", pos, json=False, timeout=60)
         assert_eq(result, str(pos))
         pos += 1
     return pos
@@ -352,6 +353,21 @@ def flask_entry_test():
     if data.get("entry_file") != "app.py":
         raise ValueError(f"expected entry_file='app.py', got {data}")
 
+@test
+def fastapi_test():
+    """Test ASGI support with FastAPI"""
+    url = 'http://localhost:5000/run/fastapi-test'
+    print("URL", url)
+    r = requests.get(url)
+    print("RESPONSE", r)
+
+    if r.status_code != 200:
+        raise ValueError(f"expected status code 200, but got {r.status_code}")
+
+    data = r.json()
+    if data != {"message": "hello world"}:
+        raise ValueError(f"expected {{'message': 'hello world'}}, but got {data}")
+
 @test
 def wsgi_entry_test():
     """Test OL_WSGI_ENTRY feature with a WSGI entry point not named 'app'"""
@@ -452,7 +468,7 @@ def run_tests():
     worker_type = get_worker_type()
     worker = worker_type()
     assert worker
-    print("Worker started")
+    print(f"Worker started at {datetime.now().strftime('%I:%M%p').lstrip('0').lower()}")
     install_examples_to_worker_registry()
     print("Examples installed")
     worker.stop()
@@ -482,6 +498,9 @@ def run_tests():
     wsgi_entry_test()
     test_http_method_restrictions()
 
+    # test ASGI support with FastAPI
+    fastapi_test()
+
     # test environment variables from ol.yaml
     env_test()
 

From 990f9cbb1365a90d0b6b7f0b6d3b04ffacf636bc Mon Sep 17 00:00:00 2001
From: Tyler Caraza-Harter <tylerharter@gmail.com>
Date: Tue, 13 Jan 2026 10:54:46 -0600
Subject: [PATCH 14/55] update ag forecast docs

---
 docs/worker/apps.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/worker/apps.md b/docs/worker/apps.md
index b651da849..f79265b52 100644
--- a/docs/worker/apps.md
+++ b/docs/worker/apps.md
@@ -33,7 +33,9 @@ triggers:
     - method: "*"
 environment:
   OL_ENTRY_FILE: app.py
+  OL_ASGI_ENTRY: app
   MEASUREMENTS_CACHE_DIR: /host/tmp/cache
+  STATIONS_CACHE_FILE: /host/tmp/cache/wisconsin_stations_cache.csv
 ```
 
 Install pip-compile and pin requirements.txt to versions suitable for OpenLambda:
@@ -50,15 +52,13 @@ Install and test:
 
 # simple test
 curl http://localhost:5000/run/ag_forecasting_api/
-```
-
-NOTE: the full app doesn't work yet (we need to make sure code and writable directories are as expected).  This fails:
 
-```bash
 # get a forecast for the ALTN station
 curl "http://localhost:5000/run/ag_forecasting_api/ag_models_wrappers/wisconet?forecasting_date=2024-07-01&risk_days=1&station_id=ALTN"
 ```
 
+Note, the first request may take minutes because OpenLambda will install all the packages in requirements.txt upon the first call.
+
 TODO: update ag_forecasting_api URLs from tylerharter fork to UW-Madison-DSI once env option is merged upstream.
 
 ## TODO: add more example apps

From 99820315ccaaed6929d1d13117289eb55f20926b Mon Sep 17 00:00:00 2001
From: Yashwanth Ranjan Singaravel <ranjansingar@wisc.edu>
Date: Tue, 3 Feb 2026 02:19:12 -0600
Subject: [PATCH 15/55] Initial fix for lambda invocation error through kafka

---
 go/worker/event/kafkaServer.go | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/go/worker/event/kafkaServer.go b/go/worker/event/kafkaServer.go
index 993dffc55..793c421ad 100644
--- a/go/worker/event/kafkaServer.go
+++ b/go/worker/event/kafkaServer.go
@@ -160,7 +160,9 @@ func (lkc *LambdaKafkaConsumer) processMessage(record *kgo.Record) {
 	defer t.T1()
 
 	// Create synthetic HTTP request from Kafka message
-	req, err := http.NewRequest("POST", "/", bytes.NewReader(record.Value))
+	// Path must be /run/<lambda-name>/ for the Python runtime to parse correctly
+	requestPath := fmt.Sprintf("/run/%s/", lkc.lambdaName)
+	req, err := http.NewRequest("POST", requestPath, bytes.NewReader(record.Value))
 	if err != nil {
 		slog.Error("Failed to create request for lambda invocation",
 			"lambda", lkc.lambdaName,
@@ -168,6 +170,8 @@ func (lkc *LambdaKafkaConsumer) processMessage(record *kgo.Record) {
 			"topic", record.Topic)
 		return
 	}
+	// RequestURI must be set explicitly for synthetic requests (http.NewRequest doesn't set it)
+	req.RequestURI = requestPath
 
 	// Set headers with Kafka metadata (The X- prefix indicates a custom non-standard header)
 	req.Header.Set("Content-Type", "application/json")

From 0392fc666c58aa17a2311cb88da74795f8bb9bb6 Mon Sep 17 00:00:00 2001
From: cblin321 <126987173+cblin321@users.noreply.github.com>
Date: Tue, 10 Feb 2026 09:12:15 -0600
Subject: [PATCH 16/55] Fix busy wait in setFreezeState (#401)

* watch event files for setFreeze

* fix implementation

* Update ci.yml

* remove goroutine

* restart on EINTR

* remove read

* linter errs

* revert ci.yml

* remove eintr err handling

* improve err msgs; improve err handling

* update setFreezeState

* update warning; update err msgs

* updates

* refactor setFreeze state; update Release; fix typo

* fix typo

* updates

* tighten up err msgs

* start

* read from same open file in setFreezeState

* bugs

* update

* reduce buffer size

* linter errs

* update ReadIntKVFromFile

* fix linter errs

* add warning for setFreezeState

* update read err handling

* update TryReadIntKVFromFile

* close file

* err handling for seek
---
 go/worker/sandbox/cgroups/cgroup.go | 43 +++++++++++++++++++++++------
 go/worker/sandbox/sock.go           |  2 +-
 2 files changed, 35 insertions(+), 10 deletions(-)

diff --git a/go/worker/sandbox/cgroups/cgroup.go b/go/worker/sandbox/cgroups/cgroup.go
index 03303e3ef..775a84c86 100644
--- a/go/worker/sandbox/cgroups/cgroup.go
+++ b/go/worker/sandbox/cgroups/cgroup.go
@@ -4,6 +4,8 @@ import (
 	"bufio"
 	"errors"
 	"fmt"
+	"golang.org/x/sys/unix"
+	"io"
 	"io/ioutil"
 	"log/slog"
 	"os"
@@ -13,7 +15,6 @@ import (
 	"time"
 
 	"github.com/open-lambda/open-lambda/go/common"
-	"golang.org/x/sys/unix"
 )
 
 type CgroupImpl struct {
@@ -138,12 +139,16 @@ func (cg *CgroupImpl) WriteString(resource string, val string) {
 	}
 }
 
-func (cg *CgroupImpl) TryReadIntKV(resource string, key string) (int64, error) {
-	raw, err := ioutil.ReadFile(cg.ResourcePath(resource))
+func (_ *CgroupImpl) TryReadIntKVFromFile(file *os.File, key string) (int64, error) {
+	_, err := file.Seek(0, io.SeekStart)
 	if err != nil {
-		return 0, err
+		return 0, fmt.Errorf("failed to seek to start of file: %w", err)
+	}
+	data, err := io.ReadAll(file)
+	if err != nil {
+		return 0, fmt.Errorf("failed to read key %s from file: %w", key, err)
 	}
-	body := string(raw)
+	body := string(data)
 	lines := strings.Split(body, "\n")
 	for i := 0; i <= len(lines); i++ {
 		parts := strings.Split(lines[i], " ")
@@ -158,6 +163,16 @@ func (cg *CgroupImpl) TryReadIntKV(resource string, key string) (int64, error) {
 	return 0, fmt.Errorf("could not find key '%s' in file: %s", key, body)
 }
 
+func (cg *CgroupImpl) TryReadIntKV(resource string, key string) (int64, error) {
+	resourcePath := cg.ResourcePath(resource)
+	file, err := os.Open(resourcePath)
+	if err != nil {
+		return 0, fmt.Errorf("failed to open file %s: %w", resourcePath, err)
+	}
+	defer file.Close()
+	return cg.TryReadIntKVFromFile(file, key)
+}
+
 func (cg *CgroupImpl) TryReadInt(resource string) (int64, error) {
 	raw, err := ioutil.ReadFile(cg.ResourcePath(resource))
 	if err != nil {
@@ -201,22 +216,30 @@ func (cg *CgroupImpl) setFreezeState(state int64) error {
 	}
 	defer eventFile.Close()
 
-	// poll(2): POLLPRI indicates "cgroup.events file modified"
+	// cgroups(7): POLLPRI indicates "cgroup.events file modified"
+	// for poll to decide a POLLPRI event occurs it maintains 2 event counters:
+	// 1. the event counter when you last read the file
+	// 2. the file's current event counter
+	// if the last read's counter is different from the current event counter poll returns POLLPRI
 	pollFDs := []unix.PollFd{
 		{
 			Fd:     int32(eventFile.Fd()),
 			Events: unix.POLLPRI,
 		},
 	}
+	pollCalls := 0
 
 	start := time.Now()
 
-	defer func(start time.Time) {
+	defer func() {
 		elapsed := time.Since(start)
 		if elapsed >= 250*time.Millisecond {
 			cg.printf("WARNING!  setFreezeState to state %v took %v to complete", state, elapsed)
 		}
-	}(start)
+		if pollCalls > 5 {
+			cg.printf("WARNING!  setFreezeState called poll %v times, could be busy waiting", pollCalls)
+		}
+	}()
 
 	cg.WriteInt("cgroup.freeze", state)
 
@@ -228,12 +251,14 @@ func (cg *CgroupImpl) setFreezeState(state int64) error {
 			return fmt.Errorf("cgroup freeze timeout after %v (expected state %v)", timeout, state)
 		}
 
+		pollCalls++
 		_, err := unix.Poll(pollFDs, int(remaining.Milliseconds()))
 		if err != nil && !errors.Is(err, unix.EINTR) {
 			return fmt.Errorf("poll syscall failed on %s: %w", resourcePath, err)
 		}
 
-		freezerState, err := cg.TryReadIntKV("cgroup.events", "frozen")
+		// read from the same file to update event counter, prevents busy wait
+		freezerState, err := cg.TryReadIntKVFromFile(eventFile, "frozen")
 		if err != nil {
 			return fmt.Errorf("failed to check self_freezing state :: %w", err)
 		}
diff --git a/go/worker/sandbox/sock.go b/go/worker/sandbox/sock.go
index bb22e0241..99fead16b 100644
--- a/go/worker/sandbox/sock.go
+++ b/go/worker/sandbox/sock.go
@@ -277,7 +277,7 @@ func (container *SOCKContainer) DestroyIfPaused(reason string) {
 }
 
 // when the count goes to zero, it means (a) this container and (b)
-// all it's descendants are destroyed. Thus, it's safe to release it's
+// all its descendants are destroyed. Thus, it's safe to release its
 // cgroups, and return the memory allocation to the memPool
 func (container *SOCKContainer) decCgRefCount() {
 	newCount := atomic.AddInt32(&container.cgRefCount, -1)

From dc141fbdf395e21eb56f323992cc538d967c9236 Mon Sep 17 00:00:00 2001
From: cblin321 <126987173+cblin321@users.noreply.github.com>
Date: Wed, 18 Feb 2026 13:20:29 -0600
Subject: [PATCH 17/55] Refactor Cgroup Kill; Update Cgroup Kill to use polling
 (#413)

* refactor freeze to use WriteEventAndWait

* fix WriteEventAndWait

* fix param

* cleanup setFreezeState; fix linter errs

* refactor Kill and Release

* update cg interface

* fix writeEventAndWait

* remove polling in Destroy

* remove process check in KillAndRelease

* update KillAndRelease
---
 go/worker/sandbox/cgroups/api.go    |   3 +-
 go/worker/sandbox/cgroups/cgroup.go | 152 +++++++++++++---------------
 go/worker/sandbox/sock.go           |   3 +-
 3 files changed, 71 insertions(+), 87 deletions(-)

diff --git a/go/worker/sandbox/cgroups/api.go b/go/worker/sandbox/cgroups/api.go
index ae41c7ba8..69fe3c6e7 100644
--- a/go/worker/sandbox/cgroups/api.go
+++ b/go/worker/sandbox/cgroups/api.go
@@ -7,10 +7,9 @@ type Cgroup interface {
 	SetMemLimitMB(mb int)
 	Pause() error
 	Unpause() error
-	Release()
 	AddPid(pid string) error
 	GetPIDs() ([]string, error)
-	KillAllProcs()
+	KillAndRelease()
 	DebugString() string
 
 	// TODO: find a way to rip this out.  Higher layers should not
diff --git a/go/worker/sandbox/cgroups/cgroup.go b/go/worker/sandbox/cgroups/cgroup.go
index 775a84c86..bf725ed54 100644
--- a/go/worker/sandbox/cgroups/cgroup.go
+++ b/go/worker/sandbox/cgroups/cgroup.go
@@ -35,27 +35,18 @@ func (cg *CgroupImpl) Name() string {
 	return cg.name
 }
 
-// Release releases the cgroup back to the pool or destroys it if the pool is full.
-func (cg *CgroupImpl) Release() {
+// KillAndRelease stops all processes inside the cgroup.
+// After releasing, the cgroup can be recycled or destroyed depending on configuration.
+// Note, the CG most be paused beforehand.
+func (cg *CgroupImpl) KillAndRelease() {
+	err := cg.WriteEventAndWait("cgroup.kill", 1, "populated", 0, 20*time.Second)
+	if err != nil {
+		panic(fmt.Errorf("can't write \"1\" to cgroup.kill: %w", err))
+	}
+
 	// if there's room in the recycled channel, add it there.
 	// Otherwise, just delete it.
 	if common.Conf.Features.Reuse_cgroups {
-		for i := 100; i >= 0; i-- {
-			pids, err := cg.GetPIDs()
-			if err != nil {
-				panic(err)
-			} else if len(pids) > 0 {
-				if i == 0 {
-					panic(fmt.Errorf("Cannot release cgroup that contains processes: %v", pids))
-				}
-
-				cg.printf("cgroup Rmdir failed, trying again in 5ms")
-				time.Sleep(5 * time.Millisecond)
-			} else {
-				break
-			}
-		}
-
 		select {
 		case cg.pool.recycled <- cg:
 			cg.printf("release and recycle")
@@ -133,6 +124,65 @@ func (cg *CgroupImpl) WriteInt(resource string, val int64) {
 	}
 }
 
+// WriteEventAndWait() writes to cgroup controller file and waits for the corresponding event in cgroup.events to be updated
+func (cg *CgroupImpl) WriteEventAndWait(controller string, controllerState int64, event string, eventState int64, timeout time.Duration) error {
+	resourcePath := cg.ResourcePath("cgroup.events")
+	eventFile, err := os.Open(resourcePath)
+	if err != nil {
+		return fmt.Errorf("failed to open %s: %w", resourcePath, err)
+	}
+
+	// cgroups(7): POLLPRI indicates "cgroup.events file modified"
+	// for poll to decide a POLLPRI event occurs it maintains 2 event counters:
+	// 1. the event counter when you last read the file
+	// 2. the file's current event counter
+	// if the last read's counter is different from the current event counter poll returns POLLPRI
+	pollFDs := []unix.PollFd{
+		{
+			Fd:     int32(eventFile.Fd()),
+			Events: unix.POLLPRI,
+		},
+	}
+	pollCalls := 0
+
+	start := time.Now()
+
+	defer func() {
+		elapsed := time.Since(start)
+		if elapsed >= 250*time.Millisecond {
+			cg.printf("WARNING!  WriteEventAndWait to state %v took %v to complete", controllerState, elapsed)
+		}
+		if pollCalls > 5 {
+			cg.printf("WARNING!  WriteEventAndWait called poll %v times, could be busy waiting", pollCalls)
+		}
+	}()
+
+	cg.WriteInt(controller, controllerState)
+	for {
+		elapsed := time.Since(start)
+
+		remaining := timeout - elapsed
+		if remaining < 0 {
+			return fmt.Errorf("%s timeout after %v (expected state %v)", controller, timeout, eventState)
+		}
+
+		pollCalls++
+		_, err := unix.Poll(pollFDs, int(remaining.Milliseconds()))
+		if err != nil && !errors.Is(err, unix.EINTR) {
+			return fmt.Errorf("poll syscall failed on %s: %w", resourcePath, err)
+		}
+
+		// read from the same file to update event counter, prevents busy wait
+		currEventState, err := cg.TryReadIntKVFromFile(eventFile, event)
+		if err != nil {
+			return fmt.Errorf("failed to check %s in %s :: %w", event, resourcePath, err)
+		}
+		if currEventState == eventState {
+			return nil
+		}
+	}
+}
+
 func (cg *CgroupImpl) WriteString(resource string, val string) {
 	if err := cg.TryWriteString(resource, val); err != nil {
 		panic(fmt.Sprintf("Error writing %v to %s: %v", val, resource, err))
@@ -207,65 +257,7 @@ func (cg *CgroupImpl) AddPid(pid string) error {
 
 func (cg *CgroupImpl) setFreezeState(state int64) error {
 	timeout := 20 * time.Second
-
-	resourcePath := cg.ResourcePath("cgroup.events")
-
-	eventFile, err := os.Open(resourcePath)
-	if err != nil {
-		return fmt.Errorf("failed to open %s: %w", resourcePath, err)
-	}
-	defer eventFile.Close()
-
-	// cgroups(7): POLLPRI indicates "cgroup.events file modified"
-	// for poll to decide a POLLPRI event occurs it maintains 2 event counters:
-	// 1. the event counter when you last read the file
-	// 2. the file's current event counter
-	// if the last read's counter is different from the current event counter poll returns POLLPRI
-	pollFDs := []unix.PollFd{
-		{
-			Fd:     int32(eventFile.Fd()),
-			Events: unix.POLLPRI,
-		},
-	}
-	pollCalls := 0
-
-	start := time.Now()
-
-	defer func() {
-		elapsed := time.Since(start)
-		if elapsed >= 250*time.Millisecond {
-			cg.printf("WARNING!  setFreezeState to state %v took %v to complete", state, elapsed)
-		}
-		if pollCalls > 5 {
-			cg.printf("WARNING!  setFreezeState called poll %v times, could be busy waiting", pollCalls)
-		}
-	}()
-
-	cg.WriteInt("cgroup.freeze", state)
-
-	for {
-		elapsed := time.Since(start)
-
-		remaining := timeout - elapsed
-		if remaining < 0 {
-			return fmt.Errorf("cgroup freeze timeout after %v (expected state %v)", timeout, state)
-		}
-
-		pollCalls++
-		_, err := unix.Poll(pollFDs, int(remaining.Milliseconds()))
-		if err != nil && !errors.Is(err, unix.EINTR) {
-			return fmt.Errorf("poll syscall failed on %s: %w", resourcePath, err)
-		}
-
-		// read from the same file to update event counter, prevents busy wait
-		freezerState, err := cg.TryReadIntKVFromFile(eventFile, "frozen")
-		if err != nil {
-			return fmt.Errorf("failed to check self_freezing state :: %w", err)
-		}
-		if freezerState == state {
-			return nil
-		}
-	}
+	return cg.WriteEventAndWait("cgroup.freeze", state, "frozen", state, timeout)
 }
 
 // get mem usage in MB
@@ -354,12 +346,6 @@ func (cg *CgroupImpl) CgroupProcsPath() string {
 	return cg.ResourcePath("cgroup.procs")
 }
 
-// KillAllProcs stops all processes inside the cgroup.
-// Note, the CG most be paused beforehand
-func (cg *CgroupImpl) KillAllProcs() {
-	cg.WriteInt("cgroup.kill", 1)
-}
-
 // DebugString returns a string representation of the cgroup's state.
 func (cg *CgroupImpl) DebugString() string {
 	s := ""
diff --git a/go/worker/sandbox/sock.go b/go/worker/sandbox/sock.go
index 99fead16b..5f813c93b 100644
--- a/go/worker/sandbox/sock.go
+++ b/go/worker/sandbox/sock.go
@@ -297,9 +297,8 @@ func (container *SOCKContainer) decCgRefCount() {
 
 		t := common.T0("Destroy()/cleanup-cgroup")
 		if container.cg != nil {
-			container.cg.KillAllProcs()
+			container.cg.KillAndRelease()
 			container.printf("killed PIDs in CG\n")
-			container.cg.Release()
 			container.pool.mem.adjustAvailableMB(container.cg.GetMemLimitMB())
 		}
 		t.T1()

From 6b5b51676d2554b6dfb3627aa9afbe9c23c69f32 Mon Sep 17 00:00:00 2001
From: Yashwanth-Ranjan-Singaravel
 <157959140+Yashwanth-Ranjan-Singaravel@users.noreply.github.com>
Date: Thu, 19 Feb 2026 10:06:27 -0600
Subject: [PATCH 18/55] minor fix for 'go test' (#410)

* Changed %w to %v when using Sprintf

* Refactored use of fmt.Sprintf()
---
 go/boss/cloudvm/local_worker.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/go/boss/cloudvm/local_worker.go b/go/boss/cloudvm/local_worker.go
index c373f73bf..8ff86c74f 100644
--- a/go/boss/cloudvm/local_worker.go
+++ b/go/boss/cloudvm/local_worker.go
@@ -31,7 +31,7 @@ func NewLocalWorkerPool() *WorkerPool {
 			// Get the worker config struct
 			defaultTemplateConfig, err := common.GetDefaultWorkerConfig("")
 			if err != nil {
-				slog.Error(fmt.Sprintf("failed to load default template config: %w", err))
+				slog.Error("failed to load default template config", "error", err)
 				os.Exit(1)
 			}
 
@@ -46,11 +46,11 @@ func NewLocalWorkerPool() *WorkerPool {
 			defaultTemplateConfig.Import_cache_tree = ""
 
 			if err := common.SaveConfig(defaultTemplateConfig, templatePath); err != nil {
-				slog.Error(fmt.Sprintf("failed to save template.json: %w", err))
+				slog.Error("failed to save template.json", "error", err)
 				os.Exit(1)
 			}
 		} else {
-			slog.Error(fmt.Sprintf("failed to stat template path: %w", err))
+			slog.Error("failed to stat template path", "error", err)
 			os.Exit(1)
 		}
 	}
@@ -99,10 +99,10 @@ func (p *LocalWorkerPoolPlatform) CreateInstance(worker *Worker) error {
 		slog.Error("Failed to get worker config", "workerId", worker.workerId, "error", err)
 		return err
 	}
-	
+
 	// Set worker-specific port
 	cfg.Worker_port = workerPort
-	
+
 	// Save to worker directory
 	configPath := filepath.Join(workerPath, "config.json")
 	if err := common.SaveConfig(cfg, configPath); err != nil {

From 5b0f23e69578a78d20701d4a4c1cc457e9d38629 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 19 Feb 2026 14:25:57 -0600
Subject: [PATCH 19/55] Bump werkzeug from 3.0.3 to 3.1.5 in
 /examples/wsgi-entry-test (#397)

Bumps [werkzeug](https://github.com/pallets/werkzeug) from 3.0.3 to 3.1.5.
- [Release notes](https://github.com/pallets/werkzeug/releases)
- [Changelog](https://github.com/pallets/werkzeug/blob/main/CHANGES.rst)
- [Commits](https://github.com/pallets/werkzeug/compare/3.0.3...3.1.5)

---
updated-dependencies:
- dependency-name: werkzeug
  dependency-version: 3.1.5
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/wsgi-entry-test/requirements.in  | 2 +-
 examples/wsgi-entry-test/requirements.txt | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/wsgi-entry-test/requirements.in b/examples/wsgi-entry-test/requirements.in
index b44f14a28..fba9b4e9e 100644
--- a/examples/wsgi-entry-test/requirements.in
+++ b/examples/wsgi-entry-test/requirements.in
@@ -1,2 +1,2 @@
 flask==2.3.2
-werkzeug==3.0.3
+werkzeug==3.1.5
diff --git a/examples/wsgi-entry-test/requirements.txt b/examples/wsgi-entry-test/requirements.txt
index 96159fc91..bb7ff3d8b 100644
--- a/examples/wsgi-entry-test/requirements.txt
+++ b/examples/wsgi-entry-test/requirements.txt
@@ -18,7 +18,7 @@ markupsafe==2.1.3
     # via
     #   jinja2
     #   werkzeug
-werkzeug==3.0.3
+werkzeug==3.1.5
     # via
     #   -r requirements.in
     #   flask

From e58b182122eeff55892956b912d9e1aaf3d82ece Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 19 Feb 2026 14:26:11 -0600
Subject: [PATCH 20/55] Bump werkzeug from 3.0.3 to 3.1.5 in
 /examples/wsgi-test (#394)

Bumps [werkzeug](https://github.com/pallets/werkzeug) from 3.0.3 to 3.1.5.
- [Release notes](https://github.com/pallets/werkzeug/releases)
- [Changelog](https://github.com/pallets/werkzeug/blob/main/CHANGES.rst)
- [Commits](https://github.com/pallets/werkzeug/compare/3.0.3...3.1.5)

---
updated-dependencies:
- dependency-name: werkzeug
  dependency-version: 3.1.5
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/wsgi-test/requirements.in  | 2 +-
 examples/wsgi-test/requirements.txt | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/wsgi-test/requirements.in b/examples/wsgi-test/requirements.in
index b44f14a28..fba9b4e9e 100644
--- a/examples/wsgi-test/requirements.in
+++ b/examples/wsgi-test/requirements.in
@@ -1,2 +1,2 @@
 flask==2.3.2
-werkzeug==3.0.3
+werkzeug==3.1.5
diff --git a/examples/wsgi-test/requirements.txt b/examples/wsgi-test/requirements.txt
index 96159fc91..bb7ff3d8b 100644
--- a/examples/wsgi-test/requirements.txt
+++ b/examples/wsgi-test/requirements.txt
@@ -18,7 +18,7 @@ markupsafe==2.1.3
     # via
     #   jinja2
     #   werkzeug
-werkzeug==3.0.3
+werkzeug==3.1.5
     # via
     #   -r requirements.in
     #   flask

From ce0d4689211927d0473f3d2a8e509e3080a19466 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 19 Feb 2026 14:26:24 -0600
Subject: [PATCH 21/55] Bump werkzeug from 3.1.4 to 3.1.5 in
 /examples/pip-compile (#392)

Bumps [werkzeug](https://github.com/pallets/werkzeug) from 3.1.4 to 3.1.5.
- [Release notes](https://github.com/pallets/werkzeug/releases)
- [Changelog](https://github.com/pallets/werkzeug/blob/main/CHANGES.rst)
- [Commits](https://github.com/pallets/werkzeug/compare/3.1.4...3.1.5)

---
updated-dependencies:
- dependency-name: werkzeug
  dependency-version: 3.1.5
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/pip-compile/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/pip-compile/requirements.txt b/examples/pip-compile/requirements.txt
index 7b76e82d5..8cb1e9143 100644
--- a/examples/pip-compile/requirements.txt
+++ b/examples/pip-compile/requirements.txt
@@ -20,5 +20,5 @@ markupsafe==3.0.3
     #   werkzeug
 pip-tools==5.5.0
     # via -r requirements.in
-werkzeug==3.1.4
+werkzeug==3.1.5
     # via flask

From 3dfc4144f9b3875d0d8f4a5df16d68b02f9d27a1 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 19 Feb 2026 14:26:42 -0600
Subject: [PATCH 22/55] Bump werkzeug from 3.0.6 to 3.1.5 in
 /examples/flask-entry-test (#391)

Bumps [werkzeug](https://github.com/pallets/werkzeug) from 3.0.6 to 3.1.5.
- [Release notes](https://github.com/pallets/werkzeug/releases)
- [Changelog](https://github.com/pallets/werkzeug/blob/main/CHANGES.rst)
- [Commits](https://github.com/pallets/werkzeug/compare/3.0.6...3.1.5)

---
updated-dependencies:
- dependency-name: werkzeug
  dependency-version: 3.1.5
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/flask-entry-test/requirements.in  | 2 +-
 examples/flask-entry-test/requirements.txt | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/flask-entry-test/requirements.in b/examples/flask-entry-test/requirements.in
index 308c5b964..944e2dae9 100644
--- a/examples/flask-entry-test/requirements.in
+++ b/examples/flask-entry-test/requirements.in
@@ -1,2 +1,2 @@
 flask
-werkzeug<3.1
+werkzeug<3.2
diff --git a/examples/flask-entry-test/requirements.txt b/examples/flask-entry-test/requirements.txt
index d29ce9e0e..8148fdc18 100644
--- a/examples/flask-entry-test/requirements.txt
+++ b/examples/flask-entry-test/requirements.txt
@@ -14,7 +14,7 @@ markupsafe==3.0.2
     # via
     #   jinja2
     #   werkzeug
-werkzeug==3.0.6
+werkzeug==3.1.5
     # via
     #   -r requirements.in
     #   flask

From e499c0a60aa24a0fcc7c744c338cbb4d632622c6 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 19 Feb 2026 14:26:54 -0600
Subject: [PATCH 23/55] Bump werkzeug from 3.0.3 to 3.1.5 in
 /examples/lambda-config-test (#390)

Bumps [werkzeug](https://github.com/pallets/werkzeug) from 3.0.3 to 3.1.5.
- [Release notes](https://github.com/pallets/werkzeug/releases)
- [Changelog](https://github.com/pallets/werkzeug/blob/main/CHANGES.rst)
- [Commits](https://github.com/pallets/werkzeug/compare/3.0.3...3.1.5)

---
updated-dependencies:
- dependency-name: werkzeug
  dependency-version: 3.1.5
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/lambda-config-test/requirements.in  | 2 +-
 examples/lambda-config-test/requirements.txt | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/lambda-config-test/requirements.in b/examples/lambda-config-test/requirements.in
index b44f14a28..fba9b4e9e 100644
--- a/examples/lambda-config-test/requirements.in
+++ b/examples/lambda-config-test/requirements.in
@@ -1,2 +1,2 @@
 flask==2.3.2
-werkzeug==3.0.3
+werkzeug==3.1.5
diff --git a/examples/lambda-config-test/requirements.txt b/examples/lambda-config-test/requirements.txt
index 96159fc91..bb7ff3d8b 100644
--- a/examples/lambda-config-test/requirements.txt
+++ b/examples/lambda-config-test/requirements.txt
@@ -18,7 +18,7 @@ markupsafe==2.1.3
     # via
     #   jinja2
     #   werkzeug
-werkzeug==3.0.3
+werkzeug==3.1.5
     # via
     #   -r requirements.in
     #   flask

From ba5a432776e06acf8e5929934b0d45b10be2fa72 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 19 Feb 2026 14:28:05 -0600
Subject: [PATCH 24/55] Bump bytes from 1.10.0 to 1.11.1 in /bin-functions
 (#408)

Bumps [bytes](https://github.com/tokio-rs/bytes) from 1.10.0 to 1.11.1.
- [Release notes](https://github.com/tokio-rs/bytes/releases)
- [Changelog](https://github.com/tokio-rs/bytes/blob/master/CHANGELOG.md)
- [Commits](https://github.com/tokio-rs/bytes/compare/v1.10.0...v1.11.1)

---
updated-dependencies:
- dependency-name: bytes
  dependency-version: 1.11.1
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 bin-functions/Cargo.lock | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin-functions/Cargo.lock b/bin-functions/Cargo.lock
index ef6be880b..b42ab06a3 100644
--- a/bin-functions/Cargo.lock
+++ b/bin-functions/Cargo.lock
@@ -135,9 +135,9 @@ checksum = "c3ac9f8b63eca6fd385229b3675f6cc0dc5c8a5c8a54a59d4f52ffd670d87b0c"
 
 [[package]]
 name = "bytes"
-version = "1.10.0"
+version = "1.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f61dac84819c6588b558454b194026eb1f09c293b9036ae9b159e74e73ab6cf9"
+checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
 
 [[package]]
 name = "cfg-if"

From 51fd4ab55c1e17bb61576fe0685544bf324237ab Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 19 Feb 2026 14:28:15 -0600
Subject: [PATCH 25/55] Bump bytes from 1.10.0 to 1.11.1 in /container-proxy
 (#407)

Bumps [bytes](https://github.com/tokio-rs/bytes) from 1.10.0 to 1.11.1.
- [Release notes](https://github.com/tokio-rs/bytes/releases)
- [Changelog](https://github.com/tokio-rs/bytes/blob/master/CHANGELOG.md)
- [Commits](https://github.com/tokio-rs/bytes/compare/v1.10.0...v1.11.1)

---
updated-dependencies:
- dependency-name: bytes
  dependency-version: 1.11.1
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 container-proxy/Cargo.lock | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/container-proxy/Cargo.lock b/container-proxy/Cargo.lock
index 9109f3bc5..85220450f 100644
--- a/container-proxy/Cargo.lock
+++ b/container-proxy/Cargo.lock
@@ -79,9 +79,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "bytes"
-version = "1.10.0"
+version = "1.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f61dac84819c6588b558454b194026eb1f09c293b9036ae9b159e74e73ab6cf9"
+checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
 
 [[package]]
 name = "cfg-if"

From 2472aaef1a1cef9d2e972f1aff9e53de506e93cb Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 19 Feb 2026 14:28:26 -0600
Subject: [PATCH 26/55] Bump jinja2 from 3.1.4 to 3.1.6 in
 /examples/wsgi-entry-test (#398)

Bumps [jinja2](https://github.com/pallets/jinja) from 3.1.4 to 3.1.6.
- [Release notes](https://github.com/pallets/jinja/releases)
- [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst)
- [Commits](https://github.com/pallets/jinja/compare/3.1.4...3.1.6)

---
updated-dependencies:
- dependency-name: jinja2
  dependency-version: 3.1.6
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/wsgi-entry-test/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/wsgi-entry-test/requirements.txt b/examples/wsgi-entry-test/requirements.txt
index bb7ff3d8b..71333ecbc 100644
--- a/examples/wsgi-entry-test/requirements.txt
+++ b/examples/wsgi-entry-test/requirements.txt
@@ -12,7 +12,7 @@ flask==2.3.2
     # via -r requirements.in
 itsdangerous==2.1.2
     # via flask
-jinja2==3.1.4
+jinja2==3.1.6
     # via flask
 markupsafe==2.1.3
     # via

From 81138d0080c7fd892584cb968a19bdf52a36c38c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 19 Feb 2026 14:28:39 -0600
Subject: [PATCH 27/55] Bump werkzeug from 3.0.3 to 3.1.5 in
 /examples/wsgi-post-echo (#389)

Bumps [werkzeug](https://github.com/pallets/werkzeug) from 3.0.3 to 3.1.5.
- [Release notes](https://github.com/pallets/werkzeug/releases)
- [Changelog](https://github.com/pallets/werkzeug/blob/main/CHANGES.rst)
- [Commits](https://github.com/pallets/werkzeug/compare/3.0.3...3.1.5)

---
updated-dependencies:
- dependency-name: werkzeug
  dependency-version: 3.1.5
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/wsgi-post-echo/requirements.txt | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/examples/wsgi-post-echo/requirements.txt b/examples/wsgi-post-echo/requirements.txt
index 96159fc91..67a1ea468 100644
--- a/examples/wsgi-post-echo/requirements.txt
+++ b/examples/wsgi-post-echo/requirements.txt
@@ -18,7 +18,5 @@ markupsafe==2.1.3
     # via
     #   jinja2
     #   werkzeug
-werkzeug==3.0.3
-    # via
-    #   -r requirements.in
-    #   flask
+werkzeug==3.1.5
+    # via flask

From 6a4eba4c26ce7b8fa40fc029849fed0e2be5d917 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 19 Feb 2026 14:28:51 -0600
Subject: [PATCH 28/55] Bump werkzeug from 3.0.3 to 3.1.5 in
 /examples/flask-test (#388)

Bumps [werkzeug](https://github.com/pallets/werkzeug) from 3.0.3 to 3.1.5.
- [Release notes](https://github.com/pallets/werkzeug/releases)
- [Changelog](https://github.com/pallets/werkzeug/blob/main/CHANGES.rst)
- [Commits](https://github.com/pallets/werkzeug/compare/3.0.3...3.1.5)

---
updated-dependencies:
- dependency-name: werkzeug
  dependency-version: 3.1.5
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/flask-test/requirements.in  | 2 +-
 examples/flask-test/requirements.txt | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/flask-test/requirements.in b/examples/flask-test/requirements.in
index b44f14a28..fba9b4e9e 100644
--- a/examples/flask-test/requirements.in
+++ b/examples/flask-test/requirements.in
@@ -1,2 +1,2 @@
 flask==2.3.2
-werkzeug==3.0.3
+werkzeug==3.1.5
diff --git a/examples/flask-test/requirements.txt b/examples/flask-test/requirements.txt
index 96159fc91..bb7ff3d8b 100644
--- a/examples/flask-test/requirements.txt
+++ b/examples/flask-test/requirements.txt
@@ -18,7 +18,7 @@ markupsafe==2.1.3
     # via
     #   jinja2
     #   werkzeug
-werkzeug==3.0.3
+werkzeug==3.1.5
     # via
     #   -r requirements.in
     #   flask

From 14345e73e661ea957366373656769779227c5b7d Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 19 Feb 2026 14:30:29 -0600
Subject: [PATCH 29/55] Bump golang.org/x/crypto from 0.39.0 to 0.45.0 in /go
 (#364)

Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.39.0 to 0.45.0.
- [Commits](https://github.com/golang/crypto/compare/v0.39.0...v0.45.0)

---
updated-dependencies:
- dependency-name: golang.org/x/crypto
  dependency-version: 0.45.0
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go/go.mod | 12 ++++++------
 go/go.sum | 24 ++++++++++++------------
 2 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/go/go.mod b/go/go.mod
index 21a70ee32..58e168923 100644
--- a/go/go.mod
+++ b/go/go.mod
@@ -1,6 +1,6 @@
 module github.com/open-lambda/open-lambda/go
 
-go 1.24
+go 1.24.0
 
 toolchain go1.24.4
 
@@ -111,12 +111,12 @@ require (
 	go.opentelemetry.io/otel/sdk v1.36.0 // indirect
 	go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect
 	go.opentelemetry.io/otel/trace v1.36.0 // indirect
-	golang.org/x/crypto v0.39.0 // indirect
-	golang.org/x/net v0.41.0 // indirect
+	golang.org/x/crypto v0.45.0 // indirect
+	golang.org/x/net v0.47.0 // indirect
 	golang.org/x/oauth2 v0.30.0 // indirect
-	golang.org/x/sync v0.15.0 // indirect
-	golang.org/x/sys v0.33.0 // indirect
-	golang.org/x/text v0.26.0 // indirect
+	golang.org/x/sync v0.18.0 // indirect
+	golang.org/x/sys v0.38.0 // indirect
+	golang.org/x/text v0.31.0 // indirect
 	golang.org/x/time v0.11.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
 	google.golang.org/api v0.235.0 // indirect
diff --git a/go/go.sum b/go/go.sum
index c080ab5ef..11c94f6c5 100644
--- a/go/go.sum
+++ b/go/go.sum
@@ -286,8 +286,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
-golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
-golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
+golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
+golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@@ -313,8 +313,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
-golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
-golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
+golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
+golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
 golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=
@@ -327,8 +327,8 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8=
-golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
+golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -344,16 +344,16 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
-golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
+golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
 golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY=
-golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg=
-golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ=
+golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
+golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
@@ -361,8 +361,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
-golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
+golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
+golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
 golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0=
 golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

From a2682880100c99aa54bdc719fecaad14d9656728 Mon Sep 17 00:00:00 2001
From: Sarthak Khattar <sarthakoct@gmail.com>
Date: Thu, 19 Feb 2026 21:58:37 -0600
Subject: [PATCH 30/55] move cgroup pool creation logic to init (#412)

* move cgroup pool creation logic to init, worker reuses it

* add loginuid check, refactored cgroups init, removed cgroup path from config

* removed extra comments

* removed more comments

* added comments and doc links for /proc/self/loginuid

* moved cgroup path logic to common, misc fixes

* removed GroupPath() wrapper, added error messages in commands

* removed loginuid check - init must be run as sudo
---
 go/common/config.go                 |  5 ++
 go/worker/commands.go               | 16 ++++--
 go/worker/helpers.go                | 14 ++----
 go/worker/sandbox/cgroups/cgroup.go |  6 +--
 go/worker/sandbox/cgroups/pool.go   | 77 +++++++++++++++--------------
 go/worker/sandbox/sockPool.go       |  3 +-
 6 files changed, 66 insertions(+), 55 deletions(-)

diff --git a/go/common/config.go b/go/common/config.go
index fa90d7188..de83d93c1 100644
--- a/go/common/config.go
+++ b/go/common/config.go
@@ -492,3 +492,8 @@ func GetOlPath(ctx *cli.Context) (string, error) {
 	}
 	return filepath.Abs(olPath)
 }
+
+// CgroupPoolPath returns the cgroup pool root path for the given OL directory.
+func CgroupPoolPath(olPath string) string {
+	return filepath.Join("/sys/fs/cgroup", filepath.Base(olPath)+"-sandboxes")
+}
diff --git a/go/worker/commands.go b/go/worker/commands.go
index b763283bc..2ef49aad9 100644
--- a/go/worker/commands.go
+++ b/go/worker/commands.go
@@ -16,6 +16,7 @@ import (
 
 	"github.com/open-lambda/open-lambda/go/common"
 	"github.com/open-lambda/open-lambda/go/worker/event"
+	"github.com/open-lambda/open-lambda/go/worker/sandbox/cgroups"
 
 	"github.com/urfave/cli/v2"
 )
@@ -39,18 +40,27 @@ func udsGet(requestPath string) (*http.Response, error) {
 
 // initCmd corresponds to the "init" command of the admin tool.
 func initCmd(ctx *cli.Context) error {
+	if os.Getuid() != 0 {
+		return fmt.Errorf("'ol worker init' must be run with sudo")
+	}
+
 	olPath, err := common.GetOlPath(ctx)
 	if err != nil {
-		return err
+		return fmt.Errorf("init failed to get OL path: %w", err)
 	}
 
 	if err := common.LoadDefaults(olPath); err != nil {
-		return err
+		return fmt.Errorf("init failed to load config defaults: %w", err)
 	}
 
 	if err := initOLDir(olPath, ctx.String("image"), ctx.Bool("newbase")); err != nil {
-		return err
+		return fmt.Errorf("init failed to create OL directory: %w", err)
+	}
+
+	if err := cgroups.InitPoolRoot(common.CgroupPoolPath(olPath)); err != nil {
+		return fmt.Errorf("init failed to create cgroup pool: %w", err)
 	}
+
 	fmt.Printf("\nYou may optionally modify the defaults here: %s\n\n",
 		filepath.Join(olPath, "config.json"))
 	fmt.Printf("Next start a worker using the \"ol worker up\" command.\n")
diff --git a/go/worker/helpers.go b/go/worker/helpers.go
index fdbdd6171..9a882e1bd 100644
--- a/go/worker/helpers.go
+++ b/go/worker/helpers.go
@@ -21,6 +21,7 @@ import (
 	"github.com/open-lambda/open-lambda/go/worker/embedded"
 )
 
+
 func initOLBaseDir(baseDir string, dockerBaseImage string) error {
 	if dockerBaseImage == "" {
 		dockerBaseImage = "ol-wasm"
@@ -279,8 +280,8 @@ func runningToStoppedClean() error {
 // It cleans up cgroups and mounts associated with the OpenLambda instance at `olPath`.
 // Returns errors encountered during cleanup operations.
 func stoppedDirtyToStoppedClean(olPath string) error {
-	// Clean up cgroups associated with sandboxes
-	cgRoot := filepath.Join("/sys", "fs", "cgroup", filepath.Base(olPath)+"-sandboxes")
+	// Clean up child cgroups, preserving the pool root
+	cgRoot := common.CgroupPoolPath(olPath)
 	fmt.Printf("Attempting to clean up cgroups at %s\n", cgRoot)
 
 	cgroupErrorCount := 0
@@ -302,7 +303,6 @@ func stoppedDirtyToStoppedClean(olPath string) error {
 		}
 		kill := filepath.Join(cgRoot, "cgroup.kill")
 		if err := os.WriteFile(kill, []byte(fmt.Sprintf("%d", 1)), os.ModeAppend); err != nil {
-			// Print an error if killing processes in the cgroup fails.
 			fmt.Printf("Could not kill processes in cgroup: %s\n", err.Error())
 			cgroupErrorCount += 1
 		}
@@ -311,17 +311,11 @@ func stoppedDirtyToStoppedClean(olPath string) error {
 				cg := filepath.Join(cgRoot, file.Name())
 				fmt.Printf("Attempting to remove %s\n", cg)
 				if err := syscall.Rmdir(cg); err != nil {
-					// Print an error if removing a cgroup fails.
-					fmt.Printf("could not remove cgroup: %s", err.Error())
+					fmt.Printf("could not remove cgroup: %s\n", err.Error())
 					cgroupErrorCount += 1
 				}
 			}
 		}
-		if err := syscall.Rmdir(cgRoot); err != nil {
-			// Print an error if removing the cgroup root directory fails.
-			fmt.Printf("could not remove cgroup root: %s", err.Error())
-			cgroupErrorCount += 1
-		}
 	}
 
 	sandboxErrorCount := 0
diff --git a/go/worker/sandbox/cgroups/cgroup.go b/go/worker/sandbox/cgroups/cgroup.go
index bf725ed54..35d18bf3a 100644
--- a/go/worker/sandbox/cgroups/cgroup.go
+++ b/go/worker/sandbox/cgroups/cgroup.go
@@ -78,9 +78,9 @@ func (cg *CgroupImpl) Destroy() {
 	}
 }
 
-// GroupPath returns the path to the Cgroup pool for OpenLambda
+// GroupPath returns the path to this cgroup directory.
 func (cg *CgroupImpl) GroupPath() string {
-	return fmt.Sprintf("%s/%s", cg.pool.GroupPath(), cg.name)
+	return fmt.Sprintf("%s/%s", cg.pool.poolPath, cg.name)
 }
 
 func (cg *CgroupImpl) MemoryEvents() map[string]int64 {
@@ -107,7 +107,7 @@ func (cg *CgroupImpl) MemoryEvents() map[string]int64 {
 
 // ResourcePath returns the path to a specific resource in this cgroup
 func (cg *CgroupImpl) ResourcePath(resource string) string {
-	return fmt.Sprintf("%s/%s/%s", cg.pool.GroupPath(), cg.name, resource)
+	return fmt.Sprintf("%s/%s/%s", cg.pool.poolPath, cg.name, resource)
 }
 
 func (cg *CgroupImpl) TryWriteInt(resource string, val int64) error {
diff --git a/go/worker/sandbox/cgroups/pool.go b/go/worker/sandbox/cgroups/pool.go
index 13383fff5..7d83a3b43 100644
--- a/go/worker/sandbox/cgroups/pool.go
+++ b/go/worker/sandbox/cgroups/pool.go
@@ -2,13 +2,12 @@ package cgroups
 
 import (
 	"fmt"
-	"io/ioutil"
 	"log/slog"
 	"os"
-	"path"
+	"path/filepath"
+	"strconv"
 	"strings"
 	"syscall"
-	"time"
 
 	"github.com/open-lambda/open-lambda/go/common"
 )
@@ -19,35 +18,56 @@ const CGROUP_RESERVE = 16
 
 type CgroupPool struct {
 	Name     string
+	poolPath string
 	ready    chan *CgroupImpl
 	recycled chan *CgroupImpl
 	quit     chan chan bool
 	nextID   int
 }
 
-// NewCgroupPool creates a new CgroupPool with the specified name.
-func NewCgroupPool(name string) (*CgroupPool, error) {
+// InitPoolRoot creates the cgroup pool root directory and enables controllers.
+func InitPoolRoot(poolPath string) error {
+
+	if err := os.MkdirAll(poolPath, 0700); err != nil {
+		return fmt.Errorf("failed to create cgroup pool root %s: %w", poolPath, err)
+	}
+
+	ctrlPath := filepath.Join(poolPath, "cgroup.subtree_control")
+	if err := os.WriteFile(ctrlPath, []byte("+pids +io +memory +cpu"), os.ModeAppend); err != nil {
+		return fmt.Errorf("failed to enable controllers at %s: %w", ctrlPath, err)
+	}
+
+	uidStr := os.Getenv("SUDO_UID")
+	if uidStr == "" {
+		return fmt.Errorf("SUDO_UID not set; worker must be run with sudo")
+	}
+	uid, err := strconv.Atoi(uidStr)
+	if err != nil {
+		return fmt.Errorf("invalid SUDO_UID value %q: %w", uidStr, err)
+	}
+	if err := os.Chown(poolPath, uid, uid); err != nil {
+		return fmt.Errorf("failed to chown cgroup pool root: %w", err)
+	}
+
+	fmt.Printf("\tCreated cgroup pool root at %s\n", poolPath)
+	return nil
+}
+
+func NewCgroupPool(name string, poolPath string) (*CgroupPool, error) {
 	pool := &CgroupPool{
-		Name:     path.Base(path.Dir(common.Conf.Worker_dir)) + "-" + name,
+		Name:     name,
+		poolPath: poolPath,
 		ready:    make(chan *CgroupImpl, CGROUP_RESERVE),
 		recycled: make(chan *CgroupImpl, CGROUP_RESERVE),
 		quit:     make(chan chan bool),
 		nextID:   0,
 	}
 
-	// create cgroup
-	groupPath := pool.GroupPath()
-	pool.printf("create %s", groupPath)
-	if err := syscall.Mkdir(groupPath, 0700); err != nil {
-		return nil, fmt.Errorf("Mkdir %s: %s", groupPath, err)
-	}
-
-	// Make controllers available to child groups
-	rpath := fmt.Sprintf("%s/cgroup.subtree_control", groupPath)
-	if err := ioutil.WriteFile(rpath, []byte("+pids +io +memory +cpu"), os.ModeAppend); err != nil {
-		panic(fmt.Sprintf("Error writing to %s: %v", rpath, err))
+	if st, err := os.Stat(poolPath); err != nil || !st.IsDir() {
+		return nil, fmt.Errorf("cgroup pool root %s does not exist.", poolPath)
 	}
 
+	pool.printf("reusing pool root %s", poolPath)
 	go pool.cgTask()
 	return pool, nil
 }
@@ -136,28 +156,14 @@ Empty:
 	done <- true
 }
 
-// Destroy this entire cgroup pool
+// Destroy drains all child cgroups but preserves the pool root.
 func (pool *CgroupPool) Destroy() {
 	// signal cgTask, then wait for it to finish
 	ch := make(chan bool)
 	pool.quit <- ch
 	<-ch
 
-	// Destroy cgroup for this entire pool
-	gpath := pool.GroupPath()
-	pool.printf("Destroying cgroup pool with path \"%s\"", gpath)
-	for i := 100; i >= 0; i-- {
-		if err := syscall.Rmdir(gpath); err != nil {
-			if i == 0 {
-				panic(fmt.Errorf("Rmdir %s: %s", gpath, err))
-			}
-
-			pool.printf("cgroup pool Rmdir failed, trying again in 5ms")
-			time.Sleep(5 * time.Millisecond)
-		} else {
-			break
-		}
-	}
+	pool.printf("destroyed all child cgroups, pool root preserved")
 }
 
 // GetCg retrieves a cgroup from the pool, setting its memory limit and CPU percentage.
@@ -178,8 +184,3 @@ func (pool *CgroupPool) GetCg(memLimitMB int, moveMemCharge bool, cpuPercent int
 
 	return cg
 }
-
-// GroupPath returns the path to the Cgroup pool for OpenLambda
-func (pool *CgroupPool) GroupPath() string {
-	return fmt.Sprintf("/sys/fs/cgroup/%s", pool.Name)
-}
diff --git a/go/worker/sandbox/sockPool.go b/go/worker/sandbox/sockPool.go
index a71f9616b..e52e8c3a3 100644
--- a/go/worker/sandbox/sockPool.go
+++ b/go/worker/sandbox/sockPool.go
@@ -34,7 +34,8 @@ type SOCKPool struct {
 
 // NewSOCKPool creates a SOCKPool.
 func NewSOCKPool(name string, mem *MemPool) (cf *SOCKPool, err error) {
-	cgPool, err := cgroups.NewCgroupPool(name)
+	olPath := filepath.Dir(common.Conf.Worker_dir)
+	cgPool, err := cgroups.NewCgroupPool(name, common.CgroupPoolPath(olPath))
 	if err != nil {
 		return nil, err
 	}

From 46888b7d66f9b8cce46c08be289706783b54edec Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 19 Feb 2026 21:59:28 -0600
Subject: [PATCH 31/55] Bump jinja2 from 3.1.4 to 3.1.6 in
 /examples/wsgi-post-echo (#419)

Bumps [jinja2](https://github.com/pallets/jinja) from 3.1.4 to 3.1.6.
- [Release notes](https://github.com/pallets/jinja/releases)
- [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst)
- [Commits](https://github.com/pallets/jinja/compare/3.1.4...3.1.6)

---
updated-dependencies:
- dependency-name: jinja2
  dependency-version: 3.1.6
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 examples/wsgi-post-echo/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/wsgi-post-echo/requirements.txt b/examples/wsgi-post-echo/requirements.txt
index 67a1ea468..6d50b78c9 100644
--- a/examples/wsgi-post-echo/requirements.txt
+++ b/examples/wsgi-post-echo/requirements.txt
@@ -12,7 +12,7 @@ flask==2.3.2
     # via -r requirements.in
 itsdangerous==2.1.2
     # via flask
-jinja2==3.1.4
+jinja2==3.1.6
     # via flask
 markupsafe==2.1.3
     # via

From 3eec940887a244774abca78072e936c8f10b53dd Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 19 Feb 2026 22:00:06 -0600
Subject: [PATCH 32/55] Bump bytes from 1.10.0 to 1.11.1 in /wasm-worker (#416)

Bumps [bytes](https://github.com/tokio-rs/bytes) from 1.10.0 to 1.11.1.
- [Release notes](https://github.com/tokio-rs/bytes/releases)
- [Changelog](https://github.com/tokio-rs/bytes/blob/master/CHANGELOG.md)
- [Commits](https://github.com/tokio-rs/bytes/compare/v1.10.0...v1.11.1)

---
updated-dependencies:
- dependency-name: bytes
  dependency-version: 1.11.1
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 wasm-worker/Cargo.lock | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/wasm-worker/Cargo.lock b/wasm-worker/Cargo.lock
index 7f57b628a..1eef641bc 100644
--- a/wasm-worker/Cargo.lock
+++ b/wasm-worker/Cargo.lock
@@ -185,9 +185,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "bytes"
-version = "1.10.0"
+version = "1.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f61dac84819c6588b558454b194026eb1f09c293b9036ae9b159e74e73ab6cf9"
+checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
 
 [[package]]
 name = "cc"

From b9f7f45d4328ea2a151f49cc1e79aa3624fa3d8d Mon Sep 17 00:00:00 2001
From: Yashwanth-Ranjan-Singaravel
 <157959140+Yashwanth-Ranjan-Singaravel@users.noreply.github.com>
Date: Wed, 4 Mar 2026 20:18:00 -0600
Subject: [PATCH 33/55] Documentation for kafka (#406)

* Initial documentation for kafka triggers

* Added kafka-configured lambdas in examples/

* Point docs to example code

* Updated docs for clarity
---
 docs/worker/README.md                    |  19 ++-
 docs/worker/kafka-triggers.md            | 198 +++++++++++++++++++++++
 docs/worker/lambda-config.md             |  33 +++-
 examples/kafka-basic/f.py                |   4 +
 examples/kafka-basic/ol.yaml             |   9 ++
 examples/kafka-metadata/f.py             |  17 ++
 examples/kafka-metadata/ol.yaml          |  10 ++
 examples/kafka-metadata/requirements.in  |   2 +
 examples/kafka-metadata/requirements.txt |  24 +++
 9 files changed, 311 insertions(+), 5 deletions(-)
 create mode 100644 docs/worker/kafka-triggers.md
 create mode 100644 examples/kafka-basic/f.py
 create mode 100644 examples/kafka-basic/ol.yaml
 create mode 100644 examples/kafka-metadata/f.py
 create mode 100644 examples/kafka-metadata/ol.yaml
 create mode 100644 examples/kafka-metadata/requirements.in
 create mode 100644 examples/kafka-metadata/requirements.txt

diff --git a/docs/worker/README.md b/docs/worker/README.md
index 0ec6cc657..fa1f409f6 100644
--- a/docs/worker/README.md
+++ b/docs/worker/README.md
@@ -64,7 +64,18 @@ Major cloud offerings (like AWS lambda) offer a variety of lambda
 triggers, such as HTTP requests, queue messages, cron, DB/S3 triggers,
 etc.
 
-This has not been a focus (so far) of OpenLambda.  The only trigger is
-an HTTP request.  Thus, all the event code is in the
-github.com/open-lambda/open-lambda/ol/worker/server package.  Requests
-to http(s)://WORKER_ADDR:PORT/run/LAMBDA_NAME invoke lambdas.
+OpenLambda currently supports three types of triggers: **HTTP
+requests**, **Kafka messages**, and **cron schedules**. The event code
+is in the github.com/open-lambda/open-lambda/ol/worker/event package.
+
+**HTTP triggers:** Requests to http(s)://WORKER_ADDR:PORT/run/LAMBDA_NAME
+invoke lambdas directly.
+
+**Kafka triggers:** Lambdas can be configured to consume from Kafka
+topics. The worker runs Kafka consumers that poll for messages and
+invoke the corresponding lambda function automatically. See
+[kafka-triggers.md](kafka-triggers.md) for details.
+
+**Cron triggers:** Lambdas can be invoked on a schedule using cron
+expressions. The boss component runs a cron scheduler that
+automatically invokes lambdas at the configured times.
diff --git a/docs/worker/kafka-triggers.md b/docs/worker/kafka-triggers.md
new file mode 100644
index 000000000..bb2045ec6
--- /dev/null
+++ b/docs/worker/kafka-triggers.md
@@ -0,0 +1,198 @@
+# Kafka Triggers
+
+Lambdas can be configured to automatically consume messages from Kafka
+topics. When a message arrives, the worker invokes the lambda with the
+message payload as the request body.
+
+## Configuration
+
+Add a `kafka` section under `triggers` in your lambda's `ol.yaml`
+(see [lambda configuration](lambda-config.md) for the full `ol.yaml`
+reference):
+
+```yaml
+triggers:
+  kafka:
+    - bootstrap_servers:
+        - "localhost:9092"
+      topics:
+        - "my-topic"
+      auto_offset_reset: "latest" # or "earliest"
+```
+
+The consumer group ID is automatically set to `lambda-<name>` based on
+the lambda name and cannot be overridden. Because each lambda gets its
+own group ID, lambdas consume from Kafka independently of one another.
+Even if multiple lambdas subscribe to the same topic, each one receives
+its own copy of every message, and offset tracking is maintained
+separately per lambda.
+
+## Quick start
+
+This walkthrough starts a Kafka broker, deploys a lambda with a Kafka
+trigger, and publishes a message to verify end-to-end.
+
+### 1. Start a Kafka broker
+
+The easiest way to get a single-node broker is with Docker. The
+[apache/kafka](https://hub.docker.com/r/apache/kafka) image bundles
+KRaft mode so no separate ZooKeeper container is needed:
+
+```bash
+docker run -d --name kafka \
+  -p 9092:9092 \
+  apache/kafka:latest
+```
+
+See the [Apache Kafka quickstart](https://kafka.apache.org/quickstart)
+for more details.
+
+### 2. Create a topic
+
+```bash
+docker exec kafka \
+  /opt/kafka/bin/kafka-topics.sh --create \
+    --topic my-topic \
+    --bootstrap-server localhost:9092
+```
+
+### 3. Write the lambda
+
+Create a directory for the lambda with two files:
+
+**f.py**
+```python
+def f(event):
+    print(f"Received: {event}")
+    return {"status": "ok"}
+```
+
+**ol.yaml**
+```yaml
+triggers:
+  kafka:
+    - bootstrap_servers:
+        - "localhost:9092"
+      topics:
+        - "my-topic"
+      auto_offset_reset: "earliest"
+```
+
+Upload the lambda to the registry. When a lambda with Kafka triggers is
+uploaded, the worker automatically starts consumers for the configured
+topics — no extra registration step is needed.
+
+### 4. Publish a test message
+
+```bash
+echo '{"hello":"world"}' | docker exec -i kafka \
+  /opt/kafka/bin/kafka-console-producer.sh \
+    --topic my-topic \
+    --bootstrap-server localhost:9092
+```
+
+The worker should pick up the message and invoke your lambda. Check the
+worker logs to confirm.
+
+## How it works
+
+1. When the worker starts in `lambda` mode, it creates a `KafkaManager`
+   alongside the `LambdaServer`.
+2. When a lambda with Kafka triggers is uploaded, the boss automatically
+   registers its Kafka consumers on the worker. Consumers can also be
+   managed manually via the `/kafka/register/<lambda-name>` HTTP
+   endpoint (POST to register, DELETE to unregister).
+3. For each trigger entry, the manager creates a `LambdaKafkaConsumer`
+   backed by a [franz-go](https://github.com/twmb/franz-go) (`kgo`)
+   client.
+4. Each consumer runs a polling loop that fetches messages with a
+   1-second timeout. On receiving a message, it builds a synthetic HTTP
+   POST request and invokes the lambda directly through the
+   `LambdaManager`.
+
+## Request format
+
+When a Kafka message triggers a lambda, the worker builds a synthetic
+HTTP POST request with the Kafka message value as the body and the
+following headers:
+
+| Header              | Description                              |
+| ------------------- | ---------------------------------------- |
+| `Content-Type`      | `application/json`                       |
+| `X-Kafka-Topic`     | The topic the message was read from.     |
+| `X-Kafka-Partition` | The partition number.                    |
+| `X-Kafka-Offset`    | The message offset within the partition. |
+| `X-Kafka-Group-Id`  | The consumer group ID.                   |
+
+### Accessing Kafka metadata in your handler
+
+The default handler type (`def f(event)`) only receives the JSON-parsed
+request body as a dict. It does **not** have access to HTTP headers,
+so the Kafka metadata headers listed above will not be available.
+
+To access Kafka metadata headers, use a **WSGI** or **ASGI** entry
+point (see [lambda configuration](lambda-config.md) for how to
+configure these).
+
+## Example lambdas
+
+Complete working examples are available in the
+[examples/](../../examples/) directory:
+
+- [kafka-basic](../../examples/kafka-basic/) — Simple `f(event)` handler
+  that processes the Kafka message body.
+- [kafka-metadata](../../examples/kafka-metadata/) — Flask WSGI handler
+  that accesses Kafka metadata headers (topic, partition, offset, group
+  ID) alongside the message body.
+
+### Simple handler (body only)
+
+The default `f(event)` handler receives the Kafka message body as a
+parsed dict, but cannot access headers
+([full example](../../examples/kafka-basic/)):
+
+```python
+def f(event):
+    # event is the JSON-parsed Kafka message value
+    print(f"Received message: {event}")
+    return {"status": "ok"}
+```
+
+### WSGI handler (body + headers)
+
+A WSGI handler can access Kafka metadata via the `environ` dict.
+HTTP headers are available with an `HTTP_` prefix, uppercased, and
+with dashes replaced by underscores
+([full example](../../examples/kafka-metadata/)):
+
+```python
+from flask import Flask, request
+
+app = Flask(__name__)
+
+@app.route("/", methods=["POST"])
+def handle():
+    topic = request.headers.get("X-Kafka-Topic", "unknown")
+    partition = request.headers.get("X-Kafka-Partition", "unknown")
+    offset = request.headers.get("X-Kafka-Offset", "unknown")
+    group_id = request.headers.get("X-Kafka-Group-Id", "unknown")
+
+    body = request.get_json()
+
+    print(f"topic={topic} partition={partition} offset={offset} group={group_id}")
+    print(f"body={body}")
+
+    return {"status": "ok"}
+```
+
+## Management API
+
+The worker exposes an HTTP endpoint for managing Kafka consumers at
+runtime:
+
+- **`POST /kafka/register/<lambda-name>`** — Reads the lambda's
+  `ol.yaml` config from the registry and starts consumers for all
+  configured Kafka triggers. Any existing consumers for that lambda are
+  cleaned up first.
+- **`DELETE /kafka/register/<lambda-name>`** — Stops and removes all
+  Kafka consumers for the given lambda.
diff --git a/docs/worker/lambda-config.md b/docs/worker/lambda-config.md
index 2e613da31..c54f5981c 100644
--- a/docs/worker/lambda-config.md
+++ b/docs/worker/lambda-config.md
@@ -13,6 +13,11 @@ triggers:
   http:
     - method: PUT
     - method: PATCH
+  kafka:
+    - bootstrap_servers:
+        - "localhost:9092"
+      topics:
+        - "my-topic"
 
 environment:
   MY_ENV_VAR1: "value1"
@@ -22,7 +27,7 @@ environment:
 ## 3. Configuration Options
 
 ### a. Triggers
-OpenLambda only supports HTTP trigger for now, but future development plans include supporting other trigger types.
+OpenLambda currently supports HTTP and Kafka triggers.
 
 #### HTTP Triggers
 Defines which HTTP methods can be used to invoke the lambda.
@@ -36,6 +41,32 @@ triggers:
 ```
 In this case, the lambda accepts GET and POST requests.
 
+#### Kafka Triggers
+Defines Kafka topics the lambda should consume from. When a message
+arrives on a configured topic, the lambda is invoked with the message
+as the request body.
+
+Example:
+```yaml
+triggers:
+  kafka:
+    - bootstrap_servers:
+        - "localhost:9092"
+      topics:
+        - "my-topic"
+      auto_offset_reset: "latest"
+```
+
+| Field               | Type       | Required | Description                                                                                 |
+| ------------------- | ---------- | -------- | ------------------------------------------------------------------------------------------- |
+| `bootstrap_servers` | `[]string` | Yes      | List of Kafka broker addresses.                                                             |
+| `topics`            | `[]string` | Yes      | Topics this lambda should consume from.                                                     |
+| `auto_offset_reset` | `string`   | No       | Where to start reading if no committed offset exists. `"latest"` (default) or `"earliest"`. |
+
+A lambda can define multiple Kafka trigger entries. Each entry creates a
+separate consumer. For more details on Kafka triggers, including how to access Kafka
+metadata headers in your handler, see [kafka-triggers.md](kafka-triggers.md).
+
 ### b. Environment Variables
 Defines environment variables that will be available to the lambda function at runtime.
 
diff --git a/examples/kafka-basic/f.py b/examples/kafka-basic/f.py
new file mode 100644
index 000000000..5009c6e03
--- /dev/null
+++ b/examples/kafka-basic/f.py
@@ -0,0 +1,4 @@
+def f(event):
+    # event is the JSON-parsed Kafka message value
+    print(f"Received message: {event}")
+    return {"status": "ok"}
diff --git a/examples/kafka-basic/ol.yaml b/examples/kafka-basic/ol.yaml
new file mode 100644
index 000000000..4e00a5664
--- /dev/null
+++ b/examples/kafka-basic/ol.yaml
@@ -0,0 +1,9 @@
+triggers:
+  http:
+    - method: POST
+  kafka:
+    - bootstrap_servers:
+        - "localhost:9092"
+      topics:
+        - "my-topic"
+      auto_offset_reset: "latest"
diff --git a/examples/kafka-metadata/f.py b/examples/kafka-metadata/f.py
new file mode 100644
index 000000000..d92f6f860
--- /dev/null
+++ b/examples/kafka-metadata/f.py
@@ -0,0 +1,17 @@
+from flask import Flask, request
+
+app = Flask(__name__)
+
+@app.route("/", methods=["POST"])
+def handle():
+    topic = request.headers.get("X-Kafka-Topic", "unknown")
+    partition = request.headers.get("X-Kafka-Partition", "unknown")
+    offset = request.headers.get("X-Kafka-Offset", "unknown")
+    group_id = request.headers.get("X-Kafka-Group-Id", "unknown")
+
+    body = request.get_json()
+
+    print(f"topic={topic} partition={partition} offset={offset} group={group_id}")
+    print(f"body={body}")
+
+    return {"status": "ok"}
diff --git a/examples/kafka-metadata/ol.yaml b/examples/kafka-metadata/ol.yaml
new file mode 100644
index 000000000..319e8bb6f
--- /dev/null
+++ b/examples/kafka-metadata/ol.yaml
@@ -0,0 +1,10 @@
+triggers:
+  http:
+    - method: POST
+  kafka:
+    - bootstrap_servers:
+        - "localhost:9092"
+      topics:
+        - "my-topic"
+      auto_offset_reset: "earliest"
+
diff --git a/examples/kafka-metadata/requirements.in b/examples/kafka-metadata/requirements.in
new file mode 100644
index 000000000..b44f14a28
--- /dev/null
+++ b/examples/kafka-metadata/requirements.in
@@ -0,0 +1,2 @@
+flask==2.3.2
+werkzeug==3.0.3
diff --git a/examples/kafka-metadata/requirements.txt b/examples/kafka-metadata/requirements.txt
new file mode 100644
index 000000000..96159fc91
--- /dev/null
+++ b/examples/kafka-metadata/requirements.txt
@@ -0,0 +1,24 @@
+#
+# This file is autogenerated by pip-compile with Python 3.13
+# by the following command:
+#
+#    pip-compile requirements.in
+#
+blinker==1.6.2
+    # via flask
+click==8.1.7
+    # via flask
+flask==2.3.2
+    # via -r requirements.in
+itsdangerous==2.1.2
+    # via flask
+jinja2==3.1.4
+    # via flask
+markupsafe==2.1.3
+    # via
+    #   jinja2
+    #   werkzeug
+werkzeug==3.0.3
+    # via
+    #   -r requirements.in
+    #   flask

From 72b4f1472d298235876dfbba086482bd2245151a Mon Sep 17 00:00:00 2001
From: Tyler Caraza-Harter <tylerharter@users.noreply.github.com>
Date: Fri, 13 Mar 2026 14:27:50 -0500
Subject: [PATCH 34/55] pin rust versions (#428)

---
 .github/workflows/ci.yml  | 2 +-
 .github/workflows/pkg.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2fbc6b696..3f602147b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -52,7 +52,7 @@ jobs:
     - name: Install revive (go linter)
       run: go install github.com/mgechev/revive@v1.5.1
     - name: Install cross
-      run: cargo install cross
+      run: cargo install cross --locked
     - name: Build OpenLambda
       run: |
         make ol imgs/ol-wasm wasm-worker wasm-functions native-functions container-proxy
diff --git a/.github/workflows/pkg.yml b/.github/workflows/pkg.yml
index d4fe13723..e219a2af8 100644
--- a/.github/workflows/pkg.yml
+++ b/.github/workflows/pkg.yml
@@ -44,7 +44,7 @@ jobs:
       with:
          go-version: '>=1.21.0'
     - name: Install cross
-      run: cargo install cross
+      run: cargo install cross --locked
     - name: Build OpenLambda
       run: |
         make ol imgs/lambda wasm-worker wasm-functions native-functions container-proxy

From 692b563d0af27013acc7f27396cc973d01498403 Mon Sep 17 00:00:00 2001
From: Yashwanth-Ranjan-Singaravel
 <157959140+Yashwanth-Ranjan-Singaravel@users.noreply.github.com>
Date: Fri, 20 Mar 2026 10:26:41 -0500
Subject: [PATCH 35/55] Added unit tests for kafkaServer implementation (#357)

* Cleaned server.go

* Removed redundant goroutine

* Changes to initializing order to fix resource cleanup

* Unit tests for kafka server

* Cleaned up tests

* Cleaned up warnings in kafka unit tests

* Modified kafka unit tests to mock and test more meaningful parts of KafkaServer.go

* Simplified unit tests

* Removed redundant nil check

* Pulled complexity out of kafka tests

* Renamed to setupConsumer + added unit tests in CI

* Pulls out harness code from tests into a helper and sufficiently comments it

* Added field in lkc to track error counts. Updated error handling test to use this field

* Better commenting of runConsumeLoop

---------

Co-authored-by: RSYashwanth <yashwanthrs1104@gmail.com>
---
 .github/workflows/ci.yml            |   4 +
 go/worker/event/kafkaServer.go      |  50 +++--
 go/worker/event/kafkaServer_test.go | 331 ++++++++++++++++++++++++++++
 3 files changed, 368 insertions(+), 17 deletions(-)
 create mode 100644 go/worker/event/kafkaServer_test.go

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3f602147b..9e958cbef 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -62,6 +62,10 @@ jobs:
       working-directory: go/common
       run: go test -v
       timeout-minutes: 5
+    - name: Test Go Worker
+      working-directory: go/worker
+      run: go test -v ./...
+      timeout-minutes: 5
     - name: Test Python (SOCK)
       run: sudo env "PATH=$PATH" ./scripts/test.py --worker_type=sock
       timeout-minutes: 20
diff --git a/go/worker/event/kafkaServer.go b/go/worker/event/kafkaServer.go
index 793c421ad..b8f5cee7d 100644
--- a/go/worker/event/kafkaServer.go
+++ b/go/worker/event/kafkaServer.go
@@ -24,21 +24,37 @@ type KafkaClient interface {
 	Close()
 }
 
+// LambdaInvoker abstracts the lambda invocation layer for testability
+type LambdaInvoker interface {
+	Invoke(lambdaName string, w http.ResponseWriter, r *http.Request)
+}
+
+// lambdaMgrInvoker wraps *lambda.LambdaMgr to implement LambdaInvoker
+type lambdaMgrInvoker struct {
+	mgr *lambda.LambdaMgr
+}
+
+func (i *lambdaMgrInvoker) Invoke(lambdaName string, w http.ResponseWriter, r *http.Request) {
+	f := i.mgr.Get(lambdaName)
+	f.Invoke(w, r)
+}
+
 // LambdaKafkaConsumer manages Kafka consumption for a specific lambda function
 type LambdaKafkaConsumer struct {
-	consumerName  string // Unique name for this consumer
-	lambdaName    string // lambda function name
-	kafkaTrigger  *common.KafkaTrigger
-	client        KafkaClient       // kgo.client implements the KafkaClient interface
-	lambdaManager *lambda.LambdaMgr // Reference to lambda manager for direct calls
-	stopChan      chan struct{}     // Shutdown signal for this consumer
+	consumerName string // Unique name for this consumer
+	lambdaName   string // lambda function name
+	kafkaTrigger *common.KafkaTrigger
+	client       KafkaClient    // kgo.client implements the KafkaClient interface
+	invoker      LambdaInvoker  // Abstraction for lambda invocation
+	stopChan     chan struct{}   // Shutdown signal for this consumer
 	// When this channel is closed, the goroutine for the consumer exits
+	errorCount   int            // Number of non-timeout Kafka client errors encountered
 }
 
 // KafkaManager manages multiple lambda-specific Kafka consumers
 type KafkaManager struct {
 	lambdaConsumers map[string]*LambdaKafkaConsumer // lambdaName -> consumer
-	lambdaManager   *lambda.LambdaMgr               // Reference to lambda manager
+	invoker         LambdaInvoker                   // Abstraction for lambda invocation
 	mu              sync.Mutex                      // Protects lambdaConsumers map
 }
 
@@ -75,12 +91,12 @@ func (km *KafkaManager) newLambdaKafkaConsumer(consumerName string, lambdaName s
 	}
 
 	return &LambdaKafkaConsumer{
-		consumerName:  consumerName,
-		lambdaName:    lambdaName,
-		kafkaTrigger:  trigger,
-		client:        client,
-		lambdaManager: km.lambdaManager,
-		stopChan:      make(chan struct{}),
+		consumerName: consumerName,
+		lambdaName:   lambdaName,
+		kafkaTrigger: trigger,
+		client:       client,
+		invoker:      km.invoker,
+		stopChan:     make(chan struct{}),
 	}, nil
 }
 
@@ -88,7 +104,7 @@ func (km *KafkaManager) newLambdaKafkaConsumer(consumerName string, lambdaName s
 func NewKafkaManager(lambdaManager *lambda.LambdaMgr) (*KafkaManager, error) {
 	manager := &KafkaManager{
 		lambdaConsumers: make(map[string]*LambdaKafkaConsumer),
-		lambdaManager:   lambdaManager,
+		invoker:         &lambdaMgrInvoker{mgr: lambdaManager},
 	}
 
 	slog.Info("Kafka manager initialized")
@@ -129,6 +145,7 @@ func (lkc *LambdaKafkaConsumer) consumeLoop() {
 						continue
 					}
 
+					lkc.errorCount++
 					// TODO: Surface Kafka consumer errors to lambda developers by invoking an error
 					// handler lambda function. Could allow lambdas to specify an onError callback in
 					// ol.yaml that gets invoked with error details.
@@ -185,9 +202,8 @@ func (lkc *LambdaKafkaConsumer) processMessage(record *kgo.Record) {
 	// for kafka triggered lambda invocations.
 	w := httptest.NewRecorder()
 
-	// Get lambda function and invoke directly
-	lambdaFunc := lkc.lambdaManager.Get(lkc.lambdaName)
-	lambdaFunc.Invoke(w, req)
+	// Invoke the lambda function directly
+	lkc.invoker.Invoke(lkc.lambdaName, w, req)
 
 	// Log the result
 	slog.Info("Kafka message processed via direct invocation",
diff --git a/go/worker/event/kafkaServer_test.go b/go/worker/event/kafkaServer_test.go
new file mode 100644
index 000000000..1f5702d56
--- /dev/null
+++ b/go/worker/event/kafkaServer_test.go
@@ -0,0 +1,331 @@
+package event
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"reflect"
+	"sync"
+	"sync/atomic"
+	"testing"
+
+	"github.com/open-lambda/open-lambda/go/common"
+	"github.com/twmb/franz-go/pkg/kgo"
+)
+
+func TestMain(m *testing.M) {
+	// Initialize common.Conf so that common.T0/T1 (latency tracking) doesn't panic
+	common.Conf = &common.Config{}
+	os.Exit(m.Run())
+}
+
+// --- Mocks ---
+
+// MockKafkaClient implements KafkaClient for testing.
+//
+// Instead of exposing pollFetchesFunc directly, tests enqueue responses via
+// Send and SendError. The mock serves them in FIFO order and returns empty
+// fetches once the queue is drained. This keeps polling/sequencing logic out
+// of individual tests.
+//
+// The Drained channel (when set) is closed the first time PollFetches is called
+// after the queue is empty. Because the consume loop calls PollFetches only
+// after finishing the previous iteration's processing, a receive on Drained
+// guarantees all enqueued records have been fully processed.
+type MockKafkaClient struct {
+	mu              sync.Mutex
+	queue           []kgo.Fetches
+	callCount       int
+	closeCalled     atomic.Bool
+	Drained         chan struct{} // closed when all queued fetches have been consumed and processed
+	drainedSignaled bool
+}
+
+// Send enqueues records that will be returned by the next PollFetches call.
+func (m *MockKafkaClient) Send(records ...*kgo.Record) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.queue = append(m.queue, makeFetches(records...))
+}
+
+// SendError enqueues a fetch error that will be returned by the next PollFetches call.
+func (m *MockKafkaClient) SendError(topic string, partition int32, err error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.queue = append(m.queue, makeErrorFetches(topic, partition, err))
+}
+
+// PollFetches returns the next queued fetch, or empty fetches if the queue is
+// drained. When the queue is empty and Drained is set, it closes Drained to
+// signal that all prior records have been processed.
+func (m *MockKafkaClient) PollFetches(ctx context.Context) kgo.Fetches {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.callCount < len(m.queue) {
+		f := m.queue[m.callCount]
+		m.callCount++
+		return f
+	}
+	if !m.drainedSignaled && m.Drained != nil {
+		close(m.Drained)
+		m.drainedSignaled = true
+	}
+	return kgo.Fetches{}
+}
+
+func (m *MockKafkaClient) Close() {
+	m.closeCalled.Store(true)
+}
+
+// MockLambdaInvoker implements LambdaInvoker for testing.
+type MockLambdaInvoker struct {
+	mu          sync.Mutex
+	invocations []invokeRecord
+}
+
+// invokeRecord captures the relevant fields from a lambda invocation in simple,
+// comparable types. Tests can build an expected invokeRecord and compare it
+// directly with reflect.DeepEqual instead of asserting each field individually.
+type invokeRecord struct {
+	LambdaName string
+	Method     string
+	Path       string
+	RequestURI string
+	Body       string
+	Headers    map[string]string
+}
+
+func (m *MockLambdaInvoker) Invoke(lambdaName string, w http.ResponseWriter, r *http.Request) {
+	body, _ := io.ReadAll(r.Body)
+
+	// Flatten headers into a simple map for easy comparison in assertions
+	headers := map[string]string{}
+	for key := range r.Header {
+		headers[key] = r.Header.Get(key)
+	}
+
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.invocations = append(m.invocations, invokeRecord{
+		LambdaName: lambdaName,
+		Method:     r.Method,
+		Path:       r.URL.Path,
+		RequestURI: r.RequestURI,
+		Body:       string(body),
+		Headers:    headers,
+	})
+	w.WriteHeader(http.StatusOK)
+}
+
+func (m *MockLambdaInvoker) getInvocations() []invokeRecord {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	cp := make([]invokeRecord, len(m.invocations))
+	copy(cp, m.invocations)
+	return cp
+}
+
+// --- Helpers ---
+
+// makeFetches converts flat kgo.Records into the nested kgo.Fetches structure.
+//
+// franz-go's PollFetches returns a deeply nested type that mirrors how Kafka
+// brokers organize data:
+//
+//	Fetches -> []Fetch -> []FetchTopic -> []FetchPartition -> []*Record
+//
+// Records are grouped by topic and then by partition. This helper handles that
+// grouping automatically so tests can think in terms of simple records rather
+// than the broker-level wire format.
+func makeFetches(records ...*kgo.Record) kgo.Fetches {
+	if len(records) == 0 {
+		return kgo.Fetches{}
+	}
+	// Group records by topic+partition
+	type key struct {
+		topic     string
+		partition int32
+	}
+	groups := map[key][]*kgo.Record{}
+	for _, r := range records {
+		k := key{r.Topic, r.Partition}
+		groups[k] = append(groups[k], r)
+	}
+
+	topicMap := map[string][]kgo.FetchPartition{}
+	for k, recs := range groups {
+		topicMap[k.topic] = append(topicMap[k.topic], kgo.FetchPartition{
+			Partition: k.partition,
+			Records:   recs,
+		})
+	}
+
+	var topics []kgo.FetchTopic
+	for topic, partitions := range topicMap {
+		topics = append(topics, kgo.FetchTopic{
+			Topic:      topic,
+			Partitions: partitions,
+		})
+	}
+	return kgo.Fetches{{Topics: topics}}
+}
+
+func makeErrorFetches(topic string, partition int32, err error) kgo.Fetches {
+	return kgo.Fetches{{
+		Topics: []kgo.FetchTopic{{
+			Topic: topic,
+			Partitions: []kgo.FetchPartition{{
+				Partition: partition,
+				Err:       err,
+			}},
+		}},
+	}}
+}
+
+// setupConsumerHarness creates the full test harness for exercising the consumer's
+// consumeLoop. It mocks both sides of the consumer:
+//
+//   - Above the consumer (Kafka broker layer): MockKafkaClient replaces the real
+//     Kafka connection so tests can enqueue records and errors without a broker.
+//   - Below the consumer (lambda invocation layer): MockLambdaInvoker replaces the
+//     real lambda invocation path so tests can capture and assert on HTTP requests.
+//
+// The consumer itself is real — it runs the actual consumeLoop logic, so tests
+// exercise the full record-processing and error-handling pipeline.
+func setupConsumerHarness(lambdaName string) (*MockKafkaClient, *MockLambdaInvoker, *LambdaKafkaConsumer) {
+	// Mock above: fake Kafka broker
+	client := &MockKafkaClient{Drained: make(chan struct{})}
+	// Mock below: fake lambda invocation
+	invoker := &MockLambdaInvoker{}
+
+	consumer := &LambdaKafkaConsumer{
+		consumerName: lambdaName + "-0",
+		lambdaName:   lambdaName,
+		kafkaTrigger: &common.KafkaTrigger{GroupId: "lambda-" + lambdaName},
+		client:       client,
+		invoker:      invoker,
+		stopChan:     make(chan struct{}),
+	}
+	return client, invoker, consumer
+}
+
+// runConsumeLoop starts consumeLoop in a goroutine and returns a stop function
+// that signals shutdown and waits for the goroutine to exit. Callers should
+// <-mockClient.Drained before stop() — Drained closes once all enqueued
+// records have been fully processed, making it safe to assert on results.
+func runConsumeLoop(consumer *LambdaKafkaConsumer) (stop func()) {
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		consumer.consumeLoop()
+	}()
+	return func() {
+		close(consumer.stopChan)
+		wg.Wait()
+	}
+}
+
+// --- Tests ---
+
+func TestConsumeLoop_ProcessesRecords(t *testing.T) {
+	mockClient, invoker, consumer := setupConsumerHarness("my-lambda")
+	mockClient.Send(&kgo.Record{
+		Topic: "orders", Partition: 3, Offset: 99,
+		Value: []byte(`{"orderId": 42}`),
+	})
+	stop := runConsumeLoop(consumer)
+	<-mockClient.Drained
+	stop()
+
+	invocations := invoker.getInvocations()
+	if len(invocations) != 1 {
+		t.Fatalf("Expected 1 invocation, got %d", len(invocations))
+	}
+
+	expected := invokeRecord{
+		LambdaName: "my-lambda",
+		Method:     "POST",
+		Path:       "/run/my-lambda/",
+		RequestURI: "/run/my-lambda/",
+		Body:       `{"orderId": 42}`,
+		Headers: map[string]string{
+			"Content-Type":      "application/json",
+			"X-Kafka-Topic":     "orders",
+			"X-Kafka-Partition": "3",
+			"X-Kafka-Offset":    "99",
+			"X-Kafka-Group-Id":  "lambda-my-lambda",
+		},
+	}
+	if !reflect.DeepEqual(invocations[0], expected) {
+		t.Errorf("Invocation mismatch:\n  got:  %+v\n  want: %+v", invocations[0], expected)
+	}
+}
+
+func TestConsumeLoop_ContinuesThroughErrors(t *testing.T) {
+	mockClient, invoker, consumer := setupConsumerHarness("test-lambda")
+	// Poll sequence: deadline-exceeded errors (silently skipped), then a real
+	// error (counted), then a valid record. The loop should survive all of them.
+	mockClient.SendError("topic", 0, context.DeadlineExceeded)
+	mockClient.SendError("topic", 0, context.DeadlineExceeded)
+	mockClient.SendError("topic", 0, fmt.Errorf("broker unreachable"))
+	mockClient.Send(&kgo.Record{
+		Topic: "topic", Partition: 0, Offset: 1, Value: []byte("survived"),
+	})
+	stop := runConsumeLoop(consumer)
+	<-mockClient.Drained
+	stop()
+
+	invocations := invoker.getInvocations()
+	if len(invocations) != 1 {
+		t.Fatalf("Expected 1 invocation after errors, got %d", len(invocations))
+	}
+
+	expected := invokeRecord{
+		LambdaName: "test-lambda",
+		Method:     "POST",
+		Path:       "/run/test-lambda/",
+		RequestURI: "/run/test-lambda/",
+		Body:       "survived",
+		Headers: map[string]string{
+			"Content-Type":      "application/json",
+			"X-Kafka-Topic":     "topic",
+			"X-Kafka-Partition": "0",
+			"X-Kafka-Offset":    "1",
+			"X-Kafka-Group-Id":  "lambda-test-lambda",
+		},
+	}
+	if !reflect.DeepEqual(invocations[0], expected) {
+		t.Errorf("Invocation mismatch:\n  got:  %+v\n  want: %+v", invocations[0], expected)
+	}
+
+	// Only real errors should be counted; DeadlineExceeded should be ignored
+	if consumer.errorCount != 1 {
+		t.Errorf("Expected 1 error counted, got %d", consumer.errorCount)
+	}
+}
+
+func TestUnregister(t *testing.T) {
+	manager := &KafkaManager{
+		lambdaConsumers: make(map[string]*LambdaKafkaConsumer),
+	}
+
+	mockClient := &MockKafkaClient{}
+	manager.lambdaConsumers["test-lambda-0"] = &LambdaKafkaConsumer{
+		consumerName: "test-lambda-0",
+		lambdaName:   "test-lambda",
+		client:       mockClient,
+		stopChan:     make(chan struct{}),
+	}
+
+	manager.UnregisterLambdaKafkaTriggers("test-lambda")
+
+	if len(manager.lambdaConsumers) != 0 {
+		t.Errorf("Expected 0 consumers, got %d", len(manager.lambdaConsumers))
+	}
+	if !mockClient.closeCalled.Load() {
+		t.Error("Expected Close to be called on client")
+	}
+}

From a388417de89910a5e3f57385a68449622ce05e4f Mon Sep 17 00:00:00 2001
From: anhdduong <154901110+anhdduong@users.noreply.github.com>
Date: Sun, 22 Mar 2026 13:49:02 -0500
Subject: [PATCH 36/55] Add per-lambda reuse-sandbox flag for fresh sandboxes
 (#405)

* Add per-lambda reuse-sandbox flag for fresh sandboxes

* Update reuseSandbox default value logic by pre-initializing config struct

* remove redundant sandbox cleanup in LambdaInstance.Task()

* Add tests for reuse-sandbox flag behavior

* Unify sandbox destruction two cases and remove extra checks to panic on invalid reuse state

* Fix CI failure

* Fix comment format

---------

Co-authored-by: Anh Duc Duong <adduong@wisc.edu>
---
 docs/worker/lambda-config.md       | 19 ++++++++++++
 go/common/lambdaConfig.go          | 20 ++++++------
 go/common/lambdaConfig_test.go     | 49 ++++++++++++++++++++++++++++++
 go/worker/lambda/lambdaInstance.go | 13 +++++++-
 scripts/boss_test.py               |  3 +-
 5 files changed, 93 insertions(+), 11 deletions(-)

diff --git a/docs/worker/lambda-config.md b/docs/worker/lambda-config.md
index c54f5981c..63462a454 100644
--- a/docs/worker/lambda-config.md
+++ b/docs/worker/lambda-config.md
@@ -100,6 +100,25 @@ With this configuration:
 
 This is useful when you want to use conventional naming (e.g., `app.py` for Flask applications) or integrate existing code without renaming files.
 
+### d. Sandbox Reuse
+
+#### reuse_sandbox
+By default, OpenLambda reuses the same sandbox across multiple invocations of a lambda function to improve performance. In some cases, such as when strict isolation is required or when avoiding state persistence between invocations, it may be desirable to create a fresh sandbox for each invocation.
+
+This behavior can be controlled using the reuse-sandbox option.
+
+Example:
+```yaml
+reuse-sandbox: false
+```
+
+With this configuration:
+
+- A new sandbox is created for each lambda invocation
+- The sandbox is destroyed after the invocation completes
+
+If reuse-sandbox is not specified, OpenLambda defaults to reusing sandboxes across invocations.
+
 ## 4. How to Use
 ### a. Define Configuration
 Create an `ol.yaml` file inside the lambda function directory with the desired configuration.
diff --git a/go/common/lambdaConfig.go b/go/common/lambdaConfig.go
index c3029960b..4ce23bd93 100644
--- a/go/common/lambdaConfig.go
+++ b/go/common/lambdaConfig.go
@@ -43,8 +43,9 @@ type KafkaTrigger struct {
 
 // LambdaConfig defines the overall configuration for the lambda function.
 type LambdaConfig struct {
-	Triggers    Triggers          `yaml:"triggers"`    // List of HTTP triggers
-	Environment map[string]string `yaml:"environment"` // Environment variables for the lambda
+	Triggers     Triggers          `yaml:"triggers"`      // List of HTTP triggers
+	Environment  map[string]string `yaml:"environment"`   // Environment variables for the lambda
+	ReuseSandbox bool              `yaml:"reuse-sandbox"` // if true, sandbox is reused across invocations
 	// Additional configurations can be added here.
 }
 
@@ -56,7 +57,8 @@ func LoadDefaultLambdaConfig() *LambdaConfig {
 				{Method: "*"}, // Default to allow all methods
 			},
 		},
-		Environment: make(map[string]string),
+		Environment:  make(map[string]string),
+		ReuseSandbox: true,
 	}
 }
 
@@ -120,15 +122,15 @@ func LoadLambdaConfig(codeDir string) (*LambdaConfig, error) {
 	}
 	defer file.Close()
 
-	var config LambdaConfig
+	config := LoadDefaultLambdaConfig()
 
 	decoder := yaml.NewDecoder(file)
-	err = decoder.Decode(&config) // Use LambdaConf instead of Conf
+	err = decoder.Decode(config) // Use LambdaConf instead of Conf
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse YAML file: %v", err)
 	}
 
-	return &config, checkLambdaConfig(&config)
+	return config, checkLambdaConfig(config)
 }
 
 func ExtractConfigFromTarGz(tarPath string) (*LambdaConfig, error) {
@@ -159,12 +161,12 @@ func ExtractConfigFromTarGz(tarPath string) (*LambdaConfig, error) {
 		// and ./ matches (./ol.yaml) as tar can encode ./ into filenames under
 		// certain conditions.
 		if filepath.Clean(header.Name) == LambdaConfigFilename {
-			var config LambdaConfig
+			config := LoadDefaultLambdaConfig()
 			decoder := yaml.NewDecoder(tr)
-			if err := decoder.Decode(&config); err != nil {
+			if err := decoder.Decode(config); err != nil {
 				return nil, fmt.Errorf("failed to parse %s: %w", LambdaConfigFilename, err)
 			}
-			return &config, checkLambdaConfig(&config)
+			return config, checkLambdaConfig(config)
 		}
 	}
 
diff --git a/go/common/lambdaConfig_test.go b/go/common/lambdaConfig_test.go
index ea776c370..843d3b30b 100644
--- a/go/common/lambdaConfig_test.go
+++ b/go/common/lambdaConfig_test.go
@@ -9,6 +9,55 @@ import (
 	"testing"
 )
 
+// TestReuseSandbox verifies that the reuse-sandbox field defaults to true
+// when not specified, and correctly reflects the value when explicitly set.
+func TestReuseSandbox(t *testing.T) {
+	tests := []struct {
+		name     string
+		yaml     string
+		expected bool
+	}{
+		{
+			name:     "no ol.yaml — defaults to true",
+			yaml:     "",
+			expected: true,
+		},
+		{
+			name:     "ol.yaml present but reuse-sandbox is not specified - defaults to true",
+			yaml:     "triggers:\n  http:\n    - method: \"GET\"\n",
+			expected: true,
+		},
+		{
+			name:     "reuse-sandbox explicitly set to false",
+			yaml:     "reuse-sandbox: false\n",
+			expected: false,
+		},
+		{
+			name:     "reuse-sandbox explicitly set to true",
+			yaml:     "reuse-sandbox: true\n",
+			expected: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			dir := t.TempDir()
+			if tt.yaml != "" {
+				if err := os.WriteFile(filepath.Join(dir, "ol.yaml"), []byte(tt.yaml), 0644); err != nil {
+					t.Fatal(err)
+				}
+			}
+			config, err := LoadLambdaConfig(dir)
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+			if config.ReuseSandbox != tt.expected {
+				t.Errorf("expected ReuseSandbox=%v, got %v", tt.expected, config.ReuseSandbox)
+			}
+		})
+	}
+}
+
 // createTestTarGz creates a tar.gz file in memory with the given files
 func createTestTarGz(t *testing.T, files map[string]string) []byte {
 	var buf bytes.Buffer
diff --git a/go/worker/lambda/lambdaInstance.go b/go/worker/lambda/lambdaInstance.go
index 36eb0cc3d..b1e4c3783 100644
--- a/go/worker/lambda/lambdaInstance.go
+++ b/go/worker/lambda/lambdaInstance.go
@@ -78,6 +78,8 @@ func (linst *LambdaInstance) Task() {
 			return
 		}
 
+		reuse := linst.meta.Config.ReuseSandbox
+
 		t := common.T0("LambdaInstance-WaitSandbox")
 		// if we have a sandbox, try unpausing it to see if it is still alive
 		if sb != nil {
@@ -192,6 +194,12 @@ func (linst *LambdaInstance) Task() {
 			}
 			f.doneChan <- req
 
+			// If reuse is disabled, destroy the sandbox after invocation.
+			if !reuse && sb != nil {
+				sb.Destroy("reuse-sandbox disabled: destroying sandbox after invocation")
+				sb = nil
+			}
+
 			// check whether we should shutdown (non-blocking)
 			select {
 			case killed := <-linst.killChan:
@@ -230,8 +238,11 @@ func (linst *LambdaInstance) Task() {
 		}
 
 		if sb != nil {
+			if !reuse {
+				panic("sb should be nil when reuse is disabled")
+			}
 			if err := sb.Pause(); err != nil {
-				f.printf("discard sandbox %s due to Pause error: %v", sb.ID(), err)
+				f.printf("Discard sandbox %s due to Pause error: %v", sb.ID(), err)
 				sb = nil
 			}
 		}
diff --git a/scripts/boss_test.py b/scripts/boss_test.py
index 60366666e..56454e69b 100644
--- a/scripts/boss_test.py
+++ b/scripts/boss_test.py
@@ -135,7 +135,8 @@ def verify_lambda_config(lambda_name):
             "Cron": None,
             "Kafka": None,
         },
-        "Environment": None,
+        "Environment": {},
+        "ReuseSandbox": True,
     }
     assert actual_config == expected_config, (
         f"Lambda config mismatch!\nExpected: {expected_config}\nActual: {actual_config}"

From 1a54ec1c08f3460604295e97db19e5b5021a5a2b Mon Sep 17 00:00:00 2001
From: Yashwanth-Ranjan-Singaravel
 <157959140+Yashwanth-Ranjan-Singaravel@users.noreply.github.com>
Date: Wed, 25 Mar 2026 10:36:39 -0500
Subject: [PATCH 37/55] Simple kafka caching (#431)

* Cleaned server.go

* Removed redundant goroutine

* Changes to initializing order to fix resource cleanup

* Unit tests for kafka server

* Cleaned up tests

* Cleaned up warnings in kafka unit tests

* Modified kafka unit tests to mock and test more meaningful parts of KafkaServer.go

* Simplified unit tests

* Removed redundant nil check

* Pulled complexity out of kafka tests

* Renamed to setupConsumer + added unit tests in CI

* Pulls out harness code from tests into a helper and sufficiently comments it

* Added field in lkc to track error counts. Updated error handling test to use this field

* Better commenting of runConsumeLoop

* Dumb cache implementation

* Added offset seeks to underlying

* Moved cached client implementation to it's own file

* Removed redundant interface fields

---------

Co-authored-by: RSYashwanth <yashwanthrs1104@gmail.com>
---
 go/worker/event/cachedKafkaClient.go | 128 +++++++++++++++++++++++++
 go/worker/event/kafkaServer.go       |  73 ++++++++++++---
 go/worker/event/kafkaServer_test.go  | 135 ++++++++++++++++++++++++++-
 3 files changed, 320 insertions(+), 16 deletions(-)
 create mode 100644 go/worker/event/cachedKafkaClient.go

diff --git a/go/worker/event/cachedKafkaClient.go b/go/worker/event/cachedKafkaClient.go
new file mode 100644
index 000000000..61d72ffc6
--- /dev/null
+++ b/go/worker/event/cachedKafkaClient.go
@@ -0,0 +1,128 @@
+package event
+
+import (
+	"context"
+	"log/slog"
+
+	"github.com/twmb/franz-go/pkg/kgo"
+)
+
+// cacheKey uniquely identifies a Kafka record by its topic, partition, and offset.
+type cacheKey struct {
+	topic     string
+	partition int32
+	offset    int64
+}
+
+// seekState tracks the current seek position when replaying from cache.
+type seekState struct {
+	topic     string
+	partition int32
+	offset    int64 // next offset to serve from cache
+}
+
+// seekRequest is returned by processMessage when the lambda requests a seek.
+type seekRequest struct {
+	offset int64
+}
+
+const defaultCacheSize = 1024
+
+// cachedKafkaClient wraps a KafkaClient and caches records in an LRU map keyed
+// by {topic, partition, offset}. When a seek is active, PollFetches serves
+// records from the cache. On cache miss, it calls Seek on the underlying
+// client so the next poll fetches from the right position.
+type cachedKafkaClient struct {
+	underlying KafkaClient
+	cache      map[cacheKey]*kgo.Record
+	evictOrder []cacheKey // front = least recently used
+	maxSize    int
+	seekTarget *seekState
+}
+
+func newCachedKafkaClient(underlying KafkaClient, maxSize int) *cachedKafkaClient {
+	return &cachedKafkaClient{
+		underlying: underlying,
+		cache:      make(map[cacheKey]*kgo.Record),
+		maxSize:    maxSize,
+	}
+}
+
+// Seek sets the seek target so that subsequent PollFetches calls serve from cache.
+func (c *cachedKafkaClient) Seek(topic string, partition int32, offset int64) {
+	c.seekTarget = &seekState{topic: topic, partition: partition, offset: offset}
+}
+
+// PollFetches serves from cache when seeking, otherwise delegates to the underlying client.
+func (c *cachedKafkaClient) PollFetches(ctx context.Context) kgo.Fetches {
+	if c.seekTarget != nil {
+		key := cacheKey{
+			topic:     c.seekTarget.topic,
+			partition: c.seekTarget.partition,
+			offset:    c.seekTarget.offset,
+		}
+		record, ok := c.cache[key]
+		if ok {
+			c.touchLRU(key)
+			c.seekTarget.offset++
+			return makeSingleRecordFetches(record)
+		}
+		// Cache miss — tell the underlying client to fetch from this offset.
+		// The next normal PollFetches will get records starting here.
+		slog.Info("Seek cache miss, setting offset on underlying client",
+			"topic", c.seekTarget.topic,
+			"partition", c.seekTarget.partition,
+			"offset", c.seekTarget.offset)
+		c.underlying.Seek(c.seekTarget.topic, c.seekTarget.partition, c.seekTarget.offset)
+		c.seekTarget = nil
+	}
+
+	fetches := c.underlying.PollFetches(ctx)
+	fetches.EachRecord(func(record *kgo.Record) {
+		c.put(cacheKey{topic: record.Topic, partition: record.Partition, offset: record.Offset}, record)
+	})
+	return fetches
+}
+
+func (c *cachedKafkaClient) Close() {
+	c.underlying.Close()
+}
+
+// put adds a record to the cache, evicting the LRU entry if at capacity.
+func (c *cachedKafkaClient) put(key cacheKey, record *kgo.Record) {
+	if _, exists := c.cache[key]; exists {
+		c.touchLRU(key)
+		return
+	}
+	if len(c.cache) >= c.maxSize {
+		evictKey := c.evictOrder[0]
+		c.evictOrder = c.evictOrder[1:]
+		delete(c.cache, evictKey)
+	}
+	c.cache[key] = record
+	c.evictOrder = append(c.evictOrder, key)
+}
+
+// touchLRU moves a key to the back of the eviction order (most recently used).
+func (c *cachedKafkaClient) touchLRU(key cacheKey) {
+	for i, k := range c.evictOrder {
+		if k == key {
+			c.evictOrder = append(c.evictOrder[:i], c.evictOrder[i+1:]...)
+			c.evictOrder = append(c.evictOrder, key)
+			return
+		}
+	}
+}
+
+// makeSingleRecordFetches wraps a single record into the kgo.Fetches structure.
+func makeSingleRecordFetches(record *kgo.Record) kgo.Fetches {
+	return kgo.Fetches{{
+		Topics: []kgo.FetchTopic{{
+			Topic: record.Topic,
+			Partitions: []kgo.FetchPartition{{
+				Partition: record.Partition,
+				Records:   []*kgo.Record{record},
+			}},
+		}},
+	}}
+}
diff --git a/go/worker/event/kafkaServer.go b/go/worker/event/kafkaServer.go
index b8f5cee7d..22d9c8c7a 100644
--- a/go/worker/event/kafkaServer.go
+++ b/go/worker/event/kafkaServer.go
@@ -9,6 +9,7 @@ import (
 	"log/slog"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -21,9 +22,29 @@ import (
 
 type KafkaClient interface {
 	PollFetches(context.Context) kgo.Fetches
+	Seek(topic string, partition int32, offset int64)
 	Close()
 }
 
+// kgoClientWrapper wraps *kgo.Client to implement KafkaClient.
+type kgoClientWrapper struct {
+	client *kgo.Client
+}
+
+func (w *kgoClientWrapper) PollFetches(ctx context.Context) kgo.Fetches {
+	return w.client.PollFetches(ctx)
+}
+
+func (w *kgoClientWrapper) Seek(topic string, partition int32, offset int64) {
+	w.client.SetOffsets(map[string]map[int32]kgo.EpochOffset{
+		topic: {partition: {Offset: offset}},
+	})
+}
+
+func (w *kgoClientWrapper) Close() {
+	w.client.Close()
+}
+
 // LambdaInvoker abstracts the lambda invocation layer for testability
 type LambdaInvoker interface {
 	Invoke(lambdaName string, w http.ResponseWriter, r *http.Request)
@@ -44,11 +65,11 @@ type LambdaKafkaConsumer struct {
 	consumerName string // Unique name for this consumer
 	lambdaName   string // lambda function name
 	kafkaTrigger *common.KafkaTrigger
-	client       KafkaClient    // kgo.client implements the KafkaClient interface
-	invoker      LambdaInvoker  // Abstraction for lambda invocation
-	stopChan     chan struct{}   // Shutdown signal for this consumer
+	client       KafkaClient          // used for PollFetches/Close/Seek
+	invoker      LambdaInvoker        // Abstraction for lambda invocation
+	stopChan     chan struct{}         // Shutdown signal for this consumer
 	// When this channel is closed, the goroutine for the consumer exits
-	errorCount   int            // Number of non-timeout Kafka client errors encountered
+	errorCount   int                  // Number of non-timeout Kafka client errors encountered
 }
 
 // KafkaManager manages multiple lambda-specific Kafka consumers
@@ -90,11 +111,12 @@ func (km *KafkaManager) newLambdaKafkaConsumer(consumerName string, lambdaName s
 		return nil, fmt.Errorf("failed to create Kafka client for lambda %s: %w", lambdaName, err)
 	}
 
+	cached := newCachedKafkaClient(&kgoClientWrapper{client: client}, defaultCacheSize)
 	return &LambdaKafkaConsumer{
 		consumerName: consumerName,
 		lambdaName:   lambdaName,
 		kafkaTrigger: trigger,
-		client:       client,
+		client:       cached,
 		invoker:      km.invoker,
 		stopChan:     make(chan struct{}),
 	}, nil
@@ -156,8 +178,9 @@ func (lkc *LambdaKafkaConsumer) consumeLoop() {
 				continue
 			}
 
-			// Process each record
-			fetches.EachRecord(func(record *kgo.Record) {
+			// Process each record. Manual iteration (instead of EachRecord) lets
+			// us break out mid-batch when a seek is requested.
+			for _, record := range fetches.Records() {
 				slog.Info("Received Kafka message for lambda",
 					"consumer", lkc.consumerName,
 					"lambda", lkc.lambdaName,
@@ -165,14 +188,18 @@ func (lkc *LambdaKafkaConsumer) consumeLoop() {
 					"partition", record.Partition,
 					"offset", record.Offset,
 					"size", len(record.Value))
-				lkc.processMessage(record)
-			})
+				if seek := lkc.processMessage(record); seek != nil {
+					lkc.client.Seek(record.Topic, record.Partition, seek.offset)
+					break // next PollFetches will serve from cache
+				}
+			}
 		}
 	}
 }
 
-// processMessage handles a single Kafka message by invoking the lambda function directly
-func (lkc *LambdaKafkaConsumer) processMessage(record *kgo.Record) {
+// processMessage handles a single Kafka message by invoking the lambda function directly.
+// If the lambda returns an X-Kafka-Seek-Offset header, the corresponding seekRequest is returned.
+func (lkc *LambdaKafkaConsumer) processMessage(record *kgo.Record) *seekRequest {
 	t := common.T0("kafka-message-processing")
 	defer t.T1()
 
@@ -185,7 +212,7 @@ func (lkc *LambdaKafkaConsumer) processMessage(record *kgo.Record) {
 			"lambda", lkc.lambdaName,
 			"error", err,
 			"topic", record.Topic)
-		return
+		return nil
 	}
 	// RequestURI must be set explicitly for synthetic requests (http.NewRequest doesn't set it)
 	req.RequestURI = requestPath
@@ -198,8 +225,6 @@ func (lkc *LambdaKafkaConsumer) processMessage(record *kgo.Record) {
 	req.Header.Set("X-Kafka-Group-Id", lkc.kafkaTrigger.GroupId)
 
 	// Create response recorder to capture lambda output.
-	// TODO: Capture and log the lambda response body using httptest's response recorder
-	// for kafka triggered lambda invocations.
 	w := httptest.NewRecorder()
 
 	// Invoke the lambda function directly
@@ -213,6 +238,26 @@ func (lkc *LambdaKafkaConsumer) processMessage(record *kgo.Record) {
 		"partition", record.Partition,
 		"offset", record.Offset,
 		"status", w.Code)
+
+	// Check if the lambda requested a seek via response header
+	if seekStr := w.Header().Get("X-Kafka-Seek-Offset"); seekStr != "" {
+		seekOffset, err := strconv.ParseInt(seekStr, 10, 64)
+		if err != nil {
+			slog.Warn("Invalid X-Kafka-Seek-Offset header",
+				"lambda", lkc.lambdaName,
+				"value", seekStr,
+				"error", err)
+			return nil
+		}
+		slog.Info("Lambda requested seek",
+			"lambda", lkc.lambdaName,
+			"topic", record.Topic,
+			"partition", record.Partition,
+			"current_offset", record.Offset,
+			"seek_offset", seekOffset)
+		return &seekRequest{offset: seekOffset}
+	}
+	return nil
 }
 
 // cleanup closes the kgo client
diff --git a/go/worker/event/kafkaServer_test.go b/go/worker/event/kafkaServer_test.go
index 1f5702d56..0b0a414cc 100644
--- a/go/worker/event/kafkaServer_test.go
+++ b/go/worker/event/kafkaServer_test.go
@@ -75,14 +75,22 @@ func (m *MockKafkaClient) PollFetches(ctx context.Context) kgo.Fetches {
 	return kgo.Fetches{}
 }
 
+func (m *MockKafkaClient) Seek(topic string, partition int32, offset int64) {
+	// No-op for mock — tests control what PollFetches returns via Send/SendError
+}
+
 func (m *MockKafkaClient) Close() {
 	m.closeCalled.Store(true)
 }
 
 // MockLambdaInvoker implements LambdaInvoker for testing.
+// When respondFunc is set, it is called with the ResponseWriter and invocation
+// index instead of the default w.WriteHeader(200). This lets tests simulate
+// custom response headers (e.g., X-Kafka-Seek-Offset) on specific invocations.
 type MockLambdaInvoker struct {
 	mu          sync.Mutex
 	invocations []invokeRecord
+	respondFunc func(w http.ResponseWriter, invocationIndex int)
 }
 
 // invokeRecord captures the relevant fields from a lambda invocation in simple,
@@ -107,7 +115,7 @@ func (m *MockLambdaInvoker) Invoke(lambdaName string, w http.ResponseWriter, r *
 	}
 
 	m.mu.Lock()
-	defer m.mu.Unlock()
+	idx := len(m.invocations)
 	m.invocations = append(m.invocations, invokeRecord{
 		LambdaName: lambdaName,
 		Method:     r.Method,
@@ -116,7 +124,14 @@ func (m *MockLambdaInvoker) Invoke(lambdaName string, w http.ResponseWriter, r *
 		Body:       string(body),
 		Headers:    headers,
 	})
-	w.WriteHeader(http.StatusOK)
+	respondFunc := m.respondFunc
+	m.mu.Unlock()
+
+	if respondFunc != nil {
+		respondFunc(w, idx)
+	} else {
+		w.WriteHeader(http.StatusOK)
+	}
 }
 
 func (m *MockLambdaInvoker) getInvocations() []invokeRecord {
@@ -329,3 +344,119 @@ func TestUnregister(t *testing.T) {
 		t.Error("Expected Close to be called on client")
 	}
 }
+
+// --- cachedKafkaClient unit tests ---
+
+func TestCachedClient_CachesRecords(t *testing.T) {
+	mock := &MockKafkaClient{Drained: make(chan struct{})}
+	mock.Send(
+		&kgo.Record{Topic: "t", Partition: 0, Offset: 0, Value: []byte("a")},
+		&kgo.Record{Topic: "t", Partition: 0, Offset: 1, Value: []byte("b")},
+	)
+
+	cached := newCachedKafkaClient(mock, 100)
+	cached.PollFetches(context.Background())
+
+	// Both records should now be in the cache
+	if _, ok := cached.cache[cacheKey{"t", 0, 0}]; !ok {
+		t.Error("Expected offset 0 to be cached")
+	}
+	if _, ok := cached.cache[cacheKey{"t", 0, 1}]; !ok {
+		t.Error("Expected offset 1 to be cached")
+	}
+}
+
+func TestCachedClient_SeekCacheHit(t *testing.T) {
+	mock := &MockKafkaClient{Drained: make(chan struct{})}
+	mock.Send(
+		&kgo.Record{Topic: "t", Partition: 0, Offset: 10, Value: []byte("ten")},
+		&kgo.Record{Topic: "t", Partition: 0, Offset: 11, Value: []byte("eleven")},
+		&kgo.Record{Topic: "t", Partition: 0, Offset: 12, Value: []byte("twelve")},
+	)
+
+	cached := newCachedKafkaClient(mock, 100)
+	// Populate the cache
+	cached.PollFetches(context.Background())
+
+	// Seek to offset 10
+	cached.Seek("t", 0, 10)
+
+	// Each PollFetches should return the next cached record
+	f1 := cached.PollFetches(context.Background())
+	records1 := f1.Records()
+	if len(records1) != 1 || records1[0].Offset != 10 {
+		t.Fatalf("Expected offset 10, got %v", records1)
+	}
+
+	f2 := cached.PollFetches(context.Background())
+	records2 := f2.Records()
+	if len(records2) != 1 || records2[0].Offset != 11 {
+		t.Fatalf("Expected offset 11, got %v", records2)
+	}
+
+	f3 := cached.PollFetches(context.Background())
+	records3 := f3.Records()
+	if len(records3) != 1 || records3[0].Offset != 12 {
+		t.Fatalf("Expected offset 12, got %v", records3)
+	}
+}
+
+func TestCachedClient_SeekCacheMiss(t *testing.T) {
+	mock := &MockKafkaClient{Drained: make(chan struct{})}
+	mock.Send(
+		&kgo.Record{Topic: "t", Partition: 0, Offset: 5, Value: []byte("five")},
+	)
+	// After SetOffset, the next PollFetches returns from the new position
+	mock.Send(
+		&kgo.Record{Topic: "t", Partition: 0, Offset: 99, Value: []byte("ninety-nine")},
+	)
+
+	cached := newCachedKafkaClient(mock, 100)
+	// Populate cache with offset 5
+	cached.PollFetches(context.Background())
+
+	// Seek to offset 99 which is not in cache
+	cached.Seek("t", 0, 99)
+
+	// Cache miss clears seek and calls SetOffset on underlying.
+	// The same PollFetches call falls through to normal polling.
+	fetches := cached.PollFetches(context.Background())
+	records := fetches.Records()
+	if len(records) != 1 || records[0].Offset != 99 {
+		t.Fatalf("Expected offset 99 from underlying after cache miss, got %v", records)
+	}
+
+	// Seek should be cleared after cache miss
+	if cached.seekTarget != nil {
+		t.Error("Expected seekTarget to be nil after cache miss")
+	}
+
+	// The fetched record should now be cached
+	if _, ok := cached.cache[cacheKey{"t", 0, 99}]; !ok {
+		t.Error("Expected offset 99 to be cached after fetch")
+	}
+}
+
+func TestCachedClient_LRUEviction(t *testing.T) {
+	mock := &MockKafkaClient{Drained: make(chan struct{})}
+	mock.Send(
+		&kgo.Record{Topic: "t", Partition: 0, Offset: 0, Value: []byte("a")},
+		&kgo.Record{Topic: "t", Partition: 0, Offset: 1, Value: []byte("b")},
+		&kgo.Record{Topic: "t", Partition: 0, Offset: 2, Value: []byte("c")},
+	)
+
+	// Cache can only hold 2 records
+	cached := newCachedKafkaClient(mock, 2)
+	cached.PollFetches(context.Background())
+
+	// Offset 0 should have been evicted (LRU), offsets 1 and 2 should remain
+	if _, ok := cached.cache[cacheKey{"t", 0, 0}]; ok {
+		t.Error("Expected offset 0 to be evicted")
+	}
+	if _, ok := cached.cache[cacheKey{"t", 0, 1}]; !ok {
+		t.Error("Expected offset 1 to be cached")
+	}
+	if _, ok := cached.cache[cacheKey{"t", 0, 2}]; !ok {
+		t.Error("Expected offset 2 to be cached")
+	}
+}

From a0273f4ee1a2510fd98a218072fe119d7473a231 Mon Sep 17 00:00:00 2001
From: Yashwanth-Ranjan-Singaravel
 <157959140+Yashwanth-Ranjan-Singaravel@users.noreply.github.com>
Date: Thu, 23 Apr 2026 11:25:51 -0500
Subject: [PATCH 38/55] A simple kafka lambda that interacts with a DB (#440)

* Initial experiment set up

* Added instructions + simplifications to f.py
---
 examples/kafka-db-sum/f.py             | 198 +++++++++++++++++++++++++
 examples/kafka-db-sum/instructions.md  | 106 +++++++++++++
 examples/kafka-db-sum/ol.yaml          |  16 ++
 examples/kafka-db-sum/produce.py       |  45 ++++++
 examples/kafka-db-sum/requirements.in  |   3 +
 examples/kafka-db-sum/requirements.txt |  26 ++++
 6 files changed, 394 insertions(+)
 create mode 100644 examples/kafka-db-sum/f.py
 create mode 100644 examples/kafka-db-sum/instructions.md
 create mode 100644 examples/kafka-db-sum/ol.yaml
 create mode 100644 examples/kafka-db-sum/produce.py
 create mode 100644 examples/kafka-db-sum/requirements.in
 create mode 100644 examples/kafka-db-sum/requirements.txt

diff --git a/examples/kafka-db-sum/f.py b/examples/kafka-db-sum/f.py
new file mode 100644
index 000000000..957b9a850
--- /dev/null
+++ b/examples/kafka-db-sum/f.py
@@ -0,0 +1,198 @@
+from flask import Flask, request, make_response, jsonify
+import os
+import random
+import psycopg2
+
+app = Flask(__name__)
+
+DATABASE_URL = os.environ.get("DATABASE_URL")
+
+# Probability (0.0–1.0) that a transaction will fail between UPDATE and COMMIT.
+# Set to 0 for normal operation; raise to stress-test seek-back recovery.
+FAIL_PROBABILITY = float(os.environ.get("FAIL_PROBABILITY", "0"))
+_db_initialized = False
+
+
+def get_db():
+    return psycopg2.connect(DATABASE_URL)
+
+
+def ensure_db():
+    """Create the running_sum table if it doesn't exist (runs once per sandbox)."""
+    global _db_initialized
+    if _db_initialized:
+        return
+    conn = get_db()
+    try:
+        with conn.cursor() as cur:
+            cur.execute(
+                """
+                CREATE TABLE IF NOT EXISTS running_sum (
+                    id INTEGER PRIMARY KEY DEFAULT 1,
+                    total BIGINT NOT NULL DEFAULT 0,
+                    last_offset BIGINT NOT NULL DEFAULT -1,
+                    message_count BIGINT NOT NULL DEFAULT 0,
+                    CHECK (id = 1)
+                )
+                """
+            )
+            cur.execute(
+                """
+                INSERT INTO running_sum (id, total, last_offset, message_count)
+                VALUES (1, 0, -1, 0)
+                ON CONFLICT (id) DO NOTHING
+                """
+            )
+        conn.commit()
+        _db_initialized = True
+    finally:
+        conn.close()
+
+
+@app.route("/reset", methods=["POST"])
+def reset():
+    """Reset running_sum to zero so the demo can be re-run cleanly."""
+    ensure_db()
+    conn = get_db()
+    try:
+        with conn.cursor() as cur:
+            cur.execute(
+                "UPDATE running_sum SET total = 0, last_offset = -1, "
+                "message_count = 0 WHERE id = 1"
+            )
+        conn.commit()
+        return jsonify({"status": "reset"})
+    finally:
+        conn.close()
+
+
+@app.route("/", methods=["GET", "POST"])
+def handle():
+    ensure_db()
+
+    # GET — return current state (useful for checking progress via HTTP)
+    if request.method == "GET":
+        conn = get_db()
+        try:
+            with conn.cursor() as cur:
+                cur.execute(
+                    "SELECT total, last_offset, message_count "
+                    "FROM running_sum WHERE id = 1"
+                )
+                row = cur.fetchone()
+                if row:
+                    return jsonify(
+                        {
+                            "running_sum": row[0],
+                            "last_offset": row[1],
+                            "message_count": row[2],
+                        }
+                    )
+                return jsonify(
+                    {"running_sum": 0, "last_offset": -1, "message_count": 0}
+                )
+        finally:
+            conn.close()
+
+    # POST — process a Kafka message containing a number
+    offset = int(request.headers.get("X-Kafka-Offset", "-1"))
+    topic = request.headers.get("X-Kafka-Topic", "unknown")
+    partition = request.headers.get("X-Kafka-Partition", "unknown")
+
+    body = request.get_json(silent=True)
+
+    # Accept {"number": N}
+    if isinstance(body, dict):
+        number = body.get("number", 0)
+
+    conn = None
+    try:
+        conn = get_db()
+        with conn.cursor() as cur:
+            # Lock the row and read last processed offset
+            cur.execute(
+                "SELECT last_offset FROM running_sum WHERE id = 1 FOR UPDATE"
+            )
+            row = cur.fetchone()
+            last_offset = row[0] if row else -1
+
+            # Idempotency: skip if this offset was already processed.
+            # This prevents double-counting after a seek-back replays
+            # messages that were already committed.
+            if offset <= last_offset:
+                conn.rollback()
+                print(f"[skip] offset={offset} already processed (last={last_offset})")
+                return jsonify(
+                    {
+                        "status": "skipped",
+                        "reason": "already processed",
+                        "offset": offset,
+                        "last_offset": last_offset,
+                    }
+                )
+
+            # Atomically add number to running sum and advance the offset
+            cur.execute(
+                """
+                UPDATE running_sum
+                SET total = total + %s,
+                    last_offset = %s,
+                    message_count = message_count + 1
+                WHERE id = 1
+                """,
+                (number, offset),
+            )
+
+            # --- Fault injection ------------------------------------------------
+            # Simulate a crash between UPDATE and COMMIT.  
+            if FAIL_PROBABILITY > 0 and random.random() < FAIL_PROBABILITY:
+                raise Exception(
+                    f"Simulated DB failure at offset {offset} "
+                    f"(FAIL_PROBABILITY={FAIL_PROBABILITY})"
+                )
+            # --------------------------------------------------------------------
+
+            conn.commit()
+
+            # Read back the new state for the response
+            cur.execute(
+                "SELECT total, last_offset, message_count "
+                "FROM running_sum WHERE id = 1"
+            )
+            total, last_off, count = cur.fetchone()
+
+        print(f"[ok] offset={offset} number={number} sum={total} count={count}")
+        return jsonify(
+            {
+                "status": "ok",
+                "offset": offset,
+                "number_added": number,
+                "running_sum": total,
+                "message_count": count,
+            }
+        )
+
+    except Exception as e:
+        print(f"[error] offset={offset} error={e}")
+        if conn:
+            try:
+                conn.rollback()
+            except Exception:
+                pass
+
+        # Tell OL's Kafka consumer to seek back to this offset and retry.
+        # The consumer's LRU cache will serve the replay without re-fetching
+        # from Kafka, and the idempotency check above prevents double-counting
+        # for any offsets that were already committed before the failure.
+        resp = make_response(
+            jsonify({"status": "error", "offset": offset, "error": str(e)}), 500
+        )
+        resp.headers["X-Kafka-Seek-Offset"] = str(offset)
+        return resp
+
+    finally:
+        if conn:
+            try:
+                conn.close()
+            except Exception:
+                pass
diff --git a/examples/kafka-db-sum/instructions.md b/examples/kafka-db-sum/instructions.md
new file mode 100644
index 000000000..e9b0872e1
--- /dev/null
+++ b/examples/kafka-db-sum/instructions.md
@@ -0,0 +1,106 @@
+# kafka-db-sum: Testing Instructions
+
+## Prerequisites
+
+- OpenLambda built (`make ol imgs/ol-min`)
+- Docker installed
+
+## 1. Start PostgreSQL
+
+```bash
+docker run -d --name ol-pg \
+  --network host \
+  -e POSTGRES_USER=ol \
+  -e POSTGRES_PASSWORD=ol \
+  -e POSTGRES_DB=ol_demo \
+  postgres:16
+```
+
+## 2. Start Kafka
+
+```bash
+docker run -d --name kafka \
+  -p 9092:9092 \
+  apache/kafka:latest
+```
+
+## 3. Create the `numbers` topic
+
+```bash
+docker exec kafka /opt/kafka/bin/kafka-topics.sh --create \
+  --topic numbers \
+  --bootstrap-server localhost:9092
+```
+
+## 4. Initialize and start the OL worker
+
+From the repository root:
+
+```bash
+sudo -A ./ol worker init -p ../default-ol -i ol-min
+sudo -A ./ol worker up -p ../default-ol
+```
+
+Run `worker up` in a separate terminal, or add `-d` for detached mode.
+The worker listens on `localhost:5000` by default.
+
+## 5. Install the lambda
+
+From the repository root:
+
+```bash
+./ol admin install examples/kafka-db-sum/
+```
+
+## 6. Register the Kafka consumer
+
+A standalone worker does not auto-register Kafka triggers on upload.
+Register manually:
+
+```bash
+curl -X POST localhost:5000/kafka/register/kafka-db-sum
+```
+
+## 7. Send test messages
+
+Python producer script (requires `pip install kafka-python`):
+
+```bash
+python examples/kafka-db-sum/produce.py 100
+```
+
+## 8. Check results
+
+```bash
+curl localhost:5000/run/kafka-db-sum/
+```
+
+Expected output (sum of 1..100 = 5050):
+
+```json
+{ "last_offset": 99, "message_count": 100, "running_sum": 5050 }
+```
+
+## 9. Reset and re-run
+
+```bash
+curl -X POST localhost:5000/run/kafka-db-sum/reset
+```
+
+Then send a fresh batch (step 7) and verify again.
+
+## Configuration
+
+In `ol.yaml`:
+
+| Variable           | Default                                     | Description                                                         |
+| ------------------ | ------------------------------------------- | ------------------------------------------------------------------- |
+| `DATABASE_URL`     | `postgresql://ol:ol@127.0.0.1:5432/ol_demo` | PostgreSQL connection string                                        |
+| `FAIL_PROBABILITY` | `0`                                         | Chance (0.0-1.0) of simulated failure. Use `0.3` to test seek-back. |
+
+## Cleanup
+
+```bash
+sudo -A ./ol worker down -p default-ol
+docker rm -f kafka ol-pg
+```
diff --git a/examples/kafka-db-sum/ol.yaml b/examples/kafka-db-sum/ol.yaml
new file mode 100644
index 000000000..28fd06747
--- /dev/null
+++ b/examples/kafka-db-sum/ol.yaml
@@ -0,0 +1,16 @@
+triggers:
+  http:
+    - method: GET
+    - method: POST
+  kafka:
+    - bootstrap_servers:
+        - "localhost:9092"
+      topics:
+        - "numbers"
+      auto_offset_reset: "earliest"
+
+environment:
+  DATABASE_URL: "postgresql://ol:ol@127.0.0.1:5432/ol_demo"
+  # Probability (0.0-1.0) of simulated DB failure between UPDATE and COMMIT.
+  # Set to "0" for normal operation. Try "0.3" to see seek-back recovery in action.
+  FAIL_PROBABILITY: "0.3"
diff --git a/examples/kafka-db-sum/produce.py b/examples/kafka-db-sum/produce.py
new file mode 100644
index 000000000..c6a3671ec
--- /dev/null
+++ b/examples/kafka-db-sum/produce.py
@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+"""
+Send numbered messages to the 'numbers' Kafka topic.
+
+Usage:
+    python produce.py              # send numbers 1..10
+    python produce.py 100          # send numbers 1..100
+    python produce.py 50 0.5       # send 1..50 with 0.5s delay between each
+"""
+
+import json
+import sys
+import time
+
+from kafka import KafkaProducer
+
+BROKER = "localhost:9092"
+TOPIC = "numbers"
+
+
+def main():
+    count = int(sys.argv[1]) if len(sys.argv) > 1 else 10
+    delay = float(sys.argv[2]) if len(sys.argv) > 2 else 0.1
+
+    producer = KafkaProducer(
+        bootstrap_servers=BROKER,
+        value_serializer=lambda v: json.dumps(v).encode("utf-8"),
+    )
+
+    expected_sum = 0
+    print(f"Sending numbers 1..{count} to topic '{TOPIC}'...")
+    for i in range(1, count + 1):
+        producer.send(TOPIC, {"number": i})
+        expected_sum += i
+        print(f"  sent {i}")
+        if delay:
+            time.sleep(delay)
+
+    producer.flush()
+    producer.close()
+    print(f"\nDone. Expected sum = {expected_sum}")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/examples/kafka-db-sum/requirements.in b/examples/kafka-db-sum/requirements.in
new file mode 100644
index 000000000..cefe02572
--- /dev/null
+++ b/examples/kafka-db-sum/requirements.in
@@ -0,0 +1,3 @@
+flask==2.3.2
+werkzeug==3.0.3
+psycopg2-binary==2.9.9
diff --git a/examples/kafka-db-sum/requirements.txt b/examples/kafka-db-sum/requirements.txt
new file mode 100644
index 000000000..869c94ee1
--- /dev/null
+++ b/examples/kafka-db-sum/requirements.txt
@@ -0,0 +1,26 @@
+#
+# This file is autogenerated by pip-compile with Python 3.13
+# by the following command:
+#
+#    pip-compile requirements.in
+#
+blinker==1.6.2
+    # via flask
+click==8.1.7
+    # via flask
+flask==2.3.2
+    # via -r requirements.in
+itsdangerous==2.1.2
+    # via flask
+jinja2==3.1.4
+    # via flask
+markupsafe==2.1.3
+    # via
+    #   jinja2
+    #   werkzeug
+psycopg2-binary==2.9.9
+    # via -r requirements.in
+werkzeug==3.0.3
+    # via
+    #   -r requirements.in
+    #   flask

From a288ce1eedd220f7424533b7ed1865c0a8a9fb63 Mon Sep 17 00:00:00 2001
From: Yashwanth-Ranjan-Singaravel
 <157959140+Yashwanth-Ranjan-Singaravel@users.noreply.github.com>
Date: Thu, 23 Apr 2026 11:28:53 -0500
Subject: [PATCH 39/55] Added worker specific kafka config (#441)

---
 go/common/config.go                  | 21 +++++++++++++++++++++
 go/worker/event/cachedKafkaClient.go |  2 --
 go/worker/event/kafkaServer.go       | 15 ++++++++++-----
 go/worker/event/kafkaServer_test.go  | 10 +++++++++-
 4 files changed, 40 insertions(+), 8 deletions(-)

diff --git a/go/common/config.go b/go/common/config.go
index de83d93c1..9243c0124 100644
--- a/go/common/config.go
+++ b/go/common/config.go
@@ -68,6 +68,20 @@ type Config struct {
 	Features        FeaturesConfig `json:"features"`
 	Trace           TraceConfig    `json:"trace"`
 	Storage         StorageConfig  `json:"storage"`
+	Kafka           KafkaConfig    `json:"kafka"`
+}
+
+type KafkaConfig struct {
+	// whether to enable the LRU message cache for seek-based replay
+	Cache_enabled bool `json:"cache_enabled"`
+	// maximum number of records held in the LRU cache
+	Cache_size int `json:"cache_size"`
+	// Kafka consumer session timeout in seconds
+	Session_timeout_sec int `json:"session_timeout_sec"`
+	// Kafka consumer heartbeat interval in seconds
+	Heartbeat_interval_sec int `json:"heartbeat_interval_sec"`
+	// poll timeout in seconds for each PollFetches call
+	Poll_timeout_sec int `json:"poll_timeout_sec"`
 }
 
 type DockerConfig struct {
@@ -316,6 +330,13 @@ func getDefaultConfigForPatching(olPath string) (*Config, error) {
 			Scratch: "",
 			Code:    "",
 		},
+		Kafka: KafkaConfig{
+			Cache_enabled:          true,
+			Cache_size:             1024,
+			Session_timeout_sec:    10,
+			Heartbeat_interval_sec: 3,
+			Poll_timeout_sec:       1,
+		},
 	}
 
 	return cfg, nil
diff --git a/go/worker/event/cachedKafkaClient.go b/go/worker/event/cachedKafkaClient.go
index 61d72ffc6..b091039aa 100644
--- a/go/worker/event/cachedKafkaClient.go
+++ b/go/worker/event/cachedKafkaClient.go
@@ -26,8 +26,6 @@ type seekRequest struct {
 	offset int64
 }
 
-const defaultCacheSize = 1024
-
 // cachedKafkaClient wraps a KafkaClient and caches records in an LRU map keyed
 // by {topic, partition, offset}. When a seek is active, PollFetches serves
 // records from the cache. On cache miss, it calls Seek on the underlying
diff --git a/go/worker/event/kafkaServer.go b/go/worker/event/kafkaServer.go
index 22d9c8c7a..7b590f984 100644
--- a/go/worker/event/kafkaServer.go
+++ b/go/worker/event/kafkaServer.go
@@ -89,13 +89,15 @@ func (km *KafkaManager) newLambdaKafkaConsumer(consumerName string, lambdaName s
 		return nil, fmt.Errorf("no topics configured for lambda %s", lambdaName)
 	}
 
+	kafkaCfg := common.Conf.Kafka
+
 	// Setup kgo client options
 	opts := []kgo.Opt{
 		kgo.SeedBrokers(trigger.BootstrapServers...),
 		kgo.ConsumerGroup(trigger.GroupId),
 		kgo.ConsumeTopics(trigger.Topics...),
-		kgo.SessionTimeout(10 * time.Second),
-		kgo.HeartbeatInterval(3 * time.Second),
+		kgo.SessionTimeout(time.Duration(kafkaCfg.Session_timeout_sec) * time.Second),
+		kgo.HeartbeatInterval(time.Duration(kafkaCfg.Heartbeat_interval_sec) * time.Second),
 	}
 
 	// Use trigger-specific offset reset or default to latest
@@ -111,12 +113,15 @@ func (km *KafkaManager) newLambdaKafkaConsumer(consumerName string, lambdaName s
 		return nil, fmt.Errorf("failed to create Kafka client for lambda %s: %w", lambdaName, err)
 	}
 
-	cached := newCachedKafkaClient(&kgoClientWrapper{client: client}, defaultCacheSize)
+	var kafkaClient KafkaClient = &kgoClientWrapper{client: client}
+	if kafkaCfg.Cache_enabled {
+		kafkaClient = newCachedKafkaClient(kafkaClient, kafkaCfg.Cache_size)
+	}
 	return &LambdaKafkaConsumer{
 		consumerName: consumerName,
 		lambdaName:   lambdaName,
 		kafkaTrigger: trigger,
-		client:       cached,
+		client:       kafkaClient,
 		invoker:      km.invoker,
 		stopChan:     make(chan struct{}),
 	}, nil
@@ -154,7 +159,7 @@ func (lkc *LambdaKafkaConsumer) consumeLoop() {
 			slog.Info("Stopping Kafka consumer for lambda", "lambda", lkc.lambdaName)
 			return
 		default:
-			ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
+			ctx, cancel := context.WithTimeout(context.Background(), time.Duration(common.Conf.Kafka.Poll_timeout_sec)*time.Second)
 			fetches := lkc.client.PollFetches(ctx)
 			cancel()
 
diff --git a/go/worker/event/kafkaServer_test.go b/go/worker/event/kafkaServer_test.go
index 0b0a414cc..4311e1514 100644
--- a/go/worker/event/kafkaServer_test.go
+++ b/go/worker/event/kafkaServer_test.go
@@ -17,7 +17,15 @@ import (
 
 func TestMain(m *testing.M) {
 	// Initialize common.Conf so that common.T0/T1 (latency tracking) doesn't panic
-	common.Conf = &common.Config{}
+	common.Conf = &common.Config{
+		Kafka: common.KafkaConfig{
+			Cache_enabled:          true,
+			Cache_size:             1024,
+			Session_timeout_sec:    10,
+			Heartbeat_interval_sec: 3,
+			Poll_timeout_sec:       1,
+		},
+	}
 	os.Exit(m.Run())
 }
 

From 826512d20aa98f52ca04813c45620ce0ea210b60 Mon Sep 17 00:00:00 2001
From: Ami Buch <amiabuch@gmail.com>
Date: Thu, 19 Feb 2026 14:09:10 -0600
Subject: [PATCH 40/55] refactor: separation of package, abstraction included,
 simpler interface and microservice like architecture

---
 go/worker/sandboxset/api.go        | 138 +++++++++++++++++++++++++++++
 go/worker/sandboxset/destroy.go    |  34 +++++++
 go/worker/sandboxset/get.go        |  92 +++++++++++++++++++
 go/worker/sandboxset/metrics.go    |  10 +++
 go/worker/sandboxset/release.go    |  38 ++++++++
 go/worker/sandboxset/sandboxset.go |  48 ++++++++++
 go/worker/sandboxset/shrink.go     |  32 +++++++
 go/worker/sandboxset/stats.go      |  21 +++++
 go/worker/sandboxset/warm.go       |  41 +++++++++
 9 files changed, 454 insertions(+)
 create mode 100644 go/worker/sandboxset/api.go
 create mode 100644 go/worker/sandboxset/destroy.go
 create mode 100644 go/worker/sandboxset/get.go
 create mode 100644 go/worker/sandboxset/metrics.go
 create mode 100644 go/worker/sandboxset/release.go
 create mode 100644 go/worker/sandboxset/sandboxset.go
 create mode 100644 go/worker/sandboxset/shrink.go
 create mode 100644 go/worker/sandboxset/stats.go
 create mode 100644 go/worker/sandboxset/warm.go

diff --git a/go/worker/sandboxset/api.go b/go/worker/sandboxset/api.go
new file mode 100644
index 000000000..6ebe48a19
--- /dev/null
+++ b/go/worker/sandboxset/api.go
@@ -0,0 +1,138 @@
+// Package sandboxset provides a thread-safe pool of sandboxes for a single
+// Lambda function.
+//
+// SandboxSet replaces per-instance goroutines with a mutex-protected slice.
+// The pool costs ~500 bytes regardless of how many sandboxes it holds.
+//
+// Sandbox lifecycle inside a SandboxSet:
+//
+//	created ──► paused (available) ──► in-use (unpaused) ──► paused (available)
+//	                  │                       │
+//	                  └───────── destroyed ◄──┘  (on error or Shrink)
+//
+// Usage:
+//
+//	cfg := &sandboxset.Config{
+//	    Pool:    myPool,
+//	    CodeDir: "/path/to/lambda",
+//	    Meta:    &sandbox.SandboxMeta{Runtime: common.RT_PYTHON},
+//	}
+//	set, err := sandboxset.New(cfg)
+//
+//	sb, err := set.GetSandbox()
+//	// ... handle request ...
+//	set.ReleaseSandbox(sb)
+package sandboxset
+
+import (
+	"time"
+
+	"github.com/open-lambda/open-lambda/go/worker/sandbox"
+)
+
+// SandboxSet is a thread-safe pool of sandboxes for a single Lambda function.
+// All methods are safe to call from multiple goroutines concurrently.
+type SandboxSet interface {
+	// GetSandbox borrows an available sandbox, creating one if needed.
+	//
+	// Returns an unpaused sandbox ready to handle a request.
+	// Caller MUST release it via ReleaseSandbox or DestroyAndRemove.
+	// Blocks up to DefaultTimeout (or WithTimeout) when at MaxSize capacity.
+	GetSandbox(opts ...GetOption) (sandbox.Sandbox, error)
+
+	// ReleaseSandbox returns a sandbox to the pool after a successful request.
+	// The sandbox is paused and made available for the next caller.
+	// On Pause failure the sandbox is destroyed automatically.
+	ReleaseSandbox(sb sandbox.Sandbox, opts ...ReleaseOption) error
+
+	// DestroyAndRemove permanently removes a sandbox from the pool.
+	// Use when a sandbox has produced an unrecoverable error.
+	DestroyAndRemove(sb sandbox.Sandbox, reason string) error
+
+	// Warm pre-creates sandboxes until at least target are paused and ready.
+	// No-op when the pool already has target or more sandboxes.
+	Warm(target int) error
+
+	// Shrink destroys idle sandboxes until at most target remain.
+	// Stops early if all remaining sandboxes are in use.
+	Shrink(target int) error
+
+	// Stats returns a snapshot of pool counters.
+	// Keys: "available", "in_use", "total".
+	Stats() map[string]int
+
+	// Metrics returns a copy of cumulative performance counters.
+	Metrics() *Metrics
+}
+
+// Config holds creation parameters for a SandboxSet.
+// Pool and CodeDir are required; all other fields have sensible zero-value defaults.
+type Config struct {
+	// Pool creates new sandboxes. Required.
+	Pool sandbox.SandboxPool
+
+	// Parent is the sandbox to fork from. Nil means create from scratch.
+	Parent sandbox.Sandbox
+
+	// IsLeaf specifies whether created sandboxes are leaves (not forkable).
+	IsLeaf bool
+
+	// CodeDir is the directory containing the Lambda function code. Required.
+	CodeDir string
+
+	// ScratchDir is the per-invocation writable directory for each sandbox.
+	ScratchDir string
+
+	// Meta holds runtime configuration (memory limits, packages, imports).
+	Meta *sandbox.SandboxMeta
+
+	// MaxSize caps the total number of sandboxes. Zero means no limit.
+	MaxSize int
+
+	// DefaultTimeout is how long GetSandbox blocks at MaxSize capacity.
+	// Zero means fail immediately when all sandboxes are in use.
+	DefaultTimeout time.Duration
+}
+
+// GetOption adjusts a single GetSandbox call.
+// Construct with WithTimeout.
+type GetOption func(*getOptions)
+
+// ReleaseOption adjusts a single ReleaseSandbox call.
+// Construct with WithoutPause.
+type ReleaseOption func(*releaseOptions)
+
+// Metrics holds cumulative counters since pool creation.
+// All fields are monotonically increasing.
+type Metrics struct {
+	Gets     int64 // total GetSandbox calls
+	Hits     int64 // Gets served from an already-available sandbox
+	Misses   int64 // Gets that required creating a new sandbox
+	Releases int64 // successful ReleaseSandbox calls
+	Destroys int64 // DestroyAndRemove calls
+	Timeouts int64 // Gets that failed due to deadline exceeded
+}
+
+// HitRate returns the fraction of Gets served from an available sandbox (0.0–1.0).
+func (m *Metrics) HitRate() float64 {
+	if m.Gets == 0 {
+		return 0.0
+	}
+	return float64(m.Hits) / float64(m.Gets)
+}
+
+// New creates a SandboxSet from cfg. Returns an error if cfg is invalid.
+func New(cfg *Config) (SandboxSet, error) {
+	return newSandboxSet(cfg)
+}
+
+// WithTimeout overrides the DefaultTimeout for a single GetSandbox call.
+func WithTimeout(d time.Duration) GetOption {
+	return func(o *getOptions) { o.timeout = d }
+}
+
+// WithoutPause skips the Pause call when returning a sandbox to the pool.
+// Use when the sandbox has already been paused by the caller.
+func WithoutPause() ReleaseOption {
+	return func(o *releaseOptions) { o.skipPause = true }
+}
diff --git a/go/worker/sandboxset/destroy.go b/go/worker/sandboxset/destroy.go
new file mode 100644
index 000000000..156067bc8
--- /dev/null
+++ b/go/worker/sandboxset/destroy.go
@@ -0,0 +1,34 @@
+package sandboxset
+
+import (
+	"fmt"
+
+	"github.com/open-lambda/open-lambda/go/worker/sandbox"
+)
+
+// DestroyAndRemove implements SandboxSet.
+//
+// The wrapper is spliced out of the pool under a short write lock using O(1)
+// swap-with-tail. Destroy is called outside the lock to keep critical sections
+// short. The sandbox is always destroyed even if it was not found in the pool.
+func (s *sandboxSetImpl) DestroyAndRemove(sb sandbox.Sandbox, reason string) error {
+	s.mu.Lock()
+	found := false
+	for i, w := range s.pool {
+		if w.sb.ID() == sb.ID() {
+			s.pool[i] = s.pool[len(s.pool)-1]
+			s.pool = s.pool[:len(s.pool)-1]
+			s.metrics.Destroys++
+			found = true
+			break
+		}
+	}
+	s.mu.Unlock()
+
+	sb.Destroy(reason)
+
+	if !found {
+		return fmt.Errorf("sandboxset: sandbox %s not found in pool (still destroyed)", sb.ID())
+	}
+	return nil
+}
diff --git a/go/worker/sandboxset/get.go b/go/worker/sandboxset/get.go
new file mode 100644
index 000000000..3be865cdf
--- /dev/null
+++ b/go/worker/sandboxset/get.go
@@ -0,0 +1,92 @@
+package sandboxset
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/open-lambda/open-lambda/go/worker/sandbox"
+)
+
+// GetSandbox implements SandboxSet.
+//
+// Fast path: an idle sandbox is claimed under a short write lock, then
+// Unpause runs outside the lock so the pool is not stalled during I/O.
+//
+// Slow path: no idle sandbox exists. If the pool has room, a new sandbox is
+// created without holding any lock, then appended. If the pool is at MaxSize
+// and nothing becomes available before the deadline, an error is returned.
+func (s *sandboxSetImpl) GetSandbox(opts ...GetOption) (sandbox.Sandbox, error) {
+	o := &getOptions{timeout: s.cfg.DefaultTimeout}
+	for _, opt := range opts {
+		opt(o)
+	}
+
+	var deadline time.Time
+	if o.timeout > 0 {
+		deadline = time.Now().Add(o.timeout)
+	}
+
+	s.mu.Lock()
+	s.metrics.Gets++
+	s.mu.Unlock()
+
+	for {
+		// Try to claim an idle sandbox.
+		s.mu.Lock()
+		var claimed sandbox.Sandbox
+		for _, w := range s.pool {
+			if !w.inUse {
+				w.inUse = true
+				claimed = w.sb
+				break
+			}
+		}
+		atCap := s.cfg.MaxSize > 0 && len(s.pool) >= s.cfg.MaxSize
+		poolSize := len(s.pool)
+		s.mu.Unlock()
+
+		if claimed != nil {
+			// Unpause outside the lock (split-lock pattern): inUse=true already
+			// prevents another goroutine from claiming this sandbox.
+			if err := claimed.Unpause(); err != nil {
+				_ = s.DestroyAndRemove(claimed, fmt.Sprintf("unpause: %v", err))
+				continue
+			}
+			s.mu.Lock()
+			s.metrics.Hits++
+			s.mu.Unlock()
+			return claimed, nil
+		}
+
+		// Pool is at capacity: wait or timeout.
+		if atCap {
+			if !deadline.IsZero() && time.Now().After(deadline) {
+				s.mu.Lock()
+				s.metrics.Timeouts++
+				s.mu.Unlock()
+				return nil, fmt.Errorf(
+					"sandboxset: all %d sandboxes in use; deadline exceeded", poolSize,
+				)
+			}
+			time.Sleep(1 * time.Millisecond)
+			continue
+		}
+
+		// Slow path: create a new sandbox without holding the lock.
+		sb, err := s.cfg.Pool.Create(
+			s.cfg.Parent, s.cfg.IsLeaf,
+			s.cfg.CodeDir, s.cfg.ScratchDir,
+			s.cfg.Meta,
+		)
+		if err != nil {
+			return nil, fmt.Errorf("sandboxset: create sandbox: %w", err)
+		}
+
+		s.mu.Lock()
+		s.pool = append(s.pool, &sandboxWrapper{sb: sb, inUse: true})
+		s.metrics.Misses++
+		s.mu.Unlock()
+
+		return sb, nil
+	}
+}
diff --git a/go/worker/sandboxset/metrics.go b/go/worker/sandboxset/metrics.go
new file mode 100644
index 000000000..8966651c7
--- /dev/null
+++ b/go/worker/sandboxset/metrics.go
@@ -0,0 +1,10 @@
+package sandboxset
+
+// Metrics implements SandboxSet. Returns a copy so callers cannot mutate
+// internal counters.
+func (s *sandboxSetImpl) Metrics() *Metrics {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	m := s.metrics
+	return &m
+}
diff --git a/go/worker/sandboxset/release.go b/go/worker/sandboxset/release.go
new file mode 100644
index 000000000..01b1613c2
--- /dev/null
+++ b/go/worker/sandboxset/release.go
@@ -0,0 +1,38 @@
+package sandboxset
+
+import (
+	"fmt"
+
+	"github.com/open-lambda/open-lambda/go/worker/sandbox"
+)
+
+// ReleaseSandbox implements SandboxSet.
+//
+// The sandbox is paused (unless WithoutPause was supplied) and its wrapper is
+// flipped back to idle. If Pause fails the sandbox is destroyed rather than
+// silently recycled — a bad sandbox should never re-enter the pool.
+func (s *sandboxSetImpl) ReleaseSandbox(sb sandbox.Sandbox, opts ...ReleaseOption) error {
+	o := &releaseOptions{}
+	for _, opt := range opts {
+		opt(o)
+	}
+
+	if !o.skipPause {
+		if err := sb.Pause(); err != nil {
+			_ = s.DestroyAndRemove(sb, fmt.Sprintf("pause failed on release: %v", err))
+			return fmt.Errorf("sandboxset: sandbox %s destroyed because Pause failed: %w", sb.ID(), err)
+		}
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	for _, w := range s.pool {
+		if w.sb.ID() == sb.ID() {
+			w.inUse = false
+			s.metrics.Releases++
+			return nil
+		}
+	}
+	return fmt.Errorf("sandboxset: sandbox %s not found in pool", sb.ID())
+}
diff --git a/go/worker/sandboxset/sandboxset.go b/go/worker/sandboxset/sandboxset.go
new file mode 100644
index 000000000..72d29ddfe
--- /dev/null
+++ b/go/worker/sandboxset/sandboxset.go
@@ -0,0 +1,48 @@
+package sandboxset
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/open-lambda/open-lambda/go/worker/sandbox"
+)
+
+// sandboxWrapper pairs a sandbox with an in-use flag.
+// Never exported; internal pool bookkeeping only.
+type sandboxWrapper struct {
+	sb    sandbox.Sandbox
+	inUse bool
+}
+
+// getOptions holds resolved settings for one GetSandbox call.
+type getOptions struct {
+	timeout time.Duration
+}
+
+// releaseOptions holds resolved settings for one ReleaseSandbox call.
+type releaseOptions struct {
+	skipPause bool
+}
+
+// sandboxSetImpl is the private concrete type returned by New.
+// All mutable state is guarded by mu.
+type sandboxSetImpl struct {
+	mu      sync.RWMutex
+	pool    []*sandboxWrapper
+	cfg     *Config
+	metrics Metrics
+}
+
+func newSandboxSet(cfg *Config) (*sandboxSetImpl, error) {
+	if cfg == nil {
+		return nil, fmt.Errorf("sandboxset: Config must not be nil")
+	}
+	if cfg.Pool == nil {
+		return nil, fmt.Errorf("sandboxset: Config.Pool must not be nil")
+	}
+	if cfg.CodeDir == "" {
+		return nil, fmt.Errorf("sandboxset: Config.CodeDir must not be empty")
+	}
+	return &sandboxSetImpl{cfg: cfg}, nil
+}
diff --git a/go/worker/sandboxset/shrink.go b/go/worker/sandboxset/shrink.go
new file mode 100644
index 000000000..c1bc053ce
--- /dev/null
+++ b/go/worker/sandboxset/shrink.go
@@ -0,0 +1,32 @@
+package sandboxset
+
+// Shrink implements SandboxSet.
+//
+// One idle sandbox is removed and destroyed per iteration so the write lock
+// is never held while destruction I/O runs.
+func (s *sandboxSetImpl) Shrink(target int) error {
+	for {
+		s.mu.Lock()
+		if len(s.pool) <= target {
+			s.mu.Unlock()
+			return nil
+		}
+
+		var victim *sandboxWrapper
+		for i, w := range s.pool {
+			if !w.inUse {
+				victim = w
+				s.pool[i] = s.pool[len(s.pool)-1]
+				s.pool = s.pool[:len(s.pool)-1]
+				break
+			}
+		}
+		s.mu.Unlock()
+
+		if victim == nil {
+			// All remaining sandboxes are actively in use.
+			return nil
+		}
+		victim.sb.Destroy("shrink")
+	}
+}
diff --git a/go/worker/sandboxset/stats.go b/go/worker/sandboxset/stats.go
new file mode 100644
index 000000000..ce76b5c79
--- /dev/null
+++ b/go/worker/sandboxset/stats.go
@@ -0,0 +1,21 @@
+package sandboxset
+
+// Stats implements SandboxSet.
+func (s *sandboxSetImpl) Stats() map[string]int {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	available, inUse := 0, 0
+	for _, w := range s.pool {
+		if w.inUse {
+			inUse++
+		} else {
+			available++
+		}
+	}
+	return map[string]int{
+		"available": available,
+		"in_use":    inUse,
+		"total":     len(s.pool),
+	}
+}
diff --git a/go/worker/sandboxset/warm.go b/go/worker/sandboxset/warm.go
new file mode 100644
index 000000000..09aa151ad
--- /dev/null
+++ b/go/worker/sandboxset/warm.go
@@ -0,0 +1,41 @@
+package sandboxset
+
+import "fmt"
+
+// Warm implements SandboxSet.
+//
+// Sandboxes are created and paused sequentially. Parallel creation could
+// overwhelm the host with container-start overhead; pools are typically small
+// (5–10 entries) so sequential creation is fast enough.
+func (s *sandboxSetImpl) Warm(target int) error {
+	for {
+		s.mu.RLock()
+		current := len(s.pool)
+		s.mu.RUnlock()
+
+		if current >= target {
+			return nil
+		}
+		if s.cfg.MaxSize > 0 && current >= s.cfg.MaxSize {
+			return nil
+		}
+
+		sb, err := s.cfg.Pool.Create(
+			s.cfg.Parent, s.cfg.IsLeaf,
+			s.cfg.CodeDir, s.cfg.ScratchDir,
+			s.cfg.Meta,
+		)
+		if err != nil {
+			return fmt.Errorf("sandboxset: Warm create[%d]: %w", current, err)
+		}
+
+		if err := sb.Pause(); err != nil {
+			sb.Destroy("pause failed during Warm")
+			return fmt.Errorf("sandboxset: Warm pause[%d]: %w", current, err)
+		}
+
+		s.mu.Lock()
+		s.pool = append(s.pool, &sandboxWrapper{sb: sb, inUse: false})
+		s.mu.Unlock()
+	}
+}

From c546be2c2aff34d839326f42b71ef0e9d6618612 Mon Sep 17 00:00:00 2001
From: Ami Buch <amiabuch@gmail.com>
Date: Sun, 1 Mar 2026 20:55:51 -0600
Subject: [PATCH 41/55] refactor: simpler, much much simpler

---
 go/worker/sandboxset/api.go        | 131 ++++++++---------------------
 go/worker/sandboxset/close.go      |  23 +++++
 go/worker/sandboxset/destroy.go    |  12 +--
 go/worker/sandboxset/get.go        | 111 +++++++++---------------
 go/worker/sandboxset/metrics.go    |  10 ---
 go/worker/sandboxset/put.go        |  30 +++++++
 go/worker/sandboxset/release.go    |  38 ---------
 go/worker/sandboxset/sandboxset.go |  23 ++---
 go/worker/sandboxset/shrink.go     |  32 -------
 go/worker/sandboxset/stats.go      |  21 -----
 go/worker/sandboxset/warm.go       |  41 ---------
 11 files changed, 139 insertions(+), 333 deletions(-)
 create mode 100644 go/worker/sandboxset/close.go
 delete mode 100644 go/worker/sandboxset/metrics.go
 create mode 100644 go/worker/sandboxset/put.go
 delete mode 100644 go/worker/sandboxset/release.go
 delete mode 100644 go/worker/sandboxset/shrink.go
 delete mode 100644 go/worker/sandboxset/stats.go
 delete mode 100644 go/worker/sandboxset/warm.go

diff --git a/go/worker/sandboxset/api.go b/go/worker/sandboxset/api.go
index 6ebe48a19..49bcb5611 100644
--- a/go/worker/sandboxset/api.go
+++ b/go/worker/sandboxset/api.go
@@ -1,72 +1,54 @@
 // Package sandboxset provides a thread-safe pool of sandboxes for a single
 // Lambda function.
 //
-// SandboxSet replaces per-instance goroutines with a mutex-protected slice.
-// The pool costs ~500 bytes regardless of how many sandboxes it holds.
-//
 // Sandbox lifecycle inside a SandboxSet:
 //
 //	created ──► paused (available) ──► in-use (unpaused) ──► paused (available)
 //	                  │                       │
-//	                  └───────── destroyed ◄──┘  (on error or Shrink)
+//	                  └───────── destroyed ◄──┘  (on error or Close)
 //
 // Usage:
 //
-//	cfg := &sandboxset.Config{
-//	    Pool:    myPool,
-//	    CodeDir: "/path/to/lambda",
-//	    Meta:    &sandbox.SandboxMeta{Runtime: common.RT_PYTHON},
-//	}
-//	set, err := sandboxset.New(cfg)
+//	set, err := sandboxset.New(&sandboxset.Config{
+//	    Pool:        myPool,
+//	    CodeDir:     "/path/to/lambda",
+//	    ScratchDirs: myScratchDirs,
+//	})
 //
-//	sb, err := set.GetSandbox()
+//	sb, err := set.Get()
 //	// ... handle request ...
-//	set.ReleaseSandbox(sb)
+//	set.Put(sb)
 package sandboxset
 
 import (
-	"time"
-
+	"github.com/open-lambda/open-lambda/go/common"
 	"github.com/open-lambda/open-lambda/go/worker/sandbox"
 )
 
 // SandboxSet is a thread-safe pool of sandboxes for a single Lambda function.
-// All methods are safe to call from multiple goroutines concurrently.
+// All methods are safe for concurrent use.
 type SandboxSet interface {
-	// GetSandbox borrows an available sandbox, creating one if needed.
-	//
-	// Returns an unpaused sandbox ready to handle a request.
-	// Caller MUST release it via ReleaseSandbox or DestroyAndRemove.
-	// Blocks up to DefaultTimeout (or WithTimeout) when at MaxSize capacity.
-	GetSandbox(opts ...GetOption) (sandbox.Sandbox, error)
-
-	// ReleaseSandbox returns a sandbox to the pool after a successful request.
-	// The sandbox is paused and made available for the next caller.
-	// On Pause failure the sandbox is destroyed automatically.
-	ReleaseSandbox(sb sandbox.Sandbox, opts ...ReleaseOption) error
-
-	// DestroyAndRemove permanently removes a sandbox from the pool.
-	// Use when a sandbox has produced an unrecoverable error.
-	DestroyAndRemove(sb sandbox.Sandbox, reason string) error
-
-	// Warm pre-creates sandboxes until at least target are paused and ready.
-	// No-op when the pool already has target or more sandboxes.
-	Warm(target int) error
-
-	// Shrink destroys idle sandboxes until at most target remain.
-	// Stops early if all remaining sandboxes are in use.
-	Shrink(target int) error
-
-	// Stats returns a snapshot of pool counters.
-	// Keys: "available", "in_use", "total".
-	Stats() map[string]int
-
-	// Metrics returns a copy of cumulative performance counters.
-	Metrics() *Metrics
+	// Get borrows a sandbox, creating one if none are idle.
+	// The returned sandbox is unpaused and ready to handle a request.
+	// Caller MUST call Put or Destroy when done.
+	Get() (sandbox.Sandbox, error)
+
+	// Put returns a sandbox to the pool after successful use.
+	// The sandbox is paused and made available for future Get calls.
+	// If pausing fails, the sandbox is destroyed automatically.
+	Put(sb sandbox.Sandbox) error
+
+	// Destroy permanently removes a sandbox from the pool and kills it.
+	// Use when a sandbox is in an unrecoverable state.
+	Destroy(sb sandbox.Sandbox, reason string) error
+
+	// Close destroys all sandboxes in the pool.
+	// After Close returns, Get/Put/Destroy return errors.
+	Close() error
 }
 
 // Config holds creation parameters for a SandboxSet.
-// Pool and CodeDir are required; all other fields have sensible zero-value defaults.
+// Pool, CodeDir, and ScratchDirs are required.
 type Config struct {
 	// Pool creates new sandboxes. Required.
 	Pool sandbox.SandboxPool
@@ -74,65 +56,20 @@ type Config struct {
 	// Parent is the sandbox to fork from. Nil means create from scratch.
 	Parent sandbox.Sandbox
 
-	// IsLeaf specifies whether created sandboxes are leaves (not forkable).
+	// IsLeaf marks created sandboxes as non-forkable.
 	IsLeaf bool
 
-	// CodeDir is the directory containing the Lambda function code. Required.
+	// CodeDir is the Lambda function code directory. Required.
 	CodeDir string
 
-	// ScratchDir is the per-invocation writable directory for each sandbox.
-	ScratchDir string
-
-	// Meta holds runtime configuration (memory limits, packages, imports).
+	// Meta holds runtime configuration. Nil means pool defaults.
 	Meta *sandbox.SandboxMeta
 
-	// MaxSize caps the total number of sandboxes. Zero means no limit.
-	MaxSize int
-
-	// DefaultTimeout is how long GetSandbox blocks at MaxSize capacity.
-	// Zero means fail immediately when all sandboxes are in use.
-	DefaultTimeout time.Duration
+	// ScratchDirs creates per-sandbox writable directories. Required.
+	ScratchDirs *common.DirMaker
 }
 
-// GetOption adjusts a single GetSandbox call.
-// Construct with WithTimeout.
-type GetOption func(*getOptions)
-
-// ReleaseOption adjusts a single ReleaseSandbox call.
-// Construct with WithoutPause.
-type ReleaseOption func(*releaseOptions)
-
-// Metrics holds cumulative counters since pool creation.
-// All fields are monotonically increasing.
-type Metrics struct {
-	Gets     int64 // total GetSandbox calls
-	Hits     int64 // Gets served from an already-available sandbox
-	Misses   int64 // Gets that required creating a new sandbox
-	Releases int64 // successful ReleaseSandbox calls
-	Destroys int64 // DestroyAndRemove calls
-	Timeouts int64 // Gets that failed due to deadline exceeded
-}
-
-// HitRate returns the fraction of Gets served from an available sandbox (0.0–1.0).
-func (m *Metrics) HitRate() float64 {
-	if m.Gets == 0 {
-		return 0.0
-	}
-	return float64(m.Hits) / float64(m.Gets)
-}
-
-// New creates a SandboxSet from cfg. Returns an error if cfg is invalid.
+// New creates a SandboxSet. Returns an error if cfg is invalid.
 func New(cfg *Config) (SandboxSet, error) {
 	return newSandboxSet(cfg)
 }
-
-// WithTimeout overrides the DefaultTimeout for a single GetSandbox call.
-func WithTimeout(d time.Duration) GetOption {
-	return func(o *getOptions) { o.timeout = d }
-}
-
-// WithoutPause skips the Pause call when returning a sandbox to the pool.
-// Use when the sandbox has already been paused by the caller.
-func WithoutPause() ReleaseOption {
-	return func(o *releaseOptions) { o.skipPause = true }
-}
diff --git a/go/worker/sandboxset/close.go b/go/worker/sandboxset/close.go
new file mode 100644
index 000000000..3762244c0
--- /dev/null
+++ b/go/worker/sandboxset/close.go
@@ -0,0 +1,23 @@
+package sandboxset
+
+import "fmt"
+
+// Close implements SandboxSet.
+//
+// All sandboxes are snapshot under the lock, then destroyed outside it.
+func (s *sandboxSetImpl) Close() error {
+	s.mu.Lock()
+	if s.closed {
+		s.mu.Unlock()
+		return fmt.Errorf("sandboxset: already closed")
+	}
+	s.closed = true
+	pool := s.pool
+	s.pool = nil
+	s.mu.Unlock()
+
+	for _, w := range pool {
+		w.sb.Destroy("sandboxset closed")
+	}
+	return nil
+}
diff --git a/go/worker/sandboxset/destroy.go b/go/worker/sandboxset/destroy.go
index 156067bc8..274c812bd 100644
--- a/go/worker/sandboxset/destroy.go
+++ b/go/worker/sandboxset/destroy.go
@@ -6,19 +6,19 @@ import (
 	"github.com/open-lambda/open-lambda/go/worker/sandbox"
 )
 
-// DestroyAndRemove implements SandboxSet.
+// Destroy implements SandboxSet.
 //
-// The wrapper is spliced out of the pool under a short write lock using O(1)
-// swap-with-tail. Destroy is called outside the lock to keep critical sections
-// short. The sandbox is always destroyed even if it was not found in the pool.
-func (s *sandboxSetImpl) DestroyAndRemove(sb sandbox.Sandbox, reason string) error {
+// The wrapper is spliced out of the pool under a short lock using O(1)
+// swap-with-tail. Destroy is called outside the lock to keep critical
+// sections short. The sandbox is always destroyed even if it was not
+// found in the pool.
+func (s *sandboxSetImpl) Destroy(sb sandbox.Sandbox, reason string) error {
 	s.mu.Lock()
 	found := false
 	for i, w := range s.pool {
 		if w.sb.ID() == sb.ID() {
 			s.pool[i] = s.pool[len(s.pool)-1]
 			s.pool = s.pool[:len(s.pool)-1]
-			s.metrics.Destroys++
 			found = true
 			break
 		}
diff --git a/go/worker/sandboxset/get.go b/go/worker/sandboxset/get.go
index 3be865cdf..7d7ede660 100644
--- a/go/worker/sandboxset/get.go
+++ b/go/worker/sandboxset/get.go
@@ -2,91 +2,58 @@ package sandboxset
 
 import (
 	"fmt"
-	"time"
 
 	"github.com/open-lambda/open-lambda/go/worker/sandbox"
 )
 
-// GetSandbox implements SandboxSet.
+// Get implements SandboxSet.
 //
-// Fast path: an idle sandbox is claimed under a short write lock, then
-// Unpause runs outside the lock so the pool is not stalled during I/O.
+// Fast path: an idle sandbox is claimed under a short lock, then Unpause
+// runs outside the lock so the pool is not stalled during I/O.
 //
-// Slow path: no idle sandbox exists. If the pool has room, a new sandbox is
-// created without holding any lock, then appended. If the pool is at MaxSize
-// and nothing becomes available before the deadline, an error is returned.
-func (s *sandboxSetImpl) GetSandbox(opts ...GetOption) (sandbox.Sandbox, error) {
-	o := &getOptions{timeout: s.cfg.DefaultTimeout}
-	for _, opt := range opts {
-		opt(o)
-	}
-
-	var deadline time.Time
-	if o.timeout > 0 {
-		deadline = time.Now().Add(o.timeout)
-	}
-
+// Slow path: no idle sandbox exists, so a new one is created without
+// holding the lock.
+func (s *sandboxSetImpl) Get() (sandbox.Sandbox, error) {
 	s.mu.Lock()
-	s.metrics.Gets++
-	s.mu.Unlock()
-
-	for {
-		// Try to claim an idle sandbox.
-		s.mu.Lock()
-		var claimed sandbox.Sandbox
-		for _, w := range s.pool {
-			if !w.inUse {
-				w.inUse = true
-				claimed = w.sb
-				break
-			}
-		}
-		atCap := s.cfg.MaxSize > 0 && len(s.pool) >= s.cfg.MaxSize
-		poolSize := len(s.pool)
+	if s.closed {
 		s.mu.Unlock()
+		return nil, fmt.Errorf("sandboxset: closed")
+	}
 
-		if claimed != nil {
-			// Unpause outside the lock (split-lock pattern): inUse=true already
-			// prevents another goroutine from claiming this sandbox.
-			if err := claimed.Unpause(); err != nil {
-				_ = s.DestroyAndRemove(claimed, fmt.Sprintf("unpause: %v", err))
-				continue
-			}
-			s.mu.Lock()
-			s.metrics.Hits++
-			s.mu.Unlock()
-			return claimed, nil
+	// Fast path: claim an idle sandbox.
+	var claimed sandbox.Sandbox
+	for _, w := range s.pool {
+		if !w.inUse {
+			w.inUse = true
+			claimed = w.sb
+			break
 		}
+	}
+	s.mu.Unlock()
 
-		// Pool is at capacity: wait or timeout.
-		if atCap {
-			if !deadline.IsZero() && time.Now().After(deadline) {
-				s.mu.Lock()
-				s.metrics.Timeouts++
-				s.mu.Unlock()
-				return nil, fmt.Errorf(
-					"sandboxset: all %d sandboxes in use; deadline exceeded", poolSize,
-				)
-			}
-			time.Sleep(1 * time.Millisecond)
-			continue
+	if claimed != nil {
+		// Unpause outside the lock (split-lock pattern).
+		if err := claimed.Unpause(); err != nil {
+			_ = s.Destroy(claimed, fmt.Sprintf("unpause: %v", err))
+			return s.Get()
 		}
+		return claimed, nil
+	}
 
-		// Slow path: create a new sandbox without holding the lock.
-		sb, err := s.cfg.Pool.Create(
-			s.cfg.Parent, s.cfg.IsLeaf,
-			s.cfg.CodeDir, s.cfg.ScratchDir,
-			s.cfg.Meta,
-		)
-		if err != nil {
-			return nil, fmt.Errorf("sandboxset: create sandbox: %w", err)
-		}
+	// Slow path: create a new sandbox without holding the lock.
+	scratchDir := s.cfg.ScratchDirs.Make("sb")
+	sb, err := s.cfg.Pool.Create(
+		s.cfg.Parent, s.cfg.IsLeaf,
+		s.cfg.CodeDir, scratchDir,
+		s.cfg.Meta,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("sandboxset: create: %w", err)
+	}
 
-		s.mu.Lock()
-		s.pool = append(s.pool, &sandboxWrapper{sb: sb, inUse: true})
-		s.metrics.Misses++
-		s.mu.Unlock()
+	s.mu.Lock()
+	s.pool = append(s.pool, &sandboxWrapper{sb: sb, inUse: true})
+	s.mu.Unlock()
 
-		return sb, nil
-	}
+	return sb, nil
 }
diff --git a/go/worker/sandboxset/metrics.go b/go/worker/sandboxset/metrics.go
deleted file mode 100644
index 8966651c7..000000000
--- a/go/worker/sandboxset/metrics.go
+++ /dev/null
@@ -1,10 +0,0 @@
-package sandboxset
-
-// Metrics implements SandboxSet. Returns a copy so callers cannot mutate
-// internal counters.
-func (s *sandboxSetImpl) Metrics() *Metrics {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
-	m := s.metrics
-	return &m
-}
diff --git a/go/worker/sandboxset/put.go b/go/worker/sandboxset/put.go
new file mode 100644
index 000000000..4f02fdfce
--- /dev/null
+++ b/go/worker/sandboxset/put.go
@@ -0,0 +1,30 @@
+package sandboxset
+
+import (
+	"fmt"
+
+	"github.com/open-lambda/open-lambda/go/worker/sandbox"
+)
+
+// Put implements SandboxSet.
+//
+// The sandbox is paused and its wrapper is flipped back to idle.
+// If Pause fails, the sandbox is destroyed rather than silently
+// recycled — a bad sandbox should never re-enter the pool.
+func (s *sandboxSetImpl) Put(sb sandbox.Sandbox) error {
+	if err := sb.Pause(); err != nil {
+		_ = s.Destroy(sb, fmt.Sprintf("pause failed: %v", err))
+		return fmt.Errorf("sandboxset: sandbox %s destroyed because Pause failed: %w", sb.ID(), err)
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	for _, w := range s.pool {
+		if w.sb.ID() == sb.ID() {
+			w.inUse = false
+			return nil
+		}
+	}
+	return fmt.Errorf("sandboxset: sandbox %s not found in pool", sb.ID())
+}
diff --git a/go/worker/sandboxset/release.go b/go/worker/sandboxset/release.go
deleted file mode 100644
index 01b1613c2..000000000
--- a/go/worker/sandboxset/release.go
+++ /dev/null
@@ -1,38 +0,0 @@
-package sandboxset
-
-import (
-	"fmt"
-
-	"github.com/open-lambda/open-lambda/go/worker/sandbox"
-)
-
-// ReleaseSandbox implements SandboxSet.
-//
-// The sandbox is paused (unless WithoutPause was supplied) and its wrapper is
-// flipped back to idle. If Pause fails the sandbox is destroyed rather than
-// silently recycled — a bad sandbox should never re-enter the pool.
-func (s *sandboxSetImpl) ReleaseSandbox(sb sandbox.Sandbox, opts ...ReleaseOption) error {
-	o := &releaseOptions{}
-	for _, opt := range opts {
-		opt(o)
-	}
-
-	if !o.skipPause {
-		if err := sb.Pause(); err != nil {
-			_ = s.DestroyAndRemove(sb, fmt.Sprintf("pause failed on release: %v", err))
-			return fmt.Errorf("sandboxset: sandbox %s destroyed because Pause failed: %w", sb.ID(), err)
-		}
-	}
-
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	for _, w := range s.pool {
-		if w.sb.ID() == sb.ID() {
-			w.inUse = false
-			s.metrics.Releases++
-			return nil
-		}
-	}
-	return fmt.Errorf("sandboxset: sandbox %s not found in pool", sb.ID())
-}
diff --git a/go/worker/sandboxset/sandboxset.go b/go/worker/sandboxset/sandboxset.go
index 72d29ddfe..96efdfbb0 100644
--- a/go/worker/sandboxset/sandboxset.go
+++ b/go/worker/sandboxset/sandboxset.go
@@ -3,35 +3,23 @@ package sandboxset
 import (
 	"fmt"
 	"sync"
-	"time"
 
 	"github.com/open-lambda/open-lambda/go/worker/sandbox"
 )
 
 // sandboxWrapper pairs a sandbox with an in-use flag.
-// Never exported; internal pool bookkeeping only.
 type sandboxWrapper struct {
 	sb    sandbox.Sandbox
 	inUse bool
 }
 
-// getOptions holds resolved settings for one GetSandbox call.
-type getOptions struct {
-	timeout time.Duration
-}
-
-// releaseOptions holds resolved settings for one ReleaseSandbox call.
-type releaseOptions struct {
-	skipPause bool
-}
-
 // sandboxSetImpl is the private concrete type returned by New.
 // All mutable state is guarded by mu.
 type sandboxSetImpl struct {
-	mu      sync.RWMutex
-	pool    []*sandboxWrapper
-	cfg     *Config
-	metrics Metrics
+	mu     sync.Mutex
+	pool   []*sandboxWrapper
+	cfg    *Config
+	closed bool
 }
 
 func newSandboxSet(cfg *Config) (*sandboxSetImpl, error) {
@@ -44,5 +32,8 @@ func newSandboxSet(cfg *Config) (*sandboxSetImpl, error) {
 	if cfg.CodeDir == "" {
 		return nil, fmt.Errorf("sandboxset: Config.CodeDir must not be empty")
 	}
+	if cfg.ScratchDirs == nil {
+		return nil, fmt.Errorf("sandboxset: Config.ScratchDirs must not be nil")
+	}
 	return &sandboxSetImpl{cfg: cfg}, nil
 }
diff --git a/go/worker/sandboxset/shrink.go b/go/worker/sandboxset/shrink.go
deleted file mode 100644
index c1bc053ce..000000000
--- a/go/worker/sandboxset/shrink.go
+++ /dev/null
@@ -1,32 +0,0 @@
-package sandboxset
-
-// Shrink implements SandboxSet.
-//
-// One idle sandbox is removed and destroyed per iteration so the write lock
-// is never held while destruction I/O runs.
-func (s *sandboxSetImpl) Shrink(target int) error {
-	for {
-		s.mu.Lock()
-		if len(s.pool) <= target {
-			s.mu.Unlock()
-			return nil
-		}
-
-		var victim *sandboxWrapper
-		for i, w := range s.pool {
-			if !w.inUse {
-				victim = w
-				s.pool[i] = s.pool[len(s.pool)-1]
-				s.pool = s.pool[:len(s.pool)-1]
-				break
-			}
-		}
-		s.mu.Unlock()
-
-		if victim == nil {
-			// All remaining sandboxes are actively in use.
-			return nil
-		}
-		victim.sb.Destroy("shrink")
-	}
-}
diff --git a/go/worker/sandboxset/stats.go b/go/worker/sandboxset/stats.go
deleted file mode 100644
index ce76b5c79..000000000
--- a/go/worker/sandboxset/stats.go
+++ /dev/null
@@ -1,21 +0,0 @@
-package sandboxset
-
-// Stats implements SandboxSet.
-func (s *sandboxSetImpl) Stats() map[string]int {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
-
-	available, inUse := 0, 0
-	for _, w := range s.pool {
-		if w.inUse {
-			inUse++
-		} else {
-			available++
-		}
-	}
-	return map[string]int{
-		"available": available,
-		"in_use":    inUse,
-		"total":     len(s.pool),
-	}
-}
diff --git a/go/worker/sandboxset/warm.go b/go/worker/sandboxset/warm.go
deleted file mode 100644
index 09aa151ad..000000000
--- a/go/worker/sandboxset/warm.go
+++ /dev/null
@@ -1,41 +0,0 @@
-package sandboxset
-
-import "fmt"
-
-// Warm implements SandboxSet.
-//
-// Sandboxes are created and paused sequentially. Parallel creation could
-// overwhelm the host with container-start overhead; pools are typically small
-// (5–10 entries) so sequential creation is fast enough.
-func (s *sandboxSetImpl) Warm(target int) error {
-	for {
-		s.mu.RLock()
-		current := len(s.pool)
-		s.mu.RUnlock()
-
-		if current >= target {
-			return nil
-		}
-		if s.cfg.MaxSize > 0 && current >= s.cfg.MaxSize {
-			return nil
-		}
-
-		sb, err := s.cfg.Pool.Create(
-			s.cfg.Parent, s.cfg.IsLeaf,
-			s.cfg.CodeDir, s.cfg.ScratchDir,
-			s.cfg.Meta,
-		)
-		if err != nil {
-			return fmt.Errorf("sandboxset: Warm create[%d]: %w", current, err)
-		}
-
-		if err := sb.Pause(); err != nil {
-			sb.Destroy("pause failed during Warm")
-			return fmt.Errorf("sandboxset: Warm pause[%d]: %w", current, err)
-		}
-
-		s.mu.Lock()
-		s.pool = append(s.pool, &sandboxWrapper{sb: sb, inUse: false})
-		s.mu.Unlock()
-	}
-}

From cb929a8551965b9243ff189a785edf66b7aa26cb Mon Sep 17 00:00:00 2001
From: Ami Buch <amiabuch@gmail.com>
Date: Sun, 1 Mar 2026 20:59:33 -0600
Subject: [PATCH 42/55] fix: simpler diagram

---
 go/worker/sandboxset/api.go | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/go/worker/sandboxset/api.go b/go/worker/sandboxset/api.go
index 49bcb5611..cea0adac5 100644
--- a/go/worker/sandboxset/api.go
+++ b/go/worker/sandboxset/api.go
@@ -3,9 +3,16 @@
 //
 // Sandbox lifecycle inside a SandboxSet:
 //
-//	created ──► paused (available) ──► in-use (unpaused) ──► paused (available)
-//	                  │                       │
-//	                  └───────── destroyed ◄──┘  (on error or Close)
+//	[created]
+//	    |
+//	    v
+//	[paused]  <---+
+//	    |         |
+//	    v         |
+//	[in-use]  ----+  (Put)
+//	    |
+//	    v
+//	[destroyed]     (Destroy / Close / error)
 //
 // Usage:
 //

From d17fee5b3f259cd712956833141ade66bd010feb Mon Sep 17 00:00:00 2001
From: Ami Buch <amiabuch@gmail.com>
Date: Tue, 3 Mar 2026 13:17:32 -0600
Subject: [PATCH 43/55] fix: commenting, some edge cases

---
 go/worker/sandbox/mock.go                     | 122 +++++
 go/worker/sandboxset/api.go                   |  86 +++-
 go/worker/sandboxset/tests/sandboxset_test.go | 449 ++++++++++++++++++
 3 files changed, 636 insertions(+), 21 deletions(-)
 create mode 100644 go/worker/sandbox/mock.go
 create mode 100644 go/worker/sandboxset/tests/sandboxset_test.go

diff --git a/go/worker/sandbox/mock.go b/go/worker/sandbox/mock.go
new file mode 100644
index 000000000..b4cbf4d66
--- /dev/null
+++ b/go/worker/sandbox/mock.go
@@ -0,0 +1,122 @@
+package sandbox
+
+import (
+	"fmt"
+	"net/http"
+	"sync"
+	"sync/atomic"
+)
+
+// MockSandbox is a test double for Sandbox.
+// Exported fields control error injection; state fields track lifecycle.
+type MockSandbox struct {
+	mu        sync.Mutex
+	id        string
+	paused    bool
+	destroyed bool
+
+	// Set these before calling Get/Put to inject errors.
+	PauseErr   error
+	UnpauseErr error
+}
+
+var mockIDCounter int64
+
+// NewMockSandbox creates a MockSandbox with the given ID.
+func NewMockSandbox(id string) *MockSandbox {
+	return &MockSandbox{id: id, paused: true}
+}
+
+func (m *MockSandbox) ID() string { return m.id }
+
+func (m *MockSandbox) Destroy(reason string) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.destroyed = true
+}
+
+func (m *MockSandbox) DestroyIfPaused(reason string) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.paused {
+		m.destroyed = true
+	}
+}
+
+func (m *MockSandbox) Pause() error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.PauseErr != nil {
+		return m.PauseErr
+	}
+	m.paused = true
+	return nil
+}
+
+func (m *MockSandbox) Unpause() error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.UnpauseErr != nil {
+		return m.UnpauseErr
+	}
+	m.paused = false
+	return nil
+}
+
+func (m *MockSandbox) Client() *http.Client   { return nil }
+func (m *MockSandbox) Meta() *SandboxMeta      { return nil }
+func (m *MockSandbox) GetRuntimeLog() string   { return "" }
+func (m *MockSandbox) GetProxyLog() string     { return "" }
+func (m *MockSandbox) DebugString() string     { return fmt.Sprintf("mock:%s", m.id) }
+func (m *MockSandbox) fork(dst Sandbox) error  { return nil }
+func (m *MockSandbox) childExit(child Sandbox) {}
+
+// IsDestroyed returns whether Destroy has been called.
+func (m *MockSandbox) IsDestroyed() bool {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.destroyed
+}
+
+// IsPaused returns the current pause state.
+func (m *MockSandbox) IsPaused() bool {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.paused
+}
+
+// MockSandboxPool is a test double for SandboxPool.
+// It creates MockSandbox instances with auto-incremented IDs.
+type MockSandboxPool struct {
+	mu      sync.Mutex
+	Created []*MockSandbox
+
+	// Set before calling Get to make pool.Create fail.
+	CreateErr error
+}
+
+func (p *MockSandboxPool) Create(parent Sandbox, isLeaf bool, codeDir, scratchDir string, meta *SandboxMeta) (Sandbox, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if p.CreateErr != nil {
+		return nil, p.CreateErr
+	}
+	id := fmt.Sprintf("mock-%d", atomic.AddInt64(&mockIDCounter, 1))
+	sb := NewMockSandbox(id)
+	sb.paused = false // Pool.Create returns unpaused sandboxes
+	p.Created = append(p.Created, sb)
+	return sb, nil
+}
+
+func (p *MockSandboxPool) Cleanup()                            {}
+func (p *MockSandboxPool) AddListener(handler SandboxEventFunc) {}
+func (p *MockSandboxPool) DebugString() string                  { return "mock-pool" }
+
+// CreatedSandboxes returns a snapshot of all sandboxes created by this pool.
+func (p *MockSandboxPool) CreatedSandboxes() []*MockSandbox {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	out := make([]*MockSandbox, len(p.Created))
+	copy(out, p.Created)
+	return out
+}
diff --git a/go/worker/sandboxset/api.go b/go/worker/sandboxset/api.go
index cea0adac5..8e9a5513e 100644
--- a/go/worker/sandboxset/api.go
+++ b/go/worker/sandboxset/api.go
@@ -1,6 +1,10 @@
 // Package sandboxset provides a thread-safe pool of sandboxes for a single
 // Lambda function.
 //
+// A SandboxSet replaces per-instance goroutines with a simple pool.
+// Callers just ask for a sandbox and don't worry about whether it is
+// freshly created or recycled from a previous request.
+//
 // Sandbox lifecycle inside a SandboxSet:
 //
 //	[created]
@@ -32,51 +36,91 @@ import (
 	"github.com/open-lambda/open-lambda/go/worker/sandbox"
 )
 
-// SandboxSet is a thread-safe pool of sandboxes for a single Lambda function.
-// All methods are safe for concurrent use.
+/*
+A SandboxSet manages a pool of sandboxes for one Lambda function.
+All methods are safe to call from multiple goroutines.
+
+The design mirrors the C process API: Get (create), Put (exit),
+Destroy (kill), Close (cleanup). There are no warm-up, shrink, or
+stats methods yet — those can be added in later PRs without
+changing the core interface.
+*/
 type SandboxSet interface {
-	// Get borrows a sandbox, creating one if none are idle.
-	// The returned sandbox is unpaused and ready to handle a request.
-	// Caller MUST call Put or Destroy when done.
+	// Return an unpaused sandbox ready to handle a request.
+	//
+	// If the pool has an idle sandbox, it is unpaused and returned.
+	// If Unpause fails (e.g., the SOCK container died while paused),
+	// that sandbox is destroyed and Get tries the next idle one or
+	// creates a fresh sandbox.
+	//
+	// A fresh scratch directory is created for each new sandbox
+	// via Config.ScratchDirs. Reused sandboxes keep their
+	// existing scratch directory from when they were first created.
 	Get() (sandbox.Sandbox, error)
 
-	// Put returns a sandbox to the pool after successful use.
-	// The sandbox is paused and made available for future Get calls.
-	// If pausing fails, the sandbox is destroyed automatically.
+	// Return a sandbox to the pool after a successful request.
+	//
+	// The sandbox is paused and becomes available for the next Get.
+	// If Pause fails (e.g., the container died during the request),
+	// the sandbox is destroyed automatically — a bad sandbox never
+	// re-enters the pool.
+	//
+	// Passing a sandbox that is not in the pool returns an error
+	// but is otherwise harmless.
 	Put(sb sandbox.Sandbox) error
 
-	// Destroy permanently removes a sandbox from the pool and kills it.
-	// Use when a sandbox is in an unrecoverable state.
+	// Permanently remove a sandbox from the pool and destroy it.
+	//
+	// Use this when a request produced an unrecoverable error and
+	// the sandbox should not be reused. "reason" is a
+	// human-readable explanation that shows up in later error
+	// messages (same convention as sandbox.Sandbox.Destroy).
+	//
+	// If the sandbox is not in the pool it is still destroyed —
+	// resources are always freed. The returned error is
+	// informational only.
 	Destroy(sb sandbox.Sandbox, reason string) error
 
-	// Close destroys all sandboxes in the pool.
-	// After Close returns, Get/Put/Destroy return errors.
+	// Destroy all sandboxes in the pool and mark the set as closed.
+	//
+	// Callers who still hold sandbox references from a previous Get
+	// will find them already dead, which is safe: per the Sandbox
+	// contract, methods on a destroyed sandbox are harmless no-ops
+	// that return errors.
+	//
+	// Calling Close a second time returns an error.
 	Close() error
 }
 
-// Config holds creation parameters for a SandboxSet.
-// Pool, CodeDir, and ScratchDirs are required.
+// Config holds the parameters needed to create a SandboxSet.
 type Config struct {
-	// Pool creates new sandboxes. Required.
+	// Pool creates and destroys the underlying sandboxes.
 	Pool sandbox.SandboxPool
 
-	// Parent is the sandbox to fork from. Nil means create from scratch.
+	// Parent sandbox to fork from (may be nil). When nil, new
+	// sandboxes are created from scratch. Not all SandboxPool
+	// implementations support forking.
 	Parent sandbox.Sandbox
 
-	// IsLeaf marks created sandboxes as non-forkable.
+	// IsLeaf marks sandboxes as non-forkable, meaning they will
+	// not be used as parents for future forks.
 	IsLeaf bool
 
-	// CodeDir is the Lambda function code directory. Required.
+	// CodeDir is the directory containing the Lambda handler code.
 	CodeDir string
 
-	// Meta holds runtime configuration. Nil means pool defaults.
+	// Meta holds runtime configuration (memory limits, packages,
+	// imports, etc.). Nil means the pool fills in defaults.
 	Meta *sandbox.SandboxMeta
 
-	// ScratchDirs creates per-sandbox writable directories. Required.
+	// ScratchDirs creates a unique writable directory for each
+	// new sandbox. The set calls ScratchDirs.Make internally
+	// so that Get can remain argument-free.
 	ScratchDirs *common.DirMaker
 }
 
-// New creates a SandboxSet. Returns an error if cfg is invalid.
+// New creates a SandboxSet from cfg. Returns an error if any of
+// Pool, CodeDir, or ScratchDirs are missing.
 func New(cfg *Config) (SandboxSet, error) {
 	return newSandboxSet(cfg)
 }
diff --git a/go/worker/sandboxset/tests/sandboxset_test.go b/go/worker/sandboxset/tests/sandboxset_test.go
new file mode 100644
index 000000000..f49d8cc78
--- /dev/null
+++ b/go/worker/sandboxset/tests/sandboxset_test.go
@@ -0,0 +1,449 @@
+package tests
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+	"testing"
+
+	"github.com/open-lambda/open-lambda/go/common"
+	"github.com/open-lambda/open-lambda/go/worker/sandbox"
+	"github.com/open-lambda/open-lambda/go/worker/sandboxset"
+)
+
+// newTestConfig returns a valid Config backed by mocks and a temp directory.
+func newTestConfig(t *testing.T) (*sandboxset.Config, *sandbox.MockSandboxPool) {
+	t.Helper()
+	tmpDir := t.TempDir()
+	common.Conf = &common.Config{Worker_dir: tmpDir}
+	scratchDirs, err := common.NewDirMaker("scratch", common.STORE_REGULAR)
+	if err != nil {
+		t.Fatal(err)
+	}
+	pool := &sandbox.MockSandboxPool{}
+	cfg := &sandboxset.Config{
+		Pool:        pool,
+		CodeDir:     tmpDir + "/code",
+		ScratchDirs: scratchDirs,
+	}
+	return cfg, pool
+}
+
+// newTestSet is a shortcut that creates a valid SandboxSet.
+func newTestSet(t *testing.T) (sandboxset.SandboxSet, *sandbox.MockSandboxPool) {
+	t.Helper()
+	cfg, pool := newTestConfig(t)
+	set, err := sandboxset.New(cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return set, pool
+}
+
+// --- Constructor tests ---
+
+func TestNew_NilConfig(t *testing.T) {
+	_, err := sandboxset.New(nil)
+	if err == nil {
+		t.Fatal("expected error for nil config")
+	}
+}
+
+func TestNew_NilPool(t *testing.T) {
+	tmpDir := t.TempDir()
+	common.Conf = &common.Config{Worker_dir: tmpDir}
+	scratchDirs, err := common.NewDirMaker("scratch", common.STORE_REGULAR)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = sandboxset.New(&sandboxset.Config{
+		CodeDir:     "/some/dir",
+		ScratchDirs: scratchDirs,
+	})
+	if err == nil {
+		t.Fatal("expected error for nil Pool")
+	}
+}
+
+func TestNew_EmptyCodeDir(t *testing.T) {
+	tmpDir := t.TempDir()
+	common.Conf = &common.Config{Worker_dir: tmpDir}
+	scratchDirs, err := common.NewDirMaker("scratch", common.STORE_REGULAR)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = sandboxset.New(&sandboxset.Config{
+		Pool:        &sandbox.MockSandboxPool{},
+		ScratchDirs: scratchDirs,
+	})
+	if err == nil {
+		t.Fatal("expected error for empty CodeDir")
+	}
+}
+
+func TestNew_NilScratchDirs(t *testing.T) {
+	_, err := sandboxset.New(&sandboxset.Config{
+		Pool:    &sandbox.MockSandboxPool{},
+		CodeDir: "/some/dir",
+	})
+	if err == nil {
+		t.Fatal("expected error for nil ScratchDirs")
+	}
+}
+
+func TestNew_Valid(t *testing.T) {
+	set, _ := newTestSet(t)
+	if set == nil {
+		t.Fatal("expected non-nil SandboxSet")
+	}
+}
+
+// --- Get tests ---
+
+func TestGet_CreatesNew(t *testing.T) {
+	set, pool := newTestSet(t)
+	sb, err := set.Get()
+	if err != nil {
+		t.Fatalf("Get: %v", err)
+	}
+	if sb == nil {
+		t.Fatal("expected non-nil sandbox")
+	}
+	if n := len(pool.CreatedSandboxes()); n != 1 {
+		t.Fatalf("expected 1 created sandbox, got %d", n)
+	}
+}
+
+func TestGet_ReusesIdle(t *testing.T) {
+	set, _ := newTestSet(t)
+
+	sb1, err := set.Get()
+	if err != nil {
+		t.Fatalf("Get: %v", err)
+	}
+	id1 := sb1.ID()
+
+	if err := set.Put(sb1); err != nil {
+		t.Fatalf("Put: %v", err)
+	}
+
+	sb2, err := set.Get()
+	if err != nil {
+		t.Fatalf("Get: %v", err)
+	}
+	if sb2.ID() != id1 {
+		t.Fatalf("expected reuse (ID %s), got new (ID %s)", id1, sb2.ID())
+	}
+}
+
+func TestGet_UnpauseFail(t *testing.T) {
+	set, pool := newTestSet(t)
+
+	sb1, err := set.Get()
+	if err != nil {
+		t.Fatalf("Get: %v", err)
+	}
+	// Inject unpause error before putting back.
+	sb1.(*sandbox.MockSandbox).UnpauseErr = errors.New("broken")
+	if err := set.Put(sb1); err != nil {
+		t.Fatalf("Put: %v", err)
+	}
+
+	// Next Get should find the bad sandbox, destroy it, and create a new one.
+	sb2, err := set.Get()
+	if err != nil {
+		t.Fatalf("Get after unpause fail: %v", err)
+	}
+	if sb2.ID() == sb1.ID() {
+		t.Fatal("expected a different sandbox after unpause failure")
+	}
+	if !sb1.(*sandbox.MockSandbox).IsDestroyed() {
+		t.Fatal("bad sandbox should have been destroyed")
+	}
+	if n := len(pool.CreatedSandboxes()); n != 2 {
+		t.Fatalf("expected 2 creates (original + retry), got %d", n)
+	}
+}
+
+func TestGet_CreateFail(t *testing.T) {
+	set, pool := newTestSet(t)
+	pool.CreateErr = errors.New("out of resources")
+
+	_, err := set.Get()
+	if err == nil {
+		t.Fatal("expected error when pool.Create fails")
+	}
+}
+
+func TestGet_AfterClose(t *testing.T) {
+	set, _ := newTestSet(t)
+	if err := set.Close(); err != nil {
+		t.Fatalf("Close: %v", err)
+	}
+	_, err := set.Get()
+	if err == nil {
+		t.Fatal("expected error after Close")
+	}
+}
+
+// --- Put tests ---
+
+func TestPut_PausesAndReturns(t *testing.T) {
+	set, _ := newTestSet(t)
+	sb, err := set.Get()
+	if err != nil {
+		t.Fatalf("Get: %v", err)
+	}
+	mock := sb.(*sandbox.MockSandbox)
+	if mock.IsPaused() {
+		t.Fatal("sandbox should be unpaused after Get")
+	}
+
+	if err := set.Put(sb); err != nil {
+		t.Fatalf("Put: %v", err)
+	}
+	if !mock.IsPaused() {
+		t.Fatal("sandbox should be paused after Put")
+	}
+}
+
+func TestPut_PauseFail(t *testing.T) {
+	set, _ := newTestSet(t)
+	sb, err := set.Get()
+	if err != nil {
+		t.Fatalf("Get: %v", err)
+	}
+	sb.(*sandbox.MockSandbox).PauseErr = errors.New("pause broken")
+
+	err = set.Put(sb)
+	if err == nil {
+		t.Fatal("expected error when Pause fails")
+	}
+	if !sb.(*sandbox.MockSandbox).IsDestroyed() {
+		t.Fatal("sandbox should be destroyed when Pause fails")
+	}
+}
+
+func TestPut_NotInPool(t *testing.T) {
+	set, _ := newTestSet(t)
+	orphan := sandbox.NewMockSandbox("orphan")
+	err := set.Put(orphan)
+	if err == nil {
+		t.Fatal("expected error for sandbox not in pool")
+	}
+}
+
+// --- Destroy tests ---
+
+func TestDestroy_RemovesFromPool(t *testing.T) {
+	set, _ := newTestSet(t)
+	sb, err := set.Get()
+	if err != nil {
+		t.Fatalf("Get: %v", err)
+	}
+
+	if err := set.Destroy(sb, "test"); err != nil {
+		t.Fatalf("Destroy: %v", err)
+	}
+	if !sb.(*sandbox.MockSandbox).IsDestroyed() {
+		t.Fatal("sandbox should be destroyed")
+	}
+
+	// Next Get should create a new one, not reuse the destroyed one.
+	sb2, err := set.Get()
+	if err != nil {
+		t.Fatalf("Get after Destroy: %v", err)
+	}
+	if sb2.ID() == sb.ID() {
+		t.Fatal("should not reuse a destroyed sandbox")
+	}
+}
+
+func TestDestroy_NotInPool(t *testing.T) {
+	set, _ := newTestSet(t)
+	orphan := sandbox.NewMockSandbox("orphan")
+	err := set.Destroy(orphan, "test")
+	if err == nil {
+		t.Fatal("expected error for sandbox not in pool")
+	}
+	if !orphan.IsDestroyed() {
+		t.Fatal("sandbox should still be destroyed even if not in pool")
+	}
+}
+
+// --- Close tests ---
+
+func TestClose_DestroysAll(t *testing.T) {
+	set, pool := newTestSet(t)
+
+	// Create 3 sandboxes: 2 in-use, 1 idle.
+	sb1, _ := set.Get()
+	sb2, _ := set.Get()
+	sb3, _ := set.Get()
+	_ = set.Put(sb3) // return one to idle
+
+	if err := set.Close(); err != nil {
+		t.Fatalf("Close: %v", err)
+	}
+
+	for _, m := range pool.CreatedSandboxes() {
+		if !m.IsDestroyed() {
+			t.Fatalf("sandbox %s should be destroyed after Close", m.ID())
+		}
+	}
+	_ = sb1
+	_ = sb2
+}
+
+func TestClose_Twice(t *testing.T) {
+	set, _ := newTestSet(t)
+	if err := set.Close(); err != nil {
+		t.Fatalf("first Close: %v", err)
+	}
+	err := set.Close()
+	if err == nil {
+		t.Fatal("expected error on second Close")
+	}
+}
+
+func TestClose_EmptyPool(t *testing.T) {
+	set, _ := newTestSet(t)
+	if err := set.Close(); err != nil {
+		t.Fatalf("Close on empty pool: %v", err)
+	}
+}
+
+// --- Lifecycle tests ---
+
+func TestLifecycle_GetPutReuse(t *testing.T) {
+	set, _ := newTestSet(t)
+
+	// Get → Put → Get should reuse.
+	sb1, _ := set.Get()
+	id := sb1.ID()
+	_ = set.Put(sb1)
+
+	sb2, _ := set.Get()
+	if sb2.ID() != id {
+		t.Fatalf("expected reuse, got different ID: %s vs %s", id, sb2.ID())
+	}
+	_ = set.Put(sb2)
+
+	_ = set.Close()
+}
+
+func TestLifecycle_GetDestroyGet(t *testing.T) {
+	set, _ := newTestSet(t)
+
+	sb1, _ := set.Get()
+	id := sb1.ID()
+	_ = set.Destroy(sb1, "bad")
+
+	sb2, _ := set.Get()
+	if sb2.ID() == id {
+		t.Fatal("expected fresh sandbox after Destroy, got same ID")
+	}
+	_ = set.Put(sb2)
+
+	_ = set.Close()
+}
+
+// --- Concurrency tests ---
+
+func TestConcurrent_Gets(t *testing.T) {
+	set, _ := newTestSet(t)
+	const n = 50
+
+	var wg sync.WaitGroup
+	sandboxes := make([]sandbox.Sandbox, n)
+	errs := make([]error, n)
+
+	for i := 0; i < n; i++ {
+		wg.Add(1)
+		go func(idx int) {
+			defer wg.Done()
+			sb, err := set.Get()
+			sandboxes[idx] = sb
+			errs[idx] = err
+		}(i)
+	}
+	wg.Wait()
+
+	for i, err := range errs {
+		if err != nil {
+			t.Fatalf("goroutine %d: Get: %v", i, err)
+		}
+	}
+
+	// Clean up: put all back then close.
+	for _, sb := range sandboxes {
+		_ = set.Put(sb)
+	}
+	_ = set.Close()
+}
+
+func TestConcurrent_GetPut(t *testing.T) {
+	set, _ := newTestSet(t)
+	const goroutines = 20
+	const iterations = 50
+
+	var wg sync.WaitGroup
+	for i := 0; i < goroutines; i++ {
+		wg.Add(1)
+		go func(id int) {
+			defer wg.Done()
+			for j := 0; j < iterations; j++ {
+				sb, err := set.Get()
+				if err != nil {
+					t.Errorf("goroutine %d iter %d: Get: %v", id, j, err)
+					return
+				}
+				if err := set.Put(sb); err != nil {
+					t.Errorf("goroutine %d iter %d: Put: %v", id, j, err)
+					return
+				}
+			}
+		}(i)
+	}
+	wg.Wait()
+	_ = set.Close()
+}
+
+func TestConcurrent_CloseWhileGet(t *testing.T) {
+	set, _ := newTestSet(t)
+	const n = 20
+
+	// Grab some sandboxes first.
+	for i := 0; i < 5; i++ {
+		sb, _ := set.Get()
+		_ = set.Put(sb)
+	}
+
+	var wg sync.WaitGroup
+	errs := make(chan error, n)
+
+	// Launch goroutines that race Get vs Close.
+	for i := 0; i < n; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			sb, err := set.Get()
+			if err != nil {
+				// Expected for some goroutines after Close.
+				return
+			}
+			errs <- set.Put(sb)
+		}()
+	}
+
+	// Close from main goroutine while Gets are racing.
+	closeErr := set.Close()
+	wg.Wait()
+	close(errs)
+
+	// Close should succeed (first call).
+	if closeErr != nil {
+		// Close might race with Get; as long as no panic, we're OK.
+		fmt.Printf("Close returned: %v (acceptable in race)\n", closeErr)
+	}
+}

From 8d1fbc3256afc589831549e821dbc7496d7ce90b Mon Sep 17 00:00:00 2001
From: Ami Buch <amiabuch@gmail.com>
Date: Thu, 5 Mar 2026 15:00:22 -0600
Subject: [PATCH 44/55] fix: recursive Get, DirMaker panic, put after close
 documented

---
 go/worker/sandboxset/api.go        |  4 ++-
 go/worker/sandboxset/get.go        | 41 ++++++++++++++++++------------
 go/worker/sandboxset/put.go        |  7 +++++
 go/worker/sandboxset/sandboxset.go | 13 ++++++++++
 4 files changed, 48 insertions(+), 17 deletions(-)

diff --git a/go/worker/sandboxset/api.go b/go/worker/sandboxset/api.go
index 8e9a5513e..b950e9d28 100644
--- a/go/worker/sandboxset/api.go
+++ b/go/worker/sandboxset/api.go
@@ -66,7 +66,9 @@ type SandboxSet interface {
 	// re-enters the pool.
 	//
 	// Passing a sandbox that is not in the pool returns an error
-	// but is otherwise harmless.
+	// but is otherwise harmless. If the set has been closed, Put
+	// returns an error immediately — the sandbox was already
+	// destroyed by Close.
 	Put(sb sandbox.Sandbox) error
 
 	// Permanently remove a sandbox from the pool and destroy it.
diff --git a/go/worker/sandboxset/get.go b/go/worker/sandboxset/get.go
index 7d7ede660..c1e2c3587 100644
--- a/go/worker/sandboxset/get.go
+++ b/go/worker/sandboxset/get.go
@@ -14,34 +14,43 @@ import (
 // Slow path: no idle sandbox exists, so a new one is created without
 // holding the lock.
 func (s *sandboxSetImpl) Get() (sandbox.Sandbox, error) {
-	s.mu.Lock()
-	if s.closed {
+	// Loop over idle sandboxes until one unpauses successfully,
+	// or the pool has no idle sandboxes left.
+	for {
+		s.mu.Lock()
+		if s.closed {
+			s.mu.Unlock()
+			return nil, fmt.Errorf("sandboxset: closed")
+		}
+
+		// Fast path: claim an idle sandbox.
+		var claimed sandbox.Sandbox
+		for _, w := range s.pool {
+			if !w.inUse {
+				w.inUse = true
+				claimed = w.sb
+				break
+			}
+		}
 		s.mu.Unlock()
-		return nil, fmt.Errorf("sandboxset: closed")
-	}
 
-	// Fast path: claim an idle sandbox.
-	var claimed sandbox.Sandbox
-	for _, w := range s.pool {
-		if !w.inUse {
-			w.inUse = true
-			claimed = w.sb
-			break
+		if claimed == nil {
+			break // no idle sandbox — fall through to Create
 		}
-	}
-	s.mu.Unlock()
 
-	if claimed != nil {
 		// Unpause outside the lock (split-lock pattern).
 		if err := claimed.Unpause(); err != nil {
 			_ = s.Destroy(claimed, fmt.Sprintf("unpause: %v", err))
-			return s.Get()
+			continue // try the next idle sandbox
 		}
 		return claimed, nil
 	}
 
 	// Slow path: create a new sandbox without holding the lock.
-	scratchDir := s.cfg.ScratchDirs.Make("sb")
+	scratchDir, err := s.makeScratchDir()
+	if err != nil {
+		return nil, fmt.Errorf("sandboxset: scratch dir: %w", err)
+	}
 	sb, err := s.cfg.Pool.Create(
 		s.cfg.Parent, s.cfg.IsLeaf,
 		s.cfg.CodeDir, scratchDir,
diff --git a/go/worker/sandboxset/put.go b/go/worker/sandboxset/put.go
index 4f02fdfce..18103bb36 100644
--- a/go/worker/sandboxset/put.go
+++ b/go/worker/sandboxset/put.go
@@ -12,6 +12,13 @@ import (
 // If Pause fails, the sandbox is destroyed rather than silently
 // recycled — a bad sandbox should never re-enter the pool.
 func (s *sandboxSetImpl) Put(sb sandbox.Sandbox) error {
+	s.mu.Lock()
+	if s.closed {
+		s.mu.Unlock()
+		return fmt.Errorf("sandboxset: closed (sandbox %s was destroyed by Close)", sb.ID())
+	}
+	s.mu.Unlock()
+
 	if err := sb.Pause(); err != nil {
 		_ = s.Destroy(sb, fmt.Sprintf("pause failed: %v", err))
 		return fmt.Errorf("sandboxset: sandbox %s destroyed because Pause failed: %w", sb.ID(), err)
diff --git a/go/worker/sandboxset/sandboxset.go b/go/worker/sandboxset/sandboxset.go
index 96efdfbb0..84f9c3b4e 100644
--- a/go/worker/sandboxset/sandboxset.go
+++ b/go/worker/sandboxset/sandboxset.go
@@ -37,3 +37,16 @@ func newSandboxSet(cfg *Config) (*sandboxSetImpl, error) {
 	}
 	return &sandboxSetImpl{cfg: cfg}, nil
 }
+
+// makeScratchDir creates a scratch directory for a new sandbox.
+// DirMaker.Make panics on failure (e.g., disk full), so we recover
+// here and return an error instead of crashing the worker.
+func (s *sandboxSetImpl) makeScratchDir() (dir string, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			err = fmt.Errorf("%v", r)
+		}
+	}()
+	dir = s.cfg.ScratchDirs.Make("sb")
+	return dir, nil
+}

From c28029d5ff8251a3e849c75a18cd85b04e315ec0 Mon Sep 17 00:00:00 2001
From: Ami Buch <amiabuch@gmail.com>
Date: Fri, 13 Mar 2026 12:19:06 -0500
Subject: [PATCH 45/55] refactor: changed the function distribution and it
 return type

---
 go/worker/sandboxset/api.go                   |  57 +++---
 go/worker/sandboxset/close.go                 |  23 ---
 go/worker/sandboxset/destroy.go               |  34 ----
 go/worker/sandboxset/get.go                   |  68 -------
 go/worker/sandboxset/put.go                   |  37 ----
 go/worker/sandboxset/sandboxset.go            | 173 +++++++++++++++++
 .../tests/sandboxset_integration_test.go      | 181 ++++++++++++++++++
 go/worker/sandboxset/tests/sandboxset_test.go | 145 +++++++-------
 8 files changed, 463 insertions(+), 255 deletions(-)
 delete mode 100644 go/worker/sandboxset/close.go
 delete mode 100644 go/worker/sandboxset/destroy.go
 delete mode 100644 go/worker/sandboxset/get.go
 delete mode 100644 go/worker/sandboxset/put.go
 create mode 100644 go/worker/sandboxset/tests/sandboxset_integration_test.go

diff --git a/go/worker/sandboxset/api.go b/go/worker/sandboxset/api.go
index b950e9d28..49bea5627 100644
--- a/go/worker/sandboxset/api.go
+++ b/go/worker/sandboxset/api.go
@@ -26,9 +26,13 @@
 //	    ScratchDirs: myScratchDirs,
 //	})
 //
-//	sb, err := set.Get()
-//	// ... handle request ...
-//	set.Put(sb)
+//	ref, err := set.GetOrCreateUnpaused()
+//	// ... use ref.Sandbox() to handle request ...
+//	if broken {
+//	    ref.Destroy("reason")
+//	} else {
+//	    ref.Put()
+//	}
 package sandboxset
 
 import (
@@ -40,35 +44,42 @@ import (
 A SandboxSet manages a pool of sandboxes for one Lambda function.
 All methods are safe to call from multiple goroutines.
 
-The design mirrors the C process API: Get (create), Put (exit),
-Destroy (kill), Close (cleanup). There are no warm-up, shrink, or
-stats methods yet — those can be added in later PRs without
-changing the core interface.
+The design mirrors the C process API: GetOrCreateUnpaused (create),
+Put (exit), Destroy (kill), Close (cleanup). There are no warm-up,
+shrink, or stats methods yet — those can be added in later PRs
+without changing the core interface.
+
+GetOrCreateUnpaused returns a *SandboxRef rather than a raw
+sandbox.Sandbox. The ref carries a health State and back-pointer
+to the parent set, so callers can use ref.Put() / ref.Destroy()
+without tracking the set. The set-level Put/Destroy methods are
+also available for callers that prefer them.
 */
 type SandboxSet interface {
-	// Return an unpaused sandbox ready to handle a request.
+	// Return an unpaused sandbox ready to handle a request,
+	// wrapped in a SandboxRef.
 	//
 	// If the pool has an idle sandbox, it is unpaused and returned.
 	// If Unpause fails (e.g., the SOCK container died while paused),
-	// that sandbox is destroyed and Get tries the next idle one or
-	// creates a fresh sandbox.
+	// that sandbox is destroyed and the next idle one is tried, or
+	// a fresh sandbox is created.
 	//
 	// A fresh scratch directory is created for each new sandbox
 	// via Config.ScratchDirs. Reused sandboxes keep their
 	// existing scratch directory from when they were first created.
-	Get() (sandbox.Sandbox, error)
+	GetOrCreateUnpaused() (*SandboxRef, error)
 
 	// Return a sandbox to the pool after a successful request.
 	//
-	// The sandbox is paused and becomes available for the next Get.
-	// If Pause fails (e.g., the container died during the request),
-	// the sandbox is destroyed automatically — a bad sandbox never
-	// re-enters the pool.
+	// The sandbox is paused and becomes available for the next
+	// GetOrCreateUnpaused. If Pause fails (e.g., the container
+	// died during the request), the sandbox is destroyed
+	// automatically — a bad sandbox never re-enters the pool.
 	//
 	// Passing a sandbox that is not in the pool returns an error
-	// but is otherwise harmless. If the set has been closed, Put
-	// returns an error immediately — the sandbox was already
-	// destroyed by Close.
+	// but is otherwise harmless.
+	//
+	// Prefer ref.Put() when you have a SandboxRef.
 	Put(sb sandbox.Sandbox) error
 
 	// Permanently remove a sandbox from the pool and destroy it.
@@ -81,14 +92,16 @@ type SandboxSet interface {
 	// If the sandbox is not in the pool it is still destroyed —
 	// resources are always freed. The returned error is
 	// informational only.
+	//
+	// Prefer ref.Destroy() when you have a SandboxRef.
 	Destroy(sb sandbox.Sandbox, reason string) error
 
 	// Destroy all sandboxes in the pool and mark the set as closed.
 	//
-	// Callers who still hold sandbox references from a previous Get
-	// will find them already dead, which is safe: per the Sandbox
-	// contract, methods on a destroyed sandbox are harmless no-ops
-	// that return errors.
+	// Callers who still hold SandboxRef values from a previous
+	// GetOrCreateUnpaused will find them already dead, which is
+	// safe: per the Sandbox contract, methods on a destroyed
+	// sandbox are harmless no-ops that return errors.
 	//
 	// Calling Close a second time returns an error.
 	Close() error
diff --git a/go/worker/sandboxset/close.go b/go/worker/sandboxset/close.go
deleted file mode 100644
index 3762244c0..000000000
--- a/go/worker/sandboxset/close.go
+++ /dev/null
@@ -1,23 +0,0 @@
-package sandboxset
-
-import "fmt"
-
-// Close implements SandboxSet.
-//
-// All sandboxes are snapshot under the lock, then destroyed outside it.
-func (s *sandboxSetImpl) Close() error {
-	s.mu.Lock()
-	if s.closed {
-		s.mu.Unlock()
-		return fmt.Errorf("sandboxset: already closed")
-	}
-	s.closed = true
-	pool := s.pool
-	s.pool = nil
-	s.mu.Unlock()
-
-	for _, w := range pool {
-		w.sb.Destroy("sandboxset closed")
-	}
-	return nil
-}
diff --git a/go/worker/sandboxset/destroy.go b/go/worker/sandboxset/destroy.go
deleted file mode 100644
index 274c812bd..000000000
--- a/go/worker/sandboxset/destroy.go
+++ /dev/null
@@ -1,34 +0,0 @@
-package sandboxset
-
-import (
-	"fmt"
-
-	"github.com/open-lambda/open-lambda/go/worker/sandbox"
-)
-
-// Destroy implements SandboxSet.
-//
-// The wrapper is spliced out of the pool under a short lock using O(1)
-// swap-with-tail. Destroy is called outside the lock to keep critical
-// sections short. The sandbox is always destroyed even if it was not
-// found in the pool.
-func (s *sandboxSetImpl) Destroy(sb sandbox.Sandbox, reason string) error {
-	s.mu.Lock()
-	found := false
-	for i, w := range s.pool {
-		if w.sb.ID() == sb.ID() {
-			s.pool[i] = s.pool[len(s.pool)-1]
-			s.pool = s.pool[:len(s.pool)-1]
-			found = true
-			break
-		}
-	}
-	s.mu.Unlock()
-
-	sb.Destroy(reason)
-
-	if !found {
-		return fmt.Errorf("sandboxset: sandbox %s not found in pool (still destroyed)", sb.ID())
-	}
-	return nil
-}
diff --git a/go/worker/sandboxset/get.go b/go/worker/sandboxset/get.go
deleted file mode 100644
index c1e2c3587..000000000
--- a/go/worker/sandboxset/get.go
+++ /dev/null
@@ -1,68 +0,0 @@
-package sandboxset
-
-import (
-	"fmt"
-
-	"github.com/open-lambda/open-lambda/go/worker/sandbox"
-)
-
-// Get implements SandboxSet.
-//
-// Fast path: an idle sandbox is claimed under a short lock, then Unpause
-// runs outside the lock so the pool is not stalled during I/O.
-//
-// Slow path: no idle sandbox exists, so a new one is created without
-// holding the lock.
-func (s *sandboxSetImpl) Get() (sandbox.Sandbox, error) {
-	// Loop over idle sandboxes until one unpauses successfully,
-	// or the pool has no idle sandboxes left.
-	for {
-		s.mu.Lock()
-		if s.closed {
-			s.mu.Unlock()
-			return nil, fmt.Errorf("sandboxset: closed")
-		}
-
-		// Fast path: claim an idle sandbox.
-		var claimed sandbox.Sandbox
-		for _, w := range s.pool {
-			if !w.inUse {
-				w.inUse = true
-				claimed = w.sb
-				break
-			}
-		}
-		s.mu.Unlock()
-
-		if claimed == nil {
-			break // no idle sandbox — fall through to Create
-		}
-
-		// Unpause outside the lock (split-lock pattern).
-		if err := claimed.Unpause(); err != nil {
-			_ = s.Destroy(claimed, fmt.Sprintf("unpause: %v", err))
-			continue // try the next idle sandbox
-		}
-		return claimed, nil
-	}
-
-	// Slow path: create a new sandbox without holding the lock.
-	scratchDir, err := s.makeScratchDir()
-	if err != nil {
-		return nil, fmt.Errorf("sandboxset: scratch dir: %w", err)
-	}
-	sb, err := s.cfg.Pool.Create(
-		s.cfg.Parent, s.cfg.IsLeaf,
-		s.cfg.CodeDir, scratchDir,
-		s.cfg.Meta,
-	)
-	if err != nil {
-		return nil, fmt.Errorf("sandboxset: create: %w", err)
-	}
-
-	s.mu.Lock()
-	s.pool = append(s.pool, &sandboxWrapper{sb: sb, inUse: true})
-	s.mu.Unlock()
-
-	return sb, nil
-}
diff --git a/go/worker/sandboxset/put.go b/go/worker/sandboxset/put.go
deleted file mode 100644
index 18103bb36..000000000
--- a/go/worker/sandboxset/put.go
+++ /dev/null
@@ -1,37 +0,0 @@
-package sandboxset
-
-import (
-	"fmt"
-
-	"github.com/open-lambda/open-lambda/go/worker/sandbox"
-)
-
-// Put implements SandboxSet.
-//
-// The sandbox is paused and its wrapper is flipped back to idle.
-// If Pause fails, the sandbox is destroyed rather than silently
-// recycled — a bad sandbox should never re-enter the pool.
-func (s *sandboxSetImpl) Put(sb sandbox.Sandbox) error {
-	s.mu.Lock()
-	if s.closed {
-		s.mu.Unlock()
-		return fmt.Errorf("sandboxset: closed (sandbox %s was destroyed by Close)", sb.ID())
-	}
-	s.mu.Unlock()
-
-	if err := sb.Pause(); err != nil {
-		_ = s.Destroy(sb, fmt.Sprintf("pause failed: %v", err))
-		return fmt.Errorf("sandboxset: sandbox %s destroyed because Pause failed: %w", sb.ID(), err)
-	}
-
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	for _, w := range s.pool {
-		if w.sb.ID() == sb.ID() {
-			w.inUse = false
-			return nil
-		}
-	}
-	return fmt.Errorf("sandboxset: sandbox %s not found in pool", sb.ID())
-}
diff --git a/go/worker/sandboxset/sandboxset.go b/go/worker/sandboxset/sandboxset.go
index 84f9c3b4e..7abfc86a5 100644
--- a/go/worker/sandboxset/sandboxset.go
+++ b/go/worker/sandboxset/sandboxset.go
@@ -7,6 +7,41 @@ import (
 	"github.com/open-lambda/open-lambda/go/worker/sandbox"
 )
 
+// SandboxState describes the health of a checked-out sandbox.
+type SandboxState int
+
+const (
+	StateReady  SandboxState = iota // healthy, usable
+	StateBroken                     // error occurred, should be destroyed
+)
+
+// SandboxRef is a handle returned by GetOrCreateUnpaused.
+// It wraps a sandbox together with a back-pointer to its parent set
+// and a health state, so the caller can Put or Destroy without
+// tracking which set the sandbox came from.
+type SandboxRef struct {
+	sb    sandbox.Sandbox
+	set   *sandboxSetImpl
+	State SandboxState
+}
+
+// Sandbox returns the underlying sandbox.
+func (r *SandboxRef) Sandbox() sandbox.Sandbox {
+	return r.sb
+}
+
+// Put returns the sandbox to its parent set.
+// This is a convenience method equivalent to set.Put(ref.Sandbox()).
+func (r *SandboxRef) Put() error {
+	return r.set.Put(r.sb)
+}
+
+// Destroy removes the sandbox from its parent set and destroys it.
+// This is a convenience method equivalent to set.Destroy(ref.Sandbox(), reason).
+func (r *SandboxRef) Destroy(reason string) error {
+	return r.set.Destroy(r.sb, reason)
+}
+
 // sandboxWrapper pairs a sandbox with an in-use flag.
 type sandboxWrapper struct {
 	sb    sandbox.Sandbox
@@ -50,3 +85,141 @@ func (s *sandboxSetImpl) makeScratchDir() (dir string, err error) {
 	dir = s.cfg.ScratchDirs.Make("sb")
 	return dir, nil
 }
+
+// GetOrCreateUnpaused implements SandboxSet.
+//
+// Fast path: an idle sandbox is claimed under a short lock, then Unpause
+// runs outside the lock so the pool is not stalled during I/O.
+//
+// Slow path: no idle sandbox exists, so a new one is created without
+// holding the lock.
+func (s *sandboxSetImpl) GetOrCreateUnpaused() (*SandboxRef, error) {
+	// Loop over idle sandboxes until one unpauses successfully,
+	// or the pool has no idle sandboxes left.
+	for {
+		s.mu.Lock()
+		if s.closed {
+			s.mu.Unlock()
+			return nil, fmt.Errorf("sandboxset: closed")
+		}
+
+		// Fast path: claim an idle sandbox.
+		var claimed sandbox.Sandbox // raw sandbox, wrapped in SandboxRef on return
+		for _, w := range s.pool {
+			if !w.inUse {
+				w.inUse = true
+				claimed = w.sb
+				break
+			}
+		}
+		s.mu.Unlock()
+
+		if claimed == nil {
+			break // no idle sandbox — fall through to Create
+		}
+
+		// Unpause outside the lock (split-lock pattern).
+		if err := claimed.Unpause(); err != nil {
+			_ = s.Destroy(claimed, fmt.Sprintf("unpause: %v", err))
+			continue // try the next idle sandbox
+		}
+		return &SandboxRef{sb: claimed, set: s, State: StateReady}, nil
+	}
+
+	// Slow path: create a new sandbox without holding the lock.
+	scratchDir, err := s.makeScratchDir()
+	if err != nil {
+		return nil, fmt.Errorf("sandboxset: scratch dir: %w", err)
+	}
+	sb, err := s.cfg.Pool.Create(
+		s.cfg.Parent, s.cfg.IsLeaf,
+		s.cfg.CodeDir, scratchDir,
+		s.cfg.Meta,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("sandboxset: create: %w", err)
+	}
+
+	s.mu.Lock()
+	s.pool = append(s.pool, &sandboxWrapper{sb: sb, inUse: true})
+	s.mu.Unlock()
+
+	return &SandboxRef{sb: sb, set: s, State: StateReady}, nil
+}
+
+// Put implements SandboxSet.
+//
+// The sandbox is paused and its wrapper is flipped back to idle.
+// If Pause fails, the sandbox is destroyed rather than silently
+// recycled — a bad sandbox should never re-enter the pool.
+func (s *sandboxSetImpl) Put(sb sandbox.Sandbox) error {
+	s.mu.Lock()
+	if s.closed {
+		s.mu.Unlock()
+		return fmt.Errorf("sandboxset: closed (sandbox %s was destroyed by Close)", sb.ID())
+	}
+	s.mu.Unlock()
+
+	if err := sb.Pause(); err != nil {
+		_ = s.Destroy(sb, fmt.Sprintf("pause failed: %v", err))
+		return fmt.Errorf("sandboxset: sandbox %s destroyed because Pause failed: %w", sb.ID(), err)
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	for _, w := range s.pool {
+		if w.sb.ID() == sb.ID() {
+			w.inUse = false
+			return nil
+		}
+	}
+	return fmt.Errorf("sandboxset: sandbox %s not found in pool", sb.ID())
+}
+
+// Destroy implements SandboxSet.
+//
+// The wrapper is spliced out of the pool under a short lock using O(1)
+// swap-with-tail. Destroy is called outside the lock to keep critical
+// sections short. The sandbox is always destroyed even if it was not
+// found in the pool.
+func (s *sandboxSetImpl) Destroy(sb sandbox.Sandbox, reason string) error {
+	s.mu.Lock()
+	found := false
+	for i, w := range s.pool {
+		if w.sb.ID() == sb.ID() {
+			s.pool[i] = s.pool[len(s.pool)-1]
+			s.pool = s.pool[:len(s.pool)-1]
+			found = true
+			break
+		}
+	}
+	s.mu.Unlock()
+
+	sb.Destroy(reason)
+
+	if !found {
+		return fmt.Errorf("sandboxset: sandbox %s not found in pool (still destroyed)", sb.ID())
+	}
+	return nil
+}
+
+// Close implements SandboxSet.
+//
+// All sandboxes are snapshot under the lock, then destroyed outside it.
+func (s *sandboxSetImpl) Close() error {
+	s.mu.Lock()
+	if s.closed {
+		s.mu.Unlock()
+		return fmt.Errorf("sandboxset: already closed")
+	}
+	s.closed = true
+	pool := s.pool
+	s.pool = nil
+	s.mu.Unlock()
+
+	for _, w := range pool {
+		w.sb.Destroy("sandboxset closed")
+	}
+	return nil
+}
diff --git a/go/worker/sandboxset/tests/sandboxset_integration_test.go b/go/worker/sandboxset/tests/sandboxset_integration_test.go
new file mode 100644
index 000000000..a7adedc58
--- /dev/null
+++ b/go/worker/sandboxset/tests/sandboxset_integration_test.go
@@ -0,0 +1,181 @@
+//go:build integration
+
+package tests
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/open-lambda/open-lambda/go/common"
+	"github.com/open-lambda/open-lambda/go/worker/sandbox"
+	"github.com/open-lambda/open-lambda/go/worker/sandboxset"
+)
+
+// newDockerSet creates a SandboxSet backed by a real DockerPool.
+// Requires Docker daemon running and the ol-min image available.
+func newDockerSet(t *testing.T) sandboxset.SandboxSet {
+	t.Helper()
+
+	tmpDir := t.TempDir()
+	workerDir := filepath.Join(tmpDir, "worker")
+	pkgsDir := filepath.Join(tmpDir, "packages")
+	codeDir := filepath.Join(tmpDir, "code")
+
+	for _, d := range []string{workerDir, pkgsDir, codeDir} {
+		if err := os.MkdirAll(d, 0755); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	common.Conf = &common.Config{
+		Worker_dir: workerDir,
+		Pkgs_dir:   pkgsDir,
+		Sandbox:    "docker",
+		Docker: common.DockerConfig{
+			Base_image: "ol-min",
+		},
+		Limits: common.LimitsConfig{
+			Procs:       10,
+			Mem_mb:      50,
+			CPU_percent: 100,
+			Swappiness:  0,
+			Runtime_sec: 30,
+		},
+	}
+
+	pool, err := sandbox.NewDockerPool("", nil)
+	if err != nil {
+		t.Fatalf("NewDockerPool: %v (is Docker running? is ol-min image built?)", err)
+	}
+
+	scratchDirs, err := common.NewDirMaker("scratch", common.STORE_REGULAR)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	set, err := sandboxset.New(&sandboxset.Config{
+		Pool:        pool,
+		IsLeaf:      true,
+		CodeDir:     codeDir,
+		ScratchDirs: scratchDirs,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Cleanup(func() {
+		_ = set.Close()
+		pool.Cleanup()
+	})
+
+	return set
+}
+
+func TestIntegration_GetCreatesRealContainer(t *testing.T) {
+	set := newDockerSet(t)
+
+	ref, err := set.GetOrCreateUnpaused()
+	if err != nil {
+		t.Fatalf("GetOrCreateUnpaused: %v", err)
+	}
+	sb := ref.Sandbox()
+
+	if sb.ID() == "" {
+		t.Fatal("expected non-empty sandbox ID")
+	}
+	t.Logf("created real container: ID=%s", sb.ID())
+	t.Logf("debug: %s", sb.DebugString())
+
+	_ = ref.Destroy("test done")
+}
+
+func TestIntegration_PutPausesAndReuses(t *testing.T) {
+	set := newDockerSet(t)
+
+	// Get a sandbox, record its ID, put it back
+	ref1, err := set.GetOrCreateUnpaused()
+	if err != nil {
+		t.Fatalf("first GetOrCreateUnpaused: %v", err)
+	}
+	id1 := ref1.Sandbox().ID()
+	t.Logf("first sandbox: ID=%s", id1)
+
+	if err := ref1.Put(); err != nil {
+		t.Fatalf("Put: %v", err)
+	}
+
+	// Get again — should reuse the same container (unpaused from paused state)
+	ref2, err := set.GetOrCreateUnpaused()
+	if err != nil {
+		t.Fatalf("second GetOrCreateUnpaused: %v", err)
+	}
+	id2 := ref2.Sandbox().ID()
+	t.Logf("second sandbox: ID=%s", id2)
+
+	if id2 != id1 {
+		t.Fatalf("expected reuse (same ID %s), got new container %s", id1, id2)
+	}
+
+	_ = ref2.Destroy("test done")
+}
+
+func TestIntegration_DestroyKillsReal(t *testing.T) {
+	set := newDockerSet(t)
+
+	ref1, err := set.GetOrCreateUnpaused()
+	if err != nil {
+		t.Fatalf("first GetOrCreateUnpaused: %v", err)
+	}
+	id1 := ref1.Sandbox().ID()
+	t.Logf("first sandbox: ID=%s", id1)
+
+	if err := ref1.Destroy("intentional destroy"); err != nil {
+		t.Fatalf("Destroy: %v", err)
+	}
+
+	// Get again — must be a different container since we destroyed the first
+	ref2, err := set.GetOrCreateUnpaused()
+	if err != nil {
+		t.Fatalf("second GetOrCreateUnpaused: %v", err)
+	}
+	id2 := ref2.Sandbox().ID()
+	t.Logf("second sandbox: ID=%s", id2)
+
+	if id2 == id1 {
+		t.Fatal("expected new container after Destroy, got same ID")
+	}
+
+	_ = ref2.Destroy("test done")
+}
+
+func TestIntegration_CloseDestroysAll(t *testing.T) {
+	set := newDockerSet(t)
+
+	// Create multiple sandboxes
+	refs := make([]*sandboxset.SandboxRef, 3)
+	for i := range refs {
+		ref, err := set.GetOrCreateUnpaused()
+		if err != nil {
+			t.Fatalf("GetOrCreateUnpaused[%d]: %v", i, err)
+		}
+		refs[i] = ref
+		t.Logf("sandbox[%d]: ID=%s", i, ref.Sandbox().ID())
+	}
+
+	// Put one back to idle so Close covers both in-use and idle
+	if err := refs[2].Put(); err != nil {
+		t.Fatalf("Put: %v", err)
+	}
+
+	// Close should destroy all
+	if err := set.Close(); err != nil {
+		t.Fatalf("Close: %v", err)
+	}
+
+	// Verify set is closed — further Gets should fail
+	_, err := set.GetOrCreateUnpaused()
+	if err == nil {
+		t.Fatal("expected error after Close")
+	}
+}
diff --git a/go/worker/sandboxset/tests/sandboxset_test.go b/go/worker/sandboxset/tests/sandboxset_test.go
index f49d8cc78..5c22cc3e7 100644
--- a/go/worker/sandboxset/tests/sandboxset_test.go
+++ b/go/worker/sandboxset/tests/sandboxset_test.go
@@ -98,15 +98,15 @@ func TestNew_Valid(t *testing.T) {
 	}
 }
 
-// --- Get tests ---
+// --- GetOrCreateUnpaused tests ---
 
 func TestGet_CreatesNew(t *testing.T) {
 	set, pool := newTestSet(t)
-	sb, err := set.Get()
+	ref, err := set.GetOrCreateUnpaused()
 	if err != nil {
-		t.Fatalf("Get: %v", err)
+		t.Fatalf("GetOrCreateUnpaused: %v", err)
 	}
-	if sb == nil {
+	if ref.Sandbox() == nil {
 		t.Fatal("expected non-nil sandbox")
 	}
 	if n := len(pool.CreatedSandboxes()); n != 1 {
@@ -117,44 +117,45 @@ func TestGet_CreatesNew(t *testing.T) {
 func TestGet_ReusesIdle(t *testing.T) {
 	set, _ := newTestSet(t)
 
-	sb1, err := set.Get()
+	ref1, err := set.GetOrCreateUnpaused()
 	if err != nil {
-		t.Fatalf("Get: %v", err)
+		t.Fatalf("GetOrCreateUnpaused: %v", err)
 	}
-	id1 := sb1.ID()
+	id1 := ref1.Sandbox().ID()
 
-	if err := set.Put(sb1); err != nil {
+	if err := ref1.Put(); err != nil {
 		t.Fatalf("Put: %v", err)
 	}
 
-	sb2, err := set.Get()
+	ref2, err := set.GetOrCreateUnpaused()
 	if err != nil {
-		t.Fatalf("Get: %v", err)
+		t.Fatalf("GetOrCreateUnpaused: %v", err)
 	}
-	if sb2.ID() != id1 {
-		t.Fatalf("expected reuse (ID %s), got new (ID %s)", id1, sb2.ID())
+	if ref2.Sandbox().ID() != id1 {
+		t.Fatalf("expected reuse (ID %s), got new (ID %s)", id1, ref2.Sandbox().ID())
 	}
 }
 
 func TestGet_UnpauseFail(t *testing.T) {
 	set, pool := newTestSet(t)
 
-	sb1, err := set.Get()
+	ref1, err := set.GetOrCreateUnpaused()
 	if err != nil {
-		t.Fatalf("Get: %v", err)
+		t.Fatalf("GetOrCreateUnpaused: %v", err)
 	}
+	sb1 := ref1.Sandbox()
 	// Inject unpause error before putting back.
 	sb1.(*sandbox.MockSandbox).UnpauseErr = errors.New("broken")
-	if err := set.Put(sb1); err != nil {
+	if err := ref1.Put(); err != nil {
 		t.Fatalf("Put: %v", err)
 	}
 
-	// Next Get should find the bad sandbox, destroy it, and create a new one.
-	sb2, err := set.Get()
+	// Next GetOrCreateUnpaused should find the bad sandbox, destroy it, and create a new one.
+	ref2, err := set.GetOrCreateUnpaused()
 	if err != nil {
-		t.Fatalf("Get after unpause fail: %v", err)
+		t.Fatalf("GetOrCreateUnpaused after unpause fail: %v", err)
 	}
-	if sb2.ID() == sb1.ID() {
+	if ref2.Sandbox().ID() == sb1.ID() {
 		t.Fatal("expected a different sandbox after unpause failure")
 	}
 	if !sb1.(*sandbox.MockSandbox).IsDestroyed() {
@@ -169,7 +170,7 @@ func TestGet_CreateFail(t *testing.T) {
 	set, pool := newTestSet(t)
 	pool.CreateErr = errors.New("out of resources")
 
-	_, err := set.Get()
+	_, err := set.GetOrCreateUnpaused()
 	if err == nil {
 		t.Fatal("expected error when pool.Create fails")
 	}
@@ -180,7 +181,7 @@ func TestGet_AfterClose(t *testing.T) {
 	if err := set.Close(); err != nil {
 		t.Fatalf("Close: %v", err)
 	}
-	_, err := set.Get()
+	_, err := set.GetOrCreateUnpaused()
 	if err == nil {
 		t.Fatal("expected error after Close")
 	}
@@ -190,16 +191,16 @@ func TestGet_AfterClose(t *testing.T) {
 
 func TestPut_PausesAndReturns(t *testing.T) {
 	set, _ := newTestSet(t)
-	sb, err := set.Get()
+	ref, err := set.GetOrCreateUnpaused()
 	if err != nil {
-		t.Fatalf("Get: %v", err)
+		t.Fatalf("GetOrCreateUnpaused: %v", err)
 	}
-	mock := sb.(*sandbox.MockSandbox)
+	mock := ref.Sandbox().(*sandbox.MockSandbox)
 	if mock.IsPaused() {
-		t.Fatal("sandbox should be unpaused after Get")
+		t.Fatal("sandbox should be unpaused after GetOrCreateUnpaused")
 	}
 
-	if err := set.Put(sb); err != nil {
+	if err := ref.Put(); err != nil {
 		t.Fatalf("Put: %v", err)
 	}
 	if !mock.IsPaused() {
@@ -209,13 +210,14 @@ func TestPut_PausesAndReturns(t *testing.T) {
 
 func TestPut_PauseFail(t *testing.T) {
 	set, _ := newTestSet(t)
-	sb, err := set.Get()
+	ref, err := set.GetOrCreateUnpaused()
 	if err != nil {
-		t.Fatalf("Get: %v", err)
+		t.Fatalf("GetOrCreateUnpaused: %v", err)
 	}
+	sb := ref.Sandbox()
 	sb.(*sandbox.MockSandbox).PauseErr = errors.New("pause broken")
 
-	err = set.Put(sb)
+	err = ref.Put()
 	if err == nil {
 		t.Fatal("expected error when Pause fails")
 	}
@@ -237,24 +239,25 @@ func TestPut_NotInPool(t *testing.T) {
 
 func TestDestroy_RemovesFromPool(t *testing.T) {
 	set, _ := newTestSet(t)
-	sb, err := set.Get()
+	ref, err := set.GetOrCreateUnpaused()
 	if err != nil {
-		t.Fatalf("Get: %v", err)
+		t.Fatalf("GetOrCreateUnpaused: %v", err)
 	}
+	sb := ref.Sandbox()
 
-	if err := set.Destroy(sb, "test"); err != nil {
+	if err := ref.Destroy("test"); err != nil {
 		t.Fatalf("Destroy: %v", err)
 	}
 	if !sb.(*sandbox.MockSandbox).IsDestroyed() {
 		t.Fatal("sandbox should be destroyed")
 	}
 
-	// Next Get should create a new one, not reuse the destroyed one.
-	sb2, err := set.Get()
+	// Next GetOrCreateUnpaused should create a new one, not reuse the destroyed one.
+	ref2, err := set.GetOrCreateUnpaused()
 	if err != nil {
-		t.Fatalf("Get after Destroy: %v", err)
+		t.Fatalf("GetOrCreateUnpaused after Destroy: %v", err)
 	}
-	if sb2.ID() == sb.ID() {
+	if ref2.Sandbox().ID() == sb.ID() {
 		t.Fatal("should not reuse a destroyed sandbox")
 	}
 }
@@ -277,10 +280,10 @@ func TestClose_DestroysAll(t *testing.T) {
 	set, pool := newTestSet(t)
 
 	// Create 3 sandboxes: 2 in-use, 1 idle.
-	sb1, _ := set.Get()
-	sb2, _ := set.Get()
-	sb3, _ := set.Get()
-	_ = set.Put(sb3) // return one to idle
+	ref1, _ := set.GetOrCreateUnpaused()
+	ref2, _ := set.GetOrCreateUnpaused()
+	ref3, _ := set.GetOrCreateUnpaused()
+	_ = ref3.Put() // return one to idle
 
 	if err := set.Close(); err != nil {
 		t.Fatalf("Close: %v", err)
@@ -291,8 +294,8 @@ func TestClose_DestroysAll(t *testing.T) {
 			t.Fatalf("sandbox %s should be destroyed after Close", m.ID())
 		}
 	}
-	_ = sb1
-	_ = sb2
+	_ = ref1
+	_ = ref2
 }
 
 func TestClose_Twice(t *testing.T) {
@@ -318,16 +321,16 @@ func TestClose_EmptyPool(t *testing.T) {
 func TestLifecycle_GetPutReuse(t *testing.T) {
 	set, _ := newTestSet(t)
 
-	// Get → Put → Get should reuse.
-	sb1, _ := set.Get()
-	id := sb1.ID()
-	_ = set.Put(sb1)
+	// GetOrCreateUnpaused → Put → GetOrCreateUnpaused should reuse.
+	ref1, _ := set.GetOrCreateUnpaused()
+	id := ref1.Sandbox().ID()
+	_ = ref1.Put()
 
-	sb2, _ := set.Get()
-	if sb2.ID() != id {
-		t.Fatalf("expected reuse, got different ID: %s vs %s", id, sb2.ID())
+	ref2, _ := set.GetOrCreateUnpaused()
+	if ref2.Sandbox().ID() != id {
+		t.Fatalf("expected reuse, got different ID: %s vs %s", id, ref2.Sandbox().ID())
 	}
-	_ = set.Put(sb2)
+	_ = ref2.Put()
 
 	_ = set.Close()
 }
@@ -335,15 +338,15 @@ func TestLifecycle_GetPutReuse(t *testing.T) {
 func TestLifecycle_GetDestroyGet(t *testing.T) {
 	set, _ := newTestSet(t)
 
-	sb1, _ := set.Get()
-	id := sb1.ID()
-	_ = set.Destroy(sb1, "bad")
+	ref1, _ := set.GetOrCreateUnpaused()
+	id := ref1.Sandbox().ID()
+	_ = ref1.Destroy("bad")
 
-	sb2, _ := set.Get()
-	if sb2.ID() == id {
+	ref2, _ := set.GetOrCreateUnpaused()
+	if ref2.Sandbox().ID() == id {
 		t.Fatal("expected fresh sandbox after Destroy, got same ID")
 	}
-	_ = set.Put(sb2)
+	_ = ref2.Put()
 
 	_ = set.Close()
 }
@@ -355,15 +358,15 @@ func TestConcurrent_Gets(t *testing.T) {
 	const n = 50
 
 	var wg sync.WaitGroup
-	sandboxes := make([]sandbox.Sandbox, n)
+	refs := make([]*sandboxset.SandboxRef, n)
 	errs := make([]error, n)
 
 	for i := 0; i < n; i++ {
 		wg.Add(1)
 		go func(idx int) {
 			defer wg.Done()
-			sb, err := set.Get()
-			sandboxes[idx] = sb
+			ref, err := set.GetOrCreateUnpaused()
+			refs[idx] = ref
 			errs[idx] = err
 		}(i)
 	}
@@ -371,13 +374,13 @@ func TestConcurrent_Gets(t *testing.T) {
 
 	for i, err := range errs {
 		if err != nil {
-			t.Fatalf("goroutine %d: Get: %v", i, err)
+			t.Fatalf("goroutine %d: GetOrCreateUnpaused: %v", i, err)
 		}
 	}
 
 	// Clean up: put all back then close.
-	for _, sb := range sandboxes {
-		_ = set.Put(sb)
+	for _, ref := range refs {
+		_ = ref.Put()
 	}
 	_ = set.Close()
 }
@@ -393,12 +396,12 @@ func TestConcurrent_GetPut(t *testing.T) {
 		go func(id int) {
 			defer wg.Done()
 			for j := 0; j < iterations; j++ {
-				sb, err := set.Get()
+				ref, err := set.GetOrCreateUnpaused()
 				if err != nil {
-					t.Errorf("goroutine %d iter %d: Get: %v", id, j, err)
+					t.Errorf("goroutine %d iter %d: GetOrCreateUnpaused: %v", id, j, err)
 					return
 				}
-				if err := set.Put(sb); err != nil {
+				if err := ref.Put(); err != nil {
 					t.Errorf("goroutine %d iter %d: Put: %v", id, j, err)
 					return
 				}
@@ -415,24 +418,24 @@ func TestConcurrent_CloseWhileGet(t *testing.T) {
 
 	// Grab some sandboxes first.
 	for i := 0; i < 5; i++ {
-		sb, _ := set.Get()
-		_ = set.Put(sb)
+		ref, _ := set.GetOrCreateUnpaused()
+		_ = ref.Put()
 	}
 
 	var wg sync.WaitGroup
 	errs := make(chan error, n)
 
-	// Launch goroutines that race Get vs Close.
+	// Launch goroutines that race GetOrCreateUnpaused vs Close.
 	for i := 0; i < n; i++ {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			sb, err := set.Get()
+			ref, err := set.GetOrCreateUnpaused()
 			if err != nil {
 				// Expected for some goroutines after Close.
 				return
 			}
-			errs <- set.Put(sb)
+			errs <- ref.Put()
 		}()
 	}
 
@@ -443,7 +446,7 @@ func TestConcurrent_CloseWhileGet(t *testing.T) {
 
 	// Close should succeed (first call).
 	if closeErr != nil {
-		// Close might race with Get; as long as no panic, we're OK.
+		// Close might race with GetOrCreateUnpaused; as long as no panic, we're OK.
 		fmt.Printf("Close returned: %v (acceptable in race)\n", closeErr)
 	}
 }

From 08b15986151ef8a2c318621a652ee8b0cf103631 Mon Sep 17 00:00:00 2001
From: Ami Buch <amiabuch@gmail.com>
Date: Fri, 20 Mar 2026 08:45:22 -0500
Subject: [PATCH 46/55] refactor: simpler interface and less comments, fixed
 race conditions too

---
 go/common/config.go                           |   2 +
 go/worker/sandbox/sandbox.go                  |   2 +
 go/worker/sandboxset/api.go                   |  84 +---
 go/worker/sandboxset/sandboxset.go            | 222 +++++-----
 .../tests/sandboxset_integration_test.go      |   2 +-
 go/worker/sandboxset/tests/sandboxset_test.go | 401 +-----------------
 6 files changed, 143 insertions(+), 570 deletions(-)

diff --git a/go/common/config.go b/go/common/config.go
index 9243c0124..4f3202183 100644
--- a/go/common/config.go
+++ b/go/common/config.go
@@ -415,6 +415,8 @@ func checkConf(cfg *Config) error {
 		if cfg.Features.Import_cache != "" {
 			return fmt.Errorf("features.import_cache must be disabled for docker Sandbox")
 		}
+	} else if cfg.Sandbox == "mock" {
+		// mock sandbox: no additional requirements
 	} else {
 		return fmt.Errorf("Unknown Sandbox type '%s'", cfg.Sandbox)
 	}
diff --git a/go/worker/sandbox/sandbox.go b/go/worker/sandbox/sandbox.go
index a6a525bf4..2d4ae17ea 100644
--- a/go/worker/sandbox/sandbox.go
+++ b/go/worker/sandbox/sandbox.go
@@ -18,6 +18,8 @@ func SandboxPoolFromConfig(name string, sizeMb int) (cf SandboxPool, err error)
 		}
 		NewSOCKEvictor(pool)
 		return pool, nil
+	} else if common.Conf.Sandbox == "mock" {
+		return &MockSandboxPool{}, nil
 	}
 
 	return nil, fmt.Errorf("invalid sandbox type: '%s'", common.Conf.Sandbox)
diff --git a/go/worker/sandboxset/api.go b/go/worker/sandboxset/api.go
index 49bea5627..af6499fb3 100644
--- a/go/worker/sandboxset/api.go
+++ b/go/worker/sandboxset/api.go
@@ -1,9 +1,4 @@
-// Package sandboxset provides a thread-safe pool of sandboxes for a single
-// Lambda function.
-//
-// A SandboxSet replaces per-instance goroutines with a simple pool.
-// Callers just ask for a sandbox and don't worry about whether it is
-// freshly created or recycled from a previous request.
+// Package sandboxset provides a thread-safe pool of sandboxes for a single Lambda function. Callers ask for a sandbox and don't worry about whether it is freshly created or recycled from a previous request.
 //
 // Sandbox lifecycle inside a SandboxSet:
 //
@@ -29,10 +24,9 @@
 //	ref, err := set.GetOrCreateUnpaused()
 //	// ... use ref.Sandbox() to handle request ...
 //	if broken {
-//	    ref.Destroy("reason")
-//	} else {
-//	    ref.Put()
+//	    ref.Broken = true
 //	}
+//	ref.Put()
 package sandboxset
 
 import (
@@ -40,70 +34,14 @@ import (
 	"github.com/open-lambda/open-lambda/go/worker/sandbox"
 )
 
-/*
-A SandboxSet manages a pool of sandboxes for one Lambda function.
-All methods are safe to call from multiple goroutines.
-
-The design mirrors the C process API: GetOrCreateUnpaused (create),
-Put (exit), Destroy (kill), Close (cleanup). There are no warm-up,
-shrink, or stats methods yet — those can be added in later PRs
-without changing the core interface.
-
-GetOrCreateUnpaused returns a *SandboxRef rather than a raw
-sandbox.Sandbox. The ref carries a health State and back-pointer
-to the parent set, so callers can use ref.Put() / ref.Destroy()
-without tracking the set. The set-level Put/Destroy methods are
-also available for callers that prefer them.
-*/
+// SandboxSet manages a pool of sandboxes for one Lambda function.
+// All methods are safe to call from multiple goroutines.
 type SandboxSet interface {
-	// Return an unpaused sandbox ready to handle a request,
-	// wrapped in a SandboxRef.
-	//
-	// If the pool has an idle sandbox, it is unpaused and returned.
-	// If Unpause fails (e.g., the SOCK container died while paused),
-	// that sandbox is destroyed and the next idle one is tried, or
-	// a fresh sandbox is created.
-	//
-	// A fresh scratch directory is created for each new sandbox
-	// via Config.ScratchDirs. Reused sandboxes keep their
-	// existing scratch directory from when they were first created.
+	// GetOrCreateUnpaused returns an unpaused sandbox ready to handle a
+	// request, wrapped in a SandboxRef.
 	GetOrCreateUnpaused() (*SandboxRef, error)
 
-	// Return a sandbox to the pool after a successful request.
-	//
-	// The sandbox is paused and becomes available for the next
-	// GetOrCreateUnpaused. If Pause fails (e.g., the container
-	// died during the request), the sandbox is destroyed
-	// automatically — a bad sandbox never re-enters the pool.
-	//
-	// Passing a sandbox that is not in the pool returns an error
-	// but is otherwise harmless.
-	//
-	// Prefer ref.Put() when you have a SandboxRef.
-	Put(sb sandbox.Sandbox) error
-
-	// Permanently remove a sandbox from the pool and destroy it.
-	//
-	// Use this when a request produced an unrecoverable error and
-	// the sandbox should not be reused. "reason" is a
-	// human-readable explanation that shows up in later error
-	// messages (same convention as sandbox.Sandbox.Destroy).
-	//
-	// If the sandbox is not in the pool it is still destroyed —
-	// resources are always freed. The returned error is
-	// informational only.
-	//
-	// Prefer ref.Destroy() when you have a SandboxRef.
-	Destroy(sb sandbox.Sandbox, reason string) error
-
-	// Destroy all sandboxes in the pool and mark the set as closed.
-	//
-	// Callers who still hold SandboxRef values from a previous
-	// GetOrCreateUnpaused will find them already dead, which is
-	// safe: per the Sandbox contract, methods on a destroyed
-	// sandbox are harmless no-ops that return errors.
-	//
-	// Calling Close a second time returns an error.
+	// Close destroys all sandboxes in the pool and marks the set as closed.
 	Close() error
 }
 
@@ -112,10 +50,10 @@ type Config struct {
 	// Pool creates and destroys the underlying sandboxes.
 	Pool sandbox.SandboxPool
 
-	// Parent sandbox to fork from (may be nil). When nil, new
+	// Parent is an optional SandboxSet to fork from. When nil, new
 	// sandboxes are created from scratch. Not all SandboxPool
 	// implementations support forking.
-	Parent sandbox.Sandbox
+	Parent SandboxSet
 
 	// IsLeaf marks sandboxes as non-forkable, meaning they will
 	// not be used as parents for future forks.
@@ -130,7 +68,7 @@ type Config struct {
 
 	// ScratchDirs creates a unique writable directory for each
 	// new sandbox. The set calls ScratchDirs.Make internally
-	// so that Get can remain argument-free.
+	// so that GetOrCreateUnpaused can remain argument-free.
 	ScratchDirs *common.DirMaker
 }
 
diff --git a/go/worker/sandboxset/sandboxset.go b/go/worker/sandboxset/sandboxset.go
index 7abfc86a5..25655be6a 100644
--- a/go/worker/sandboxset/sandboxset.go
+++ b/go/worker/sandboxset/sandboxset.go
@@ -3,26 +3,20 @@ package sandboxset
 import (
 	"fmt"
 	"sync"
+	"sync/atomic"
 
 	"github.com/open-lambda/open-lambda/go/worker/sandbox"
 )
 
-// SandboxState describes the health of a checked-out sandbox.
-type SandboxState int
-
-const (
-	StateReady  SandboxState = iota // healthy, usable
-	StateBroken                     // error occurred, should be destroyed
-)
-
 // SandboxRef is a handle returned by GetOrCreateUnpaused.
-// It wraps a sandbox together with a back-pointer to its parent set
-// and a health state, so the caller can Put or Destroy without
-// tracking which set the sandbox came from.
+// It wraps a sandbox with a back-pointer to its parent set.
+// Set Broken = true before calling Put if the sandbox should not be recycled.
 type SandboxRef struct {
-	sb    sandbox.Sandbox
-	set   *sandboxSetImpl
-	State SandboxState
+	sb        sandbox.Sandbox
+	set       *sandboxSetImpl
+	Broken    bool        // public: caller sets true if request failed; Put will destroy instead of recycle
+	inUse     bool        // true when checked out; false when idle in pool
+	destroyed atomic.Bool // set atomically after Destroy(); guards against concurrent Close + Put
 }
 
 // Sandbox returns the underlying sandbox.
@@ -30,29 +24,30 @@ func (r *SandboxRef) Sandbox() sandbox.Sandbox {
 	return r.sb
 }
 
-// Put returns the sandbox to its parent set.
-// This is a convenience method equivalent to set.Put(ref.Sandbox()).
+// Put returns the sandbox to its parent set, or destroys it if Broken is true.
 func (r *SandboxRef) Put() error {
-	return r.set.Put(r.sb)
+	if r.destroyed.Load() {
+		return fmt.Errorf("sandboxset: sandbox %s already destroyed", r.sb.ID())
+	}
+	if r.Broken {
+		return r.set.destroy(r, "state marked broken")
+	}
+	return r.set.put(r)
 }
 
 // Destroy removes the sandbox from its parent set and destroys it.
-// This is a convenience method equivalent to set.Destroy(ref.Sandbox(), reason).
 func (r *SandboxRef) Destroy(reason string) error {
-	return r.set.Destroy(r.sb, reason)
-}
-
-// sandboxWrapper pairs a sandbox with an in-use flag.
-type sandboxWrapper struct {
-	sb    sandbox.Sandbox
-	inUse bool
+	if r.destroyed.Load() {
+		return nil
+	}
+	return r.set.destroy(r, reason)
 }
 
 // sandboxSetImpl is the private concrete type returned by New.
 // All mutable state is guarded by mu.
 type sandboxSetImpl struct {
 	mu     sync.Mutex
-	pool   []*sandboxWrapper
+	pool   []*SandboxRef
 	cfg    *Config
 	closed bool
 }
@@ -73,66 +68,69 @@ func newSandboxSet(cfg *Config) (*sandboxSetImpl, error) {
 	return &sandboxSetImpl{cfg: cfg}, nil
 }
 
-// makeScratchDir creates a scratch directory for a new sandbox.
-// DirMaker.Make panics on failure (e.g., disk full), so we recover
-// here and return an error instead of crashing the worker.
-func (s *sandboxSetImpl) makeScratchDir() (dir string, err error) {
-	defer func() {
-		if r := recover(); r != nil {
-			err = fmt.Errorf("%v", r)
+// claimIdle returns an idle ref from the pool, marking it inUse.
+// Caller must hold s.mu.
+func (s *sandboxSetImpl) claimIdle() *SandboxRef {
+	for _, ref := range s.pool {
+		if !ref.inUse {
+			ref.inUse = true
+			return ref
 		}
-	}()
-	dir = s.cfg.ScratchDirs.Make("sb")
-	return dir, nil
+	}
+	return nil
+}
+
+// tryClaimIdle acquires the lock for its full duration, checks closed,
+// and returns an idle ref (or nil if none available).
+func (s *sandboxSetImpl) tryClaimIdle() (*SandboxRef, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.closed {
+		return nil, fmt.Errorf("sandboxset: closed")
+	}
+	return s.claimIdle(), nil
 }
 
 // GetOrCreateUnpaused implements SandboxSet.
 //
-// Fast path: an idle sandbox is claimed under a short lock, then Unpause
-// runs outside the lock so the pool is not stalled during I/O.
+// Fast path: claim an idle ref via tryClaimIdle (which holds the lock for
+// its full duration), then Unpause outside the lock so the pool is not
+// stalled during I/O.
 //
-// Slow path: no idle sandbox exists, so a new one is created without
+// Slow path: no idle ref exists, so a new sandbox is created without
 // holding the lock.
 func (s *sandboxSetImpl) GetOrCreateUnpaused() (*SandboxRef, error) {
-	// Loop over idle sandboxes until one unpauses successfully,
-	// or the pool has no idle sandboxes left.
 	for {
-		s.mu.Lock()
-		if s.closed {
-			s.mu.Unlock()
-			return nil, fmt.Errorf("sandboxset: closed")
+		claimed, err := s.tryClaimIdle()
+		if err != nil {
+			return nil, err
 		}
-
-		// Fast path: claim an idle sandbox.
-		var claimed sandbox.Sandbox // raw sandbox, wrapped in SandboxRef on return
-		for _, w := range s.pool {
-			if !w.inUse {
-				w.inUse = true
-				claimed = w.sb
-				break
-			}
-		}
-		s.mu.Unlock()
-
 		if claimed == nil {
 			break // no idle sandbox — fall through to Create
 		}
 
-		// Unpause outside the lock (split-lock pattern).
-		if err := claimed.Unpause(); err != nil {
-			_ = s.Destroy(claimed, fmt.Sprintf("unpause: %v", err))
+		// Unpause outside the lock.
+		if err := claimed.sb.Unpause(); err != nil {
+			_ = s.destroy(claimed, fmt.Sprintf("unpause: %v", err))
 			continue // try the next idle sandbox
 		}
-		return &SandboxRef{sb: claimed, set: s, State: StateReady}, nil
+		return claimed, nil
 	}
 
 	// Slow path: create a new sandbox without holding the lock.
-	scratchDir, err := s.makeScratchDir()
-	if err != nil {
-		return nil, fmt.Errorf("sandboxset: scratch dir: %w", err)
+	var parentSb sandbox.Sandbox
+	if s.cfg.Parent != nil {
+		parentRef, err := s.cfg.Parent.GetOrCreateUnpaused()
+		if err != nil {
+			return nil, fmt.Errorf("sandboxset: parent get: %w", err)
+		}
+		parentSb = parentRef.Sandbox()
+		defer parentRef.Put()
 	}
+
+	scratchDir := s.cfg.ScratchDirs.Make("sb")
 	sb, err := s.cfg.Pool.Create(
-		s.cfg.Parent, s.cfg.IsLeaf,
+		parentSb, s.cfg.IsLeaf,
 		s.cfg.CodeDir, scratchDir,
 		s.cfg.Meta,
 	)
@@ -140,86 +138,86 @@ func (s *sandboxSetImpl) GetOrCreateUnpaused() (*SandboxRef, error) {
 		return nil, fmt.Errorf("sandboxset: create: %w", err)
 	}
 
-	s.mu.Lock()
-	s.pool = append(s.pool, &sandboxWrapper{sb: sb, inUse: true})
-	s.mu.Unlock()
-
-	return &SandboxRef{sb: sb, set: s, State: StateReady}, nil
+	ref := &SandboxRef{sb: sb, set: s, inUse: true}
+	if !s.appendToPool(ref) {
+		sb.Destroy("set closed during create")
+		return nil, fmt.Errorf("sandboxset: closed")
+	}
+	return ref, nil
 }
 
-// Put implements SandboxSet.
-//
-// The sandbox is paused and its wrapper is flipped back to idle.
-// If Pause fails, the sandbox is destroyed rather than silently
-// recycled — a bad sandbox should never re-enter the pool.
-func (s *sandboxSetImpl) Put(sb sandbox.Sandbox) error {
+// appendToPool appends ref to the pool if the set is not closed.
+// Returns false if the set is closed. Caller must not hold s.mu.
+func (s *sandboxSetImpl) appendToPool(ref *SandboxRef) bool {
 	s.mu.Lock()
+	defer s.mu.Unlock()
 	if s.closed {
-		s.mu.Unlock()
-		return fmt.Errorf("sandboxset: closed (sandbox %s was destroyed by Close)", sb.ID())
+		return false
 	}
-	s.mu.Unlock()
+	s.pool = append(s.pool, ref)
+	return true
+}
 
-	if err := sb.Pause(); err != nil {
-		_ = s.Destroy(sb, fmt.Sprintf("pause failed: %v", err))
-		return fmt.Errorf("sandboxset: sandbox %s destroyed because Pause failed: %w", sb.ID(), err)
+// put pauses the sandbox and returns it to the idle pool.
+// Pause is called before acquiring the lock to avoid blocking pool access during I/O.
+func (s *sandboxSetImpl) put(ref *SandboxRef) error {
+	if err := ref.sb.Pause(); err != nil {
+		_ = s.destroy(ref, fmt.Sprintf("pause failed: %v", err))
+		return fmt.Errorf("sandboxset: sandbox %s destroyed because Pause failed: %w", ref.sb.ID(), err)
 	}
 
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
-	for _, w := range s.pool {
-		if w.sb.ID() == sb.ID() {
-			w.inUse = false
-			return nil
-		}
+	if s.closed {
+		return fmt.Errorf("sandboxset: closed (sandbox %s was destroyed by Close)", ref.sb.ID())
 	}
-	return fmt.Errorf("sandboxset: sandbox %s not found in pool", sb.ID())
+
+	ref.inUse = false
+	return nil
 }
 
-// Destroy implements SandboxSet.
-//
-// The wrapper is spliced out of the pool under a short lock using O(1)
-// swap-with-tail. Destroy is called outside the lock to keep critical
-// sections short. The sandbox is always destroyed even if it was not
-// found in the pool.
-func (s *sandboxSetImpl) Destroy(sb sandbox.Sandbox, reason string) error {
+// tryRemoveFromPool removes ref from the pool using O(1) swap-with-tail.
+// Returns true if found. Caller must not hold s.mu.
+func (s *sandboxSetImpl) tryRemoveFromPool(ref *SandboxRef) bool {
 	s.mu.Lock()
-	found := false
-	for i, w := range s.pool {
-		if w.sb.ID() == sb.ID() {
+	defer s.mu.Unlock()
+	for i, r := range s.pool {
+		if r == ref {
 			s.pool[i] = s.pool[len(s.pool)-1]
 			s.pool = s.pool[:len(s.pool)-1]
-			found = true
-			break
+			return true
 		}
 	}
-	s.mu.Unlock()
-
-	sb.Destroy(reason)
+	return false
+}
 
+// destroy removes ref from the pool and destroys the underlying sandbox.
+// The sandbox is always destroyed even if not found in the pool.
+func (s *sandboxSetImpl) destroy(ref *SandboxRef, reason string) error {
+	found := s.tryRemoveFromPool(ref)
+	ref.sb.Destroy(reason) // I/O outside lock
+	ref.destroyed.Store(true)
 	if !found {
-		return fmt.Errorf("sandboxset: sandbox %s not found in pool (still destroyed)", sb.ID())
+		return fmt.Errorf("sandboxset: sandbox %s not found in pool (still destroyed)", ref.sb.ID())
 	}
 	return nil
 }
 
 // Close implements SandboxSet.
-//
-// All sandboxes are snapshot under the lock, then destroyed outside it.
 func (s *sandboxSetImpl) Close() error {
 	s.mu.Lock()
+	defer s.mu.Unlock()
+
 	if s.closed {
-		s.mu.Unlock()
 		return fmt.Errorf("sandboxset: already closed")
 	}
 	s.closed = true
-	pool := s.pool
-	s.pool = nil
-	s.mu.Unlock()
 
-	for _, w := range pool {
-		w.sb.Destroy("sandboxset closed")
+	for _, ref := range s.pool {
+		ref.sb.Destroy("sandboxset closed")
+		ref.destroyed.Store(true)
 	}
+	s.pool = nil
 	return nil
 }
diff --git a/go/worker/sandboxset/tests/sandboxset_integration_test.go b/go/worker/sandboxset/tests/sandboxset_integration_test.go
index a7adedc58..b05ff16d0 100644
--- a/go/worker/sandboxset/tests/sandboxset_integration_test.go
+++ b/go/worker/sandboxset/tests/sandboxset_integration_test.go
@@ -178,4 +178,4 @@ func TestIntegration_CloseDestroysAll(t *testing.T) {
 	if err == nil {
 		t.Fatal("expected error after Close")
 	}
-}
+}
\ No newline at end of file
diff --git a/go/worker/sandboxset/tests/sandboxset_test.go b/go/worker/sandboxset/tests/sandboxset_test.go
index 5c22cc3e7..87e16067d 100644
--- a/go/worker/sandboxset/tests/sandboxset_test.go
+++ b/go/worker/sandboxset/tests/sandboxset_test.go
@@ -1,9 +1,6 @@
 package tests
 
 import (
-	"errors"
-	"fmt"
-	"sync"
 	"testing"
 
 	"github.com/open-lambda/open-lambda/go/common"
@@ -11,8 +8,8 @@ import (
 	"github.com/open-lambda/open-lambda/go/worker/sandboxset"
 )
 
-// newTestConfig returns a valid Config backed by mocks and a temp directory.
-func newTestConfig(t *testing.T) (*sandboxset.Config, *sandbox.MockSandboxPool) {
+// newTestSet creates a valid SandboxSet backed by a MockSandboxPool.
+func newTestSet(t *testing.T) (sandboxset.SandboxSet, *sandbox.MockSandboxPool) {
 	t.Helper()
 	tmpDir := t.TempDir()
 	common.Conf = &common.Config{Worker_dir: tmpDir}
@@ -21,87 +18,22 @@ func newTestConfig(t *testing.T) (*sandboxset.Config, *sandbox.MockSandboxPool)
 		t.Fatal(err)
 	}
 	pool := &sandbox.MockSandboxPool{}
-	cfg := &sandboxset.Config{
+	set, err := sandboxset.New(&sandboxset.Config{
 		Pool:        pool,
 		CodeDir:     tmpDir + "/code",
 		ScratchDirs: scratchDirs,
-	}
-	return cfg, pool
-}
-
-// newTestSet is a shortcut that creates a valid SandboxSet.
-func newTestSet(t *testing.T) (sandboxset.SandboxSet, *sandbox.MockSandboxPool) {
-	t.Helper()
-	cfg, pool := newTestConfig(t)
-	set, err := sandboxset.New(cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-	return set, pool
-}
-
-// --- Constructor tests ---
-
-func TestNew_NilConfig(t *testing.T) {
-	_, err := sandboxset.New(nil)
-	if err == nil {
-		t.Fatal("expected error for nil config")
-	}
-}
-
-func TestNew_NilPool(t *testing.T) {
-	tmpDir := t.TempDir()
-	common.Conf = &common.Config{Worker_dir: tmpDir}
-	scratchDirs, err := common.NewDirMaker("scratch", common.STORE_REGULAR)
-	if err != nil {
-		t.Fatal(err)
-	}
-	_, err = sandboxset.New(&sandboxset.Config{
-		CodeDir:     "/some/dir",
-		ScratchDirs: scratchDirs,
 	})
-	if err == nil {
-		t.Fatal("expected error for nil Pool")
-	}
-}
-
-func TestNew_EmptyCodeDir(t *testing.T) {
-	tmpDir := t.TempDir()
-	common.Conf = &common.Config{Worker_dir: tmpDir}
-	scratchDirs, err := common.NewDirMaker("scratch", common.STORE_REGULAR)
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = sandboxset.New(&sandboxset.Config{
-		Pool:        &sandbox.MockSandboxPool{},
-		ScratchDirs: scratchDirs,
-	})
-	if err == nil {
-		t.Fatal("expected error for empty CodeDir")
-	}
-}
-
-func TestNew_NilScratchDirs(t *testing.T) {
-	_, err := sandboxset.New(&sandboxset.Config{
-		Pool:    &sandbox.MockSandboxPool{},
-		CodeDir: "/some/dir",
-	})
-	if err == nil {
-		t.Fatal("expected error for nil ScratchDirs")
-	}
-}
-
-func TestNew_Valid(t *testing.T) {
-	set, _ := newTestSet(t)
-	if set == nil {
-		t.Fatal("expected non-nil SandboxSet")
-	}
+	return set, pool
 }
 
-// --- GetOrCreateUnpaused tests ---
-
+// TestGet_CreatesNew verifies that GetOrCreateUnpaused creates a new sandbox
+// when the pool is empty.
 func TestGet_CreatesNew(t *testing.T) {
 	set, pool := newTestSet(t)
+
 	ref, err := set.GetOrCreateUnpaused()
 	if err != nil {
 		t.Fatalf("GetOrCreateUnpaused: %v", err)
@@ -114,70 +46,35 @@ func TestGet_CreatesNew(t *testing.T) {
 	}
 }
 
-func TestGet_ReusesIdle(t *testing.T) {
+// TestLifecycle_GetPutReuse verifies the full create → put → reuse cycle.
+func TestLifecycle_GetPutReuse(t *testing.T) {
 	set, _ := newTestSet(t)
+	defer set.Close()
 
 	ref1, err := set.GetOrCreateUnpaused()
 	if err != nil {
 		t.Fatalf("GetOrCreateUnpaused: %v", err)
 	}
-	id1 := ref1.Sandbox().ID()
-
-	if err := ref1.Put(); err != nil {
-		t.Fatalf("Put: %v", err)
-	}
-
-	ref2, err := set.GetOrCreateUnpaused()
-	if err != nil {
-		t.Fatalf("GetOrCreateUnpaused: %v", err)
-	}
-	if ref2.Sandbox().ID() != id1 {
-		t.Fatalf("expected reuse (ID %s), got new (ID %s)", id1, ref2.Sandbox().ID())
-	}
-}
-
-func TestGet_UnpauseFail(t *testing.T) {
-	set, pool := newTestSet(t)
+	id := ref1.Sandbox().ID()
 
-	ref1, err := set.GetOrCreateUnpaused()
-	if err != nil {
-		t.Fatalf("GetOrCreateUnpaused: %v", err)
-	}
-	sb1 := ref1.Sandbox()
-	// Inject unpause error before putting back.
-	sb1.(*sandbox.MockSandbox).UnpauseErr = errors.New("broken")
 	if err := ref1.Put(); err != nil {
 		t.Fatalf("Put: %v", err)
 	}
 
-	// Next GetOrCreateUnpaused should find the bad sandbox, destroy it, and create a new one.
 	ref2, err := set.GetOrCreateUnpaused()
 	if err != nil {
-		t.Fatalf("GetOrCreateUnpaused after unpause fail: %v", err)
-	}
-	if ref2.Sandbox().ID() == sb1.ID() {
-		t.Fatal("expected a different sandbox after unpause failure")
-	}
-	if !sb1.(*sandbox.MockSandbox).IsDestroyed() {
-		t.Fatal("bad sandbox should have been destroyed")
-	}
-	if n := len(pool.CreatedSandboxes()); n != 2 {
-		t.Fatalf("expected 2 creates (original + retry), got %d", n)
+		t.Fatalf("second GetOrCreateUnpaused: %v", err)
 	}
-}
-
-func TestGet_CreateFail(t *testing.T) {
-	set, pool := newTestSet(t)
-	pool.CreateErr = errors.New("out of resources")
-
-	_, err := set.GetOrCreateUnpaused()
-	if err == nil {
-		t.Fatal("expected error when pool.Create fails")
+	if ref2.Sandbox().ID() != id {
+		t.Fatalf("expected reuse (ID %s), got new (ID %s)", id, ref2.Sandbox().ID())
 	}
+	_ = ref2.Put()
 }
 
+// TestGet_AfterClose verifies that GetOrCreateUnpaused returns an error after Close.
 func TestGet_AfterClose(t *testing.T) {
 	set, _ := newTestSet(t)
+
 	if err := set.Close(); err != nil {
 		t.Fatalf("Close: %v", err)
 	}
@@ -186,267 +83,3 @@ func TestGet_AfterClose(t *testing.T) {
 		t.Fatal("expected error after Close")
 	}
 }
-
-// --- Put tests ---
-
-func TestPut_PausesAndReturns(t *testing.T) {
-	set, _ := newTestSet(t)
-	ref, err := set.GetOrCreateUnpaused()
-	if err != nil {
-		t.Fatalf("GetOrCreateUnpaused: %v", err)
-	}
-	mock := ref.Sandbox().(*sandbox.MockSandbox)
-	if mock.IsPaused() {
-		t.Fatal("sandbox should be unpaused after GetOrCreateUnpaused")
-	}
-
-	if err := ref.Put(); err != nil {
-		t.Fatalf("Put: %v", err)
-	}
-	if !mock.IsPaused() {
-		t.Fatal("sandbox should be paused after Put")
-	}
-}
-
-func TestPut_PauseFail(t *testing.T) {
-	set, _ := newTestSet(t)
-	ref, err := set.GetOrCreateUnpaused()
-	if err != nil {
-		t.Fatalf("GetOrCreateUnpaused: %v", err)
-	}
-	sb := ref.Sandbox()
-	sb.(*sandbox.MockSandbox).PauseErr = errors.New("pause broken")
-
-	err = ref.Put()
-	if err == nil {
-		t.Fatal("expected error when Pause fails")
-	}
-	if !sb.(*sandbox.MockSandbox).IsDestroyed() {
-		t.Fatal("sandbox should be destroyed when Pause fails")
-	}
-}
-
-func TestPut_NotInPool(t *testing.T) {
-	set, _ := newTestSet(t)
-	orphan := sandbox.NewMockSandbox("orphan")
-	err := set.Put(orphan)
-	if err == nil {
-		t.Fatal("expected error for sandbox not in pool")
-	}
-}
-
-// --- Destroy tests ---
-
-func TestDestroy_RemovesFromPool(t *testing.T) {
-	set, _ := newTestSet(t)
-	ref, err := set.GetOrCreateUnpaused()
-	if err != nil {
-		t.Fatalf("GetOrCreateUnpaused: %v", err)
-	}
-	sb := ref.Sandbox()
-
-	if err := ref.Destroy("test"); err != nil {
-		t.Fatalf("Destroy: %v", err)
-	}
-	if !sb.(*sandbox.MockSandbox).IsDestroyed() {
-		t.Fatal("sandbox should be destroyed")
-	}
-
-	// Next GetOrCreateUnpaused should create a new one, not reuse the destroyed one.
-	ref2, err := set.GetOrCreateUnpaused()
-	if err != nil {
-		t.Fatalf("GetOrCreateUnpaused after Destroy: %v", err)
-	}
-	if ref2.Sandbox().ID() == sb.ID() {
-		t.Fatal("should not reuse a destroyed sandbox")
-	}
-}
-
-func TestDestroy_NotInPool(t *testing.T) {
-	set, _ := newTestSet(t)
-	orphan := sandbox.NewMockSandbox("orphan")
-	err := set.Destroy(orphan, "test")
-	if err == nil {
-		t.Fatal("expected error for sandbox not in pool")
-	}
-	if !orphan.IsDestroyed() {
-		t.Fatal("sandbox should still be destroyed even if not in pool")
-	}
-}
-
-// --- Close tests ---
-
-func TestClose_DestroysAll(t *testing.T) {
-	set, pool := newTestSet(t)
-
-	// Create 3 sandboxes: 2 in-use, 1 idle.
-	ref1, _ := set.GetOrCreateUnpaused()
-	ref2, _ := set.GetOrCreateUnpaused()
-	ref3, _ := set.GetOrCreateUnpaused()
-	_ = ref3.Put() // return one to idle
-
-	if err := set.Close(); err != nil {
-		t.Fatalf("Close: %v", err)
-	}
-
-	for _, m := range pool.CreatedSandboxes() {
-		if !m.IsDestroyed() {
-			t.Fatalf("sandbox %s should be destroyed after Close", m.ID())
-		}
-	}
-	_ = ref1
-	_ = ref2
-}
-
-func TestClose_Twice(t *testing.T) {
-	set, _ := newTestSet(t)
-	if err := set.Close(); err != nil {
-		t.Fatalf("first Close: %v", err)
-	}
-	err := set.Close()
-	if err == nil {
-		t.Fatal("expected error on second Close")
-	}
-}
-
-func TestClose_EmptyPool(t *testing.T) {
-	set, _ := newTestSet(t)
-	if err := set.Close(); err != nil {
-		t.Fatalf("Close on empty pool: %v", err)
-	}
-}
-
-// --- Lifecycle tests ---
-
-func TestLifecycle_GetPutReuse(t *testing.T) {
-	set, _ := newTestSet(t)
-
-	// GetOrCreateUnpaused → Put → GetOrCreateUnpaused should reuse.
-	ref1, _ := set.GetOrCreateUnpaused()
-	id := ref1.Sandbox().ID()
-	_ = ref1.Put()
-
-	ref2, _ := set.GetOrCreateUnpaused()
-	if ref2.Sandbox().ID() != id {
-		t.Fatalf("expected reuse, got different ID: %s vs %s", id, ref2.Sandbox().ID())
-	}
-	_ = ref2.Put()
-
-	_ = set.Close()
-}
-
-func TestLifecycle_GetDestroyGet(t *testing.T) {
-	set, _ := newTestSet(t)
-
-	ref1, _ := set.GetOrCreateUnpaused()
-	id := ref1.Sandbox().ID()
-	_ = ref1.Destroy("bad")
-
-	ref2, _ := set.GetOrCreateUnpaused()
-	if ref2.Sandbox().ID() == id {
-		t.Fatal("expected fresh sandbox after Destroy, got same ID")
-	}
-	_ = ref2.Put()
-
-	_ = set.Close()
-}
-
-// --- Concurrency tests ---
-
-func TestConcurrent_Gets(t *testing.T) {
-	set, _ := newTestSet(t)
-	const n = 50
-
-	var wg sync.WaitGroup
-	refs := make([]*sandboxset.SandboxRef, n)
-	errs := make([]error, n)
-
-	for i := 0; i < n; i++ {
-		wg.Add(1)
-		go func(idx int) {
-			defer wg.Done()
-			ref, err := set.GetOrCreateUnpaused()
-			refs[idx] = ref
-			errs[idx] = err
-		}(i)
-	}
-	wg.Wait()
-
-	for i, err := range errs {
-		if err != nil {
-			t.Fatalf("goroutine %d: GetOrCreateUnpaused: %v", i, err)
-		}
-	}
-
-	// Clean up: put all back then close.
-	for _, ref := range refs {
-		_ = ref.Put()
-	}
-	_ = set.Close()
-}
-
-func TestConcurrent_GetPut(t *testing.T) {
-	set, _ := newTestSet(t)
-	const goroutines = 20
-	const iterations = 50
-
-	var wg sync.WaitGroup
-	for i := 0; i < goroutines; i++ {
-		wg.Add(1)
-		go func(id int) {
-			defer wg.Done()
-			for j := 0; j < iterations; j++ {
-				ref, err := set.GetOrCreateUnpaused()
-				if err != nil {
-					t.Errorf("goroutine %d iter %d: GetOrCreateUnpaused: %v", id, j, err)
-					return
-				}
-				if err := ref.Put(); err != nil {
-					t.Errorf("goroutine %d iter %d: Put: %v", id, j, err)
-					return
-				}
-			}
-		}(i)
-	}
-	wg.Wait()
-	_ = set.Close()
-}
-
-func TestConcurrent_CloseWhileGet(t *testing.T) {
-	set, _ := newTestSet(t)
-	const n = 20
-
-	// Grab some sandboxes first.
-	for i := 0; i < 5; i++ {
-		ref, _ := set.GetOrCreateUnpaused()
-		_ = ref.Put()
-	}
-
-	var wg sync.WaitGroup
-	errs := make(chan error, n)
-
-	// Launch goroutines that race GetOrCreateUnpaused vs Close.
-	for i := 0; i < n; i++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			ref, err := set.GetOrCreateUnpaused()
-			if err != nil {
-				// Expected for some goroutines after Close.
-				return
-			}
-			errs <- ref.Put()
-		}()
-	}
-
-	// Close from main goroutine while Gets are racing.
-	closeErr := set.Close()
-	wg.Wait()
-	close(errs)
-
-	// Close should succeed (first call).
-	if closeErr != nil {
-		// Close might race with GetOrCreateUnpaused; as long as no panic, we're OK.
-		fmt.Printf("Close returned: %v (acceptable in race)\n", closeErr)
-	}
-}

From 1ac1d796f0a1eca9977d759e0c792d837c19317d Mon Sep 17 00:00:00 2001
From: Ami Buch <amiabuch@gmail.com>
Date: Sun, 22 Mar 2026 22:54:40 -0500
Subject: [PATCH 47/55] fix: removed unnecessary fields

---
 go/worker/sandboxset/sandboxset.go            | 216 ++++++++++--------
 go/worker/sandboxset/tests/sandboxset_test.go |  29 +++
 2 files changed, 156 insertions(+), 89 deletions(-)

diff --git a/go/worker/sandboxset/sandboxset.go b/go/worker/sandboxset/sandboxset.go
index 25655be6a..f513d8f07 100644
--- a/go/worker/sandboxset/sandboxset.go
+++ b/go/worker/sandboxset/sandboxset.go
@@ -3,20 +3,21 @@ package sandboxset
 import (
 	"fmt"
 	"sync"
-	"sync/atomic"
 
 	"github.com/open-lambda/open-lambda/go/worker/sandbox"
 )
 
 // SandboxRef is a handle returned by GetOrCreateUnpaused.
-// It wraps a sandbox with a back-pointer to its parent set.
 // Set Broken = true before calling Put if the sandbox should not be recycled.
+// sb == nil means the ref has no live sandbox (destroyed or not yet created).
+
 type SandboxRef struct {
-	sb        sandbox.Sandbox
-	set       *sandboxSetImpl
-	Broken    bool        // public: caller sets true if request failed; Put will destroy instead of recycle
-	inUse     bool        // true when checked out; false when idle in pool
-	destroyed atomic.Bool // set atomically after Destroy(); guards against concurrent Close + Put
+	set    *sandboxSetImpl
+	Broken bool
+
+	// protected by set.mu
+	sb    sandbox.Sandbox
+	inUse bool
 }
 
 // Sandbox returns the underlying sandbox.
@@ -26,8 +27,11 @@ func (r *SandboxRef) Sandbox() sandbox.Sandbox {
 
 // Put returns the sandbox to its parent set, or destroys it if Broken is true.
 func (r *SandboxRef) Put() error {
-	if r.destroyed.Load() {
-		return fmt.Errorf("sandboxset: sandbox %s already destroyed", r.sb.ID())
+	r.set.mu.Lock()
+	already := r.sb == nil
+	r.set.mu.Unlock()
+	if already {
+		return fmt.Errorf("sandboxset: sandbox already destroyed")
 	}
 	if r.Broken {
 		return r.set.destroy(r, "state marked broken")
@@ -37,18 +41,21 @@ func (r *SandboxRef) Put() error {
 
 // Destroy removes the sandbox from its parent set and destroys it.
 func (r *SandboxRef) Destroy(reason string) error {
-	if r.destroyed.Load() {
+	r.set.mu.Lock()
+	already := r.sb == nil
+	r.set.mu.Unlock()
+	if already {
 		return nil
 	}
 	return r.set.destroy(r, reason)
 }
 
 // sandboxSetImpl is the private concrete type returned by New.
-// All mutable state is guarded by mu.
 type sandboxSetImpl struct {
-	mu     sync.Mutex
+	cfg *Config
+
+	mu     sync.Mutex // protects below fields
 	pool   []*SandboxRef
-	cfg    *Config
 	closed bool
 }
 
@@ -69,19 +76,29 @@ func newSandboxSet(cfg *Config) (*sandboxSetImpl, error) {
 }
 
 // claimIdle returns an idle ref from the pool, marking it inUse.
+// Prefers refs with an existing sandbox (avoids a Create), falls back to nil refs.
 // Caller must hold s.mu.
 func (s *sandboxSetImpl) claimIdle() *SandboxRef {
+	var nilRef *SandboxRef
 	for _, ref := range s.pool {
-		if !ref.inUse {
+		if ref.inUse {
+			continue
+		}
+		if ref.sb != nil {
 			ref.inUse = true
 			return ref
 		}
+		if nilRef == nil {
+			nilRef = ref
+		}
 	}
-	return nil
+	if nilRef != nil {
+		nilRef.inUse = true
+	}
+	return nilRef
 }
 
-// tryClaimIdle acquires the lock for its full duration, checks closed,
-// and returns an idle ref (or nil if none available).
+// tryClaimIdle acquires the lock, checks closed, and returns an idle ref (or nil if none).
 func (s *sandboxSetImpl) tryClaimIdle() (*SandboxRef, error) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
@@ -91,33 +108,22 @@ func (s *sandboxSetImpl) tryClaimIdle() (*SandboxRef, error) {
 	return s.claimIdle(), nil
 }
 
-// GetOrCreateUnpaused implements SandboxSet.
-//
-// Fast path: claim an idle ref via tryClaimIdle (which holds the lock for
-// its full duration), then Unpause outside the lock so the pool is not
-// stalled during I/O.
-//
-// Slow path: no idle ref exists, so a new sandbox is created without
-// holding the lock.
-func (s *sandboxSetImpl) GetOrCreateUnpaused() (*SandboxRef, error) {
-	for {
-		claimed, err := s.tryClaimIdle()
-		if err != nil {
-			return nil, err
-		}
-		if claimed == nil {
-			break // no idle sandbox — fall through to Create
-		}
-
-		// Unpause outside the lock.
-		if err := claimed.sb.Unpause(); err != nil {
-			_ = s.destroy(claimed, fmt.Sprintf("unpause: %v", err))
-			continue // try the next idle sandbox
-		}
-		return claimed, nil
+// appendNilRef adds a new nil-sb ref to the pool (inUse=true) and returns it.
+// Returns an error if the set is closed.
+func (s *sandboxSetImpl) appendNilRef() (*SandboxRef, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.closed {
+		return nil, fmt.Errorf("sandboxset: closed")
 	}
+	ref := &SandboxRef{set: s, inUse: true}
+	s.pool = append(s.pool, ref)
+	return ref, nil
+}
 
-	// Slow path: create a new sandbox without holding the lock.
+// createSandbox creates a new underlying sandbox, handling parent forking if configured.
+// Must be called without holding s.mu.
+func (s *sandboxSetImpl) createSandbox() (sandbox.Sandbox, error) {
 	var parentSb sandbox.Sandbox
 	if s.cfg.Parent != nil {
 		parentRef, err := s.cfg.Parent.GetOrCreateUnpaused()
@@ -135,89 +141,121 @@ func (s *sandboxSetImpl) GetOrCreateUnpaused() (*SandboxRef, error) {
 		s.cfg.Meta,
 	)
 	if err != nil {
-		return nil, fmt.Errorf("sandboxset: create: %w", err)
+		return nil, err
 	}
-
-	ref := &SandboxRef{sb: sb, set: s, inUse: true}
-	if !s.appendToPool(ref) {
-		sb.Destroy("set closed during create")
-		return nil, fmt.Errorf("sandboxset: closed")
-	}
-	return ref, nil
+	return sb, nil
 }
 
-// appendToPool appends ref to the pool if the set is not closed.
-// Returns false if the set is closed. Caller must not hold s.mu.
-func (s *sandboxSetImpl) appendToPool(ref *SandboxRef) bool {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	if s.closed {
-		return false
+// GetOrCreateUnpaused implements SandboxSet.
+// Fast path: claim an idle ref with an existing sandbox via tryClaimIdle, then Unpause outside the lock.
+// Slow path (nil ref): the claimed ref has no sandbox (either destroyed or newly added). A new sandbox is created outside the lock and assigned to the ref.
+func (s *sandboxSetImpl) GetOrCreateUnpaused() (*SandboxRef, error) {
+	for {
+		ref, err := s.tryClaimIdle()
+		if err != nil {
+			return nil, err
+		}
+
+		if ref == nil {
+			// No idle ref — add a new nil ref to the pool.
+			ref, err = s.appendNilRef()
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		s.mu.Lock()
+		sb := ref.sb
+		s.mu.Unlock()
+
+		if sb != nil {
+			// Path 1: existing paused sandbox — just unpause.
+			if err := sb.Unpause(); err != nil {
+				_ = s.destroy(ref, fmt.Sprintf("unpause: %v", err))
+				continue
+			}
+			return ref, nil
+		}
+
+		// Path 2/3: nil ref — create a new sandbox for it.
+		newSb, err := s.createSandbox()
+		if err != nil {
+			s.mu.Lock()
+			ref.inUse = false // release back as idle nil ref
+			s.mu.Unlock()
+			return nil, fmt.Errorf("sandboxset: create: %w", err)
+		}
+
+		s.mu.Lock()
+		ref.sb = newSb
+		s.mu.Unlock()
+		return ref, nil
 	}
-	s.pool = append(s.pool, ref)
-	return true
 }
 
-// put pauses the sandbox and returns it to the idle pool.
-// Pause is called before acquiring the lock to avoid blocking pool access during I/O.
+// put pauses the sandbox and returns the ref to the idle pool.
+// Pause is called before re-acquiring the lock to avoid blocking pool access during I/O.
 func (s *sandboxSetImpl) put(ref *SandboxRef) error {
-	if err := ref.sb.Pause(); err != nil {
+	s.mu.Lock()
+	sb := ref.sb
+	s.mu.Unlock()
+
+	if sb == nil {
+		return fmt.Errorf("sandboxset: sandbox already destroyed")
+	}
+	if err := sb.Pause(); err != nil {
 		_ = s.destroy(ref, fmt.Sprintf("pause failed: %v", err))
-		return fmt.Errorf("sandboxset: sandbox %s destroyed because Pause failed: %w", ref.sb.ID(), err)
+		return fmt.Errorf("sandboxset: sandbox destroyed because Pause failed: %w", err)
 	}
 
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
 	if s.closed {
-		return fmt.Errorf("sandboxset: closed (sandbox %s was destroyed by Close)", ref.sb.ID())
+		return fmt.Errorf("sandboxset: closed (sandbox destroyed by Close)")
 	}
 
 	ref.inUse = false
 	return nil
 }
 
-// tryRemoveFromPool removes ref from the pool using O(1) swap-with-tail.
-// Returns true if found. Caller must not hold s.mu.
-func (s *sandboxSetImpl) tryRemoveFromPool(ref *SandboxRef) bool {
+// destroy nils out ref.sb under the lock and destroys the underlying sandbox outside it.
+// The ref remains in the pool as an idle nil ref, ready to receive a new sandbox.
+func (s *sandboxSetImpl) destroy(ref *SandboxRef, reason string) error {
 	s.mu.Lock()
-	defer s.mu.Unlock()
-	for i, r := range s.pool {
-		if r == ref {
-			s.pool[i] = s.pool[len(s.pool)-1]
-			s.pool = s.pool[:len(s.pool)-1]
-			return true
-		}
-	}
-	return false
-}
+	sb := ref.sb
+	ref.sb = nil
+	ref.inUse = false
+	s.mu.Unlock()
 
-// destroy removes ref from the pool and destroys the underlying sandbox.
-// The sandbox is always destroyed even if not found in the pool.
-func (s *sandboxSetImpl) destroy(ref *SandboxRef, reason string) error {
-	found := s.tryRemoveFromPool(ref)
-	ref.sb.Destroy(reason) // I/O outside lock
-	ref.destroyed.Store(true)
-	if !found {
-		return fmt.Errorf("sandboxset: sandbox %s not found in pool (still destroyed)", ref.sb.ID())
+	if sb == nil {
+		return fmt.Errorf("sandboxset: sandbox already destroyed")
 	}
+	sb.Destroy(reason) // I/O outside lock
 	return nil
 }
 
 // Close implements SandboxSet.
 func (s *sandboxSetImpl) Close() error {
 	s.mu.Lock()
-	defer s.mu.Unlock()
-
 	if s.closed {
+		s.mu.Unlock()
 		return fmt.Errorf("sandboxset: already closed")
 	}
 	s.closed = true
 
+	var toDestroy []sandbox.Sandbox
 	for _, ref := range s.pool {
-		ref.sb.Destroy("sandboxset closed")
-		ref.destroyed.Store(true)
+		if ref.sb != nil {
+			toDestroy = append(toDestroy, ref.sb)
+			ref.sb = nil
+		}
 	}
 	s.pool = nil
+	s.mu.Unlock()
+
+	for _, sb := range toDestroy {
+		sb.Destroy("sandboxset closed")
+	}
 	return nil
 }
diff --git a/go/worker/sandboxset/tests/sandboxset_test.go b/go/worker/sandboxset/tests/sandboxset_test.go
index 87e16067d..b151ef11b 100644
--- a/go/worker/sandboxset/tests/sandboxset_test.go
+++ b/go/worker/sandboxset/tests/sandboxset_test.go
@@ -71,6 +71,35 @@ func TestLifecycle_GetPutReuse(t *testing.T) {
 	_ = ref2.Put()
 }
 
+// TestDestroy_NilRefReused verifies that after Destroy the ref stays in the pool
+// as a nil ref, and the next GetOrCreateUnpaused reuses it (pool size stays 1).
+func TestDestroy_NilRefReused(t *testing.T) {
+	set, pool := newTestSet(t)
+	defer set.Close()
+
+	ref1, err := set.GetOrCreateUnpaused()
+	if err != nil {
+		t.Fatalf("GetOrCreateUnpaused: %v", err)
+	}
+	id1 := ref1.Sandbox().ID()
+
+	if err := ref1.Destroy("test"); err != nil {
+		t.Fatalf("Destroy: %v", err)
+	}
+
+	ref2, err := set.GetOrCreateUnpaused()
+	if err != nil {
+		t.Fatalf("second GetOrCreateUnpaused: %v", err)
+	}
+	if ref2.Sandbox().ID() == id1 {
+		t.Fatal("expected a new sandbox after Destroy, got the same ID")
+	}
+	if n := len(pool.CreatedSandboxes()); n != 2 {
+		t.Fatalf("expected 2 created sandboxes total, got %d", n)
+	}
+	_ = ref2.Put()
+}
+
 // TestGet_AfterClose verifies that GetOrCreateUnpaused returns an error after Close.
 func TestGet_AfterClose(t *testing.T) {
 	set, _ := newTestSet(t)

From d71e56bf089e0209824d23a6441d7138eb960f99 Mon Sep 17 00:00:00 2001
From: Ami Buch <amiabuch@gmail.com>
Date: Sat, 4 Apr 2026 10:59:44 -0500
Subject: [PATCH 48/55] fix: locking and nomenclature

---
 go/worker/sandboxset/api.go                   |  21 +--
 go/worker/sandboxset/sandboxset.go            | 135 ++++++------------
 .../tests/sandboxset_integration_test.go      |  19 +--
 go/worker/sandboxset/tests/sandboxset_test.go |  10 +-
 4 files changed, 64 insertions(+), 121 deletions(-)

diff --git a/go/worker/sandboxset/api.go b/go/worker/sandboxset/api.go
index af6499fb3..cce262c5b 100644
--- a/go/worker/sandboxset/api.go
+++ b/go/worker/sandboxset/api.go
@@ -1,21 +1,8 @@
 // Package sandboxset provides a thread-safe pool of sandboxes for a single Lambda function. Callers ask for a sandbox and don't worry about whether it is freshly created or recycled from a previous request.
 //
-// Sandbox lifecycle inside a SandboxSet:
-//
-//	[created]
-//	    |
-//	    v
-//	[paused]  <---+
-//	    |         |
-//	    v         |
-//	[in-use]  ----+  (Put)
-//	    |
-//	    v
-//	[destroyed]     (Destroy / Close / error)
-//
 // Usage:
 //
-//	set, err := sandboxset.New(&sandboxset.Config{
+//	set := sandboxset.New(&sandboxset.Config{
 //	    Pool:        myPool,
 //	    CodeDir:     "/path/to/lambda",
 //	    ScratchDirs: myScratchDirs,
@@ -72,8 +59,8 @@ type Config struct {
 	ScratchDirs *common.DirMaker
 }
 
-// New creates a SandboxSet from cfg. Returns an error if any of
-// Pool, CodeDir, or ScratchDirs are missing.
-func New(cfg *Config) (SandboxSet, error) {
+// New creates a SandboxSet from cfg.
+// Panics if Pool, CodeDir, or ScratchDirs are missing.
+func New(cfg *Config) SandboxSet {
 	return newSandboxSet(cfg)
 }
diff --git a/go/worker/sandboxset/sandboxset.go b/go/worker/sandboxset/sandboxset.go
index f513d8f07..a4966cef9 100644
--- a/go/worker/sandboxset/sandboxset.go
+++ b/go/worker/sandboxset/sandboxset.go
@@ -39,17 +39,6 @@ func (r *SandboxRef) Put() error {
 	return r.set.put(r)
 }
 
-// Destroy removes the sandbox from its parent set and destroys it.
-func (r *SandboxRef) Destroy(reason string) error {
-	r.set.mu.Lock()
-	already := r.sb == nil
-	r.set.mu.Unlock()
-	if already {
-		return nil
-	}
-	return r.set.destroy(r, reason)
-}
-
 // sandboxSetImpl is the private concrete type returned by New.
 type sandboxSetImpl struct {
 	cfg *Config
@@ -59,63 +48,47 @@ type sandboxSetImpl struct {
 	closed bool
 }
 
-func newSandboxSet(cfg *Config) (*sandboxSetImpl, error) {
+func newSandboxSet(cfg *Config) *sandboxSetImpl {
 	if cfg == nil {
-		return nil, fmt.Errorf("sandboxset: Config must not be nil")
+		panic("sandboxset: Config must not be nil")
 	}
 	if cfg.Pool == nil {
-		return nil, fmt.Errorf("sandboxset: Config.Pool must not be nil")
+		panic("sandboxset: Config.Pool must not be nil")
 	}
 	if cfg.CodeDir == "" {
-		return nil, fmt.Errorf("sandboxset: Config.CodeDir must not be empty")
+		panic("sandboxset: Config.CodeDir must not be empty")
 	}
 	if cfg.ScratchDirs == nil {
-		return nil, fmt.Errorf("sandboxset: Config.ScratchDirs must not be nil")
-	}
-	return &sandboxSetImpl{cfg: cfg}, nil
-}
-
-// claimIdle returns an idle ref from the pool, marking it inUse.
-// Prefers refs with an existing sandbox (avoids a Create), falls back to nil refs.
-// Caller must hold s.mu.
-func (s *sandboxSetImpl) claimIdle() *SandboxRef {
-	var nilRef *SandboxRef
-	for _, ref := range s.pool {
-		if ref.inUse {
-			continue
-		}
-		if ref.sb != nil {
-			ref.inUse = true
-			return ref
-		}
-		if nilRef == nil {
-			nilRef = ref
-		}
-	}
-	if nilRef != nil {
-		nilRef.inUse = true
+		panic("sandboxset: Config.ScratchDirs must not be nil")
 	}
-	return nilRef
+	return &sandboxSetImpl{cfg: cfg}
 }
 
-// tryClaimIdle acquires the lock, checks closed, and returns an idle ref (or nil if none).
-func (s *sandboxSetImpl) tryClaimIdle() (*SandboxRef, error) {
+// claimIdle acquires the lock and returns an inUse ref.
+// It prefers a ref with an existing sandbox, falls back to an empty ref,
+// and appends a new ref to the pool if none are idle.
+// Always returns a non-nil ref or an error.
+func (s *sandboxSetImpl) claimIdle() (*SandboxRef, error) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	if s.closed {
 		return nil, fmt.Errorf("sandboxset: closed")
 	}
-	return s.claimIdle(), nil
-}
-
-// appendNilRef adds a new nil-sb ref to the pool (inUse=true) and returns it.
-// Returns an error if the set is closed.
-func (s *sandboxSetImpl) appendNilRef() (*SandboxRef, error) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	if s.closed {
-		return nil, fmt.Errorf("sandboxset: closed")
+	// Path 1: prefer a ref with an existing sandbox.
+	for _, ref := range s.pool {
+		if !ref.inUse && ref.sb != nil {
+			ref.inUse = true
+			return ref, nil
+		}
 	}
+	// Path 2: fall back to a ref without a sandbox.
+	for _, ref := range s.pool {
+		if !ref.inUse {
+			ref.inUse = true
+			return ref, nil
+		}
+	}
+	// Path 3: no idle ref — create and append a new one.
 	ref := &SandboxRef{set: s, inUse: true}
 	s.pool = append(s.pool, ref)
 	return ref, nil
@@ -147,50 +120,34 @@ func (s *sandboxSetImpl) createSandbox() (sandbox.Sandbox, error) {
 }
 
 // GetOrCreateUnpaused implements SandboxSet.
-// Fast path: claim an idle ref with an existing sandbox via tryClaimIdle, then Unpause outside the lock.
-// Slow path (nil ref): the claimed ref has no sandbox (either destroyed or newly added). A new sandbox is created outside the lock and assigned to the ref.
+// Step 1: claim a ref from the pool (requires locking).
+// Step 2: ensure the ref has a healthy, unpaused sandbox (no locking).
 func (s *sandboxSetImpl) GetOrCreateUnpaused() (*SandboxRef, error) {
-	for {
-		ref, err := s.tryClaimIdle()
-		if err != nil {
-			return nil, err
-		}
-
-		if ref == nil {
-			// No idle ref — add a new nil ref to the pool.
-			ref, err = s.appendNilRef()
-			if err != nil {
-				return nil, err
-			}
-		}
-
-		s.mu.Lock()
-		sb := ref.sb
-		s.mu.Unlock()
-
-		if sb != nil {
-			// Path 1: existing paused sandbox — just unpause.
-			if err := sb.Unpause(); err != nil {
-				_ = s.destroy(ref, fmt.Sprintf("unpause: %v", err))
-				continue
-			}
-			return ref, nil
-		}
+	// Step 1: get a SandboxRef (with or without a Sandbox).
+	ref, err := s.claimIdle()
+	if err != nil {
+		return nil, err
+	}
 
-		// Path 2/3: nil ref — create a new sandbox for it.
-		newSb, err := s.createSandbox()
-		if err != nil {
-			s.mu.Lock()
-			ref.inUse = false // release back as idle nil ref
-			s.mu.Unlock()
-			return nil, fmt.Errorf("sandboxset: create: %w", err)
+	// Step 2: make sure the ref has a healthy, unpaused Sandbox.
+	if ref.sb != nil {
+		if err := ref.sb.Unpause(); err != nil {
+			_ = s.destroy(ref, fmt.Sprintf("unpause: %v", err))
+			return nil, fmt.Errorf("sandboxset: unpause: %w", err)
 		}
+		return ref, nil
+	}
 
+	newSb, err := s.createSandbox()
+	if err != nil {
 		s.mu.Lock()
-		ref.sb = newSb
+		ref.inUse = false
 		s.mu.Unlock()
-		return ref, nil
+		return nil, fmt.Errorf("sandboxset: create: %w", err)
 	}
+
+	ref.sb = newSb
+	return ref, nil
 }
 
 // put pauses the sandbox and returns the ref to the idle pool.
diff --git a/go/worker/sandboxset/tests/sandboxset_integration_test.go b/go/worker/sandboxset/tests/sandboxset_integration_test.go
index b05ff16d0..93ff34e26 100644
--- a/go/worker/sandboxset/tests/sandboxset_integration_test.go
+++ b/go/worker/sandboxset/tests/sandboxset_integration_test.go
@@ -54,15 +54,12 @@ func newDockerSet(t *testing.T) sandboxset.SandboxSet {
 		t.Fatal(err)
 	}
 
-	set, err := sandboxset.New(&sandboxset.Config{
+	set := sandboxset.New(&sandboxset.Config{
 		Pool:        pool,
 		IsLeaf:      true,
 		CodeDir:     codeDir,
 		ScratchDirs: scratchDirs,
 	})
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	t.Cleanup(func() {
 		_ = set.Close()
@@ -87,7 +84,8 @@ func TestIntegration_GetCreatesRealContainer(t *testing.T) {
 	t.Logf("created real container: ID=%s", sb.ID())
 	t.Logf("debug: %s", sb.DebugString())
 
-	_ = ref.Destroy("test done")
+	ref.Broken = true
+	_ = ref.Put()
 }
 
 func TestIntegration_PutPausesAndReuses(t *testing.T) {
@@ -117,7 +115,8 @@ func TestIntegration_PutPausesAndReuses(t *testing.T) {
 		t.Fatalf("expected reuse (same ID %s), got new container %s", id1, id2)
 	}
 
-	_ = ref2.Destroy("test done")
+	ref2.Broken = true
+	_ = ref2.Put()
 }
 
 func TestIntegration_DestroyKillsReal(t *testing.T) {
@@ -130,8 +129,9 @@ func TestIntegration_DestroyKillsReal(t *testing.T) {
 	id1 := ref1.Sandbox().ID()
 	t.Logf("first sandbox: ID=%s", id1)
 
-	if err := ref1.Destroy("intentional destroy"); err != nil {
-		t.Fatalf("Destroy: %v", err)
+	ref1.Broken = true
+	if err := ref1.Put(); err != nil {
+		t.Fatalf("Put (broken): %v", err)
 	}
 
 	// Get again — must be a different container since we destroyed the first
@@ -146,7 +146,8 @@ func TestIntegration_DestroyKillsReal(t *testing.T) {
 		t.Fatal("expected new container after Destroy, got same ID")
 	}
 
-	_ = ref2.Destroy("test done")
+	ref2.Broken = true
+	_ = ref2.Put()
 }
 
 func TestIntegration_CloseDestroysAll(t *testing.T) {
diff --git a/go/worker/sandboxset/tests/sandboxset_test.go b/go/worker/sandboxset/tests/sandboxset_test.go
index b151ef11b..30db1a43d 100644
--- a/go/worker/sandboxset/tests/sandboxset_test.go
+++ b/go/worker/sandboxset/tests/sandboxset_test.go
@@ -18,14 +18,11 @@ func newTestSet(t *testing.T) (sandboxset.SandboxSet, *sandbox.MockSandboxPool)
 		t.Fatal(err)
 	}
 	pool := &sandbox.MockSandboxPool{}
-	set, err := sandboxset.New(&sandboxset.Config{
+	set := sandboxset.New(&sandboxset.Config{
 		Pool:        pool,
 		CodeDir:     tmpDir + "/code",
 		ScratchDirs: scratchDirs,
 	})
-	if err != nil {
-		t.Fatal(err)
-	}
 	return set, pool
 }
 
@@ -83,8 +80,9 @@ func TestDestroy_NilRefReused(t *testing.T) {
 	}
 	id1 := ref1.Sandbox().ID()
 
-	if err := ref1.Destroy("test"); err != nil {
-		t.Fatalf("Destroy: %v", err)
+	ref1.Broken = true
+	if err := ref1.Put(); err != nil {
+		t.Fatalf("Put (broken): %v", err)
 	}
 
 	ref2, err := set.GetOrCreateUnpaused()

From 868296cb8d276fa297aa9b25bf7f0c9bd44c176d Mon Sep 17 00:00:00 2001
From: Ami Buch <amiabuch@gmail.com>
Date: Sat, 11 Apr 2026 00:17:37 -0500
Subject: [PATCH 49/55] fix: redundant locking, return inconsistencies

---
 go/worker/sandboxset/sandboxset.go            | 76 ++++++++++---------
 .../tests/sandboxset_integration_test.go      | 13 +++-
 2 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/go/worker/sandboxset/sandboxset.go b/go/worker/sandboxset/sandboxset.go
index a4966cef9..c4874e1ce 100644
--- a/go/worker/sandboxset/sandboxset.go
+++ b/go/worker/sandboxset/sandboxset.go
@@ -8,14 +8,23 @@ import (
 )
 
 // SandboxRef is a handle returned by GetOrCreateUnpaused.
-// Set Broken = true before calling Put if the sandbox should not be recycled.
-// sb == nil means the ref has no live sandbox (destroyed or not yet created).
-
+//
+// Contract: a ref with inUse == true is owned by exactly one goroutine; the
+// holder can read and write sb without locking. When inUse is false, sb and
+// inUse are both protected by set.mu.
+//
+// Broken: callers MUST set Broken = true before Put() on any failed request.
+// The set can self-heal from hard container failures (Pause/Unpause errors),
+// but it cannot detect soft failures — a live container with a wedged runtime,
+// a crashed handler, or any error returned by the sandbox's HTTP client. Only
+// the caller knows whether their request succeeded. Forgetting to mark Broken
+// recycles the bad sandbox back to the pool and the next caller inherits it.
 type SandboxRef struct {
 	set    *sandboxSetImpl
 	Broken bool
 
-	// protected by set.mu
+	// sb is owned by the holder while inUse == true (no locking needed).
+	// When inUse == false, both sb and inUse are protected by set.mu.
 	sb    sandbox.Sandbox
 	inUse bool
 }
@@ -27,15 +36,12 @@ func (r *SandboxRef) Sandbox() sandbox.Sandbox {
 
 // Put returns the sandbox to its parent set, or destroys it if Broken is true.
 func (r *SandboxRef) Put() error {
-	r.set.mu.Lock()
-	already := r.sb == nil
-	r.set.mu.Unlock()
-	if already {
-		return fmt.Errorf("sandboxset: sandbox already destroyed")
-	}
 	if r.Broken {
 		return r.set.destroy(r, "state marked broken")
 	}
+	if r.sb == nil {
+		return fmt.Errorf("sandboxset: sandbox already destroyed")
+	}
 	return r.set.put(r)
 }
 
@@ -65,8 +71,6 @@ func newSandboxSet(cfg *Config) *sandboxSetImpl {
 }
 
 // claimIdle acquires the lock and returns an inUse ref.
-// It prefers a ref with an existing sandbox, falls back to an empty ref,
-// and appends a new ref to the pool if none are idle.
 // Always returns a non-nil ref or an error.
 func (s *sandboxSetImpl) claimIdle() (*SandboxRef, error) {
 	s.mu.Lock()
@@ -114,7 +118,7 @@ func (s *sandboxSetImpl) createSandbox() (sandbox.Sandbox, error) {
 		s.cfg.Meta,
 	)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("sandboxset: pool create: %w", err)
 	}
 	return sb, nil
 }
@@ -132,18 +136,18 @@ func (s *sandboxSetImpl) GetOrCreateUnpaused() (*SandboxRef, error) {
 	// Step 2: make sure the ref has a healthy, unpaused Sandbox.
 	if ref.sb != nil {
 		if err := ref.sb.Unpause(); err != nil {
-			_ = s.destroy(ref, fmt.Sprintf("unpause: %v", err))
-			return nil, fmt.Errorf("sandboxset: unpause: %w", err)
+			ref.sb.Destroy(fmt.Sprintf("unpause: %v", err))
+			ref.sb = nil
+		} else {
+			return ref, nil
 		}
-		return ref, nil
 	}
 
 	newSb, err := s.createSandbox()
 	if err != nil {
-		s.mu.Lock()
-		ref.inUse = false
-		s.mu.Unlock()
-		return nil, fmt.Errorf("sandboxset: create: %w", err)
+		ref.Broken = true
+		_ = ref.Put()
+		return nil, err
 	}
 
 	ref.sb = newSb
@@ -151,12 +155,9 @@ func (s *sandboxSetImpl) GetOrCreateUnpaused() (*SandboxRef, error) {
 }
 
 // put pauses the sandbox and returns the ref to the idle pool.
-// Pause is called before re-acquiring the lock to avoid blocking pool access during I/O.
+
 func (s *sandboxSetImpl) put(ref *SandboxRef) error {
-	s.mu.Lock()
 	sb := ref.sb
-	s.mu.Unlock()
-
 	if sb == nil {
 		return fmt.Errorf("sandboxset: sandbox already destroyed")
 	}
@@ -166,13 +167,15 @@ func (s *sandboxSetImpl) put(ref *SandboxRef) error {
 	}
 
 	s.mu.Lock()
-	defer s.mu.Unlock()
-
 	if s.closed {
-		return fmt.Errorf("sandboxset: closed (sandbox destroyed by Close)")
+		ref.sb = nil
+		ref.inUse = false
+		s.mu.Unlock()
+		sb.Destroy("sandboxset closed during put")
+		return fmt.Errorf("sandboxset: closed")
 	}
-
 	ref.inUse = false
+	s.mu.Unlock()
 	return nil
 }
 
@@ -193,26 +196,25 @@ func (s *sandboxSetImpl) destroy(ref *SandboxRef, reason string) error {
 }
 
 // Close implements SandboxSet.
+// Close only destroys idle sandboxes; in-use sandboxes are left to their holders,
+// whose put() will see s.closed and destroy them on release. This preserves the
+// "inUse ref is owned by exactly one goroutine" invariant — Close never touches
+// a held ref's sb field.
 func (s *sandboxSetImpl) Close() error {
 	s.mu.Lock()
+	defer s.mu.Unlock()
+
 	if s.closed {
-		s.mu.Unlock()
 		return fmt.Errorf("sandboxset: already closed")
 	}
 	s.closed = true
 
-	var toDestroy []sandbox.Sandbox
 	for _, ref := range s.pool {
-		if ref.sb != nil {
-			toDestroy = append(toDestroy, ref.sb)
+		if ref.sb != nil && !ref.inUse {
+			ref.sb.Destroy("sandboxset closed")
 			ref.sb = nil
 		}
 	}
 	s.pool = nil
-	s.mu.Unlock()
-
-	for _, sb := range toDestroy {
-		sb.Destroy("sandboxset closed")
-	}
 	return nil
 }
diff --git a/go/worker/sandboxset/tests/sandboxset_integration_test.go b/go/worker/sandboxset/tests/sandboxset_integration_test.go
index 93ff34e26..3fabd335d 100644
--- a/go/worker/sandboxset/tests/sandboxset_integration_test.go
+++ b/go/worker/sandboxset/tests/sandboxset_integration_test.go
@@ -164,16 +164,25 @@ func TestIntegration_CloseDestroysAll(t *testing.T) {
 		t.Logf("sandbox[%d]: ID=%s", i, ref.Sandbox().ID())
 	}
 
-	// Put one back to idle so Close covers both in-use and idle
+	// Put one back to idle so Close covers the idle path.
 	if err := refs[2].Put(); err != nil {
 		t.Fatalf("Put: %v", err)
 	}
 
-	// Close should destroy all
+	// Close destroys idle sandboxes; the two in-use ones are left to be
+	// destroyed by put()'s closed-branch when their holders release them.
 	if err := set.Close(); err != nil {
 		t.Fatalf("Close: %v", err)
 	}
 
+	// Releasing in-use refs after Close should destroy their sandboxes via
+	// put()'s closed branch and return an error indicating the set is closed.
+	for i := 0; i < 2; i++ {
+		if err := refs[i].Put(); err == nil {
+			t.Fatalf("expected error from Put[%d] after Close, got nil", i)
+		}
+	}
+
 	// Verify set is closed — further Gets should fail
 	_, err := set.GetOrCreateUnpaused()
 	if err == nil {

From a0ded0cc7a83bd305f0563437e780f0704a98e06 Mon Sep 17 00:00:00 2001
From: Ami Buch <amiabuch@gmail.com>
Date: Thu, 23 Apr 2026 22:58:35 -0500
Subject: [PATCH 50/55] refactor: heavy refactoring, simpler flow

---
 go/worker/sandboxset/api.go                   |   4 +-
 go/worker/sandboxset/sandboxset.go            | 173 +++++++++---------
 .../tests/sandboxset_integration_test.go      |  52 +++---
 go/worker/sandboxset/tests/sandboxset_test.go |  63 +++++--
 4 files changed, 160 insertions(+), 132 deletions(-)

diff --git a/go/worker/sandboxset/api.go b/go/worker/sandboxset/api.go
index cce262c5b..f02bee361 100644
--- a/go/worker/sandboxset/api.go
+++ b/go/worker/sandboxset/api.go
@@ -11,7 +11,7 @@
 //	ref, err := set.GetOrCreateUnpaused()
 //	// ... use ref.Sandbox() to handle request ...
 //	if broken {
-//	    ref.Broken = true
+//	    ref.MarkDead()
 //	}
 //	ref.Put()
 package sandboxset
@@ -39,7 +39,7 @@ type Config struct {
 
 	// Parent is an optional SandboxSet to fork from. When nil, new
 	// sandboxes are created from scratch. Not all SandboxPool
-	// implementations support forking.
+	// implementations support forking. The parent must outlive this child.
 	Parent SandboxSet
 
 	// IsLeaf marks sandboxes as non-forkable, meaning they will
diff --git a/go/worker/sandboxset/sandboxset.go b/go/worker/sandboxset/sandboxset.go
index c4874e1ce..a0f01ebad 100644
--- a/go/worker/sandboxset/sandboxset.go
+++ b/go/worker/sandboxset/sandboxset.go
@@ -2,54 +2,46 @@ package sandboxset
 
 import (
 	"fmt"
+	"log/slog"
 	"sync"
 
 	"github.com/open-lambda/open-lambda/go/worker/sandbox"
 )
 
-// SandboxRef is a handle returned by GetOrCreateUnpaused.
-//
-// Contract: a ref with inUse == true is owned by exactly one goroutine; the
-// holder can read and write sb without locking. When inUse is false, sb and
-// inUse are both protected by set.mu.
-//
-// Broken: callers MUST set Broken = true before Put() on any failed request.
-// The set can self-heal from hard container failures (Pause/Unpause errors),
-// but it cannot detect soft failures — a live container with a wedged runtime,
-// a crashed handler, or any error returned by the sandbox's HTTP client. Only
-// the caller knows whether their request succeeded. Forgetting to mark Broken
-// recycles the bad sandbox back to the pool and the next caller inherits it.
+// SandboxRef is a handle returned by GetOrCreateUnpaused. While inUse is true
+// the holder owns sb; otherwise sb and inUse are protected by set.mu.
+// Callers signal a dead sandbox by calling MarkDead before Put. The set never
+// destroys sandboxes — lifecycle is owned upstream.
+// A ref must not be shared across goroutines; one goroutine holds it at a time.
 type SandboxRef struct {
-	set    *sandboxSetImpl
-	Broken bool
-
-	// sb is owned by the holder while inUse == true (no locking needed).
-	// When inUse == false, both sb and inUse are protected by set.mu.
+	set   *sandboxSetImpl
 	sb    sandbox.Sandbox
 	inUse bool
 }
 
-// Sandbox returns the underlying sandbox.
-func (r *SandboxRef) Sandbox() sandbox.Sandbox {
-	return r.sb
-}
+func (r *SandboxRef) Sandbox() sandbox.Sandbox { return r.sb }
 
-// Put returns the sandbox to its parent set, or destroys it if Broken is true.
-func (r *SandboxRef) Put() error {
-	if r.Broken {
-		return r.set.destroy(r, "state marked broken")
+func (r *SandboxRef) MarkDead() {
+	r.set.mu.Lock()
+	defer r.set.mu.Unlock()
+	if !r.inUse {
+		panic(fmt.Sprintf("sandboxset: MarkDead on ref %p not currently held (inUse=%v)", r, r.inUse))
 	}
+	r.sb = nil
+}
+
+func (r *SandboxRef) Put() {
 	if r.sb == nil {
-		return fmt.Errorf("sandboxset: sandbox already destroyed")
+		r.set.releaseSlot(r)
+	} else {
+		r.set.put(r)
 	}
-	return r.set.put(r)
 }
 
-// sandboxSetImpl is the private concrete type returned by New.
 type sandboxSetImpl struct {
 	cfg *Config
 
-	mu     sync.Mutex // protects below fields
+	mu     sync.Mutex
 	pool   []*SandboxRef
 	closed bool
 }
@@ -70,42 +62,45 @@ func newSandboxSet(cfg *Config) *sandboxSetImpl {
 	return &sandboxSetImpl{cfg: cfg}
 }
 
-// claimIdle acquires the lock and returns an inUse ref.
-// Always returns a non-nil ref or an error.
 func (s *sandboxSetImpl) claimIdle() (*SandboxRef, error) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
+
 	if s.closed {
 		return nil, fmt.Errorf("sandboxset: closed")
 	}
-	// Path 1: prefer a ref with an existing sandbox.
+
+	var empty *SandboxRef
 	for _, ref := range s.pool {
-		if !ref.inUse && ref.sb != nil {
-			ref.inUse = true
-			return ref, nil
+		if ref.inUse {
+			continue
 		}
-	}
-	// Path 2: fall back to a ref without a sandbox.
-	for _, ref := range s.pool {
-		if !ref.inUse {
+		if ref.sb != nil {
 			ref.inUse = true
 			return ref, nil
 		}
+		if empty == nil {
+			empty = ref
+		}
 	}
-	// Path 3: no idle ref — create and append a new one.
+
+	if empty != nil {
+		empty.inUse = true
+		return empty, nil
+	}
+
 	ref := &SandboxRef{set: s, inUse: true}
 	s.pool = append(s.pool, ref)
 	return ref, nil
 }
 
-// createSandbox creates a new underlying sandbox, handling parent forking if configured.
-// Must be called without holding s.mu.
+// createSandbox must be called without holding s.mu.
 func (s *sandboxSetImpl) createSandbox() (sandbox.Sandbox, error) {
 	var parentSb sandbox.Sandbox
 	if s.cfg.Parent != nil {
 		parentRef, err := s.cfg.Parent.GetOrCreateUnpaused()
 		if err != nil {
-			return nil, fmt.Errorf("sandboxset: parent get: %w", err)
+			return nil, err
 		}
 		parentSb = parentRef.Sandbox()
 		defer parentRef.Put()
@@ -123,83 +118,80 @@ func (s *sandboxSetImpl) createSandbox() (sandbox.Sandbox, error) {
 	return sb, nil
 }
 
-// GetOrCreateUnpaused implements SandboxSet.
-// Step 1: claim a ref from the pool (requires locking).
-// Step 2: ensure the ref has a healthy, unpaused sandbox (no locking).
 func (s *sandboxSetImpl) GetOrCreateUnpaused() (*SandboxRef, error) {
-	// Step 1: get a SandboxRef (with or without a Sandbox).
 	ref, err := s.claimIdle()
 	if err != nil {
 		return nil, err
 	}
 
-	// Step 2: make sure the ref has a healthy, unpaused Sandbox.
 	if ref.sb != nil {
 		if err := ref.sb.Unpause(); err != nil {
-			ref.sb.Destroy(fmt.Sprintf("unpause: %v", err))
+			slog.Warn("sandboxset: unpause failed, discarding sandbox", "err", err)
 			ref.sb = nil
-		} else {
-			return ref, nil
 		}
 	}
 
-	newSb, err := s.createSandbox()
-	if err != nil {
-		ref.Broken = true
-		_ = ref.Put()
-		return nil, err
+	if ref.sb == nil {
+		newSb, err := s.createSandbox()
+		if err != nil {
+			s.releaseSlot(ref)
+			return nil, err
+		}
+		ref.sb = newSb
 	}
 
-	ref.sb = newSb
 	return ref, nil
 }
 
-// put pauses the sandbox and returns the ref to the idle pool.
-
-func (s *sandboxSetImpl) put(ref *SandboxRef) error {
-	sb := ref.sb
-	if sb == nil {
-		return fmt.Errorf("sandboxset: sandbox already destroyed")
+// put relies on Sandbox.Pause being no-op-safe after external death (see sandbox/api.go).
+func (s *sandboxSetImpl) put(ref *SandboxRef) {
+	s.mu.Lock()
+	if !ref.inUse {
+		s.mu.Unlock()
+		panic(fmt.Sprintf("sandboxset: put on ref %p not currently held (inUse=%v)", ref, ref.inUse))
+	}
+	closed := s.closed
+	if closed {
+		s.releaseSlotLocked(ref)
+	}
+	s.mu.Unlock()
+	if closed {
+		return
 	}
-	if err := sb.Pause(); err != nil {
-		_ = s.destroy(ref, fmt.Sprintf("pause failed: %v", err))
-		return fmt.Errorf("sandboxset: sandbox destroyed because Pause failed: %w", err)
+
+	if err := ref.sb.Pause(); err != nil {
+		s.releaseSlot(ref)
+		return
 	}
 
 	s.mu.Lock()
+	defer s.mu.Unlock()
 	if s.closed {
-		ref.sb = nil
-		ref.inUse = false
-		s.mu.Unlock()
-		sb.Destroy("sandboxset closed during put")
-		return fmt.Errorf("sandboxset: closed")
+		// rare: closed raced in during Pause; sandbox is paused, caller owns lifecycle
+		s.releaseSlotLocked(ref)
+		return
 	}
 	ref.inUse = false
-	s.mu.Unlock()
-	return nil
 }
 
-// destroy nils out ref.sb under the lock and destroys the underlying sandbox outside it.
-// The ref remains in the pool as an idle nil ref, ready to receive a new sandbox.
-func (s *sandboxSetImpl) destroy(ref *SandboxRef, reason string) error {
-	s.mu.Lock()
-	sb := ref.sb
+// releaseSlotLocked clears sb and inUse. Caller must hold s.mu.
+func (s *sandboxSetImpl) releaseSlotLocked(ref *SandboxRef) {
 	ref.sb = nil
 	ref.inUse = false
-	s.mu.Unlock()
+}
 
-	if sb == nil {
-		return fmt.Errorf("sandboxset: sandbox already destroyed")
+func (s *sandboxSetImpl) releaseSlot(ref *SandboxRef) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if !ref.inUse {
+		panic(fmt.Sprintf("sandboxset: releaseSlot on ref %p not currently held (inUse=%v)", ref, ref.inUse))
 	}
-	sb.Destroy(reason) // I/O outside lock
-	return nil
+	s.releaseSlotLocked(ref)
 }
 
-// Close implements SandboxSet.
-// Close only destroys idle sandboxes; in-use sandboxes are left to their holders,
-// whose put() will see s.closed and destroy them on release. This preserves the
-// "inUse ref is owned by exactly one goroutine" invariant — Close never touches
-// a held ref's sb field.
+// Close clears idle slots; in-use refs are left to their holders, whose put()
+// will see s.closed and release them. Never touches a held ref's sb.
+// Best-effort: if a holder never calls Put, the slot is not reclaimed.
 func (s *sandboxSetImpl) Close() error {
 	s.mu.Lock()
 	defer s.mu.Unlock()
@@ -210,9 +202,8 @@ func (s *sandboxSetImpl) Close() error {
 	s.closed = true
 
 	for _, ref := range s.pool {
-		if ref.sb != nil && !ref.inUse {
-			ref.sb.Destroy("sandboxset closed")
-			ref.sb = nil
+		if !ref.inUse {
+			s.releaseSlotLocked(ref)
 		}
 	}
 	s.pool = nil
diff --git a/go/worker/sandboxset/tests/sandboxset_integration_test.go b/go/worker/sandboxset/tests/sandboxset_integration_test.go
index 3fabd335d..1c81d4f31 100644
--- a/go/worker/sandboxset/tests/sandboxset_integration_test.go
+++ b/go/worker/sandboxset/tests/sandboxset_integration_test.go
@@ -84,8 +84,10 @@ func TestIntegration_GetCreatesRealContainer(t *testing.T) {
 	t.Logf("created real container: ID=%s", sb.ID())
 	t.Logf("debug: %s", sb.DebugString())
 
-	ref.Broken = true
-	_ = ref.Put()
+	// Caller owns lifecycle: destroy the sandbox, mark the ref dead, release.
+	ref.Sandbox().Destroy("test cleanup")
+	ref.MarkDead()
+	ref.Put()
 }
 
 func TestIntegration_PutPausesAndReuses(t *testing.T) {
@@ -99,9 +101,7 @@ func TestIntegration_PutPausesAndReuses(t *testing.T) {
 	id1 := ref1.Sandbox().ID()
 	t.Logf("first sandbox: ID=%s", id1)
 
-	if err := ref1.Put(); err != nil {
-		t.Fatalf("Put: %v", err)
-	}
+	ref1.Put()
 
 	// Get again — should reuse the same container (unpaused from paused state)
 	ref2, err := set.GetOrCreateUnpaused()
@@ -115,11 +115,12 @@ func TestIntegration_PutPausesAndReuses(t *testing.T) {
 		t.Fatalf("expected reuse (same ID %s), got new container %s", id1, id2)
 	}
 
-	ref2.Broken = true
-	_ = ref2.Put()
+	ref2.Sandbox().Destroy("test cleanup")
+	ref2.MarkDead()
+	ref2.Put()
 }
 
-func TestIntegration_DestroyKillsReal(t *testing.T) {
+func TestIntegration_MarkDeadGetsNew(t *testing.T) {
 	set := newDockerSet(t)
 
 	ref1, err := set.GetOrCreateUnpaused()
@@ -129,12 +130,12 @@ func TestIntegration_DestroyKillsReal(t *testing.T) {
 	id1 := ref1.Sandbox().ID()
 	t.Logf("first sandbox: ID=%s", id1)
 
-	ref1.Broken = true
-	if err := ref1.Put(); err != nil {
-		t.Fatalf("Put (broken): %v", err)
-	}
+	// Caller-owned destroy + MarkDead + Put releases the slot without a sandbox.
+	ref1.Sandbox().Destroy("test: simulate handler failure")
+	ref1.MarkDead()
+	ref1.Put()
 
-	// Get again — must be a different container since we destroyed the first
+	// Get again — must be a different container since the slot is empty
 	ref2, err := set.GetOrCreateUnpaused()
 	if err != nil {
 		t.Fatalf("second GetOrCreateUnpaused: %v", err)
@@ -143,11 +144,12 @@ func TestIntegration_DestroyKillsReal(t *testing.T) {
 	t.Logf("second sandbox: ID=%s", id2)
 
 	if id2 == id1 {
-		t.Fatal("expected new container after Destroy, got same ID")
+		t.Fatal("expected new container after MarkDead, got same ID")
 	}
 
-	ref2.Broken = true
-	_ = ref2.Put()
+	ref2.Sandbox().Destroy("test cleanup")
+	ref2.MarkDead()
+	ref2.Put()
 }
 
 func TestIntegration_CloseDestroysAll(t *testing.T) {
@@ -165,22 +167,20 @@ func TestIntegration_CloseDestroysAll(t *testing.T) {
 	}
 
 	// Put one back to idle so Close covers the idle path.
-	if err := refs[2].Put(); err != nil {
-		t.Fatalf("Put: %v", err)
-	}
+	refs[2].Put()
 
-	// Close destroys idle sandboxes; the two in-use ones are left to be
-	// destroyed by put()'s closed-branch when their holders release them.
+	// Close clears idle slots; in-use refs are left to their holders.
+	// The set does not destroy any sandbox — caller owns lifecycle.
 	if err := set.Close(); err != nil {
 		t.Fatalf("Close: %v", err)
 	}
 
-	// Releasing in-use refs after Close should destroy their sandboxes via
-	// put()'s closed branch and return an error indicating the set is closed.
+	// Caller destroys the live sandboxes and releases the refs.
+	// Put after Close routes through put()'s closed branch (void; clears the slot).
 	for i := 0; i < 2; i++ {
-		if err := refs[i].Put(); err == nil {
-			t.Fatalf("expected error from Put[%d] after Close, got nil", i)
-		}
+		refs[i].Sandbox().Destroy("test cleanup after close")
+		refs[i].MarkDead()
+		refs[i].Put()
 	}
 
 	// Verify set is closed — further Gets should fail
diff --git a/go/worker/sandboxset/tests/sandboxset_test.go b/go/worker/sandboxset/tests/sandboxset_test.go
index 30db1a43d..1a25f01cf 100644
--- a/go/worker/sandboxset/tests/sandboxset_test.go
+++ b/go/worker/sandboxset/tests/sandboxset_test.go
@@ -54,9 +54,7 @@ func TestLifecycle_GetPutReuse(t *testing.T) {
 	}
 	id := ref1.Sandbox().ID()
 
-	if err := ref1.Put(); err != nil {
-		t.Fatalf("Put: %v", err)
-	}
+	ref1.Put()
 
 	ref2, err := set.GetOrCreateUnpaused()
 	if err != nil {
@@ -65,12 +63,13 @@ func TestLifecycle_GetPutReuse(t *testing.T) {
 	if ref2.Sandbox().ID() != id {
 		t.Fatalf("expected reuse (ID %s), got new (ID %s)", id, ref2.Sandbox().ID())
 	}
-	_ = ref2.Put()
+	ref2.Put()
 }
 
-// TestDestroy_NilRefReused verifies that after Destroy the ref stays in the pool
-// as a nil ref, and the next GetOrCreateUnpaused reuses it (pool size stays 1).
-func TestDestroy_NilRefReused(t *testing.T) {
+// TestMarkDead_NewSandboxOnNextGet verifies that after MarkDead+Put the slot
+// is empty, and the next GetOrCreateUnpaused creates a fresh sandbox in it
+// (pool size stays 1, total created sandboxes grows to 2).
+func TestMarkDead_NewSandboxOnNextGet(t *testing.T) {
 	set, pool := newTestSet(t)
 	defer set.Close()
 
@@ -80,22 +79,20 @@ func TestDestroy_NilRefReused(t *testing.T) {
 	}
 	id1 := ref1.Sandbox().ID()
 
-	ref1.Broken = true
-	if err := ref1.Put(); err != nil {
-		t.Fatalf("Put (broken): %v", err)
-	}
+	ref1.MarkDead()
+	ref1.Put()
 
 	ref2, err := set.GetOrCreateUnpaused()
 	if err != nil {
 		t.Fatalf("second GetOrCreateUnpaused: %v", err)
 	}
 	if ref2.Sandbox().ID() == id1 {
-		t.Fatal("expected a new sandbox after Destroy, got the same ID")
+		t.Fatal("expected a new sandbox after MarkDead, got the same ID")
 	}
 	if n := len(pool.CreatedSandboxes()); n != 2 {
 		t.Fatalf("expected 2 created sandboxes total, got %d", n)
 	}
-	_ = ref2.Put()
+	ref2.Put()
 }
 
 // TestGet_AfterClose verifies that GetOrCreateUnpaused returns an error after Close.
@@ -110,3 +107,43 @@ func TestGet_AfterClose(t *testing.T) {
 		t.Fatal("expected error after Close")
 	}
 }
+
+// TestPut_Twice_Panics verifies the double-Put guard: the second Put on a
+// ref that's already been returned must panic rather than silently corrupt.
+func TestPut_Twice_Panics(t *testing.T) {
+	set, _ := newTestSet(t)
+	defer set.Close()
+
+	ref, err := set.GetOrCreateUnpaused()
+	if err != nil {
+		t.Fatalf("GetOrCreateUnpaused: %v", err)
+	}
+	ref.Put()
+
+	defer func() {
+		if recover() == nil {
+			t.Fatal("expected panic on double Put")
+		}
+	}()
+	ref.Put()
+}
+
+// TestMarkDead_AfterPut_Panics verifies MarkDead is rejected on a ref that
+// is no longer held.
+func TestMarkDead_AfterPut_Panics(t *testing.T) {
+	set, _ := newTestSet(t)
+	defer set.Close()
+
+	ref, err := set.GetOrCreateUnpaused()
+	if err != nil {
+		t.Fatalf("GetOrCreateUnpaused: %v", err)
+	}
+	ref.Put()
+
+	defer func() {
+		if recover() == nil {
+			t.Fatal("expected panic on MarkDead after Put")
+		}
+	}()
+	ref.MarkDead()
+}

From dd93d9b3594ac33870c278407a0e10b599d6cf8b Mon Sep 17 00:00:00 2001
From: Ami Buch <amiabuch@gmail.com>
Date: Thu, 23 Apr 2026 23:26:33 -0500
Subject: [PATCH 51/55] doc: added reasoning for design/eror handling
 inconsistencies

---
 go/worker/sandboxset/sandboxset.go | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/go/worker/sandboxset/sandboxset.go b/go/worker/sandboxset/sandboxset.go
index a0f01ebad..94c8c2297 100644
--- a/go/worker/sandboxset/sandboxset.go
+++ b/go/worker/sandboxset/sandboxset.go
@@ -1,6 +1,7 @@
 package sandboxset
 
 import (
+	"errors"
 	"fmt"
 	"log/slog"
 	"sync"
@@ -8,17 +9,23 @@ import (
 	"github.com/open-lambda/open-lambda/go/worker/sandbox"
 )
 
+// ErrClosed is returned by operations on a closed SandboxSet. Match with errors.Is.
+var ErrClosed = errors.New("sandboxset: closed")
+
 // SandboxRef is a handle returned by GetOrCreateUnpaused. While inUse is true
 // the holder owns sb; otherwise sb and inUse are protected by set.mu.
 // Callers signal a dead sandbox by calling MarkDead before Put. The set never
 // destroys sandboxes — lifecycle is owned upstream.
 // A ref must not be shared across goroutines; one goroutine holds it at a time.
+// Guards that lock s.mu to panic on !inUse use the lock only for the inUse
+// check, not to protect sb — sb access is governed by the single-owner rule.
 type SandboxRef struct {
 	set   *sandboxSetImpl
 	sb    sandbox.Sandbox
 	inUse bool
 }
 
+// Sandbox returns the underlying sandbox. No inUse guard: hot path; misuse is caught by Put/MarkDead guards instead.
 func (r *SandboxRef) Sandbox() sandbox.Sandbox { return r.sb }
 
 func (r *SandboxRef) MarkDead() {
@@ -67,7 +74,7 @@ func (s *sandboxSetImpl) claimIdle() (*SandboxRef, error) {
 	defer s.mu.Unlock()
 
 	if s.closed {
-		return nil, fmt.Errorf("sandboxset: closed")
+		return nil, fmt.Errorf("claimIdle: %w", ErrClosed)
 	}
 
 	var empty *SandboxRef
@@ -167,7 +174,7 @@ func (s *sandboxSetImpl) put(ref *SandboxRef) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	if s.closed {
-		// rare: closed raced in during Pause; sandbox is paused, caller owns lifecycle
+		// rare Close-during-Pause race: sandbox is paused, unreachable, leaked until pool.Cleanup at process exit
 		s.releaseSlotLocked(ref)
 		return
 	}
@@ -197,7 +204,7 @@ func (s *sandboxSetImpl) Close() error {
 	defer s.mu.Unlock()
 
 	if s.closed {
-		return fmt.Errorf("sandboxset: already closed")
+		return fmt.Errorf("Close: already %w", ErrClosed)
 	}
 	s.closed = true
 

From f578e42ffa683f83f77084aa2e74b82907ad06fa Mon Sep 17 00:00:00 2001
From: Ami Buch <amiabuch@gmail.com>
Date: Sun, 26 Apr 2026 20:01:21 -0500
Subject: [PATCH 52/55] fix: more streamlined

---
 go/worker/sandboxset/api.go                   |  3 +-
 go/worker/sandboxset/sandboxset.go            | 79 ++++++-------------
 .../tests/sandboxset_integration_test.go      | 10 +--
 go/worker/sandboxset/tests/sandboxset_test.go | 62 +++++++++++++++
 4 files changed, 90 insertions(+), 64 deletions(-)

diff --git a/go/worker/sandboxset/api.go b/go/worker/sandboxset/api.go
index f02bee361..f78d22791 100644
--- a/go/worker/sandboxset/api.go
+++ b/go/worker/sandboxset/api.go
@@ -28,7 +28,8 @@ type SandboxSet interface {
 	// request, wrapped in a SandboxRef.
 	GetOrCreateUnpaused() (*SandboxRef, error)
 
-	// Close destroys all sandboxes in the pool and marks the set as closed.
+	// Close marks the set closed and destroys idle sandboxes. In-use refs
+	// are destroyed when their holder returns them via Put.
 	Close() error
 }
 
diff --git a/go/worker/sandboxset/sandboxset.go b/go/worker/sandboxset/sandboxset.go
index 94c8c2297..33bf01e6b 100644
--- a/go/worker/sandboxset/sandboxset.go
+++ b/go/worker/sandboxset/sandboxset.go
@@ -12,13 +12,6 @@ import (
 // ErrClosed is returned by operations on a closed SandboxSet. Match with errors.Is.
 var ErrClosed = errors.New("sandboxset: closed")
 
-// SandboxRef is a handle returned by GetOrCreateUnpaused. While inUse is true
-// the holder owns sb; otherwise sb and inUse are protected by set.mu.
-// Callers signal a dead sandbox by calling MarkDead before Put. The set never
-// destroys sandboxes — lifecycle is owned upstream.
-// A ref must not be shared across goroutines; one goroutine holds it at a time.
-// Guards that lock s.mu to panic on !inUse use the lock only for the inUse
-// check, not to protect sb — sb access is governed by the single-owner rule.
 type SandboxRef struct {
 	set   *sandboxSetImpl
 	sb    sandbox.Sandbox
@@ -29,21 +22,13 @@ type SandboxRef struct {
 func (r *SandboxRef) Sandbox() sandbox.Sandbox { return r.sb }
 
 func (r *SandboxRef) MarkDead() {
-	r.set.mu.Lock()
-	defer r.set.mu.Unlock()
 	if !r.inUse {
 		panic(fmt.Sprintf("sandboxset: MarkDead on ref %p not currently held (inUse=%v)", r, r.inUse))
 	}
 	r.sb = nil
 }
 
-func (r *SandboxRef) Put() {
-	if r.sb == nil {
-		r.set.releaseSlot(r)
-	} else {
-		r.set.put(r)
-	}
-}
+func (r *SandboxRef) Put() { r.set.put(r) }
 
 type sandboxSetImpl struct {
 	cfg *Config
@@ -141,7 +126,9 @@ func (s *sandboxSetImpl) GetOrCreateUnpaused() (*SandboxRef, error) {
 	if ref.sb == nil {
 		newSb, err := s.createSandbox()
 		if err != nil {
-			s.releaseSlot(ref)
+			s.mu.Lock()
+			ref.inUse = false
+			s.mu.Unlock()
 			return nil, err
 		}
 		ref.sb = newSb
@@ -150,55 +137,33 @@ func (s *sandboxSetImpl) GetOrCreateUnpaused() (*SandboxRef, error) {
 	return ref, nil
 }
 
-// put relies on Sandbox.Pause being no-op-safe after external death (see sandbox/api.go).
 func (s *sandboxSetImpl) put(ref *SandboxRef) {
-	s.mu.Lock()
-	if !ref.inUse {
-		s.mu.Unlock()
-		panic(fmt.Sprintf("sandboxset: put on ref %p not currently held (inUse=%v)", ref, ref.inUse))
-	}
-	closed := s.closed
-	if closed {
-		s.releaseSlotLocked(ref)
-	}
-	s.mu.Unlock()
-	if closed {
-		return
-	}
-
-	if err := ref.sb.Pause(); err != nil {
-		s.releaseSlot(ref)
-		return
+	if ref.sb != nil {
+		if err := ref.sb.Pause(); err != nil {
+			ref.sb.Destroy("sandboxset: pause failed")
+			ref.sb = nil
+		}
 	}
 
 	s.mu.Lock()
 	defer s.mu.Unlock()
+
+	if !ref.inUse {
+		panic(fmt.Sprintf("sandboxset: put on ref %p not currently held (inUse=%v)", ref, ref.inUse))
+	}
 	if s.closed {
-		// rare Close-during-Pause race: sandbox is paused, unreachable, leaked until pool.Cleanup at process exit
-		s.releaseSlotLocked(ref)
+		sb := ref.sb
+		ref.sb = nil
+		ref.inUse = false
+		if sb != nil {
+			sb.Destroy("sandboxset: closed during put")
+		}
 		return
 	}
 	ref.inUse = false
 }
 
-// releaseSlotLocked clears sb and inUse. Caller must hold s.mu.
-func (s *sandboxSetImpl) releaseSlotLocked(ref *SandboxRef) {
-	ref.sb = nil
-	ref.inUse = false
-}
-
-func (s *sandboxSetImpl) releaseSlot(ref *SandboxRef) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	if !ref.inUse {
-		panic(fmt.Sprintf("sandboxset: releaseSlot on ref %p not currently held (inUse=%v)", ref, ref.inUse))
-	}
-	s.releaseSlotLocked(ref)
-}
 
-// Close clears idle slots; in-use refs are left to their holders, whose put()
-// will see s.closed and release them. Never touches a held ref's sb.
-// Best-effort: if a holder never calls Put, the slot is not reclaimed.
 func (s *sandboxSetImpl) Close() error {
 	s.mu.Lock()
 	defer s.mu.Unlock()
@@ -209,10 +174,10 @@ func (s *sandboxSetImpl) Close() error {
 	s.closed = true
 
 	for _, ref := range s.pool {
-		if !ref.inUse {
-			s.releaseSlotLocked(ref)
+		if !ref.inUse && ref.sb != nil {
+			ref.sb.Destroy("sandboxset: closed")
+			ref.sb = nil
 		}
 	}
-	s.pool = nil
 	return nil
 }
diff --git a/go/worker/sandboxset/tests/sandboxset_integration_test.go b/go/worker/sandboxset/tests/sandboxset_integration_test.go
index 1c81d4f31..a699da323 100644
--- a/go/worker/sandboxset/tests/sandboxset_integration_test.go
+++ b/go/worker/sandboxset/tests/sandboxset_integration_test.go
@@ -169,17 +169,15 @@ func TestIntegration_CloseDestroysAll(t *testing.T) {
 	// Put one back to idle so Close covers the idle path.
 	refs[2].Put()
 
-	// Close clears idle slots; in-use refs are left to their holders.
-	// The set does not destroy any sandbox — caller owns lifecycle.
+	// Close destroys the idle sandbox; in-use refs are destroyed by put()
+	// when their holders return them below.
 	if err := set.Close(); err != nil {
 		t.Fatalf("Close: %v", err)
 	}
 
-	// Caller destroys the live sandboxes and releases the refs.
-	// Put after Close routes through put()'s closed branch (void; clears the slot).
+	// Put after Close routes through put()'s closed branch, which destroys
+	// the sandbox. The caller does not Destroy here.
 	for i := 0; i < 2; i++ {
-		refs[i].Sandbox().Destroy("test cleanup after close")
-		refs[i].MarkDead()
 		refs[i].Put()
 	}
 
diff --git a/go/worker/sandboxset/tests/sandboxset_test.go b/go/worker/sandboxset/tests/sandboxset_test.go
index 1a25f01cf..e9442b409 100644
--- a/go/worker/sandboxset/tests/sandboxset_test.go
+++ b/go/worker/sandboxset/tests/sandboxset_test.go
@@ -1,6 +1,7 @@
 package tests
 
 import (
+	"errors"
 	"testing"
 
 	"github.com/open-lambda/open-lambda/go/common"
@@ -128,6 +129,67 @@ func TestPut_Twice_Panics(t *testing.T) {
 	ref.Put()
 }
 
+// TestPut_PauseFailure_DestroysSandbox verifies that when Pause fails inside
+// put(), the orphaned sandbox is destroyed by the set rather than leaked.
+func TestPut_PauseFailure_DestroysSandbox(t *testing.T) {
+	set, pool := newTestSet(t)
+	defer set.Close()
+
+	ref, err := set.GetOrCreateUnpaused()
+	if err != nil {
+		t.Fatalf("GetOrCreateUnpaused: %v", err)
+	}
+	sb := pool.CreatedSandboxes()[0]
+	sb.PauseErr = errors.New("simulated pause failure")
+
+	ref.Put()
+
+	if !sb.IsDestroyed() {
+		t.Fatal("expected sandbox to be destroyed after Pause failure in Put")
+	}
+}
+
+// TestClose_DestroysIdleSandbox verifies Close destroys idle sandboxes
+// (which the set is the only holder of).
+func TestClose_DestroysIdleSandbox(t *testing.T) {
+	set, pool := newTestSet(t)
+
+	ref, err := set.GetOrCreateUnpaused()
+	if err != nil {
+		t.Fatalf("GetOrCreateUnpaused: %v", err)
+	}
+	sb := pool.CreatedSandboxes()[0]
+	ref.Put() // ref now idle in the pool
+
+	if err := set.Close(); err != nil {
+		t.Fatalf("Close: %v", err)
+	}
+	if !sb.IsDestroyed() {
+		t.Fatal("expected idle sandbox to be destroyed by Close")
+	}
+}
+
+// TestPut_AfterClose_DestroysSandbox verifies that a Put arriving after Close
+// destroys the returned sandbox (the set is the only remaining holder).
+func TestPut_AfterClose_DestroysSandbox(t *testing.T) {
+	set, pool := newTestSet(t)
+
+	ref, err := set.GetOrCreateUnpaused()
+	if err != nil {
+		t.Fatalf("GetOrCreateUnpaused: %v", err)
+	}
+	sb := pool.CreatedSandboxes()[0]
+
+	if err := set.Close(); err != nil {
+		t.Fatalf("Close: %v", err)
+	}
+	ref.Put()
+
+	if !sb.IsDestroyed() {
+		t.Fatal("expected sandbox to be destroyed by Put after Close")
+	}
+}
+
 // TestMarkDead_AfterPut_Panics verifies MarkDead is rejected on a ref that
 // is no longer held.
 func TestMarkDead_AfterPut_Panics(t *testing.T) {

From e12a6c2eada6e07c2c7f38f242097f1e507838bf Mon Sep 17 00:00:00 2001
From: Ami Buch <amiabuch@gmail.com>
Date: Thu, 7 May 2026 10:12:04 -0500
Subject: [PATCH 53/55] fix: redundant destroy on unpause error

---
 go/worker/sandboxset/sandboxset.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/go/worker/sandboxset/sandboxset.go b/go/worker/sandboxset/sandboxset.go
index 33bf01e6b..f4fd89abe 100644
--- a/go/worker/sandboxset/sandboxset.go
+++ b/go/worker/sandboxset/sandboxset.go
@@ -140,7 +140,6 @@ func (s *sandboxSetImpl) GetOrCreateUnpaused() (*SandboxRef, error) {
 func (s *sandboxSetImpl) put(ref *SandboxRef) {
 	if ref.sb != nil {
 		if err := ref.sb.Pause(); err != nil {
-			ref.sb.Destroy("sandboxset: pause failed")
 			ref.sb = nil
 		}
 	}

From d6777162a2b264327a84565b076876fbc1186123 Mon Sep 17 00:00:00 2001
From: Tyler Caraza-Harter <tylerharter@gmail.com>
Date: Mon, 11 May 2026 10:40:27 -0500
Subject: [PATCH 54/55] bump up versions to fix js 20 => 24

---
 .github/workflows/ci.yml  | 6 +++---
 .github/workflows/pkg.yml | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 9e958cbef..97a2c8d75 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -19,7 +19,7 @@ jobs:
         shell: bash
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v5
     - name: Install Docker and Prereqs
       # This uses a workaround for a known issue with docker. See here: https://github.com/actions/virtual-environments/issues/5490#issuecomment-1118328567
       run: |
@@ -40,11 +40,11 @@ jobs:
       with:
          workspaces: "wasm-worker-> target"
     - name: Install golang
-      uses: actions/setup-go@v5
+      uses: actions/setup-go@v6
       with:
          go-version: '>=1.21'
     - name: Set up Python 3
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
          python-version: '3.12'
     - name: Install pylint
diff --git a/.github/workflows/pkg.yml b/.github/workflows/pkg.yml
index e219a2af8..7d0347dd2 100644
--- a/.github/workflows/pkg.yml
+++ b/.github/workflows/pkg.yml
@@ -15,7 +15,7 @@ jobs:
         shell: bash
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v5
     - name: Install Docker and Prereqs
       # This uses a workaround for a known issue with docker. See here: https://github.com/actions/virtual-environments/issues/5490#issuecomment-1118328567
       run: |
@@ -40,7 +40,7 @@ jobs:
       with:
          working-directory: wasm-worker
     - name: Install golang
-      uses: actions/setup-go@v3
+      uses: actions/setup-go@v6
       with:
          go-version: '>=1.21.0'
     - name: Install cross

From 5a1a912f0791a04394f4d6e565e6e4673e273d90 Mon Sep 17 00:00:00 2001
From: Tyler Caraza-Harter <tylerharter@gmail.com>
Date: Mon, 11 May 2026 11:00:56 -0500
Subject: [PATCH 55/55] drop bad test

---
 go/worker/sandboxset/tests/sandboxset_test.go | 21 -------------------
 1 file changed, 21 deletions(-)

diff --git a/go/worker/sandboxset/tests/sandboxset_test.go b/go/worker/sandboxset/tests/sandboxset_test.go
index e9442b409..647adf8f2 100644
--- a/go/worker/sandboxset/tests/sandboxset_test.go
+++ b/go/worker/sandboxset/tests/sandboxset_test.go
@@ -1,7 +1,6 @@
 package tests
 
 import (
-	"errors"
 	"testing"
 
 	"github.com/open-lambda/open-lambda/go/common"
@@ -129,26 +128,6 @@ func TestPut_Twice_Panics(t *testing.T) {
 	ref.Put()
 }
 
-// TestPut_PauseFailure_DestroysSandbox verifies that when Pause fails inside
-// put(), the orphaned sandbox is destroyed by the set rather than leaked.
-func TestPut_PauseFailure_DestroysSandbox(t *testing.T) {
-	set, pool := newTestSet(t)
-	defer set.Close()
-
-	ref, err := set.GetOrCreateUnpaused()
-	if err != nil {
-		t.Fatalf("GetOrCreateUnpaused: %v", err)
-	}
-	sb := pool.CreatedSandboxes()[0]
-	sb.PauseErr = errors.New("simulated pause failure")
-
-	ref.Put()
-
-	if !sb.IsDestroyed() {
-		t.Fatal("expected sandbox to be destroyed after Pause failure in Put")
-	}
-}
-
 // TestClose_DestroysIdleSandbox verifies Close destroys idle sandboxes
 // (which the set is the only holder of).
 func TestClose_DestroysIdleSandbox(t *testing.T) {