Skip to content

Commit d6f81c8

Browse files
thomastaylor312thomastaylor312
committed
feat(tests): Adds Windows e2e tests
These tests require a bit more setup and hand holding, but they should work. It still isn't super nice to do locally (as you have to modify your kube config each time you spin up a new kind cluster), but now we will at least be exercising functionality in Windows as well
1 parent 8f7d24a commit d6f81c8

9 files changed

Lines changed: 167 additions & 37 deletions

File tree

‎.github/workflows/build.yml‎

Lines changed: 51 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ jobs:
5353
}
5454
steps:
5555
- uses: actions/checkout@v2
56-
- uses: engineerd/configurator@v0.0.5
56+
- uses: engineerd/configurator@v0.0.7
5757
with:
5858
name: ${{ matrix.config.name }}
5959
url: ${{ matrix.config.url }}
@@ -100,18 +100,63 @@ jobs:
100100
run: |
101101
just --justfile justfile-windows build
102102
just --justfile justfile-windows test
103-
# TODO: Figure out how to get kind or minikube running on a windows test host and see how we can
104-
# get things working with rustls
105-
# windows-e2e:
106-
# runs-on: windows-latest
103+
windows-e2e:
104+
env:
105+
# Because we are on a shared build machine, we need to use a different directory than the default homedir
106+
KRUSTLET_DATA_DIR: ".krustlet"
107+
CONFIG_DIR: '.krustlet\config'
108+
runs-on: [self-hosted, windows, x64]
109+
steps:
110+
- uses: actions/checkout@v2
111+
- uses: engineerd/configurator@v0.0.7
112+
with:
113+
name: just.exe
114+
url: "https://github.com/casey/just/releases/download/v0.9.4/just-v0.9.4-x86_64-pc-windows-msvc.zip"
115+
pathInArchive: just.exe
116+
- uses: engineerd/configurator@v0.0.7
117+
with:
118+
name: kind.exe
119+
url: "https://kind.sigs.k8s.io/dl/v0.11.1/kind-windows-amd64"
120+
- name: Ensure Docker is running
121+
run: .\tests\windows\ensure-docker.ps1
122+
- name: Setup kind cluster
123+
# The image flag is a workaround to force to 1.20 until we add support for projected volumes
124+
run: kind create cluster --config .\tests\windows\kind-config.yaml --name kind-${{ github.run_id }} --image kindest/node:v1.20.7@sha256:cbeaf907fc78ac97ce7b625e4bf0de16e3ea725daf6b04f930bd14c67c671ff9
125+
# Because Windows uses rustls, it can't use a bare IP address. This
126+
# switches the kubeconfig file to use localhost instead
127+
- name: Modify kubeconfig
128+
run: |
129+
kubectl config view -o jsonpath='{.clusters[?(@.name == \"kind-kind-${{ github.run_id }}\")].cluster.server}' | % {$_.replace("127.0.0.1", "localhost")} | % {kubectl config set clusters.kind-kind-${{ github.run_id }}.server $_}
130+
- name: Get NODE_IP
131+
run: |
132+
$addr = (Get-NetIPAddress -AddressFamily IPV4 -InterfaceAlias 'vEthernet (WSL)').IPAddress
133+
echo "KRUSTLET_NODE_IP=$addr" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
134+
- name: Run e2e tests (full)
135+
if: ${{ github.event_name == 'push' }}
136+
env:
137+
KRUSTLET_TEST_ENV: "ci"
138+
KRUSTLET_E2E_IMAGE_PULL_SECRET: ${{ secrets.KRUSTLET_E2E_IMAGE_PULL_SECRET }}
139+
KRUSTLET_NODE_IP: ${{ env.KRUSTLET_NODE_IP }}
140+
run: just --justfile justfile-windows test-e2e-standalone
141+
- name: Run e2e tests (PR)
142+
if: ${{ github.event_name == 'pull_request' }}
143+
run: just --justfile justfile-windows test-e2e-standalone
144+
- uses: actions/upload-artifact@v2
145+
if: ${{ always() }}
146+
with:
147+
name: e2e-logs-windows
148+
path: oneclick-logs/
149+
- name: Cleanup kind cluster
150+
if: ${{ always() }}
151+
run: kind delete cluster --name kind-${{ github.run_id }}
107152
e2e:
108153
runs-on: ubuntu-latest
109154
steps:
110155
- uses: actions/checkout@v2
111156
- uses: engineerd/setup-kind@v0.5.0
112157
with:
113158
version: "v0.10.0"
114-
- uses: engineerd/configurator@v0.0.5
159+
- uses: engineerd/configurator@v0.0.7
115160
with:
116161
name: just
117162
url: https://github.com/casey/just/releases/download/v0.5.11/just-v0.5.11-x86_64-unknown-linux-musl.tar.gz

‎docs/howto/assets/bootstrap.ps1‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,10 @@ stringData:
1919
usage-bootstrap-signing: "true"
2020
"@ | kubectl.exe apply -f -
2121

22-
if (!$env:CONFIG_DIR -or -not (Test-Path $env:CONFIG_DIR)) {
23-
$config_dir = "$HOME\.krustlet\config"
24-
}
25-
else {
26-
$config_dir = $env:CONFIG_DIR
22+
if (-not (Test-Path env:CONFIG_DIR)) {
23+
$env:CONFIG_DIR = '$HOME\.krustlet\config'
2724
}
25+
$config_dir = $env:CONFIG_DIR
2826

2927
mkdir $config_dir -ErrorAction SilentlyContinue > $null
3028

‎justfile-windows‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
set shell := ["powershell.exe", "-c"]
22

3-
export RUST_LOG := "wasi_provider=debug,main=debug"
3+
export RUST_LOG := "wasi_provider=debug,main=debug,kubelet=debug"
44
export PFX_PASSWORD := "testing"
55
export CONFIG_DIR := env_var_or_default('CONFIG_DIR', '$HOME\.krustlet\config')
66

@@ -25,6 +25,15 @@ test:
2525
test-e2e +FLAGS='--no-default-features --features rustls-tls,kubelet/derive':
2626
cargo test --test integration_tests {{FLAGS}}
2727

28+
test-e2e-standalone +FLAGS='--no-default-features --features rustls-tls,kubelet/derive':
29+
cargo run --bin oneclick {{FLAGS}}
30+
31+
test-e2e-ci +FLAGS='--no-default-features --features rustls-tls,kubelet/derive':
32+
KRUSTLET_TEST_ENV=ci cargo test --test integration_tests {{FLAGS}}
33+
34+
test-e2e-standalone-ci +FLAGS='--no-default-features --features rustls-tls,kubelet/derive':
35+
KRUSTLET_TEST_ENV=ci cargo run --bin oneclick {{FLAGS}}
36+
2837
run +FLAGS='--no-default-features --features rustls-tls,kubelet/derive': bootstrap
2938
$env:KUBECONFIG = "$(Invoke-Expression "echo $env:CONFIG_DIR")\kubeconfig-wasi"; cargo run --bin krustlet-wasi {{FLAGS}} -- --node-name krustlet-wasi --port 3001 --bootstrap-file "$(Invoke-Expression "echo $env:CONFIG_DIR")\bootstrap.conf" --cert-file "$(Invoke-Expression "echo $env:CONFIG_DIR")\krustlet-wasi.crt" --private-key-file "$(Invoke-Expression "echo $env:CONFIG_DIR")\krustlet-wasi.key"
3039

‎tests/assert.rs‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -124,10 +124,7 @@ pub async fn container_file_contains(
124124
file_error: &str,
125125
) -> anyhow::Result<()> {
126126
let pod_dir_name = format!("{}-{}", pod_name, pod_namespace);
127-
let file_path_base = dirs::home_dir()
128-
.expect("home dir does not exist")
129-
.join(".krustlet/volumes")
130-
.join(pod_dir_name);
127+
let file_path_base = data_dir().join("volumes").join(pod_dir_name);
131128
let container_file_bytes = tokio::fs::read(file_path_base.join(container_file_path))
132129
.await
133130
.expect(file_error);
@@ -138,6 +135,16 @@ pub async fn container_file_contains(
138135
Ok(())
139136
}
140137

138+
fn data_dir() -> std::path::PathBuf {
139+
match std::env::var("KRUSTLET_DATA_DIR") {
140+
Ok(data_dir) => std::path::PathBuf::from(data_dir),
141+
_ => {
142+
let home_dir = dirs::home_dir().expect("Can't get home dir");
143+
home_dir.join(".krustlet")
144+
}
145+
}
146+
}
147+
141148
pub async fn try_dump_pod_logs(pods: &Api<Pod>, pod_name: &str) {
142149
let logs = pods.logs(pod_name, &LogParams::default()).await;
143150

‎tests/csi/README.md‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,9 @@ in the future
1111

1212
## Why not use the Docker images for these?
1313

14-
Those would work here for our e2e tests since we are running a KinD node. However, we want these
15-
tests to reflect more real world usage and there is no guarantee that a Krustlet node will have a
16-
container runtime (nor do we want to require one). There is also the additional benefit of having
17-
this be some simple instructions of how you can build some of the key components to get CSI running
14+
Those would work here for our e2e tests since we are running a KinD node.
15+
However, we want these tests to reflect more real world usage and there is no
16+
guarantee that a Krustlet node will have a container runtime (nor do we want to
17+
require one). There is also the additional benefit of having this be some simple
18+
instructions of how you can build some of the key components to get CSI running
1819
for a Real World™ cluster.

‎tests/integration_tests.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
use k8s_openapi::api::core::v1::{Node, Pod, Taint};
2-
use kube::api::{Api, DeleteParams, PostParams};
2+
#[cfg(target_os = "linux")]
3+
use kube::api::DeleteParams;
4+
use kube::api::{Api, PostParams};
35
use serde_json::json;
46

57
mod assert;

‎tests/oneclick/src/main.rs‎

Lines changed: 43 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
1+
use std::env;
12
use std::io::BufRead;
2-
use std::path::Path;
3+
use std::path::{Path, PathBuf};
34

45
enum BootstrapReadiness {
56
AlreadyBootstrapped,
@@ -74,18 +75,31 @@ fn main() {
7475
}
7576

7677
fn config_dir() -> std::path::PathBuf {
77-
let home_dir = dirs::home_dir().expect("Can't get home dir"); // TODO: allow override of config dir
78-
home_dir.join(".krustlet/config")
78+
match env::var("KRUSTLET_DATA_DIR") {
79+
Ok(config_dir) => PathBuf::from(config_dir),
80+
_ => {
81+
let home_dir = dirs::home_dir().expect("Can't get home dir");
82+
home_dir.join(".krustlet/config")
83+
}
84+
}
7985
}
8086

8187
fn config_file_path_str(file_name: impl AsRef<std::path::Path>) -> String {
8288
config_dir().join(file_name).to_str().unwrap().to_owned()
8389
}
8490

8591
fn build_workspace() -> anyhow::Result<()> {
86-
let build_result = std::process::Command::new("cargo")
87-
.args(&["build"])
88-
.output()?;
92+
let mut cmd = std::process::Command::new("cargo");
93+
#[cfg(target_family = "unix")]
94+
cmd.args(&["build"]);
95+
#[cfg(target_family = "windows")]
96+
cmd.args(&[
97+
"build",
98+
"--no-default-features",
99+
"--features",
100+
"rustls-tls,kubelet/derive",
101+
]);
102+
let build_result = cmd.output()?;
89103

90104
if build_result.status.success() {
91105
Ok(())
@@ -220,11 +234,10 @@ fn is_resource_gone(kubectl_output: &std::process::Output) -> bool {
220234
}
221235

222236
fn run_bootstrap() -> anyhow::Result<()> {
223-
let (shell, ext) = match std::env::consts::OS {
224-
"windows" => Ok(("powershell.exe", "ps1")),
225-
"linux" | "macos" => Ok(("bash", "sh")),
226-
os => Err(anyhow::anyhow!("Unsupported OS {}", os)),
227-
}?;
237+
#[cfg(target_family = "unix")]
238+
let (shell, ext) = ("bash", "sh");
239+
#[cfg(target_family = "windows")]
240+
let (shell, ext) = ("powershell.exe", "ps1");
228241

229242
let repo_root = std::env!("CARGO_MANIFEST_DIR");
230243

@@ -266,7 +279,12 @@ fn launch_kubelet(
266279
let port_arg = format!("{}", kubelet_port);
267280

268281
let repo_root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
282+
#[cfg(target_family = "unix")]
269283
let bin_path = repo_root.join("target/debug").join(name);
284+
#[cfg(target_family = "windows")]
285+
let bin_path = repo_root
286+
.join("target/debug")
287+
.join(name.to_owned() + ".exe");
270288

271289
let stderr = std::fs::File::create(Path::new(LOG_DIR).join(format!("{}.stderr", name)))?;
272290

@@ -483,11 +501,20 @@ fn run_test_suite(krustlet_process: &mut OwnedChildProcess) -> anyhow::Result<()
483501
let stdout = std::fs::File::create(Path::new(LOG_DIR).join("integration_tests.stdout"))?;
484502
let stderr = std::fs::File::create(Path::new(LOG_DIR).join("integration_tests.stderr"))?;
485503

486-
let mut test_process = std::process::Command::new("cargo")
487-
.args(&["test", "--test", "integration_tests"])
488-
.stderr(stdout)
489-
.stdout(stderr)
490-
.spawn()?;
504+
let mut cmd = std::process::Command::new("cargo");
505+
#[cfg(target_family = "unix")]
506+
cmd.args(&["test", "--test", "integration_tests"]);
507+
#[cfg(target_family = "windows")]
508+
cmd.args(&[
509+
"test",
510+
"--test",
511+
"integration_tests",
512+
"--no-default-features",
513+
"--features",
514+
"rustls-tls,kubelet/derive",
515+
]);
516+
517+
let mut test_process = cmd.stderr(stdout).stdout(stderr).spawn()?;
491518
println!("Integration tests running");
492519
let start = std::time::Instant::now();
493520
loop {

‎tests/windows/ensure-docker.ps1‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# If Docker isn't started, start it
2+
Get-Process 'Docker Desktop' -ErrorVariable NotRunning -ErrorAction SilentlyContinue 2>$null 1>$null
3+
4+
if ($NotRunning) {
5+
Write-Host "Docker process is not running, starting process"
6+
Start-Process "C:\Program Files\Docker\Docker\Docker Desktop.exe"
7+
}
8+
else {
9+
Write-Host "Docker process is running"
10+
}
11+
12+
# Even if it is running, make sure it is responding
13+
$startTime = Get-Date
14+
Write-Host "Ensuring that Docker is ready for commands"
15+
Do {
16+
$ErrorActionPreference = 'SilentlyContinue'
17+
docker ps 2>$null 1>$null
18+
$exitCode = $LASTEXITCODE
19+
Write-Host "Docker is not ready, will retry in 5s"
20+
Start-Sleep 5
21+
} Until ($exitCode -eq 0 -or (Get-Date) -gt $startTime.AddMinutes(3))
22+
23+
if ((Get-Date) -gt $startTime.AddMinutes(3)) {
24+
Write-Error "Docker never went ready" -ErrorAction Stop
25+
}
26+
27+
Write-Host "Docker is now ready"

‎tests/windows/kind-config.yaml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
kind: Cluster
2+
apiVersion: kind.x-k8s.io/v1alpha4
3+
networking:
4+
ipFamily: ipv6
5+
apiServerAddress: 127.0.0.1
6+
nodes:
7+
- role: control-plane
8+
kubeadmConfigPatches:
9+
- |
10+
kind: ClusterConfiguration
11+
apiServer:
12+
certSANs:
13+
- localhost
14+
- 127.0.0.1

0 commit comments

Comments
 (0)