feat: add scaletest Runner for dynamicparameters load gen #19890
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| package dynamicparameters | ||
|
|
||
| import "github.com/google/uuid" | ||
|
|
||
| type Config struct { | ||
| TemplateVersion uuid.UUID `json:"template_version"` | ||
| SessionToken string `json:"session_token"` | ||
| Metrics *Metrics `json:"-"` | ||
| MetricLabelValues []string `json:"metric_label_values"` | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| package dynamicparameters | ||
|
|
||
| import "github.com/prometheus/client_golang/prometheus" | ||
|
|
||
| type Metrics struct { | ||
| LatencyInitialResponseSeconds prometheus.HistogramVec | ||
| LatencyChangeResponseSeconds prometheus.HistogramVec | ||
| } | ||
|
|
||
| func NewMetrics(reg prometheus.Registerer, labelNames ...string) *Metrics { | ||
| m := &Metrics{ | ||
| LatencyInitialResponseSeconds: *prometheus.NewHistogramVec(prometheus.HistogramOpts{ | ||
| Namespace: "coderd", | ||
| Subsystem: "scaletest", | ||
| Name: "dynamic_parameters_latency_initial_response_seconds", | ||
| Help: "Time in seconds to get the initial dynamic parameters response from start of request.", | ||
| }, labelNames), | ||
| LatencyChangeResponseSeconds: *prometheus.NewHistogramVec(prometheus.HistogramOpts{ | ||
| Namespace: "coderd", | ||
| Subsystem: "scaletest", | ||
| Name: "dynamic_parameters_latency_change_response_seconds", | ||
| Help: "Time in seconds to between sending a dynamic parameters change request and receiving the response.", | ||
| }, labelNames), | ||
| } | ||
| reg.MustRegister(m.LatencyInitialResponseSeconds) | ||
| reg.MustRegister(m.LatencyChangeResponseSeconds) | ||
| return m | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,114 @@ | ||
| package dynamicparameters | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "io" | ||
| "slices" | ||
| "time" | ||
|
|
||
| "golang.org/x/xerrors" | ||
|
|
||
| "github.com/coder/coder/v2/codersdk" | ||
| "github.com/coder/coder/v2/scaletest/harness" | ||
| "github.com/coder/websocket" | ||
| ) | ||
|
|
||
| type Runner struct { | ||
| client *codersdk.Client | ||
| cfg Config | ||
| } | ||
|
|
||
| var _ harness.Runnable = &Runner{} | ||
|
|
||
| func NewRunner(client *codersdk.Client, cfg Config) *Runner { | ||
| clone := codersdk.New(client.URL) | ||
| clone.HTTPClient = client.HTTPClient | ||
| clone.SetLogger(client.Logger()) | ||
| clone.SetSessionToken(cfg.SessionToken) | ||
| return &Runner{ | ||
| client: clone, | ||
| cfg: cfg, | ||
| } | ||
| } | ||
|
|
||
| // Run executes the dynamic parameters test, which: | ||
| // | ||
| // 1. connects to the dynamic parameters stream | ||
| // 2. waits for the initial response | ||
| // 3. sends a change request | ||
| // 4. waits for the change response | ||
| // 5. closes the stream | ||
| func (r *Runner) Run(ctx context.Context, _ string, logs io.Writer) (retErr error) { | ||
| startTime := time.Now() | ||
| stream, err := r.client.TemplateVersionDynamicParameters(ctx, codersdk.Me, r.cfg.TemplateVersion) | ||
| if err != nil { | ||
| return xerrors.Errorf("connect to dynamic parameters stream: %w", err) | ||
| } | ||
| defer stream.Close(websocket.StatusNormalClosure) | ||
| respCh := stream.Chan() | ||
|
|
||
| var initTime time.Time | ||
| select { | ||
| case <-ctx.Done(): | ||
| return ctx.Err() | ||
| case resp, ok := <-respCh: | ||
| if !ok { | ||
| return xerrors.Errorf("dynamic parameters stream closed before initial response") | ||
| } | ||
| initTime = time.Now() | ||
| r.cfg.Metrics.LatencyInitialResponseSeconds. | ||
| WithLabelValues(r.cfg.MetricLabelValues...). | ||
| Observe(initTime.Sub(startTime).Seconds()) | ||
| _, _ = fmt.Fprintf(logs, "initial response: %+v\n", resp) | ||
| if !slices.ContainsFunc(resp.Parameters, func(p codersdk.PreviewParameter) bool { | ||
| return p.Name == "zero" | ||
| }) { | ||
| return xerrors.Errorf("missing expected parameter: 'zero'") | ||
| } | ||
| if err := checkNoDiagnostics(resp); err != nil { | ||
| return xerrors.Errorf("unexpected initial response diagnostics: %w", err) | ||
| } | ||
| } | ||
|
|
||
| err = stream.Send(codersdk.DynamicParametersRequest{ | ||
| ID: 1, | ||
| Inputs: map[string]string{ | ||
| "zero": "B", | ||
| }, | ||
| }) | ||
| if err != nil { | ||
| return xerrors.Errorf("send change request: %w", err) | ||
| } | ||
| select { | ||
| case <-ctx.Done(): | ||
| return ctx.Err() | ||
| case resp, ok := <-respCh: | ||
| if !ok { | ||
| return xerrors.Errorf("dynamic parameters stream closed before change response") | ||
| } | ||
| _, _ = fmt.Fprintf(logs, "change response: %+v\n", resp) | ||
| r.cfg.Metrics.LatencyChangeResponseSeconds. | ||
| WithLabelValues(r.cfg.MetricLabelValues...). | ||
| Observe(time.Since(initTime).Seconds()) | ||
| if resp.ID != 1 { | ||
| return xerrors.Errorf("unexpected response ID: %d", resp.ID) | ||
| } | ||
| if err := checkNoDiagnostics(resp); err != nil { | ||
| return xerrors.Errorf("unexpected change response diagnostics: %w", err) | ||
| } | ||
| return nil | ||
|
Comment on lines
+86
to
+100
Member
There was a problem hiding this comment. We do not assert any param condition on the response. I wonder if we can refactor the coder/coderd/coderdtest/dynamicparameters.go Line 122 in 0b6d03c It really streamlines the assertion code to something like: coderdtest.AssertParameter(t, "groups", resp.Parameters).
Exists().
Options(database.EveryoneGroup, "admin", "auditor").
Value("admin")It just currently only works in unit tests.
Contributor
Author
There was a problem hiding this comment. I feel like this is missing the point --- we are scale testing, not functional testing. I'm not looking to check that the response we get is "correct" in any detailed sense of the term, just that the response has not thrown errors and is doing the computation we want it to. |
||
| } | ||
| } | ||
|
|
||
| func checkNoDiagnostics(resp codersdk.DynamicParametersResponse) error { | ||
| if len(resp.Diagnostics) != 0 { | ||
| return xerrors.Errorf("unexpected response diagnostics: %v", resp.Diagnostics) | ||
| } | ||
| for _, param := range resp.Parameters { | ||
| if len(param.Diagnostics) != 0 { | ||
| return xerrors.Errorf("unexpected parameter diagnostics for '%s': %v", param.Name, param.Diagnostics) | ||
| } | ||
| } | ||
| return nil | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| package dynamicparameters_test | ||
|
|
||
| import ( | ||
| "strings" | ||
| "testing" | ||
|
|
||
| "github.com/prometheus/client_golang/prometheus" | ||
| "github.com/stretchr/testify/require" | ||
|
|
||
| "cdr.dev/slog" | ||
| "github.com/coder/coder/v2/coderd/coderdtest" | ||
| "github.com/coder/coder/v2/scaletest/dynamicparameters" | ||
| "github.com/coder/coder/v2/testutil" | ||
| ) | ||
|
|
||
| func TestRun(t *testing.T) { | ||
| t.Parallel() | ||
| ctx := testutil.Context(t, testutil.WaitLong) | ||
|
|
||
| client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerDaemon: true}) | ||
| client.SetLogger(testutil.Logger(t).Leveled(slog.LevelDebug)) | ||
| first := coderdtest.CreateFirstUser(t, client) | ||
| userClient, _ := coderdtest.CreateAnotherUser(t, client, first.OrganizationID) | ||
| orgID := first.OrganizationID | ||
|
|
||
| dynamicParametersTerraformSource, err := dynamicparameters.TemplateContent() | ||
| require.NoError(t, err) | ||
|
|
||
| template, version := coderdtest.DynamicParameterTemplate(t, client, orgID, coderdtest.DynamicParameterTemplateParams{ | ||
| MainTF: dynamicParametersTerraformSource, | ||
| Plan: nil, | ||
| ModulesArchive: nil, | ||
| StaticParams: nil, | ||
| ExtraFiles: dynamicparameters.GetModuleFiles(), | ||
| }) | ||
|
|
||
| reg := prometheus.NewRegistry() | ||
| cfg := dynamicparameters.Config{ | ||
| TemplateVersion: version.ID, | ||
| SessionToken: userClient.SessionToken(), | ||
| Metrics: dynamicparameters.NewMetrics(reg, "template", "test_label_name"), | ||
| MetricLabelValues: []string{template.Name, "test_label_value"}, | ||
| } | ||
| runner := dynamicparameters.NewRunner(userClient, cfg) | ||
| var logs strings.Builder | ||
| err = runner.Run(ctx, t.Name(), &logs) | ||
| t.Log("Runner logs:\n\n" + logs.String()) | ||
| require.NoError(t, err) | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| package dynamicparameters | ||
|
|
||
| import ( | ||
| _ "embed" | ||
| "encoding/json" | ||
| "strings" | ||
| "text/template" | ||
|
|
||
| "github.com/coder/coder/v2/cryptorand" | ||
| ) | ||
|
|
||
| //go:embed tf/main.tf | ||
| var templateContent string | ||
|
|
||
| func TemplateContent() (string, error) { | ||
| randomString, err := cryptorand.String(8) | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
| tmpl, err := template.New("workspace-template").Parse(templateContent) | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
| var result strings.Builder | ||
| err = tmpl.Execute(&result, map[string]string{ | ||
| "RandomString": randomString, | ||
| }) | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
| return result.String(), nil | ||
| } | ||
|
|
||
| //go:embed tf/modules/two/main.tf | ||
| var moduleTwoMainTF string | ||
|
|
||
| // GetModuleFiles returns a map of module files to be used with ExtraFiles | ||
| func GetModuleFiles() map[string][]byte { | ||
| // Create the modules.json that Terraform needs to see the module | ||
| modulesJSON := struct { | ||
| Modules []struct { | ||
| Key string `json:"Key"` | ||
| Source string `json:"Source"` | ||
| Dir string `json:"Dir"` | ||
| } `json:"Modules"` | ||
| }{ | ||
| Modules: []struct { | ||
| Key string `json:"Key"` | ||
| Source string `json:"Source"` | ||
| Dir string `json:"Dir"` | ||
| }{ | ||
| { | ||
| Key: "", | ||
| Source: "", | ||
| Dir: ".", | ||
| }, | ||
| { | ||
| Key: "two", | ||
| Source: "./modules/two", | ||
| Dir: "modules/two", | ||
| }, | ||
| }, | ||
| } | ||
|
|
||
| modulesJSONBytes, err := json.Marshal(modulesJSON) | ||
| if err != nil { | ||
| panic(err) // This should never happen with static data | ||
| } | ||
|
|
||
| return map[string][]byte{ | ||
| "modules/two/main.tf": []byte(moduleTwoMainTF), | ||
| ".terraform/modules/modules.json": modulesJSONBytes, | ||
| } | ||
| } |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Optional: could use a Validate function here, I've found the one on my load generator somewhat useful, such as for checking that my time.Duration timeouts aren't zero.