Skip to content

Commit ef09b51

Browse files
johnstcnjohnstcn
authored
fix(coderd): extract provisionerdserver.StaleInterval to 90 seconds (#15643)
Follow-up from #15578 Extracts `provisionerdserver.StaleInterval` and sets it to 90 seconds by default
1 parent 7a7db9f commit ef09b51

5 files changed

Lines changed: 91 additions & 32 deletions

File tree

‎coderd/coderd.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -628,7 +628,8 @@ func New(options *Options) *API {
628628
CurrentVersion: buildinfo.Version(),
629629
CurrentAPIMajorVersion: proto.CurrentMajor,
630630
Store: options.Database,
631-
// TimeNow and StaleInterval set to defaults, see healthcheck/provisioner.go
631+
StaleInterval: provisionerdserver.StaleInterval,
632+
// TimeNow set to default, see healthcheck/provisioner.go
632633
},
633634
})
634635
}

‎coderd/healthcheck/provisioner.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ func (r *ProvisionerDaemonsReport) Run(ctx context.Context, opts *ProvisionerDae
5050
now := opts.TimeNow()
5151

5252
if opts.StaleInterval == 0 {
53-
opts.StaleInterval = provisionerdserver.DefaultHeartbeatInterval * 3
53+
opts.StaleInterval = provisionerdserver.StaleInterval
5454
}
5555

5656
if opts.CurrentVersion == "" {

‎coderd/healthcheck/provisioner_test.go‎

Lines changed: 80 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -15,15 +15,21 @@ import (
1515
"github.com/coder/coder/v2/coderd/database/dbtime"
1616
"github.com/coder/coder/v2/coderd/healthcheck"
1717
"github.com/coder/coder/v2/coderd/healthcheck/health"
18+
"github.com/coder/coder/v2/coderd/provisionerdserver"
1819
"github.com/coder/coder/v2/codersdk"
1920
"github.com/coder/coder/v2/codersdk/healthsdk"
2021
"github.com/coder/coder/v2/provisionerd/proto"
22+
"github.com/coder/coder/v2/testutil"
2123
)
2224

2325
func TestProvisionerDaemonReport(t *testing.T) {
2426
t.Parallel()
2527

26-
now := dbtime.Now()
28+
var (
29+
now = dbtime.Now()
30+
oneHourAgo = now.Add(-time.Hour)
31+
staleThreshold = now.Add(-provisionerdserver.StaleInterval).Add(-time.Second)
32+
)
2733

2834
for _, tt := range []struct {
2935
name string
@@ -65,7 +71,9 @@ func TestProvisionerDaemonReport(t *testing.T) {
6571
currentVersion: "v1.2.3",
6672
currentAPIMajorVersion: proto.CurrentMajor,
6773
expectedSeverity: health.SeverityOK,
68-
provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-ok", "v1.2.3", "1.0", now)},
74+
provisionerDaemons: []database.ProvisionerDaemon{
75+
fakeProvisionerDaemon(t, withName("pd-ok"), withVersion("v1.2.3"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)),
76+
},
6977
expectedItems: []healthsdk.ProvisionerDaemonsReportItem{
7078
{
7179
ProvisionerDaemon: codersdk.ProvisionerDaemon{
@@ -88,7 +96,9 @@ func TestProvisionerDaemonReport(t *testing.T) {
8896
currentAPIMajorVersion: proto.CurrentMajor,
8997
expectedSeverity: health.SeverityWarning,
9098
expectedWarningCode: health.CodeProvisionerDaemonVersionMismatch,
91-
provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-old", "v1.1.2", "1.0", now)},
99+
provisionerDaemons: []database.ProvisionerDaemon{
100+
fakeProvisionerDaemon(t, withName("pd-old"), withVersion("v1.1.2"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)),
101+
},
92102
expectedItems: []healthsdk.ProvisionerDaemonsReportItem{
93103
{
94104
ProvisionerDaemon: codersdk.ProvisionerDaemon{
@@ -116,7 +126,9 @@ func TestProvisionerDaemonReport(t *testing.T) {
116126
currentAPIMajorVersion: proto.CurrentMajor,
117127
expectedSeverity: health.SeverityError,
118128
expectedWarningCode: health.CodeUnknown,
119-
provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-invalid-version", "invalid", "1.0", now)},
129+
provisionerDaemons: []database.ProvisionerDaemon{
130+
fakeProvisionerDaemon(t, withName("pd-invalid-version"), withVersion("invalid"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)),
131+
},
120132
expectedItems: []healthsdk.ProvisionerDaemonsReportItem{
121133
{
122134
ProvisionerDaemon: codersdk.ProvisionerDaemon{
@@ -144,7 +156,9 @@ func TestProvisionerDaemonReport(t *testing.T) {
144156
currentAPIMajorVersion: proto.CurrentMajor,
145157
expectedSeverity: health.SeverityError,
146158
expectedWarningCode: health.CodeUnknown,
147-
provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-invalid-api", "v1.2.3", "invalid", now)},
159+
provisionerDaemons: []database.ProvisionerDaemon{
160+
fakeProvisionerDaemon(t, withName("pd-invalid-api"), withVersion("v1.2.3"), withAPIVersion("invalid"), withCreatedAt(now), withLastSeenAt(now)),
161+
},
148162
expectedItems: []healthsdk.ProvisionerDaemonsReportItem{
149163
{
150164
ProvisionerDaemon: codersdk.ProvisionerDaemon{
@@ -172,7 +186,9 @@ func TestProvisionerDaemonReport(t *testing.T) {
172186
currentAPIMajorVersion: 2,
173187
expectedSeverity: health.SeverityWarning,
174188
expectedWarningCode: health.CodeProvisionerDaemonAPIMajorVersionDeprecated,
175-
provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-old-api", "v2.3.4", "1.0", now)},
189+
provisionerDaemons: []database.ProvisionerDaemon{
190+
fakeProvisionerDaemon(t, withName("pd-old-api"), withVersion("v2.3.4"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)),
191+
},
176192
expectedItems: []healthsdk.ProvisionerDaemonsReportItem{
177193
{
178194
ProvisionerDaemon: codersdk.ProvisionerDaemon{
@@ -200,7 +216,10 @@ func TestProvisionerDaemonReport(t *testing.T) {
200216
currentAPIMajorVersion: proto.CurrentMajor,
201217
expectedSeverity: health.SeverityWarning,
202218
expectedWarningCode: health.CodeProvisionerDaemonVersionMismatch,
203-
provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-ok", "v1.2.3", "1.0", now), fakeProvisionerDaemon(t, "pd-old", "v1.1.2", "1.0", now)},
219+
provisionerDaemons: []database.ProvisionerDaemon{
220+
fakeProvisionerDaemon(t, withName("pd-ok"), withVersion("v1.2.3"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)),
221+
fakeProvisionerDaemon(t, withName("pd-old"), withVersion("v1.1.2"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)),
222+
},
204223
expectedItems: []healthsdk.ProvisionerDaemonsReportItem{
205224
{
206225
ProvisionerDaemon: codersdk.ProvisionerDaemon{
@@ -241,7 +260,10 @@ func TestProvisionerDaemonReport(t *testing.T) {
241260
currentAPIMajorVersion: proto.CurrentMajor,
242261
expectedSeverity: health.SeverityWarning,
243262
expectedWarningCode: health.CodeProvisionerDaemonVersionMismatch,
244-
provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-ok", "v1.2.3", "1.0", now), fakeProvisionerDaemon(t, "pd-new", "v2.3.4", "1.0", now)},
263+
provisionerDaemons: []database.ProvisionerDaemon{
264+
fakeProvisionerDaemon(t, withName("pd-ok"), withVersion("v1.2.3"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)),
265+
fakeProvisionerDaemon(t, withName("pd-new"), withVersion("v2.3.4"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)),
266+
},
245267
expectedItems: []healthsdk.ProvisionerDaemonsReportItem{
246268
{
247269
ProvisionerDaemon: codersdk.ProvisionerDaemon{
@@ -281,7 +303,10 @@ func TestProvisionerDaemonReport(t *testing.T) {
281303
currentVersion: "v2.3.4",
282304
currentAPIMajorVersion: proto.CurrentMajor,
283305
expectedSeverity: health.SeverityOK,
284-
provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemonStale(t, "pd-stale", "v1.2.3", "0.9", now.Add(-5*time.Minute), now), fakeProvisionerDaemon(t, "pd-ok", "v2.3.4", "1.0", now)},
306+
provisionerDaemons: []database.ProvisionerDaemon{
307+
fakeProvisionerDaemon(t, withName("pd-stale"), withVersion("v1.2.3"), withAPIVersion("0.9"), withCreatedAt(oneHourAgo), withLastSeenAt(staleThreshold)),
308+
fakeProvisionerDaemon(t, withName("pd-ok"), withVersion("v2.3.4"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)),
309+
},
285310
expectedItems: []healthsdk.ProvisionerDaemonsReportItem{
286311
{
287312
ProvisionerDaemon: codersdk.ProvisionerDaemon{
@@ -304,8 +329,10 @@ func TestProvisionerDaemonReport(t *testing.T) {
304329
currentAPIMajorVersion: proto.CurrentMajor,
305330
expectedSeverity: health.SeverityError,
306331
expectedWarningCode: health.CodeProvisionerDaemonsNoProvisionerDaemons,
307-
provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemonStale(t, "pd-ok", "v1.2.3", "0.9", now.Add(-5*time.Minute), now)},
308-
expectedItems: []healthsdk.ProvisionerDaemonsReportItem{},
332+
provisionerDaemons: []database.ProvisionerDaemon{
333+
fakeProvisionerDaemon(t, withName("pd-stale"), withVersion("v1.2.3"), withAPIVersion("0.9"), withCreatedAt(oneHourAgo), withLastSeenAt(staleThreshold)),
334+
},
335+
expectedItems: []healthsdk.ProvisionerDaemonsReportItem{},
309336
},
310337
} {
311338
tt := tt
@@ -353,25 +380,52 @@ func TestProvisionerDaemonReport(t *testing.T) {
353380
}
354381
}
355382

356-
func fakeProvisionerDaemon(t *testing.T, name, version, apiVersion string, now time.Time) database.ProvisionerDaemon {
383+
func withName(s string) func(*database.ProvisionerDaemon) {
384+
return func(pd *database.ProvisionerDaemon) {
385+
pd.Name = s
386+
}
387+
}
388+
389+
func withCreatedAt(at time.Time) func(*database.ProvisionerDaemon) {
390+
return func(pd *database.ProvisionerDaemon) {
391+
pd.CreatedAt = at
392+
}
393+
}
394+
395+
func withLastSeenAt(at time.Time) func(*database.ProvisionerDaemon) {
396+
return func(pd *database.ProvisionerDaemon) {
397+
pd.LastSeenAt.Valid = true
398+
pd.LastSeenAt.Time = at
399+
}
400+
}
401+
402+
func withVersion(v string) func(*database.ProvisionerDaemon) {
403+
return func(pd *database.ProvisionerDaemon) {
404+
pd.Version = v
405+
}
406+
}
407+
408+
func withAPIVersion(v string) func(*database.ProvisionerDaemon) {
409+
return func(pd *database.ProvisionerDaemon) {
410+
pd.APIVersion = v
411+
}
412+
}
413+
414+
func fakeProvisionerDaemon(t *testing.T, opts ...func(*database.ProvisionerDaemon)) database.ProvisionerDaemon {
357415
t.Helper()
358-
return database.ProvisionerDaemon{
416+
pd := database.ProvisionerDaemon{
359417
ID: uuid.Nil,
360-
Name: name,
361-
CreatedAt: now,
362-
LastSeenAt: sql.NullTime{Time: now, Valid: true},
418+
Name: testutil.GetRandomName(t),
419+
CreatedAt: time.Time{},
420+
LastSeenAt: sql.NullTime{},
363421
Provisioners: []database.ProvisionerType{database.ProvisionerTypeEcho, database.ProvisionerTypeTerraform},
364422
ReplicaID: uuid.NullUUID{},
365423
Tags: map[string]string{},
366-
Version: version,
367-
APIVersion: apiVersion,
424+
Version: "",
425+
APIVersion: "",
368426
}
369-
}
370-
371-
func fakeProvisionerDaemonStale(t *testing.T, name, version, apiVersion string, lastSeenAt, now time.Time) database.ProvisionerDaemon {
372-
t.Helper()
373-
d := fakeProvisionerDaemon(t, name, version, apiVersion, now)
374-
d.LastSeenAt.Valid = true
375-
d.LastSeenAt.Time = lastSeenAt
376-
return d
427+
for _, o := range opts {
428+
o(&pd)
429+
}
430+
return pd
377431
}

‎coderd/provisionerdserver/provisionerdserver.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,10 @@ const (
5757
// DefaultHeartbeatInterval is the interval at which the provisioner daemon
5858
// will update its last seen at timestamp in the database.
5959
DefaultHeartbeatInterval = time.Minute
60+
61+
// StaleInterval is the amount of time after the last heartbeat for which
62+
// the provisioner will be reported as 'stale'.
63+
StaleInterval = 90 * time.Second
6064
)
6165

6266
type Options struct {

‎coderd/templateversions.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1515,7 +1515,7 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht
15151515

15161516
// Check for eligible provisioners. This allows us to log a message warning deployment administrators
15171517
// of users submitting jobs for which no provisioners are available.
1518-
matchedProvisioners, err = checkProvisioners(ctx, tx, organization.ID, tags, api.DeploymentValues.Provisioner.DaemonPollInterval.Value())
1518+
matchedProvisioners, err = checkProvisioners(ctx, tx, organization.ID, tags)
15191519
if err != nil {
15201520
api.Logger.Error(ctx, "failed to check eligible provisioner daemons for job", slog.Error(err))
15211521
} else if matchedProvisioners.Count == 0 {
@@ -1823,7 +1823,7 @@ func (api *API) publishTemplateUpdate(ctx context.Context, templateID uuid.UUID)
18231823
}
18241824
}
18251825

1826-
func checkProvisioners(ctx context.Context, store database.Store, orgID uuid.UUID, wantTags map[string]string, pollInterval time.Duration) (codersdk.MatchedProvisioners, error) {
1826+
func checkProvisioners(ctx context.Context, store database.Store, orgID uuid.UUID, wantTags map[string]string) (codersdk.MatchedProvisioners, error) {
18271827
// Check for eligible provisioners. This allows us to return a warning to the user if they
18281828
// submit a job for which no provisioner is available.
18291829
eligibleProvisioners, err := store.GetProvisionerDaemonsByOrganization(ctx, database.GetProvisionerDaemonsByOrganizationParams{
@@ -1835,15 +1835,15 @@ func checkProvisioners(ctx context.Context, store database.Store, orgID uuid.UUI
18351835
return codersdk.MatchedProvisioners{}, xerrors.Errorf("provisioner daemons by organization: %w", err)
18361836
}
18371837

1838-
threePollsAgo := time.Now().Add(-3 * pollInterval)
1838+
staleInterval := time.Now().Add(-provisionerdserver.StaleInterval)
18391839
mostRecentlySeen := codersdk.NullTime{}
18401840
var matched codersdk.MatchedProvisioners
18411841
for _, provisioner := range eligibleProvisioners {
18421842
if !provisioner.LastSeenAt.Valid {
18431843
continue
18441844
}
18451845
matched.Count++
1846-
if provisioner.LastSeenAt.Time.After(threePollsAgo) {
1846+
if provisioner.LastSeenAt.Time.After(staleInterval) {
18471847
matched.Available++
18481848
}
18491849
if provisioner.LastSeenAt.Time.After(mostRecentlySeen.Time) {

0 commit comments

Comments
 (0)