Skip to content

Commit 1871b09

Browse files
spikecurtisspikecurtis
authored
feat: in-process provisionerd connection (coder#1568)
* in-process provisionerd connection Signed-off-by: Spike Curtis <spike@coder.com> * disable lint for server.go/newProvisionerDaemon Signed-off-by: Spike Curtis <spike@coder.com>
1 parent 376c681 commit 1871b09

32 files changed

Lines changed: 344 additions & 304 deletions

‎cli/agent_test.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,10 @@ func TestWorkspaceAgent(t *testing.T) {
1919
instanceID := "instanceidentifier"
2020
certificates, metadataClient := coderdtest.NewAzureInstanceIdentity(t, instanceID)
2121
client := coderdtest.New(t, &coderdtest.Options{
22-
AzureCertificates: certificates,
22+
AzureCertificates: certificates,
23+
IncludeProvisionerD: true,
2324
})
2425
user := coderdtest.CreateFirstUser(t, client)
25-
coderdtest.NewProvisionerDaemon(t, client)
2626
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
2727
Parse: echo.ParseComplete,
2828
Provision: []*proto.Provision_Response{{
@@ -73,10 +73,10 @@ func TestWorkspaceAgent(t *testing.T) {
7373
instanceID := "instanceidentifier"
7474
certificates, metadataClient := coderdtest.NewAWSInstanceIdentity(t, instanceID)
7575
client := coderdtest.New(t, &coderdtest.Options{
76-
AWSCertificates: certificates,
76+
AWSCertificates: certificates,
77+
IncludeProvisionerD: true,
7778
})
7879
user := coderdtest.CreateFirstUser(t, client)
79-
coderdtest.NewProvisionerDaemon(t, client)
8080
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
8181
Parse: echo.ParseComplete,
8282
Provision: []*proto.Provision_Response{{
@@ -128,9 +128,9 @@ func TestWorkspaceAgent(t *testing.T) {
128128
validator, metadata := coderdtest.NewGoogleInstanceIdentity(t, instanceID, false)
129129
client := coderdtest.New(t, &coderdtest.Options{
130130
GoogleTokenValidator: validator,
131+
IncludeProvisionerD: true,
131132
})
132133
user := coderdtest.CreateFirstUser(t, client)
133-
coderdtest.NewProvisionerDaemon(t, client)
134134
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
135135
Parse: echo.ParseComplete,
136136
Provision: []*proto.Provision_Response{{

‎cli/autostart_test.go‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,7 @@ func TestAutostart(t *testing.T) {
2222

2323
var (
2424
ctx = context.Background()
25-
client = coderdtest.New(t, nil)
26-
_ = coderdtest.NewProvisionerDaemon(t, client)
25+
client = coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
2726
user = coderdtest.CreateFirstUser(t, client)
2827
version = coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
2928
_ = coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
@@ -54,8 +53,7 @@ func TestAutostart(t *testing.T) {
5453

5554
var (
5655
ctx = context.Background()
57-
client = coderdtest.New(t, nil)
58-
_ = coderdtest.NewProvisionerDaemon(t, client)
56+
client = coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
5957
user = coderdtest.CreateFirstUser(t, client)
6058
version = coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
6159
_ = coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
@@ -99,8 +97,7 @@ func TestAutostart(t *testing.T) {
9997
t.Parallel()
10098

10199
var (
102-
client = coderdtest.New(t, nil)
103-
_ = coderdtest.NewProvisionerDaemon(t, client)
100+
client = coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
104101
user = coderdtest.CreateFirstUser(t, client)
105102
version = coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
106103
_ = coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
@@ -117,8 +114,7 @@ func TestAutostart(t *testing.T) {
117114
t.Parallel()
118115

119116
var (
120-
client = coderdtest.New(t, nil)
121-
_ = coderdtest.NewProvisionerDaemon(t, client)
117+
client = coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
122118
user = coderdtest.CreateFirstUser(t, client)
123119
version = coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
124120
_ = coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
@@ -136,8 +132,7 @@ func TestAutostart(t *testing.T) {
136132

137133
var (
138134
ctx = context.Background()
139-
client = coderdtest.New(t, nil)
140-
_ = coderdtest.NewProvisionerDaemon(t, client)
135+
client = coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
141136
user = coderdtest.CreateFirstUser(t, client)
142137
version = coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
143138
_ = coderdtest.AwaitTemplateVersionJob(t, client, version.ID)

‎cli/configssh_test.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,8 @@ import (
2727
func TestConfigSSH(t *testing.T) {
2828
t.Parallel()
2929

30-
client := coderdtest.New(t, nil)
30+
client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
3131
user := coderdtest.CreateFirstUser(t, client)
32-
coderdtest.NewProvisionerDaemon(t, client)
3332
authToken := uuid.NewString()
3433
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
3534
Parse: echo.ParseComplete,

‎cli/create_test.go‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,8 @@ func TestCreate(t *testing.T) {
1717
t.Parallel()
1818
t.Run("Create", func(t *testing.T) {
1919
t.Parallel()
20-
client := coderdtest.New(t, nil)
20+
client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
2121
user := coderdtest.CreateFirstUser(t, client)
22-
coderdtest.NewProvisionerDaemon(t, client)
2322
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
2423
coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
2524
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
@@ -48,9 +47,8 @@ func TestCreate(t *testing.T) {
4847

4948
t.Run("CreateFromList", func(t *testing.T) {
5049
t.Parallel()
51-
client := coderdtest.New(t, nil)
50+
client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
5251
user := coderdtest.CreateFirstUser(t, client)
53-
coderdtest.NewProvisionerDaemon(t, client)
5452
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
5553
coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
5654
_ = coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
@@ -79,9 +77,8 @@ func TestCreate(t *testing.T) {
7977

8078
t.Run("FromNothing", func(t *testing.T) {
8179
t.Parallel()
82-
client := coderdtest.New(t, nil)
80+
client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
8381
user := coderdtest.CreateFirstUser(t, client)
84-
coderdtest.NewProvisionerDaemon(t, client)
8582
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
8683
coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
8784
_ = coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
@@ -111,9 +108,8 @@ func TestCreate(t *testing.T) {
111108

112109
t.Run("WithParameter", func(t *testing.T) {
113110
t.Parallel()
114-
client := coderdtest.New(t, nil)
111+
client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
115112
user := coderdtest.CreateFirstUser(t, client)
116-
coderdtest.NewProvisionerDaemon(t, client)
117113

118114
defaultValue := "something"
119115
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{

‎cli/gitssh_test.go‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ import (
2222
func TestGitSSH(t *testing.T) {
2323
t.Parallel()
2424
t.Run("Dial", func(t *testing.T) {
25-
client := coderdtest.New(t, nil)
25+
client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
2626
user := coderdtest.CreateFirstUser(t, client)
2727

2828
// get user public key
@@ -31,9 +31,8 @@ func TestGitSSH(t *testing.T) {
3131
publicKey, _, _, _, err := gossh.ParseAuthorizedKey([]byte(keypair.PublicKey))
3232
require.NoError(t, err)
3333

34-
// setup provisioner
34+
// setup template
3535
agentToken := uuid.NewString()
36-
coderdtest.NewProvisionerDaemon(t, client)
3736
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
3837
Parse: echo.ParseComplete,
3938
ProvisionDryRun: echo.ProvisionComplete,

‎cli/list_test.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,8 @@ func TestList(t *testing.T) {
1414
t.Parallel()
1515
t.Run("Single", func(t *testing.T) {
1616
t.Parallel()
17-
client := coderdtest.New(t, nil)
17+
client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
1818
user := coderdtest.CreateFirstUser(t, client)
19-
coderdtest.NewProvisionerDaemon(t, client)
2019
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
2120
coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
2221
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)

‎cli/portforward_test.go‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ func TestPortForward(t *testing.T) {
149149
t.Run("OnePort", func(t *testing.T) {
150150
t.Parallel()
151151
var (
152-
client = coderdtest.New(t, nil)
152+
client = coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
153153
user = coderdtest.CreateFirstUser(t, client)
154154
_, workspace = runAgent(t, client, user.UserID)
155155
l1, p1 = setupTestListener(t, c.setupRemote(t))
@@ -193,7 +193,7 @@ func TestPortForward(t *testing.T) {
193193
t.Run("TwoPorts", func(t *testing.T) {
194194
t.Parallel()
195195
var (
196-
client = coderdtest.New(t, nil)
196+
client = coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
197197
user = coderdtest.CreateFirstUser(t, client)
198198
_, workspace = runAgent(t, client, user.UserID)
199199
l1, p1 = setupTestListener(t, c.setupRemote(t))
@@ -244,7 +244,7 @@ func TestPortForward(t *testing.T) {
244244
t.Run("TCP2Unix", func(t *testing.T) {
245245
t.Parallel()
246246
var (
247-
client = coderdtest.New(t, nil)
247+
client = coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
248248
user = coderdtest.CreateFirstUser(t, client)
249249
_, workspace = runAgent(t, client, user.UserID)
250250

@@ -297,7 +297,7 @@ func TestPortForward(t *testing.T) {
297297
t.Run("All", func(t *testing.T) {
298298
t.Parallel()
299299
var (
300-
client = coderdtest.New(t, nil)
300+
client = coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
301301
user = coderdtest.CreateFirstUser(t, client)
302302
_, workspace = runAgent(t, client, user.UserID)
303303
// These aren't fixed size because we exclude Unix on Windows.
@@ -371,9 +371,8 @@ func runAgent(t *testing.T, client *codersdk.Client, userID uuid.UUID) ([]coders
371371
require.Greater(t, len(user.OrganizationIDs), 0, "user has no organizations")
372372
orgID := user.OrganizationIDs[0]
373373

374-
// Setup echo provisioner
374+
// Setup template
375375
agentToken := uuid.NewString()
376-
coderdtest.NewProvisionerDaemon(t, client)
377376
version := coderdtest.CreateTemplateVersion(t, client, orgID, &echo.Responses{
378377
Parse: echo.ParseComplete,
379378
ProvisionDryRun: echo.ProvisionComplete,

‎cli/server.go‎

Lines changed: 28 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ import (
1919
"path/filepath"
2020
"time"
2121

22+
"github.com/coder/coder/provisioner/echo"
23+
2224
"github.com/briandowns/spinner"
2325
"github.com/coreos/go-systemd/daemon"
2426
"github.com/google/go-github/v43/github"
@@ -261,7 +263,7 @@ func server() *cobra.Command {
261263
}
262264
}
263265

264-
handler, closeCoderd := coderd.New(options)
266+
coderDaemon := coderd.New(options)
265267
client := codersdk.New(localURL)
266268
if tlsEnable {
267269
// Secure transport isn't needed for locally communicating!
@@ -287,7 +289,7 @@ func server() *cobra.Command {
287289
errCh := make(chan error, 1)
288290
provisionerDaemons := make([]*provisionerd.Server, 0)
289291
for i := 0; uint8(i) < provisionerDaemonCount; i++ {
290-
daemonClose, err := newProvisionerDaemon(cmd.Context(), client, logger, cacheDir, errCh)
292+
daemonClose, err := newProvisionerDaemon(cmd.Context(), coderDaemon, logger, cacheDir, errCh, dev)
291293
if err != nil {
292294
return xerrors.Errorf("create provisioner daemon: %w", err)
293295
}
@@ -307,7 +309,7 @@ func server() *cobra.Command {
307309
// These errors are typically noise like "TLS: EOF". Vault does similar:
308310
// https://github.com/hashicorp/vault/blob/e2490059d0711635e529a4efcbaa1b26998d6e1c/command/server.go#L2714
309311
ErrorLog: log.New(io.Discard, "", 0),
310-
Handler: handler,
312+
Handler: coderDaemon.Handler(),
311313
BaseContext: func(_ net.Listener) context.Context {
312314
return shutdownConnsCtx
313315
},
@@ -375,15 +377,15 @@ func server() *cobra.Command {
375377
signal.Notify(stopChan, os.Interrupt)
376378
select {
377379
case <-cmd.Context().Done():
378-
closeCoderd()
380+
coderDaemon.CloseWait()
379381
return cmd.Context().Err()
380382
case err := <-tunnelErrChan:
381383
if err != nil {
382384
return err
383385
}
384386
case err := <-errCh:
385387
shutdownConns()
386-
closeCoderd()
388+
coderDaemon.CloseWait()
387389
return err
388390
case <-stopChan:
389391
}
@@ -447,7 +449,7 @@ func server() *cobra.Command {
447449

448450
_, _ = fmt.Fprintf(cmd.OutOrStdout(), cliui.Styles.Prompt.String()+"Waiting for WebSocket connections to close...\n")
449451
shutdownConns()
450-
closeCoderd()
452+
coderDaemon.CloseWait()
451453
return nil
452454
},
453455
}
@@ -541,7 +543,9 @@ func createFirstUser(cmd *cobra.Command, client *codersdk.Client, cfg config.Roo
541543
return nil
542544
}
543545

544-
func newProvisionerDaemon(ctx context.Context, client *codersdk.Client, logger slog.Logger, cacheDir string, errChan chan error) (*provisionerd.Server, error) {
546+
// nolint:revive
547+
func newProvisionerDaemon(ctx context.Context, coderDaemon coderd.CoderD,
548+
logger slog.Logger, cacheDir string, errChan chan error, dev bool) (*provisionerd.Server, error) {
545549
err := os.MkdirAll(cacheDir, 0700)
546550
if err != nil {
547551
return nil, xerrors.Errorf("mkdir %q: %w", cacheDir, err)
@@ -566,14 +570,26 @@ func newProvisionerDaemon(ctx context.Context, client *codersdk.Client, logger s
566570
return nil, err
567571
}
568572

569-
return provisionerd.New(client.ListenProvisionerDaemon, &provisionerd.Options{
573+
provisioners := provisionerd.Provisioners{
574+
string(database.ProvisionerTypeTerraform): proto.NewDRPCProvisionerClient(provisionersdk.Conn(terraformClient)),
575+
}
576+
// include echo provisioner when in dev mode
577+
if dev {
578+
echoClient, echoServer := provisionersdk.TransportPipe()
579+
go func() {
580+
err := echo.Serve(ctx, &provisionersdk.ServeOptions{Listener: echoServer})
581+
if err != nil {
582+
errChan <- err
583+
}
584+
}()
585+
provisioners[string(database.ProvisionerTypeEcho)] = proto.NewDRPCProvisionerClient(provisionersdk.Conn(echoClient))
586+
}
587+
return provisionerd.New(coderDaemon.ListenProvisionerDaemon, &provisionerd.Options{
570588
Logger: logger,
571589
PollInterval: 500 * time.Millisecond,
572590
UpdateInterval: 500 * time.Millisecond,
573-
Provisioners: provisionerd.Provisioners{
574-
string(database.ProvisionerTypeTerraform): proto.NewDRPCProvisionerClient(provisionersdk.Conn(terraformClient)),
575-
},
576-
WorkDirectory: tempDir,
591+
Provisioners: provisioners,
592+
WorkDirectory: tempDir,
577593
}), nil
578594
}
579595

‎cli/server_test.go‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -236,12 +236,11 @@ func TestServer(t *testing.T) {
236236
}
237237
ctx, cancelFunc := context.WithCancel(context.Background())
238238
defer cancelFunc()
239-
root, cfg := clitest.New(t, "server", "--dev", "--tunnel=false", "--address", ":0", "--provisioner-daemons", "0")
240-
done := make(chan struct{})
239+
root, cfg := clitest.New(t, "server", "--dev", "--tunnel=false", "--address", ":0", "--provisioner-daemons", "1")
240+
serverErr := make(chan error)
241241
go func() {
242-
defer close(done)
243242
err := root.ExecuteContext(ctx)
244-
require.NoError(t, err)
243+
serverErr <- err
245244
}()
246245
var token string
247246
require.Eventually(t, func() bool {
@@ -258,7 +257,6 @@ func TestServer(t *testing.T) {
258257
client.SessionToken = token
259258
orgs, err := client.OrganizationsByUser(ctx, codersdk.Me)
260259
require.NoError(t, err)
261-
coderdtest.NewProvisionerDaemon(t, client)
262260

263261
// Create a workspace so the cleanup occurs!
264262
version := coderdtest.CreateTemplateVersion(t, client, orgs[0].ID, nil)
@@ -278,7 +276,8 @@ func TestServer(t *testing.T) {
278276
require.NoError(t, err)
279277
err = currentProcess.Signal(os.Interrupt)
280278
require.NoError(t, err)
281-
<-done
279+
err = <-serverErr
280+
require.NoError(t, err)
282281
})
283282
t.Run("TracerNoLeak", func(t *testing.T) {
284283
t.Parallel()

‎cli/ssh_test.go‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,8 @@ func TestSSH(t *testing.T) {
2626
t.Parallel()
2727
t.Run("ImmediateExit", func(t *testing.T) {
2828
t.Parallel()
29-
client := coderdtest.New(t, nil)
29+
client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
3030
user := coderdtest.CreateFirstUser(t, client)
31-
coderdtest.NewProvisionerDaemon(t, client)
3231
agentToken := uuid.NewString()
3332
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
3433
Parse: echo.ParseComplete,
@@ -81,9 +80,8 @@ func TestSSH(t *testing.T) {
8180
})
8281
t.Run("Stdio", func(t *testing.T) {
8382
t.Parallel()
84-
client := coderdtest.New(t, nil)
83+
client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
8584
user := coderdtest.CreateFirstUser(t, client)
86-
coderdtest.NewProvisionerDaemon(t, client)
8785
agentToken := uuid.NewString()
8886
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
8987
Parse: echo.ParseComplete,

0 commit comments

Comments
 (0)