Skip to content

Commit cd890aa

Browse files
sreyasreya
authored
feat: enable key rotation (#15066)
This PR contains the remaining logic necessary to hook up key rotation to the product.
1 parent ccfffc6 commit cd890aa

54 files changed

Lines changed: 1410 additions & 1127 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎cli/server.go‎

Lines changed: 21 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ import (
1010
"crypto/tls"
1111
"crypto/x509"
1212
"database/sql"
13-
"encoding/hex"
1413
"errors"
1514
"flag"
1615
"fmt"
@@ -62,6 +61,7 @@ import (
6261
"github.com/coder/serpent"
6362
"github.com/coder/wgtunnel/tunnelsdk"
6463

64+
"github.com/coder/coder/v2/coderd/cryptokeys"
6565
"github.com/coder/coder/v2/coderd/entitlements"
6666
"github.com/coder/coder/v2/coderd/notifications/reports"
6767
"github.com/coder/coder/v2/coderd/runtimeconfig"
@@ -97,7 +97,6 @@ import (
9797
"github.com/coder/coder/v2/coderd/updatecheck"
9898
"github.com/coder/coder/v2/coderd/util/slice"
9999
stringutil "github.com/coder/coder/v2/coderd/util/strings"
100-
"github.com/coder/coder/v2/coderd/workspaceapps"
101100
"github.com/coder/coder/v2/coderd/workspaceapps/appurl"
102101
"github.com/coder/coder/v2/coderd/workspacestats"
103102
"github.com/coder/coder/v2/codersdk"
@@ -743,90 +742,31 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
743742
return xerrors.Errorf("set deployment id: %w", err)
744743
}
745744
}
746-
747-
// Read the app signing key from the DB. We store it hex encoded
748-
// since the config table uses strings for the value and we
749-
// don't want to deal with automatic encoding issues.
750-
appSecurityKeyStr, err := tx.GetAppSecurityKey(ctx)
751-
if err != nil && !xerrors.Is(err, sql.ErrNoRows) {
752-
return xerrors.Errorf("get app signing key: %w", err)
753-
}
754-
// If the string in the DB is an invalid hex string or the
755-
// length is not equal to the current key length, generate a new
756-
// one.
757-
//
758-
// If the key is regenerated, old signed tokens and encrypted
759-
// strings will become invalid. New signed app tokens will be
760-
// generated automatically on failure. Any workspace app token
761-
// smuggling operations in progress may fail, although with a
762-
// helpful error.
763-
if decoded, err := hex.DecodeString(appSecurityKeyStr); err != nil || len(decoded) != len(workspaceapps.SecurityKey{}) {
764-
b := make([]byte, len(workspaceapps.SecurityKey{}))
765-
_, err := rand.Read(b)
766-
if err != nil {
767-
return xerrors.Errorf("generate fresh app signing key: %w", err)
768-
}
769-
770-
appSecurityKeyStr = hex.EncodeToString(b)
771-
err = tx.UpsertAppSecurityKey(ctx, appSecurityKeyStr)
772-
if err != nil {
773-
return xerrors.Errorf("insert freshly generated app signing key to database: %w", err)
774-
}
775-
}
776-
777-
appSecurityKey, err := workspaceapps.KeyFromString(appSecurityKeyStr)
778-
if err != nil {
779-
return xerrors.Errorf("decode app signing key from database: %w", err)
780-
}
781-
782-
options.AppSecurityKey = appSecurityKey
783-
784-
// Read the oauth signing key from the database. Like the app security, generate a new one
785-
// if it is invalid for any reason.
786-
oauthSigningKeyStr, err := tx.GetOAuthSigningKey(ctx)
787-
if err != nil && !xerrors.Is(err, sql.ErrNoRows) {
788-
return xerrors.Errorf("get app oauth signing key: %w", err)
789-
}
790-
if decoded, err := hex.DecodeString(oauthSigningKeyStr); err != nil || len(decoded) != len(options.OAuthSigningKey) {
791-
b := make([]byte, len(options.OAuthSigningKey))
792-
_, err := rand.Read(b)
793-
if err != nil {
794-
return xerrors.Errorf("generate fresh oauth signing key: %w", err)
795-
}
796-
797-
oauthSigningKeyStr = hex.EncodeToString(b)
798-
err = tx.UpsertOAuthSigningKey(ctx, oauthSigningKeyStr)
799-
if err != nil {
800-
return xerrors.Errorf("insert freshly generated oauth signing key to database: %w", err)
801-
}
802-
}
803-
804-
oauthKeyBytes, err := hex.DecodeString(oauthSigningKeyStr)
805-
if err != nil {
806-
return xerrors.Errorf("decode oauth signing key from database: %w", err)
807-
}
808-
if len(oauthKeyBytes) != len(options.OAuthSigningKey) {
809-
return xerrors.Errorf("oauth signing key in database is not the correct length, expect %d got %d", len(options.OAuthSigningKey), len(oauthKeyBytes))
810-
}
811-
copy(options.OAuthSigningKey[:], oauthKeyBytes)
812-
if options.OAuthSigningKey == [32]byte{} {
813-
return xerrors.Errorf("oauth signing key in database is empty")
814-
}
815-
816-
// Read the coordinator resume token signing key from the
817-
// database.
818-
resumeTokenKey, err := tailnet.ResumeTokenSigningKeyFromDatabase(ctx, tx)
819-
if err != nil {
820-
return xerrors.Errorf("get coordinator resume token key from database: %w", err)
821-
}
822-
options.CoordinatorResumeTokenProvider = tailnet.NewResumeTokenKeyProvider(resumeTokenKey, quartz.NewReal(), tailnet.DefaultResumeTokenExpiry)
823-
824745
return nil
825746
}, nil)
826747
if err != nil {
827-
return err
748+
return xerrors.Errorf("set deployment id: %w", err)
749+
}
750+
751+
fetcher := &cryptokeys.DBFetcher{
752+
DB: options.Database,
753+
}
754+
755+
resumeKeycache, err := cryptokeys.NewSigningCache(ctx,
756+
logger,
757+
fetcher,
758+
codersdk.CryptoKeyFeatureTailnetResume,
759+
)
760+
if err != nil {
761+
logger.Critical(ctx, "failed to properly instantiate tailnet resume signing cache", slog.Error(err))
828762
}
829763

764+
options.CoordinatorResumeTokenProvider = tailnet.NewResumeTokenKeyProvider(
765+
resumeKeycache,
766+
quartz.NewReal(),
767+
tailnet.DefaultResumeTokenExpiry,
768+
)
769+
830770
options.RuntimeConfig = runtimeconfig.NewManager()
831771

832772
// This should be output before the logs start streaming.

‎coderd/apidoc/docs.go‎

Lines changed: 13 additions & 5 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/apidoc/swagger.json‎

Lines changed: 17 additions & 5 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/coderd.go‎

Lines changed: 61 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ import (
4040
"github.com/coder/quartz"
4141
"github.com/coder/serpent"
4242

43+
"github.com/coder/coder/v2/coderd/cryptokeys"
4344
"github.com/coder/coder/v2/coderd/entitlements"
4445
"github.com/coder/coder/v2/coderd/idpsync"
4546
"github.com/coder/coder/v2/coderd/runtimeconfig"
@@ -185,9 +186,6 @@ type Options struct {
185186
TemplateScheduleStore *atomic.Pointer[schedule.TemplateScheduleStore]
186187
UserQuietHoursScheduleStore *atomic.Pointer[schedule.UserQuietHoursScheduleStore]
187188
AccessControlStore *atomic.Pointer[dbauthz.AccessControlStore]
188-
// AppSecurityKey is the crypto key used to sign and encrypt tokens related to
189-
// workspace applications. It consists of both a signing and encryption key.
190-
AppSecurityKey workspaceapps.SecurityKey
191189
// CoordinatorResumeTokenProvider is used to provide and validate resume
192190
// tokens issued by and passed to the coordinator DRPC API.
193191
CoordinatorResumeTokenProvider tailnet.ResumeTokenProvider
@@ -251,6 +249,12 @@ type Options struct {
251249

252250
// OneTimePasscodeValidityPeriod specifies how long a one time passcode should be valid for.
253251
OneTimePasscodeValidityPeriod time.Duration
252+
253+
// Keycaches
254+
AppSigningKeyCache cryptokeys.SigningKeycache
255+
AppEncryptionKeyCache cryptokeys.EncryptionKeycache
256+
OIDCConvertKeyCache cryptokeys.SigningKeycache
257+
Clock quartz.Clock
254258
}
255259

256260
// @title Coder API
@@ -352,6 +356,9 @@ func New(options *Options) *API {
352356
if options.PrometheusRegistry == nil {
353357
options.PrometheusRegistry = prometheus.NewRegistry()
354358
}
359+
if options.Clock == nil {
360+
options.Clock = quartz.NewReal()
361+
}
355362
if options.DERPServer == nil && options.DeploymentValues.DERP.Server.Enable {
356363
options.DERPServer = derp.NewServer(key.NewNode(), tailnet.Logger(options.Logger.Named("derp")))
357364
}
@@ -444,6 +451,49 @@ func New(options *Options) *API {
444451
if err != nil {
445452
panic(xerrors.Errorf("get deployment ID: %w", err))
446453
}
454+
455+
fetcher := &cryptokeys.DBFetcher{
456+
DB: options.Database,
457+
}
458+
459+
if options.OIDCConvertKeyCache == nil {
460+
options.OIDCConvertKeyCache, err = cryptokeys.NewSigningCache(ctx,
461+
options.Logger.Named("oidc_convert_keycache"),
462+
fetcher,
463+
codersdk.CryptoKeyFeatureOIDCConvert,
464+
)
465+
if err != nil {
466+
options.Logger.Critical(ctx, "failed to properly instantiate oidc convert signing cache", slog.Error(err))
467+
}
468+
}
469+
470+
if options.AppSigningKeyCache == nil {
471+
options.AppSigningKeyCache, err = cryptokeys.NewSigningCache(ctx,
472+
options.Logger.Named("app_signing_keycache"),
473+
fetcher,
474+
codersdk.CryptoKeyFeatureWorkspaceAppsToken,
475+
)
476+
if err != nil {
477+
options.Logger.Critical(ctx, "failed to properly instantiate app signing key cache", slog.Error(err))
478+
}
479+
}
480+
481+
if options.AppEncryptionKeyCache == nil {
482+
options.AppEncryptionKeyCache, err = cryptokeys.NewEncryptionCache(ctx,
483+
options.Logger,
484+
fetcher,
485+
codersdk.CryptoKeyFeatureWorkspaceAppsAPIKey,
486+
)
487+
if err != nil {
488+
options.Logger.Critical(ctx, "failed to properly instantiate app encryption key cache", slog.Error(err))
489+
}
490+
}
491+
492+
// Start a background process that rotates keys. We intentionally start this after the caches
493+
// are created to force initial requests for a key to populate the caches. This helps catch
494+
// bugs that may only occur when a key isn't precached in tests and the latency cost is minimal.
495+
cryptokeys.StartRotator(ctx, options.Logger, options.Database)
496+
447497
api := &API{
448498
ctx: ctx,
449499
cancel: cancel,
@@ -464,7 +514,7 @@ func New(options *Options) *API {
464514
options.DeploymentValues,
465515
oauthConfigs,
466516
options.AgentInactiveDisconnectTimeout,
467-
options.AppSecurityKey,
517+
options.AppSigningKeyCache,
468518
),
469519
metricsCache: metricsCache,
470520
Auditor: atomic.Pointer[audit.Auditor]{},
@@ -606,7 +656,7 @@ func New(options *Options) *API {
606656
ResumeTokenProvider: api.Options.CoordinatorResumeTokenProvider,
607657
})
608658
if err != nil {
609-
api.Logger.Fatal(api.ctx, "failed to initialize tailnet client service", slog.Error(err))
659+
api.Logger.Fatal(context.Background(), "failed to initialize tailnet client service", slog.Error(err))
610660
}
611661

612662
api.statsReporter = workspacestats.NewReporter(workspacestats.ReporterOptions{
@@ -628,9 +678,6 @@ func New(options *Options) *API {
628678
options.WorkspaceAppsStatsCollectorOptions.Reporter = api.statsReporter
629679
}
630680

631-
if options.AppSecurityKey.IsZero() {
632-
api.Logger.Fatal(api.ctx, "app security key cannot be zero")
633-
}
634681
api.workspaceAppServer = &workspaceapps.Server{
635682
Logger: workspaceAppsLogger,
636683

@@ -642,11 +689,11 @@ func New(options *Options) *API {
642689

643690
SignedTokenProvider: api.WorkspaceAppsProvider,
644691
AgentProvider: api.agentProvider,
645-
AppSecurityKey: options.AppSecurityKey,
646692
StatsCollector: workspaceapps.NewStatsCollector(options.WorkspaceAppsStatsCollectorOptions),
647693

648-
DisablePathApps: options.DeploymentValues.DisablePathApps.Value(),
649-
SecureAuthCookie: options.DeploymentValues.SecureAuthCookie.Value(),
694+
DisablePathApps: options.DeploymentValues.DisablePathApps.Value(),
695+
SecureAuthCookie: options.DeploymentValues.SecureAuthCookie.Value(),
696+
APIKeyEncryptionKeycache: options.AppEncryptionKeyCache,
650697
}
651698

652699
apiKeyMiddleware := httpmw.ExtractAPIKeyMW(httpmw.ExtractAPIKeyConfig{
@@ -1434,6 +1481,9 @@ func (api *API) Close() error {
14341481
_ = api.agentProvider.Close()
14351482
_ = api.statsReporter.Close()
14361483
_ = api.NetworkTelemetryBatcher.Close()
1484+
_ = api.OIDCConvertKeyCache.Close()
1485+
_ = api.AppSigningKeyCache.Close()
1486+
_ = api.AppEncryptionKeyCache.Close()
14371487
return nil
14381488
}
14391489

0 commit comments

Comments
 (0)