Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ require (
github.com/spf13/cobra v1.10.2
github.com/spf13/pflag v1.0.10
github.com/spf13/viper v1.10.1
github.com/stackql-labs/omnisdk v0.1.4-alpha05
github.com/stackql-labs/omnisdk v0.1.5-alpha03
github.com/stackql/any-sdk v0.6.1-alpha03
github.com/stackql/go-suffix-map v0.0.1-alpha01
github.com/stackql/psql-wire v0.1.3-beta01
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -368,8 +368,8 @@ github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk=
github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU=
github.com/spiffe/go-spiffe/v2 v2.7.0 h1:uXe1MflJoHw58wAUvxVlcM7WpKtijWG7I1UidcGh6g4=
github.com/spiffe/go-spiffe/v2 v2.7.0/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U=
github.com/stackql-labs/omnisdk v0.1.4-alpha05 h1:Z22XGi99mjKPJEAAYv3qkW1J0iqqFRd9OANK+9cw4Xs=
github.com/stackql-labs/omnisdk v0.1.4-alpha05/go.mod h1:2P2vFi+c/wEeQ3kFgIt0QuIlefpuzIoTm6bPJAiLAvQ=
github.com/stackql-labs/omnisdk v0.1.5-alpha03 h1:zk6G8Fn3SrKBqPnvNl2alGvHnCjb9Em/oEtKQDQuQVA=
github.com/stackql-labs/omnisdk v0.1.5-alpha03/go.mod h1:2P2vFi+c/wEeQ3kFgIt0QuIlefpuzIoTm6bPJAiLAvQ=
github.com/stackql/any-sdk v0.6.1-alpha03 h1:8WPgE+MftTnLNsR/WbDkyujnuyaQbM1PvQTgEsuNbR4=
github.com/stackql/any-sdk v0.6.1-alpha03/go.mod h1:ahkRgwHHEn7RTmSfKrshPVk6i4X1BdHknSeBflSyRac=
github.com/stackql/go-suffix-map v0.0.1-alpha01 h1:TDUDS8bySu41Oo9p0eniUeCm43mnRM6zFEd6j6VUaz8=
Expand Down
2 changes: 1 addition & 1 deletion internal/stackql/intrinsic/doc.go
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,7 @@ func openDocQuery(ctx queryContext, translated docQuery) (omnisdk.Rows, string,
return nil, "", argsErr
}
args.Tuning.Limit = translated.getLimit()
plan, planErr := omnisdk.NewGraphSelectQuery(registry, res.Graph(), args)
plan, planErr := omnisdk.NewGraphQuery(registry, res.Graph(), args)
if planErr != nil {
return nil, "", planErr
}
Expand Down
122 changes: 95 additions & 27 deletions internal/stackql/intrinsic/iac.go
Original file line number Diff line number Diff line change
@@ -1,15 +1,16 @@
package intrinsic

// stackql_preview.iac presents omnisdk's converge run as a relation, and the
// blueprints that render one as a catalogue alongside it. A run is issued as an imperative - these
// resources, under this collection name - and idempotence, ordering, locking and
// stackql_preview.iac presents omnisdk's converge run and destroy as relations, and the
// blueprints that render a run as a catalogue alongside them. A run is issued as an imperative -
// these resources, under this collection name - and idempotence, ordering, locking and
// compensation are the SDK's problem, not stackql's.

import (
"encoding/json"
"fmt"
"path/filepath"
"strings"
"time"

"github.com/stackql-labs/omnisdk/pkg/omnisdk"
"github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto"
Expand All @@ -22,6 +23,9 @@ import (
// disk: O_EXCL and link are unreliable on a network share.
const stateDirName = "iac"

// runIDLayout stamps a run that names no run id, as omnisdk's Converge always has.
const runIDLayout = "20060102T150405Z"

// resourceDTO is the wire shape of one entry in the 'resources' predicate. It
// is omnisdk's ManagedResource with the interface flattened into data.
type resourceDTO struct {
Expand Down Expand Up @@ -53,57 +57,121 @@ func iacSelectFunc(
node *sqlparser.Select,
resource string,
) (func() internaldto.ExecutorOutput, bool) {
if !strings.EqualFold(resource, convergeRelation) {
return blueprintSelectFunc(resource)
switch {
case strings.EqualFold(resource, convergeRelation):
return convergeSelectFunc(ctx, node)
case strings.EqualFold(resource, destroyRelation):
return destroySelectFunc(ctx, node)
}
return blueprintSelectFunc(resource)
}

// convergeSelectFunc routes a SELECT over the converge relation.
func convergeSelectFunc(ctx queryContext, node *sqlparser.Select) (func() internaldto.ExecutorOutput, bool) {
relation := relationAddress(iacService, convergeRelation)
params, refusal := previewPredicates(node, relation)
params, ledger, refusal := ledgerPredicates(ctx, node, relation)
if refusal != nil {
return refusal, true
}
collection := popPredicate(params, collectionPredicate)
if strings.TrimSpace(collection) == "" {
return refuse(fmt.Errorf(
"relation '%s' needs a '%s' predicate; it is the ledger key prefix, the lease scope "+
"and the correlation stamp, and it is how a later run addresses the same resources",
relation, collectionPredicate)), true
}
handle := popPredicate(params, blueprintPredicate)
spec := popPredicate(params, resourcesPredicate)
state := popPredicate(params, statePredicate)
runID := popPredicate(params, runIDPredicate)
if (handle == "") == (spec == "") {
file := popPredicate(params, resourcesFilePredicate)
if countSupplied(handle, spec, file) != 1 {
return refuse(fmt.Errorf(
"relation '%s' needs exactly one of '%s' or '%s'; "+
"a blueprint renders the resources, a specification states them",
relation, blueprintPredicate, resourcesPredicate)), true
}
if state == "" {
state = filepath.Join(ctx.GetRuntimeContext().ApplicationFilesRootPath, stateDirName)
"relation '%s' needs exactly one of '%s', '%s' or '%s'; "+
"a blueprint renders the resources, a specification or a file states them",
relation, blueprintPredicate, resourcesPredicate, resourcesFilePredicate)), true
}
return func() internaldto.ExecutorOutput {
resources, err := convergeResources(handle, spec, params)
resources, err := convergeResources(handle, spec, file, params)
if err != nil {
return internaldto.NewErroneousExecutorOutput(err)
}
args, argsErr := previewArgs(ctx, convergeCloud(resources), params)
if argsErr != nil {
return internaldto.NewErroneousExecutorOutput(argsErr)
}
plan, planErr := omnisdk.Converge(registryRoot(ctx), collection, state, runID, resources, args)
plan, planErr := omnisdk.Prepare(registryRoot(ctx), omnisdk.DesiredState(resources), ledger, args)
if planErr != nil {
return internaldto.NewErroneousExecutorOutput(planErr)
}
return streamPlan(ctx, plan, convergeRelation, node.SelectExprs)
}, true
}

// convergeResources renders what a run converges, from whichever of the two
// destroySelectFunc routes a SELECT over the destroy relation. What the collection holds comes
// from its ledger alone, so the remaining predicates are only the scope the deletes run in. With
// no resources to name a provider, credentials come from each cloud's canonical environment
// variables.
func destroySelectFunc(ctx queryContext, node *sqlparser.Select) (func() internaldto.ExecutorOutput, bool) {
relation := relationAddress(iacService, destroyRelation)
params, ledger, refusal := ledgerPredicates(ctx, node, relation)
if refusal != nil {
return refusal, true
}
return func() internaldto.ExecutorOutput {
args, argsErr := previewArgs(ctx, "", params)
if argsErr != nil {
return internaldto.NewErroneousExecutorOutput(argsErr)
}
plan, planErr := omnisdk.Prepare(registryRoot(ctx), omnisdk.Destroy(), ledger, args)
if planErr != nil {
return internaldto.NewErroneousExecutorOutput(planErr)
}
return streamPlan(ctx, plan, destroyRelation, node.SelectExprs)
}, true
}

// ledgerPredicates reads the predicates every ledger backed relation takes: the collection it
// addresses, and the state directory and run id, each defaulted when absent. The predicates left
// over are returned for the caller.
func ledgerPredicates(
ctx queryContext,
node *sqlparser.Select,
relation string,
) (map[string]string, omnisdk.WALConfig, func() internaldto.ExecutorOutput) {
params, refusal := previewPredicates(node, relation)
if refusal != nil {
return nil, nil, refusal
}
collection := popPredicate(params, collectionPredicate)
if strings.TrimSpace(collection) == "" {
return nil, nil, refuse(fmt.Errorf(
"relation '%s' needs a '%s' predicate; it is the ledger key prefix, the lease scope "+
"and the correlation stamp, and it is how a later run addresses the same resources",
relation, collectionPredicate))
}
state := popPredicate(params, statePredicate)
runID := popPredicate(params, runIDPredicate)
if state == "" {
state = filepath.Join(ctx.GetRuntimeContext().ApplicationFilesRootPath, stateDirName)
}
if runID == "" {
runID = time.Now().UTC().Format(runIDLayout)
}
return params, omnisdk.NewLedger(collection, state, runID), nil
}

// countSupplied counts the non-empty values.
func countSupplied(values ...string) int {
count := 0
for _, value := range values {
if value != "" {
count++
}
}
return count
}

// convergeResources renders what a run converges, from whichever of the three
// predicates was supplied.
func convergeResources(
handle, spec string, params map[string]string) ([]omnisdk.ManagedResource, error) {
if handle != "" {
handle, spec, file string, params map[string]string) ([]omnisdk.ManagedResource, error) {
switch {
case handle != "":
return blueprintResources(handle, params)
case file != "":
return omnisdk.LoadResources(file)
}
return specResources(spec)
}
Expand Down
28 changes: 18 additions & 10 deletions internal/stackql/intrinsic/preview_services.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,20 +33,22 @@ const (
const (
graphRelation = "query"
convergeRelation = "run"
destroyRelation = "destroy"
)

const (
dynamicGraphTitle = "queries wired across several document-declared exchanges"
iacTitle = "idempotent converge runs, and the blueprints that render one"
iacTitle = "idempotent converge runs and destroys, and the blueprints that render a run"
)

const (
specPredicate = "spec"
collectionPredicate = "collection"
blueprintPredicate = "blueprint"
resourcesPredicate = "resources"
statePredicate = "state"
runIDPredicate = "run_id"
specPredicate = "spec"
collectionPredicate = "collection"
blueprintPredicate = "blueprint"
resourcesPredicate = "resources"
resourcesFilePredicate = "resources_file"
statePredicate = "state"
runIDPredicate = "run_id"
)

// previewService canonicalises one of the extended stackql_preview services, or
Expand Down Expand Up @@ -261,8 +263,8 @@ func showPreviewMethods(ctx queryContext, extended bool) internaldto.ExecutorOut
}

// previewTables presents an extended service's relations. The iac service holds
// the converge relation alongside the blueprints, because a blueprint is a way
// of building a run's resources rather than a thing of its own.
// the converge and destroy relations alongside the blueprints, because a blueprint
// is a way of building a run's resources rather than a thing of its own.
func previewTables(service string) []table {
if service == dynamicGraphService {
return []table{{
Expand All @@ -277,6 +279,11 @@ func previewTables(service string) []table {
name: convergeRelation,
isData: true,
description: "an idempotent converge run over a named collection",
}, {
service: iacService,
name: destroyRelation,
isData: true,
description: "every resource a named collection holds, deleted",
}}
return append(out, blueprintTables()...)
}
Expand All @@ -297,7 +304,8 @@ func describePreviewTableFunc(
}
resource := node.Table.Name.GetRawVal()
extended := isExtended(node.Extended)
if service == iacService && !strings.EqualFold(resource, convergeRelation) {
if service == iacService && !strings.EqualFold(resource, convergeRelation) &&
!strings.EqualFold(resource, destroyRelation) {
blueprint, ok := blueprintFor(resource)
if !ok {
return refuse(fmt.Errorf(
Expand Down
1 change: 1 addition & 0 deletions test/assets/input/iac/resource-without-provider.json
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"resources": [{"key": "a/b/c"}]}
56 changes: 52 additions & 4 deletions test/robot/functional/stackql_mocked_from_cmd_line.robot
Original file line number Diff line number Diff line change
Expand Up @@ -11896,7 +11896,7 @@ Preview Show Services Includes Dynamic Graph And Iac
... id,name,title
... audit:internal,audit,omnisdk audited resources
... dynamic_graph:internal,dynamic_graph,queries wired across several document-declared exchanges
... iac:internal,iac,"idempotent converge runs, and the blueprints that render one"
... iac:internal,iac,"idempotent converge runs and destroys, and the blueprints that render a run"
Should StackQL Exec Inline Equal
... ${STACKQL_EXE}
... ${OKTA_SECRET_STR}
Expand Down Expand Up @@ -11937,6 +11937,7 @@ Preview Iac Show Resources Lists Run And Blueprint Handles
${expected} = Catenate SEPARATOR=\n
... name,id
... run,stackql_preview.iac.run
... destroy,stackql_preview.iac.destroy
... aws_vpc_subnet,stackql_preview.iac.aws_vpc_subnet
Should StackQL Exec Inline Equal
... ${STACKQL_EXE}
Expand Down Expand Up @@ -12024,9 +12025,9 @@ Preview Iac Converge Needs Exactly One Of Blueprint Or Resources
${query} = Catenate SEPARATOR=${SPACE}
... select * from stackql_preview.iac.run where collection = 'scratch';
${expected} = Catenate SEPARATOR=${SPACE}
... relation 'stackql_preview.iac.run' needs exactly one of 'blueprint'
... or 'resources'; a blueprint renders the resources, a specification
... states them
... relation 'stackql_preview.iac.run' needs exactly one of 'blueprint',
... 'resources' or 'resources_file'; a blueprint renders the resources,
... a specification or a file states them
Should StackQL Exec Inline Equal Stderr
... ${STACKQL_EXE}
... ${OKTA_SECRET_STR}
Expand Down Expand Up @@ -12133,3 +12134,50 @@ Preview Iac Resource Without Provider Is Refused By Omnisdk
... --preview\=${preview}
... stdout=${CURDIR}${/}tmp${/}Preview-Iac-Resource-No-Provider.tmp
... stderr=${CURDIR}${/}tmp${/}Preview-Iac-Resource-No-Provider-stderr.tmp

Preview Iac Resources File Without Provider Is Refused By Omnisdk
[Documentation] The 'resources_file' predicate is read by omnisdk's LoadResources,
... which also takes an object whose 'resources' is the list; a
... malformed entry is caught before any effect is attempted.
${preview} = Set Variable {"unstable":true}
${nativeResourcesFile} = Replace String
... ${REPOSITORY_ROOT}${/}test${/}assets${/}input${/}iac${/}resource-without-provider.json \\ /
${resourcesFile} = Set Variable If "${EXECUTION_PLATFORM}" == "docker"
... /opt/stackql/input/iac/resource-without-provider.json
... ${nativeResourcesFile}
${query} = Catenate SEPARATOR=${SPACE}
... select * from stackql_preview.iac.run where collection = 'scratch'
... and resources_file = '${resourcesFile}';
Should StackQL Exec Inline Equal Stderr
... ${STACKQL_EXE}
... ${OKTA_SECRET_STR}
... ${GITHUB_SECRET_STR}
... ${K8S_SECRET_STR}
... ${REGISTRY_NO_VERIFY_CFG_STR}
... ${AUTH_CFG_STR}
... ${SQL_BACKEND_CFG_STR_CANONICAL}
... ${query}
... omnisdk: resource "a/b/c" names no provider or address
... --preview\=${preview}
... stdout=${CURDIR}${/}tmp${/}Preview-Iac-Resources-File-No-Provider.tmp
... stderr=${CURDIR}${/}tmp${/}Preview-Iac-Resources-File-No-Provider-stderr.tmp

Preview Iac Destroy Without Collection Is Refused
${preview} = Set Variable {"unstable":true}
${expected} = Catenate SEPARATOR=${SPACE}
... relation 'stackql_preview.iac.destroy' needs a 'collection' predicate;
... it is the ledger key prefix, the lease scope and the correlation
... stamp, and it is how a later run addresses the same resources
Should StackQL Exec Inline Equal Stderr
... ${STACKQL_EXE}
... ${OKTA_SECRET_STR}
... ${GITHUB_SECRET_STR}
... ${K8S_SECRET_STR}
... ${REGISTRY_NO_VERIFY_CFG_STR}
... ${AUTH_CFG_STR}
... ${SQL_BACKEND_CFG_STR_CANONICAL}
... select * from stackql_preview.iac.destroy;
... ${expected}
... --preview\=${preview}
... stdout=${CURDIR}${/}tmp${/}Preview-Iac-Destroy-No-Collection.tmp
... stderr=${CURDIR}${/}tmp${/}Preview-Iac-Destroy-No-Collection-stderr.tmp
Loading