Skip to content

feat: Add per table sync metrics file flag #18397

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kodiakhq merged 8 commits into from
Jun 27, 2024
Merged

feat: Add per table sync metrics file flag #18397

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
b63d160
feat: Support OTEL traces and metrics collection in the CLI
erezrokah Jun 25, 2024
8b186ae
chore: Cleanup
erezrokah Jun 25, 2024
c08cdc5
chore: Cleanup
erezrokah Jun 25, 2024
9abf09e
feat: CLI option
erezrokah Jun 26, 2024
20704b3
style: Fix lint
erezrokah Jun 26, 2024
0f79d15
chore: cleanup
erezrokah Jun 27, 2024
2a79995
chore: Hide flag
erezrokah Jun 27, 2024
0ffc9c4
style: Fix lint
erezrokah Jun 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions cli/cmd/sync.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"strings"

"github.com/cloudquery/cloudquery/cli/internal/auth"
"github.com/cloudquery/cloudquery/cli/internal/otel"
"github.com/cloudquery/cloudquery/cli/internal/specs/v0"
"github.com/cloudquery/plugin-pb-go/managedplugin"
"github.com/rs/zerolog/log"
Expand Down Expand Up @@ -35,6 +36,9 @@ func NewCmdSync() *cobra.Command {
cmd.Flags().Bool("no-migrate", false, "Disable auto-migration before sync. By default, sync runs a migration before syncing resources.")
cmd.Flags().String("license", "", "set offline license file")
cmd.Flags().String("summary-location", "", "Sync summary file location. This feature is in Preview. Please provide feedback to help us improve it.")
cmd.Flags().String("tables-metrics-location", "", "Tables metrics file location. This feature is in Preview. Please provide feedback to help us improve it.")
// Hide the flag until we release all plugins with https://github.com/cloudquery/plugin-sdk/releases/tag/v4.49.0
_ = cmd.Flags().MarkHidden("tables-metrics-location")

return cmd
}
Expand Down Expand Up @@ -123,7 +127,23 @@ func sync(cmd *cobra.Command, args []string) error {
// in a cloud sync environment, we pass only the relevant environment variables to the plugin
osEnviron := os.Environ()

tableMetricsLocation, err := cmd.Flags().GetString("tables-metrics-location")
if err != nil {
return err
}
var otelReceiver *otel.OtelReceiver
if tableMetricsLocation != "" {
otelReceiver, err = otel.StartOtelReceiver(ctx, otel.WithMetricsFilename(tableMetricsLocation))
if err == nil {
defer otelReceiver.Shutdown(ctx)
}
}

for _, source := range sources {
if source.OtelEndpoint == "" && otelReceiver != nil {
source.OtelEndpoint = otelReceiver.Endpoint
source.OtelEndpointInsecure = true
}
opts := []managedplugin.Option{
managedplugin.WithLogger(log.Logger),
managedplugin.WithOtelEndpoint(source.OtelEndpoint),
Expand Down
68 changes: 50 additions & 18 deletions cli/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ require (
github.com/google/go-cmp v0.6.0
github.com/google/uuid v1.6.0
github.com/invopop/jsonschema v0.12.0
github.com/jedib0t/go-pretty/v6 v6.5.9
github.com/opencontainers/go-digest v1.0.0
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c
github.com/rs/zerolog v1.33.0
Expand All @@ -30,10 +31,18 @@ require (
github.com/vnteamopen/godebouncer v1.1.1-0.20230626172639-4b59d27e1b8c
github.com/wk8/go-ordered-map/v2 v2.1.8
github.com/yuin/goldmark v1.6.0
go.opentelemetry.io/collector/component v0.103.0
go.opentelemetry.io/collector/consumer v0.103.0
go.opentelemetry.io/collector/pdata v1.10.0
go.opentelemetry.io/collector/receiver v0.103.0
go.opentelemetry.io/collector/receiver/otlpreceiver v0.103.0
go.opentelemetry.io/otel/metric v1.27.0
go.opentelemetry.io/otel/trace v1.27.0
go.uber.org/zap v1.27.0
golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc
golang.org/x/net v0.25.0
golang.org/x/net v0.26.0
golang.org/x/sync v0.7.0
golang.org/x/term v0.20.0
golang.org/x/term v0.21.0
google.golang.org/grpc v1.64.0
google.golang.org/protobuf v1.34.2
gopkg.in/yaml.v3 v3.0.1
Expand All @@ -56,7 +65,7 @@ require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/buger/jsonparser v1.1.1 // indirect
github.com/bytedance/sonic v1.11.2 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
github.com/chenzhuoyu/iasm v0.9.1 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
Expand All @@ -69,6 +78,7 @@ require (
github.com/fatih/structs v1.1.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/flosch/pongo2/v4 v4.0.2 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/gin-gonic/gin v1.9.1 // indirect
Expand All @@ -77,16 +87,17 @@ require (
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.18.0 // indirect
github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect
github.com/goccy/go-json v0.10.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/gomarkdown/markdown v0.0.0-20231222211730-1d6d20845b47 // indirect
github.com/google/flatbuffers v24.3.25+incompatible // indirect
github.com/gorilla/css v1.0.1 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
github.com/hashicorp/go-version v1.7.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/iris-contrib/schema v0.0.6 // indirect
github.com/josharian/intern v1.0.0 // indirect
Expand All @@ -97,31 +108,38 @@ require (
github.com/kataras/pio v0.0.13 // indirect
github.com/kataras/sitemap v0.0.6 // indirect
github.com/kataras/tunnel v0.0.4 // indirect
github.com/klauspost/compress v1.17.7 // indirect
github.com/klauspost/compress v1.17.8 // indirect
github.com/klauspost/cpuid/v2 v2.2.7 // indirect
github.com/knadh/koanf/maps v0.1.1 // indirect
github.com/knadh/koanf/providers/confmap v0.1.0 // indirect
github.com/knadh/koanf/v2 v2.1.1 // indirect
github.com/labstack/echo/v4 v4.11.4 // indirect
github.com/labstack/gommon v0.4.2 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/mailgun/raymond/v2 v2.0.48 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/microcosm-cc/bluemonday v1.0.26 // indirect
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/mostynb/go-grpc-compression v1.2.3 // indirect
github.com/opencontainers/image-spec v1.1.0 // indirect
github.com/pelletier/go-toml/v2 v2.1.1 // indirect
github.com/pierrec/lz4/v4 v4.1.21 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.11.1 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.26.0 // indirect
github.com/prometheus/procfs v0.6.0 // indirect
github.com/prometheus/client_golang v1.19.1 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.54.0 // indirect
github.com/prometheus/procfs v0.15.0 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/rs/cors v1.10.1 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/schollz/closestmatch v2.1.0+incompatible // indirect
github.com/segmentio/backo-go v1.1.0 // indirect
Expand All @@ -140,19 +158,33 @@ require (
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/yosssi/ace v0.0.5 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
go.opentelemetry.io/collector v0.103.0 // indirect
go.opentelemetry.io/collector/config/configauth v0.103.0 // indirect
go.opentelemetry.io/collector/config/configcompression v1.10.0 // indirect
go.opentelemetry.io/collector/config/configgrpc v0.103.0 // indirect
go.opentelemetry.io/collector/config/confighttp v0.103.0 // indirect
go.opentelemetry.io/collector/config/confignet v0.103.0 // indirect
go.opentelemetry.io/collector/config/configopaque v1.10.0 // indirect
go.opentelemetry.io/collector/config/configtelemetry v0.103.0 // indirect
go.opentelemetry.io/collector/config/configtls v0.103.0 // indirect
go.opentelemetry.io/collector/config/internal v0.103.0 // indirect
go.opentelemetry.io/collector/confmap v0.103.0 // indirect
go.opentelemetry.io/collector/extension v0.103.0 // indirect
go.opentelemetry.io/collector/extension/auth v0.103.0 // indirect
go.opentelemetry.io/collector/featuregate v1.10.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.52.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 // indirect
go.opentelemetry.io/otel v1.27.0 // indirect
go.opentelemetry.io/otel/metric v1.27.0 // indirect
go.opentelemetry.io/otel/trace v1.27.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/arch v0.7.0 // indirect
golang.org/x/crypto v0.23.0 // indirect
golang.org/x/crypto v0.24.0 // indirect
golang.org/x/mod v0.17.0 // indirect
golang.org/x/sys v0.20.0 // indirect
golang.org/x/text v0.15.0 // indirect
golang.org/x/sys v0.21.0 // indirect
golang.org/x/text v0.16.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/tools v0.21.0 // indirect
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240520151616-dc85e6b867a5 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)
Expand Down
Loading