cloudquery · disq · Apr 7, 2022 · Apr 7, 2022 · Apr 7, 2022 · Apr 7, 2022
@@ -5,15 +5,15 @@ import (
 	"os"
 	"strings"
 
-	"github.com/cloudquery/cloudquery/pkg/client"
-
-	"github.com/thoas/go-funk"
-
 	"github.com/cloudquery/cloudquery/internal/logging"
+	"github.com/cloudquery/cloudquery/pkg/client"
+	"github.com/cloudquery/cloudquery/pkg/ui"
 
 	zerolog "github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
+	"github.com/thoas/go-funk"
 )
 
 // This is copied from https://github.com/spf13/cobra/blob/master/command.go#L491
@@ -163,5 +163,26 @@ func initLogging() {
 	if funk.ContainsString(os.Args, "completion") {
 		return
 	}
+	if !ui.IsTerminal() {
+		loggerConfig.ConsoleLoggingEnabled = true // always true when no terminal
+	}
+
 	zerolog.Logger = logging.Configure(loggerConfig)
 }
+
+func logInvocationParams(cmd *cobra.Command, args []string) {
+	l := zerolog.Info().Str("core_version", client.Version)
+	rootCmd.Flags().Visit(func(f *pflag.Flag) {
+		if f.Name == "dsn" {
+			l = l.Str("pflag:"+f.Name, "(redacted)")
+			return
+		}
+
+		l = l.Str("pflag:"+f.Name, f.Value.String())
+	})
+	cmd.Flags().Visit(func(f *pflag.Flag) {
+		l = l.Str("flag:"+f.Name, f.Value.String())
+	})
+
+	l.Str("command", cmd.CommandPath()).Strs("args", args).Msg("Invocation parameters")
+}
@@ -72,8 +72,9 @@ func handleCommand(f func(context.Context, *console.Client, *cobra.Command, []st
 }
 
 func handleConsole(ctx context.Context, tele *telemetry.Client, cmd *cobra.Command, args []string, f func(context.Context, *console.Client, *cobra.Command, []string) error) error {
-	cfgPath := viper.GetString("configPath")
+	logInvocationParams(cmd, args)
 
+	cfgPath := viper.GetString("configPath")
 	ctx, _ = signalcontext.WithInterrupt(ctx, logging.NewZHcLog(&log.Logger, ""))
 	var c *console.Client
 

@@ -53,7 +53,7 @@ type Config struct {
 func Configure(config Config) zerolog.Logger {
 	var writers []io.Writer
 
-	if config.ConsoleLoggingEnabled || !ui.IsTerminal() {
+	if config.ConsoleLoggingEnabled {
 		if config.EncodeLogsAsJson {
 			writers = append(writers, zerolog.ConsoleWriter{FormatLevel: formatLevel(config.ConsoleNoColor), Out: os.Stdout, NoColor: config.ConsoleNoColor})
 		} else {

@@ -43,7 +43,7 @@ import (
 type Client struct {
 	c       *client.Client
 	cfg     *config.Config
-	updater *Progress
+	updater ui.Progress
 }
 
 func CreateClient(ctx context.Context, configPath string, configMutator func(*config.Config) error, opts ...client.Option) (*Client, error) {
@@ -62,16 +62,17 @@ func CreateClient(ctx context.Context, configPath string, configMutator func(*co
 }
 
 func CreateClientFromConfig(ctx context.Context, cfg *config.Config, opts ...client.Option) (*Client, error) {
-	progressUpdater := NewProgress(ctx, func(o *ProgressOptions) {
-		o.AppendDecorators = []decor.Decorator{decor.Percentage()}
-	})
+	var progressUpdater ui.Progress
+	if ui.DoProgress() {
+		progressUpdater = NewProgress(ctx, func(o *ProgressOptions) {
+			o.AppendDecorators = []decor.Decorator{decor.Percentage()}
+		})
+	}
 	if cfg.CloudQuery.Connection == nil {
 		return nil, errors.New("connection configuration is not set")
 	}
 	opts = append(opts, func(c *client.Client) {
-		if ui.IsTerminal() {
-			c.HubProgressUpdater = progressUpdater
-		}
+		c.HubProgressUpdater = progressUpdater
 		c.Providers = cfg.CloudQuery.Providers
 		c.NoVerify = viper.GetBool("no-verify")
 		c.PluginDirectory = cfg.CloudQuery.PluginDirectory
@@ -92,15 +93,16 @@ func CreateClientFromConfig(ctx context.Context, cfg *config.Config, opts ...cli
 }
 
 func CreateNullClient(ctx context.Context, opts ...client.Option) (*Client, error) {
-	progressUpdater := NewProgress(ctx, func(o *ProgressOptions) {
-		o.AppendDecorators = []decor.Decorator{decor.Percentage()}
-	})
+	var progressUpdater ui.Progress
+	if ui.DoProgress() {
+		progressUpdater = NewProgress(ctx, func(o *ProgressOptions) {
+			o.AppendDecorators = []decor.Decorator{decor.Percentage()}
+		})
+	}
 	opts = append(opts, func(c *client.Client) {
-		if ui.IsTerminal() {
-			c.HubProgressUpdater = progressUpdater
-			c.PluginDirectory = "./.cq/providers"
-			c.PolicyDirectory = "./.cq/policies"
-		}
+		c.HubProgressUpdater = progressUpdater
+		c.PluginDirectory = "./.cq/providers"
+		c.PolicyDirectory = "./.cq/policies"
 	})
 	c, err := client.New(ctx, opts...)
 	if err != nil {
@@ -125,13 +127,11 @@ func (c Client) DownloadProviders(ctx context.Context) error {
 	ui.ColorizedOutput(ui.ColorProgress, "Initializing CloudQuery Providers...\n\n")
 	err := c.c.DownloadProviders(ctx)
 	if err != nil {
-		time.Sleep(100 * time.Millisecond)
+		ui.SleepBeforeError(ctx)
 		ui.ColorizedOutput(ui.ColorError, "❌ Failed to initialize provider: %s.\n\n", err.Error())
 		return err
 	}
-	// sleep some extra 500 milliseconds for progress refresh
-	if ui.IsTerminal() {
-		time.Sleep(500 * time.Millisecond)
+	if c.updater != nil {
 		c.updater.Wait()
 	}
 	ui.ColorizedOutput(ui.ColorProgress, "Finished provider initialization...\n\n")
@@ -156,10 +156,10 @@ func (c Client) Fetch(ctx context.Context, failOnError bool) error {
 		return err
 	}
 	ui.ColorizedOutput(ui.ColorProgress, "Starting provider fetch...\n\n")
-	var fetchProgress *Progress
+	var fetchProgress ui.Progress
 	var fetchCallback client.FetchUpdateCallback
 
-	if ui.IsTerminal() {
+	if ui.DoProgress() {
 		fetchProgress, fetchCallback = buildFetchProgress(ctx, c.cfg.Providers)
 	}
 	request := client.FetchRequest{
@@ -175,11 +175,11 @@ func (c Client) Fetch(ctx context.Context, failOnError bool) error {
 		}
 	}
 
-	if ui.IsTerminal() && fetchProgress != nil {
+	if fetchProgress != nil {
 		fetchProgress.MarkAllDone()
 		fetchProgress.Wait()
-		printFetchResponse(response, viper.GetBool("redact-diags"), viper.GetBool("verbose"))
 	}
+	printFetchResponse(response, viper.GetBool("redact-diags"), viper.GetBool("verbose"))
 
 	if response == nil {
 		ui.ColorizedOutput(ui.ColorProgress, "Provider fetch canceled.\n\n")
@@ -215,13 +215,11 @@ func (c Client) DownloadPolicy(ctx context.Context, args []string) error {
 	ui.ColorizedOutput(ui.ColorProgress, "Downloading CloudQuery Policy...\n")
 	p, err := c.c.LoadPolicy(ctx, "policy", args[0])
 	if err != nil {
-		time.Sleep(100 * time.Millisecond)
+		ui.SleepBeforeError(ctx)
 		ui.ColorizedOutput(ui.ColorError, "❌ Failed to Download policy: %s.\n\n", err.Error())
 		return err
 	}
-	// sleep some extra 300 milliseconds for progress refresh
-	if ui.IsTerminal() {
-		time.Sleep(300 * time.Millisecond)
+	if c.updater != nil {
 		c.updater.Wait()
 	}
 	ui.ColorizedOutput(ui.ColorProgress, "Finished downloading policy...\n")
@@ -251,11 +249,11 @@ func (c Client) RunPolicies(ctx context.Context, policySource, outputDir string,
 
 	ui.ColorizedOutput(ui.ColorProgress, "Starting policies run...\n\n")
 
-	var policyRunProgress *Progress
+	var policyRunProgress ui.Progress
 	var policyRunCallback policy.UpdateCallback
 
 	// if we are running in a terminal, build the progress bar
-	if ui.IsTerminal() {
+	if ui.DoProgress() {
 		policyRunProgress, policyRunCallback = buildPolicyRunProgress(ctx, policiesToRun)
 	}
 	// Policies run request
@@ -266,18 +264,16 @@ func (c Client) RunPolicies(ctx context.Context, policySource, outputDir string,
 	}
 	results, err := c.c.RunPolicies(ctx, req)
 
-	if ui.IsTerminal() && policyRunProgress != nil {
+	if policyRunProgress != nil {
 		policyRunProgress.MarkAllDone()
-		// sleep some extra 500 milliseconds for progress refresh
-		time.Sleep(500 * time.Millisecond)
 		policyRunProgress.Wait()
-		if !noResults {
-			printPolicyResponse(results)
-		}
+	}
+	if !noResults {
+		printPolicyResponse(results)
 	}
 
 	if err != nil {
-		time.Sleep(100 * time.Millisecond)
+		ui.SleepBeforeError(ctx)
 		ui.ColorizedOutput(ui.ColorError, "❌ Failed to run policies: %s.\n\n", err.Error())
 		return err
 	}
@@ -365,7 +361,7 @@ func (c Client) CallModule(ctx context.Context, req ModuleCallRequest) error {
 	}
 	out, err := c.c.ExecuteModule(ctx, runReq)
 	if err != nil {
-		time.Sleep(100 * time.Millisecond)
+		ui.SleepBeforeError(ctx)
 		ui.ColorizedOutput(ui.ColorError, "❌ Failed to execute module: %s.\n\n", err.Error())
 		return err
 	} else if out == nil {

@@ -69,7 +69,7 @@ func getNestedPolicyExample(p *policy.Policy, policyPath string) string {
 }
 
 func printPolicyResponse(results []*policy.ExecutionResult) {
-	if results == nil {
+	if len(results) == 0 {
 		return
 	}
 	for _, execResult := range results {

@@ -147,7 +147,7 @@ func (u *Progress) AttachReader(name string, data io.Reader) io.Reader {
 }
 
 func (u *Progress) Wait() {
-	time.Sleep(100 * time.Millisecond)
+	time.Sleep(600 * time.Millisecond)
 	u.p.Wait()
 	fmt.Println()
 }
@@ -168,7 +168,9 @@ func (u *Progress) AbortAll() {
 	for _, b := range u.bars {
 		b.b.Abort(true)
 	}
-	u.Wait()
+	time.Sleep(100 * time.Millisecond)
+	u.p.Wait()
+	fmt.Println()
 }
 
 func (u *Progress) MarkAllDone() {

@@ -1,9 +1,11 @@
 package ui
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/fatih/color"
 	"github.com/mattn/go-isatty"
@@ -15,21 +17,35 @@ import (
 // ColorizedOutput outputs a colored message directly to the terminal.
 // The remaining arguments should be interpolations for the format string.
 func ColorizedOutput(c *color.Color, msg string, values ...interface{}) {
-	if !IsTerminal() {
+	if viper.GetBool("enable-console-log") {
 		// Print output to log
-		log.Info().Msgf(strings.ReplaceAll(msg, "\n", ""), values...)
+		if logMsg := strings.ReplaceAll(msg, "\n", ""); logMsg != "" {
+			log.Info().Msgf(logMsg, values...)
+		}
 		return
 	}
 	_, _ = c.Printf(msg, values...)
 }
 
 func IsTerminal() bool {
-	if viper.GetBool("enable-console-log") {
-		return false
-	}
 	return isatty.IsTerminal(os.Stdout.Fd()) && term.IsTerminal(int(os.Stdout.Fd()))
 }
 
+func DoProgress() bool {
+	return IsTerminal() && !viper.GetBool("enable-console-log")
+}
+
+func SleepBeforeError(ctx context.Context) {
+	if !IsTerminal() {
+		return
+	}
+
+	select {
+	case <-ctx.Done():
+	case <-time.After(100 * time.Millisecond):
+	}
+}
+
 func Colorize(c *color.Color, noColor bool, msg string, values ...interface{}) string {
 	if noColor {
 		return fmt.Sprintf(msg, values...)