From 05f1988bec7eb6ec3f1ea9c1d5961bc8180a0938 Mon Sep 17 00:00:00 2001 From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com> Date: Sat, 17 Dec 2022 19:37:28 +0200 Subject: [PATCH 1/3] fix: Add better logging/metric per table --- plugins/source/scheduler_dfs.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/plugins/source/scheduler_dfs.go b/plugins/source/scheduler_dfs.go index 42961b1c81..20b2fdac42 100644 --- a/plugins/source/scheduler_dfs.go +++ b/plugins/source/scheduler_dfs.go @@ -74,15 +74,24 @@ func (p *Plugin) syncDfs(ctx context.Context, spec specs.Source, client schema.C wg.Wait() } +func (p *Plugin) logTablesMetrics(tables schema.Tables, client schema.ClientMeta) { + clientName := client.ID() + for _, table := range tables { + metrics := p.metrics.TableClient[table.Name][clientName] + p.logger.Info().Str("table", table.Name).Str("client", clientName).Uint64("resources", metrics.Resources).Uint64("errors", metrics.Errors).Msg("table sync finished") + p.logTablesMetrics(table.Relations, client) + } +} + func (p *Plugin) resolveTableDfs(ctx context.Context, table *schema.Table, client schema.ClientMeta, parent *schema.Resource, resolvedResources chan<- *schema.Resource, depth int) { clientName := client.ID() logger := p.logger.With().Str("table", table.Name).Str("client", clientName).Logger() if parent == nil { // Log only for root tables, otherwise we spam too much. - logger.Info().Msg("table resolver started") + logger.Info().Msg("root table resolver started") } - tableMetrics := p.metrics.TableClient[table.Name][clientName] + res := make(chan interface{}) go func() { defer func() { @@ -109,8 +118,8 @@ func (p *Plugin) resolveTableDfs(ctx context.Context, table *schema.Table, clien } // we don't need any waitgroups here because we are waiting for the channel to close - if parent == nil { // Log only for root tables, otherwise we spam too much. - logger.Info().Uint64("resources", tableMetrics.Resources).Uint64("errors", tableMetrics.Errors).Msg("fetch table finished") + if parent == nil { // Log only for root tables and relations only after resolving is done, otherwise we spam per object instead of per table. + logger.Info().Uint64("resources", tableMetrics.Resources).Uint64("errors", tableMetrics.Errors).Msg("table sync finished") } } From d3eb8c6f28babedbd028947175d50dbd7d8152aa Mon Sep 17 00:00:00 2001 From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com> Date: Sat, 17 Dec 2022 19:45:59 +0200 Subject: [PATCH 2/3] call fmt --- plugins/source/scheduler_dfs.go | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/source/scheduler_dfs.go b/plugins/source/scheduler_dfs.go index 20b2fdac42..d171cb520c 100644 --- a/plugins/source/scheduler_dfs.go +++ b/plugins/source/scheduler_dfs.go @@ -120,6 +120,7 @@ func (p *Plugin) resolveTableDfs(ctx context.Context, table *schema.Table, clien // we don't need any waitgroups here because we are waiting for the channel to close if parent == nil { // Log only for root tables and relations only after resolving is done, otherwise we spam per object instead of per table. logger.Info().Uint64("resources", tableMetrics.Resources).Uint64("errors", tableMetrics.Errors).Msg("table sync finished") + p.logTablesMetrics(table.Relations, client) } } From 55af7e3acee5d143445e383673270ea540906bee Mon Sep 17 00:00:00 2001 From: Yevgeny Pats Date: Sun, 18 Dec 2022 09:45:16 +0200 Subject: [PATCH 3/3] Update plugins/source_scheduler_dfs.go Co-authored-by: Erez Rokah --- plugins/source/scheduler_dfs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/source/scheduler_dfs.go b/plugins/source/scheduler_dfs.go index d171cb520c..e0b44a142a 100644 --- a/plugins/source/scheduler_dfs.go +++ b/plugins/source/scheduler_dfs.go @@ -88,7 +88,7 @@ func (p *Plugin) resolveTableDfs(ctx context.Context, table *schema.Table, clien logger := p.logger.With().Str("table", table.Name).Str("client", clientName).Logger() if parent == nil { // Log only for root tables, otherwise we spam too much. - logger.Info().Msg("root table resolver started") + logger.Info().Msg("top level table resolver started") } tableMetrics := p.metrics.TableClient[table.Name][clientName]