From c0a865179fc0a8e5c955a697fb38ef57bad2264e Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Tue, 8 Mar 2022 15:00:20 +0000 Subject: [PATCH 1/3] feat: Store cq_fetch_id in meta --- go.mod | 2 +- go.sum | 4 ++-- pkg/client/client.go | 10 ++++++++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4b60a6a83c247d..675f435269d67e 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/aws/aws-lambda-go v1.23.0 - github.com/cloudquery/cq-provider-sdk v0.8.6 + github.com/cloudquery/cq-provider-sdk v0.8.7-0.20220308145730-9e5839d9eb28 github.com/fatih/color v1.13.0 github.com/fsnotify/fsnotify v1.4.9 github.com/golang-migrate/migrate/v4 v4.15.0 diff --git a/go.sum b/go.sum index 49bb9d59c484fa..7a174550fd78a8 100644 --- a/go.sum +++ b/go.sum @@ -208,8 +208,8 @@ github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= -github.com/cloudquery/cq-provider-sdk v0.8.6 h1:6W/5S1YhoFisAEB3oHxRAdsdWjtXJH+LdQvlXiUzWkE= -github.com/cloudquery/cq-provider-sdk v0.8.6/go.mod h1:IHxqY7TOttWhNQhMRqYl1vBo2JS2szLAf5Mhg78MwTQ= +github.com/cloudquery/cq-provider-sdk v0.8.7-0.20220308145730-9e5839d9eb28 h1:1zpiSte9rSTvsCfYgQDoNzQdNreronbQVNMMSnm0dOk= +github.com/cloudquery/cq-provider-sdk v0.8.7-0.20220308145730-9e5839d9eb28/go.mod h1:IHxqY7TOttWhNQhMRqYl1vBo2JS2szLAf5Mhg78MwTQ= github.com/cloudquery/faker/v3 v3.7.4 h1:cCcU3r0yHpS0gqKj9rRKAGS0/hY33fBxbqCNFtDD4ec= github.com/cloudquery/faker/v3 v3.7.4/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= diff --git a/pkg/client/client.go b/pkg/client/client.go index 3058e4959be363..d2a66abe601709 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -433,9 +433,18 @@ func (c *Client) Fetch(ctx context.Context, request FetchRequest) (res *FetchRes defer saveFetchSummary() pLog := c.Logger.With("provider", providerConfig.Name, "alias", providerConfig.Alias, "version", providerPlugin.Version()) pLog.Info("requesting provider to configure") + + metadata := map[string]interface{}{ + "cq_fetch_id": fetchId.String(), + } + if c.HistoryCfg != nil { fd := c.HistoryCfg.FetchDate() pLog.Info("history enabled adding fetch date", "fetch_date", fd.Format(time.RFC3339)) + metadata["cq_fetch_date"] = fd + + // TODO Remove(Compatibility): Code below is for providers using the old SDK version, where metadata isn't available in FetchRequest + // Removing this without updating provider will set cq_fetch_date to the time of execution start, which HistoryCfg.TimeTruncation doesn't apply if request.ExtraFields == nil { request.ExtraFields = make(map[string]interface{}) } @@ -464,6 +473,7 @@ func (c *Client) Fetch(ctx context.Context, request FetchRequest) (res *FetchRes PartialFetchingEnabled: true, ParallelFetchingLimit: providerConfig.MaxParallelResourceFetchLimit, MaxGoroutines: providerConfig.MaxGoroutines, + Metadata: metadata, }) if err != nil { return err From eab14e5e24524bea521a4fc9795aefaca9b98f0d Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Tue, 8 Mar 2022 15:06:06 +0000 Subject: [PATCH 2/3] Update SDK --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 675f435269d67e..d2d38766ea6b56 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/aws/aws-lambda-go v1.23.0 - github.com/cloudquery/cq-provider-sdk v0.8.7-0.20220308145730-9e5839d9eb28 + github.com/cloudquery/cq-provider-sdk v0.8.7-0.20220308150447-441c2cfd92ac github.com/fatih/color v1.13.0 github.com/fsnotify/fsnotify v1.4.9 github.com/golang-migrate/migrate/v4 v4.15.0 diff --git a/go.sum b/go.sum index 7a174550fd78a8..6ca37cb5b04806 100644 --- a/go.sum +++ b/go.sum @@ -208,8 +208,8 @@ github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= -github.com/cloudquery/cq-provider-sdk v0.8.7-0.20220308145730-9e5839d9eb28 h1:1zpiSte9rSTvsCfYgQDoNzQdNreronbQVNMMSnm0dOk= -github.com/cloudquery/cq-provider-sdk v0.8.7-0.20220308145730-9e5839d9eb28/go.mod h1:IHxqY7TOttWhNQhMRqYl1vBo2JS2szLAf5Mhg78MwTQ= +github.com/cloudquery/cq-provider-sdk v0.8.7-0.20220308150447-441c2cfd92ac h1:B4oK7Dk5fF4I2/W4MJl9JboJvOuSk+xTIJVls/GScQw= +github.com/cloudquery/cq-provider-sdk v0.8.7-0.20220308150447-441c2cfd92ac/go.mod h1:IHxqY7TOttWhNQhMRqYl1vBo2JS2szLAf5Mhg78MwTQ= github.com/cloudquery/faker/v3 v3.7.4 h1:cCcU3r0yHpS0gqKj9rRKAGS0/hY33fBxbqCNFtDD4ec= github.com/cloudquery/faker/v3 v3.7.4/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= From 74d47a77717554143c1133c521b096c8f5b5202e Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Thu, 10 Mar 2022 12:36:56 +0000 Subject: [PATCH 3/3] update SDK --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d2d38766ea6b56..90c8caff814737 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/aws/aws-lambda-go v1.23.0 - github.com/cloudquery/cq-provider-sdk v0.8.7-0.20220308150447-441c2cfd92ac + github.com/cloudquery/cq-provider-sdk v0.8.7 github.com/fatih/color v1.13.0 github.com/fsnotify/fsnotify v1.4.9 github.com/golang-migrate/migrate/v4 v4.15.0 diff --git a/go.sum b/go.sum index 6ca37cb5b04806..970cb5c392000f 100644 --- a/go.sum +++ b/go.sum @@ -208,8 +208,8 @@ github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= -github.com/cloudquery/cq-provider-sdk v0.8.7-0.20220308150447-441c2cfd92ac h1:B4oK7Dk5fF4I2/W4MJl9JboJvOuSk+xTIJVls/GScQw= -github.com/cloudquery/cq-provider-sdk v0.8.7-0.20220308150447-441c2cfd92ac/go.mod h1:IHxqY7TOttWhNQhMRqYl1vBo2JS2szLAf5Mhg78MwTQ= +github.com/cloudquery/cq-provider-sdk v0.8.7 h1:lVduh/Im+WvMBD/BeHbPfBXdaH4CwSrBKyczYm3GQC0= +github.com/cloudquery/cq-provider-sdk v0.8.7/go.mod h1:IHxqY7TOttWhNQhMRqYl1vBo2JS2szLAf5Mhg78MwTQ= github.com/cloudquery/faker/v3 v3.7.4 h1:cCcU3r0yHpS0gqKj9rRKAGS0/hY33fBxbqCNFtDD4ec= github.com/cloudquery/faker/v3 v3.7.4/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=