From 56604aec9369fad26641c0d3194672779f807a54 Mon Sep 17 00:00:00 2001 From: CloudQuery Bot <102256036+cq-bot@users.noreply.github.com> Date: Fri, 23 Dec 2022 13:34:15 +0200 Subject: [PATCH 1/8] chore(main): Release plugins-destination-mongodb v1.0.0 --- .release-please-manifest.json | 3 ++- plugins/destination/mongodb/CHANGELOG.md | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index fa8b11bc150dfd..3862eb98a78a71 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -47,5 +47,6 @@ "plugins/source/slack+FILLER": "0.0.0", "plugins/source/crowdstrike": "1.0.0", "plugins/source/crowdstrike+FILLER": "0.0.0", - "plugins/source/pagerduty": "1.1.0" + "plugins/source/pagerduty": "1.1.0", + "plugins/destination/mongodb": "1.0.0" } diff --git a/plugins/destination/mongodb/CHANGELOG.md b/plugins/destination/mongodb/CHANGELOG.md index 4dc68c6ff8e4ba..bbab8953ffb472 100644 --- a/plugins/destination/mongodb/CHANGELOG.md +++ b/plugins/destination/mongodb/CHANGELOG.md @@ -1,2 +1,10 @@ # Changelog +## 1.0.0 (2022-12-23) + + +### Features + +* MongoDB destination ([#5930](https://github.com/cloudquery/cloudquery/issues/5930)) ([e9ef1f5](https://github.com/cloudquery/cloudquery/commit/e9ef1f5f334874b6b6d47c6ad30b7cca46613018)) + +## Changelog From 7639dcc81b0c4915a91644f094650dd549f39621 Mon Sep 17 00:00:00 2001 From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com> Date: Fri, 23 Dec 2022 13:50:37 +0200 Subject: [PATCH 2/8] fix mongodb website --- website/pages/docs/plugins/destinations/overview.mdx | 1 + website/versions/destination-mongodb.json | 1 + 2 files changed, 2 insertions(+) create mode 100644 website/versions/destination-mongodb.json diff --git a/website/pages/docs/plugins/destinations/overview.mdx b/website/pages/docs/plugins/destinations/overview.mdx index 343281249f1723..70179fad2f77e3 100644 --- a/website/pages/docs/plugins/destinations/overview.mdx +++ b/website/pages/docs/plugins/destinations/overview.mdx @@ -16,6 +16,7 @@ This is a list of all official and community destination plugins. [ { name: "BigQuery", stage:"Preview" }, { name: "CSV", stage:"Preview" }, + { name: "MongoDB", stage:"Preview" }, { name: "PostgreSQL", stage:"GA", meta: () => * }, { name: "SQLite", stage:"Preview" }, { name: "Snowflake", stage:"Preview" }, diff --git a/website/versions/destination-mongodb.json b/website/versions/destination-mongodb.json new file mode 100644 index 00000000000000..279059bcb7004d --- /dev/null +++ b/website/versions/destination-mongodb.json @@ -0,0 +1 @@ +{ "latest": "plugins-destination-mongodb-v1.0.0" } From 19c72e17f2ec606775734133e31c6fdeee031792 Mon Sep 17 00:00:00 2001 From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com> Date: Fri, 23 Dec 2022 16:30:53 +0200 Subject: [PATCH 3/8] feat(azure): Add virtual_machine_instance_views --- .../codegen0/internal/azparser/parser.go | 4 + .../source/azure/codegen1/recipes/compute.go | 9 -- plugins/source/azure/codegen2/main.go | 7 +- .../source/azure/codegen2/recipes/compute.go | 12 -- .../azure/codegen2/recipes/compute_e.go | 33 ++++++ .../source/azure/resources/plugin/tables.go | 2 +- .../compute/virtual_machine_instance_views.go | 103 ++++++++++++++++++ .../virtual_machine_instance_views_fetch.go | 29 +++++ ...irtual_machine_instance_views_mock_test.go | 31 ++++++ .../services/compute/virtual_machines.go | 4 + .../compute/virtual_machines_mock_test.go | 1 + 11 files changed, 211 insertions(+), 24 deletions(-) create mode 100644 plugins/source/azure/resources/services/compute/virtual_machine_instance_views.go create mode 100644 plugins/source/azure/resources/services/compute/virtual_machine_instance_views_fetch.go create mode 100644 plugins/source/azure/resources/services/compute/virtual_machine_instance_views_mock_test.go diff --git a/plugins/source/azure/codegen0/internal/azparser/parser.go b/plugins/source/azure/codegen0/internal/azparser/parser.go index c4a445992667f8..d82ff2353db6ed 100644 --- a/plugins/source/azure/codegen0/internal/azparser/parser.go +++ b/plugins/source/azure/codegen0/internal/azparser/parser.go @@ -33,6 +33,10 @@ var newFuncToSkipPerPackage = map[string]map[string]bool{ "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/sql/armsql": { "NewDeletedServersClient": true, }, + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute": { + // we migrated this to manual written client as it has childs + "NewVirtualMachinesClient": true, + }, "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/cosmos/armcosmos": { "NewDatabaseAccountsClient": true, }, diff --git a/plugins/source/azure/codegen1/recipes/compute.go b/plugins/source/azure/codegen1/recipes/compute.go index 07c1d2d58d6f5d..b62736b2298118 100644 --- a/plugins/source/azure/codegen1/recipes/compute.go +++ b/plugins/source/azure/codegen1/recipes/compute.go @@ -95,15 +95,6 @@ func Armcompute() []*Table { Pager: `NewListAllPager`, ResponseStruct: "VirtualMachineScaleSetsClientListAllResponse", }, - { - NewFunc: armcompute.NewVirtualMachinesClient, - PkgPath: "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute", - URL: "/subscriptions/{subscriptionId}/providers/Microsoft.Compute/virtualMachines", - Namespace: "microsoft.compute", - Multiplex: `client.SubscriptionMultiplexRegisteredNamespace(client.Namespacemicrosoft_compute)`, - Pager: `NewListAllPager`, - ResponseStruct: "VirtualMachinesClientListAllResponse", - }, } return tables } diff --git a/plugins/source/azure/codegen2/main.go b/plugins/source/azure/codegen2/main.go index 5ff72e017b2d3b..78e8e20790d234 100644 --- a/plugins/source/azure/codegen2/main.go +++ b/plugins/source/azure/codegen2/main.go @@ -145,7 +145,9 @@ func initTable(parent *recipes.Table, r *recipes.Table) error { if r.Relations != nil { for _, relation := range r.Relations { r.Table.Relations = append(r.Table.Relations, relation.Name+"()") - initTable(r, relation) + if err := initTable(r, relation); err != nil { + return err + } } } return nil @@ -162,7 +164,8 @@ func generateTable(parent *recipes.Table, r *recipes.Table) error { } var buff bytes.Buffer if err := tpl.Execute(&buff, r); err != nil { - return fmt.Errorf("failed to execute template for %s: %w", r.Table.Name, err) + return fmt.Errorf("failed to execute template for %w", err) + // return fmt.Errorf("failed to execute template for %s: %w", r.Table.Name, err) } filePath := path.Join(currentDir, "../resources/services", r.PackageName) diff --git a/plugins/source/azure/codegen2/recipes/compute.go b/plugins/source/azure/codegen2/recipes/compute.go index fd1b45d014a178..e1e173518d78b4 100644 --- a/plugins/source/azure/codegen2/recipes/compute.go +++ b/plugins/source/azure/codegen2/recipes/compute.go @@ -113,18 +113,6 @@ func init() { Multiplex: `client.SubscriptionMultiplexRegisteredNamespace(client.Namespacemicrosoft_compute)`, ExtraColumns: DefaultExtraColumns, }, - { - Service: "armcompute", - Name: "virtual_machines", - Struct: &armcompute.VirtualMachine{}, - ResponseStruct: &armcompute.VirtualMachinesClientListAllResponse{}, - Client: &armcompute.VirtualMachinesClient{}, - ListFunc: (&armcompute.VirtualMachinesClient{}).NewListAllPager, - NewFunc: armcompute.NewVirtualMachinesClient, - URL: "/subscriptions/{subscriptionId}/providers/Microsoft.Compute/virtualMachines", - Multiplex: `client.SubscriptionMultiplexRegisteredNamespace(client.Namespacemicrosoft_compute)`, - ExtraColumns: DefaultExtraColumns, - }, } Tables = append(Tables, tables...) } diff --git a/plugins/source/azure/codegen2/recipes/compute_e.go b/plugins/source/azure/codegen2/recipes/compute_e.go index 001ddf23fb13d0..c30ae78e1c8ea7 100644 --- a/plugins/source/azure/codegen2/recipes/compute_e.go +++ b/plugins/source/azure/codegen2/recipes/compute_e.go @@ -26,6 +26,39 @@ func init() { }, }, }, + { + Service: "armcompute", + Name: "virtual_machines", + Struct: &armcompute.VirtualMachine{}, + ResponseStruct: &armcompute.VirtualMachinesClientListAllResponse{}, + Client: &armcompute.VirtualMachinesClient{}, + ListFunc: (&armcompute.VirtualMachinesClient{}).NewListAllPager, + NewFunc: armcompute.NewVirtualMachinesClient, + URL: "/subscriptions/{subscriptionId}/providers/Microsoft.Compute/virtualMachines", + Multiplex: `client.SubscriptionMultiplexRegisteredNamespace(client.Namespacemicrosoft_compute)`, + ExtraColumns: DefaultExtraColumns, + Relations: []*Table{ + { + Name: "virtual_machine_instance_views", + Service: "armcompute", + Struct: &armcompute.VirtualMachineInstanceView{}, + // ResponseStruct is just a stub for codegen not to fail + ResponseStruct: &armcompute.VirtualMachinesClientInstanceViewResponse{}, + Client: &armcompute.VirtualMachinesClient{}, + ListFunc: (&armcompute.VirtualMachinesClient{}).InstanceView, + NewFunc: armcompute.NewVirtualMachinesClient, + URL: "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Compute/virtualMachines/{vmName}/instanceView", + SkipFetch: true, + ExtraColumns: []codegen.ColumnDefinition{ + { + Name: "id", + Type: schema.TypeString, + Resolver: `schema.PathResolver("ComputerName")`, + }, + }, + }, + }, + }, } Tables = append(Tables, tables...) } diff --git a/plugins/source/azure/resources/plugin/tables.go b/plugins/source/azure/resources/plugin/tables.go index 2be73b48dae2ab..8de8cb3b325ef2 100644 --- a/plugins/source/azure/resources/plugin/tables.go +++ b/plugins/source/azure/resources/plugin/tables.go @@ -143,8 +143,8 @@ func generatedTables() []*schema.Table { compute.RestorePointCollections(), compute.Snapshots(), compute.VirtualMachineScaleSets(), - compute.VirtualMachines(), compute.Skus(), + compute.VirtualMachines(), confluent.MarketplaceAgreements(), connectedvmware.Clusters(), connectedvmware.Datastores(), diff --git a/plugins/source/azure/resources/services/compute/virtual_machine_instance_views.go b/plugins/source/azure/resources/services/compute/virtual_machine_instance_views.go new file mode 100644 index 00000000000000..d275e43cbf1f06 --- /dev/null +++ b/plugins/source/azure/resources/services/compute/virtual_machine_instance_views.go @@ -0,0 +1,103 @@ +// Code generated by codegen2; DO NOT EDIT. +package compute + +import ( + "github.com/cloudquery/plugin-sdk/schema" +) + +func virtual_machine_instance_views() *schema.Table { + return &schema.Table{ + Name: "azure_compute_virtual_machine_instance_views", + Resolver: fetchVirtualMachineInstanceViews, + Columns: []schema.Column{ + { + Name: "id", + Type: schema.TypeString, + Resolver: schema.PathResolver("ComputerName"), + CreationOptions: schema.ColumnCreationOptions{ + PrimaryKey: true, + }, + }, + { + Name: "boot_diagnostics", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("BootDiagnostics"), + }, + { + Name: "computer_name", + Type: schema.TypeString, + Resolver: schema.PathResolver("ComputerName"), + }, + { + Name: "disks", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("Disks"), + }, + { + Name: "extensions", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("Extensions"), + }, + { + Name: "hyper_v_generation", + Type: schema.TypeString, + Resolver: schema.PathResolver("HyperVGeneration"), + }, + { + Name: "maintenance_redeploy_status", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("MaintenanceRedeployStatus"), + }, + { + Name: "os_name", + Type: schema.TypeString, + Resolver: schema.PathResolver("OSName"), + }, + { + Name: "os_version", + Type: schema.TypeString, + Resolver: schema.PathResolver("OSVersion"), + }, + { + Name: "patch_status", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("PatchStatus"), + }, + { + Name: "platform_fault_domain", + Type: schema.TypeInt, + Resolver: schema.PathResolver("PlatformFaultDomain"), + }, + { + Name: "platform_update_domain", + Type: schema.TypeInt, + Resolver: schema.PathResolver("PlatformUpdateDomain"), + }, + { + Name: "rdp_thumb_print", + Type: schema.TypeString, + Resolver: schema.PathResolver("RdpThumbPrint"), + }, + { + Name: "statuses", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("Statuses"), + }, + { + Name: "vm_agent", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("VMAgent"), + }, + { + Name: "assigned_host", + Type: schema.TypeString, + Resolver: schema.PathResolver("AssignedHost"), + }, + { + Name: "vm_health", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("VMHealth"), + }, + }, + } +} diff --git a/plugins/source/azure/resources/services/compute/virtual_machine_instance_views_fetch.go b/plugins/source/azure/resources/services/compute/virtual_machine_instance_views_fetch.go new file mode 100644 index 00000000000000..0671c7ca122bf4 --- /dev/null +++ b/plugins/source/azure/resources/services/compute/virtual_machine_instance_views_fetch.go @@ -0,0 +1,29 @@ +package compute + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" + "github.com/cloudquery/cloudquery/plugins/source/azure/client" + "github.com/cloudquery/plugin-sdk/schema" +) + +func fetchVirtualMachineInstanceViews(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { + cl := meta.(*client.Client) + svc, err := armcompute.NewVirtualMachinesClient(cl.SubscriptionId, cl.Creds, cl.Options) + if err != nil { + return err + } + item := parent.Item.(*armcompute.VirtualMachine) + group, err := client.ParseResourceGroup(*item.ID) + if err != nil { + return err + } + instanceView, err := svc.InstanceView(ctx, group, *item.Name, nil) + if err != nil { + return err + } + // views := []any{instanceView} + res <- instanceView.VirtualMachineInstanceView + return nil +} diff --git a/plugins/source/azure/resources/services/compute/virtual_machine_instance_views_mock_test.go b/plugins/source/azure/resources/services/compute/virtual_machine_instance_views_mock_test.go new file mode 100644 index 00000000000000..3734bbabecc26b --- /dev/null +++ b/plugins/source/azure/resources/services/compute/virtual_machine_instance_views_mock_test.go @@ -0,0 +1,31 @@ +// Code generated by codegen2; DO NOT EDIT. +package compute + +import ( + "encoding/json" + "net/http" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" + "github.com/cloudquery/plugin-sdk/faker" + "github.com/gorilla/mux" +) + +func createVirtualMachineInstanceViews(router *mux.Router) error { + var item armcompute.VirtualMachinesClientInstanceViewResponse + if err := faker.FakeObject(&item); err != nil { + return err + } + + router.HandleFunc("/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Compute/virtualMachines/{vmName}/instanceView", func(w http.ResponseWriter, r *http.Request) { + b, err := json.Marshal(&item) + if err != nil { + http.Error(w, "unable to marshal request: "+err.Error(), http.StatusBadRequest) + return + } + if _, err := w.Write(b); err != nil { + http.Error(w, "failed to write", http.StatusBadRequest) + return + } + }) + return nil +} diff --git a/plugins/source/azure/resources/services/compute/virtual_machines.go b/plugins/source/azure/resources/services/compute/virtual_machines.go index 4d66b8153f7c17..300160a0c28455 100644 --- a/plugins/source/azure/resources/services/compute/virtual_machines.go +++ b/plugins/source/azure/resources/services/compute/virtual_machines.go @@ -78,6 +78,10 @@ func VirtualMachines() *schema.Table { Resolver: schema.PathResolver("Type"), }, }, + + Relations: []*schema.Table{ + virtual_machine_instance_views(), + }, } } diff --git a/plugins/source/azure/resources/services/compute/virtual_machines_mock_test.go b/plugins/source/azure/resources/services/compute/virtual_machines_mock_test.go index 49831c05a24a0f..07211e7353ba02 100644 --- a/plugins/source/azure/resources/services/compute/virtual_machines_mock_test.go +++ b/plugins/source/azure/resources/services/compute/virtual_machines_mock_test.go @@ -32,6 +32,7 @@ func createVirtualMachines(router *mux.Router) error { return } }) + createVirtualMachineInstanceViews(router) return nil } From 7750f05cba209e2c1598204185bc776724bb62fa Mon Sep 17 00:00:00 2001 From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com> Date: Fri, 23 Dec 2022 16:49:07 +0200 Subject: [PATCH 4/8] fmt --- plugins/source/azure/codegen2/recipes/compute_e.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/source/azure/codegen2/recipes/compute_e.go b/plugins/source/azure/codegen2/recipes/compute_e.go index c30ae78e1c8ea7..4a8a94f38f2f16 100644 --- a/plugins/source/azure/codegen2/recipes/compute_e.go +++ b/plugins/source/azure/codegen2/recipes/compute_e.go @@ -39,9 +39,9 @@ func init() { ExtraColumns: DefaultExtraColumns, Relations: []*Table{ { - Name: "virtual_machine_instance_views", - Service: "armcompute", - Struct: &armcompute.VirtualMachineInstanceView{}, + Name: "virtual_machine_instance_views", + Service: "armcompute", + Struct: &armcompute.VirtualMachineInstanceView{}, // ResponseStruct is just a stub for codegen not to fail ResponseStruct: &armcompute.VirtualMachinesClientInstanceViewResponse{}, Client: &armcompute.VirtualMachinesClient{}, From 8190815b3e50bcd859b5eaf98776725014f28014 Mon Sep 17 00:00:00 2001 From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com> Date: Sat, 24 Dec 2022 10:10:13 +0200 Subject: [PATCH 5/8] turn instance_view into column --- .../azure/codegen2/recipes/compute_e.go | 47 ++++---- .../compute/virtual_machine_instance_views.go | 103 ------------------ .../virtual_machine_instance_views_fetch.go | 8 +- ...irtual_machine_instance_views_mock_test.go | 31 ------ .../services/compute/virtual_machines.go | 9 +- .../compute/virtual_machines_mock_test.go | 22 +++- 6 files changed, 55 insertions(+), 165 deletions(-) delete mode 100644 plugins/source/azure/resources/services/compute/virtual_machine_instance_views.go delete mode 100644 plugins/source/azure/resources/services/compute/virtual_machine_instance_views_mock_test.go diff --git a/plugins/source/azure/codegen2/recipes/compute_e.go b/plugins/source/azure/codegen2/recipes/compute_e.go index 4a8a94f38f2f16..0248d5ed1bf0ec 100644 --- a/plugins/source/azure/codegen2/recipes/compute_e.go +++ b/plugins/source/azure/codegen2/recipes/compute_e.go @@ -36,28 +36,37 @@ func init() { NewFunc: armcompute.NewVirtualMachinesClient, URL: "/subscriptions/{subscriptionId}/providers/Microsoft.Compute/virtualMachines", Multiplex: `client.SubscriptionMultiplexRegisteredNamespace(client.Namespacemicrosoft_compute)`, - ExtraColumns: DefaultExtraColumns, - Relations: []*Table{ + SkipMock: true, + ExtraColumns: []codegen.ColumnDefinition{ + SubscriptionIdColumn, { - Name: "virtual_machine_instance_views", - Service: "armcompute", - Struct: &armcompute.VirtualMachineInstanceView{}, - // ResponseStruct is just a stub for codegen not to fail - ResponseStruct: &armcompute.VirtualMachinesClientInstanceViewResponse{}, - Client: &armcompute.VirtualMachinesClient{}, - ListFunc: (&armcompute.VirtualMachinesClient{}).InstanceView, - NewFunc: armcompute.NewVirtualMachinesClient, - URL: "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Compute/virtualMachines/{vmName}/instanceView", - SkipFetch: true, - ExtraColumns: []codegen.ColumnDefinition{ - { - Name: "id", - Type: schema.TypeString, - Resolver: `schema.PathResolver("ComputerName")`, - }, - }, + Name: "instance_view", + Type: schema.TypeJSON, + Resolver: `getInstanceView`, }, }, + // ExtraColumns: DefaultExtraColumns, + // Relations: []*Table{ + // { + // Name: "virtual_machine_instance_views", + // Service: "armcompute", + // Struct: &armcompute.VirtualMachineInstanceView{}, + // // ResponseStruct is just a stub for codegen not to fail + // ResponseStruct: &armcompute.VirtualMachinesClientInstanceViewResponse{}, + // Client: &armcompute.VirtualMachinesClient{}, + // ListFunc: (&armcompute.VirtualMachinesClient{}).InstanceView, + // NewFunc: armcompute.NewVirtualMachinesClient, + // URL: "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Compute/virtualMachines/{vmName}/instanceView", + // SkipFetch: true, + // ExtraColumns: []codegen.ColumnDefinition{ + // { + // Name: "id", + // Type: schema.TypeString, + // Resolver: `schema.PathResolver("ComputerName")`, + // }, + // }, + // }, + // }, }, } Tables = append(Tables, tables...) diff --git a/plugins/source/azure/resources/services/compute/virtual_machine_instance_views.go b/plugins/source/azure/resources/services/compute/virtual_machine_instance_views.go deleted file mode 100644 index d275e43cbf1f06..00000000000000 --- a/plugins/source/azure/resources/services/compute/virtual_machine_instance_views.go +++ /dev/null @@ -1,103 +0,0 @@ -// Code generated by codegen2; DO NOT EDIT. -package compute - -import ( - "github.com/cloudquery/plugin-sdk/schema" -) - -func virtual_machine_instance_views() *schema.Table { - return &schema.Table{ - Name: "azure_compute_virtual_machine_instance_views", - Resolver: fetchVirtualMachineInstanceViews, - Columns: []schema.Column{ - { - Name: "id", - Type: schema.TypeString, - Resolver: schema.PathResolver("ComputerName"), - CreationOptions: schema.ColumnCreationOptions{ - PrimaryKey: true, - }, - }, - { - Name: "boot_diagnostics", - Type: schema.TypeJSON, - Resolver: schema.PathResolver("BootDiagnostics"), - }, - { - Name: "computer_name", - Type: schema.TypeString, - Resolver: schema.PathResolver("ComputerName"), - }, - { - Name: "disks", - Type: schema.TypeJSON, - Resolver: schema.PathResolver("Disks"), - }, - { - Name: "extensions", - Type: schema.TypeJSON, - Resolver: schema.PathResolver("Extensions"), - }, - { - Name: "hyper_v_generation", - Type: schema.TypeString, - Resolver: schema.PathResolver("HyperVGeneration"), - }, - { - Name: "maintenance_redeploy_status", - Type: schema.TypeJSON, - Resolver: schema.PathResolver("MaintenanceRedeployStatus"), - }, - { - Name: "os_name", - Type: schema.TypeString, - Resolver: schema.PathResolver("OSName"), - }, - { - Name: "os_version", - Type: schema.TypeString, - Resolver: schema.PathResolver("OSVersion"), - }, - { - Name: "patch_status", - Type: schema.TypeJSON, - Resolver: schema.PathResolver("PatchStatus"), - }, - { - Name: "platform_fault_domain", - Type: schema.TypeInt, - Resolver: schema.PathResolver("PlatformFaultDomain"), - }, - { - Name: "platform_update_domain", - Type: schema.TypeInt, - Resolver: schema.PathResolver("PlatformUpdateDomain"), - }, - { - Name: "rdp_thumb_print", - Type: schema.TypeString, - Resolver: schema.PathResolver("RdpThumbPrint"), - }, - { - Name: "statuses", - Type: schema.TypeJSON, - Resolver: schema.PathResolver("Statuses"), - }, - { - Name: "vm_agent", - Type: schema.TypeJSON, - Resolver: schema.PathResolver("VMAgent"), - }, - { - Name: "assigned_host", - Type: schema.TypeString, - Resolver: schema.PathResolver("AssignedHost"), - }, - { - Name: "vm_health", - Type: schema.TypeJSON, - Resolver: schema.PathResolver("VMHealth"), - }, - }, - } -} diff --git a/plugins/source/azure/resources/services/compute/virtual_machine_instance_views_fetch.go b/plugins/source/azure/resources/services/compute/virtual_machine_instance_views_fetch.go index 0671c7ca122bf4..0b96df615c6d6c 100644 --- a/plugins/source/azure/resources/services/compute/virtual_machine_instance_views_fetch.go +++ b/plugins/source/azure/resources/services/compute/virtual_machine_instance_views_fetch.go @@ -8,13 +8,13 @@ import ( "github.com/cloudquery/plugin-sdk/schema" ) -func fetchVirtualMachineInstanceViews(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error { +func getInstanceView(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { cl := meta.(*client.Client) svc, err := armcompute.NewVirtualMachinesClient(cl.SubscriptionId, cl.Creds, cl.Options) if err != nil { return err } - item := parent.Item.(*armcompute.VirtualMachine) + item := resource.Item.(*armcompute.VirtualMachine) group, err := client.ParseResourceGroup(*item.ID) if err != nil { return err @@ -23,7 +23,5 @@ func fetchVirtualMachineInstanceViews(ctx context.Context, meta schema.ClientMet if err != nil { return err } - // views := []any{instanceView} - res <- instanceView.VirtualMachineInstanceView - return nil + return resource.Set(c.Name, instanceView.VirtualMachineInstanceView) } diff --git a/plugins/source/azure/resources/services/compute/virtual_machine_instance_views_mock_test.go b/plugins/source/azure/resources/services/compute/virtual_machine_instance_views_mock_test.go deleted file mode 100644 index 3734bbabecc26b..00000000000000 --- a/plugins/source/azure/resources/services/compute/virtual_machine_instance_views_mock_test.go +++ /dev/null @@ -1,31 +0,0 @@ -// Code generated by codegen2; DO NOT EDIT. -package compute - -import ( - "encoding/json" - "net/http" - - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" - "github.com/cloudquery/plugin-sdk/faker" - "github.com/gorilla/mux" -) - -func createVirtualMachineInstanceViews(router *mux.Router) error { - var item armcompute.VirtualMachinesClientInstanceViewResponse - if err := faker.FakeObject(&item); err != nil { - return err - } - - router.HandleFunc("/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Compute/virtualMachines/{vmName}/instanceView", func(w http.ResponseWriter, r *http.Request) { - b, err := json.Marshal(&item) - if err != nil { - http.Error(w, "unable to marshal request: "+err.Error(), http.StatusBadRequest) - return - } - if _, err := w.Write(b); err != nil { - http.Error(w, "failed to write", http.StatusBadRequest) - return - } - }) - return nil -} diff --git a/plugins/source/azure/resources/services/compute/virtual_machines.go b/plugins/source/azure/resources/services/compute/virtual_machines.go index 300160a0c28455..31b83dcc82c3bc 100644 --- a/plugins/source/azure/resources/services/compute/virtual_machines.go +++ b/plugins/source/azure/resources/services/compute/virtual_machines.go @@ -19,6 +19,11 @@ func VirtualMachines() *schema.Table { Type: schema.TypeString, Resolver: client.ResolveAzureSubscription, }, + { + Name: "instance_view", + Type: schema.TypeJSON, + Resolver: getInstanceView, + }, { Name: "location", Type: schema.TypeString, @@ -78,10 +83,6 @@ func VirtualMachines() *schema.Table { Resolver: schema.PathResolver("Type"), }, }, - - Relations: []*schema.Table{ - virtual_machine_instance_views(), - }, } } diff --git a/plugins/source/azure/resources/services/compute/virtual_machines_mock_test.go b/plugins/source/azure/resources/services/compute/virtual_machines_mock_test.go index 07211e7353ba02..d8c865a3d645bf 100644 --- a/plugins/source/azure/resources/services/compute/virtual_machines_mock_test.go +++ b/plugins/source/azure/resources/services/compute/virtual_machines_mock_test.go @@ -1,12 +1,12 @@ -// Code generated by codegen2; DO NOT EDIT. package compute import ( "encoding/json" - "github.com/cloudquery/cloudquery/plugins/source/azure/client" "net/http" "testing" + "github.com/cloudquery/cloudquery/plugins/source/azure/client" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" "github.com/cloudquery/plugin-sdk/faker" "github.com/gorilla/mux" @@ -32,7 +32,23 @@ func createVirtualMachines(router *mux.Router) error { return } }) - createVirtualMachineInstanceViews(router) + + var instanceViewItem armcompute.VirtualMachinesClientInstanceViewResponse + if err := faker.FakeObject(&instanceViewItem); err != nil { + return err + } + // /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Compute/virtualMachines/{vmName}/instanceView + router.HandleFunc("/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Compute/virtualMachines/{vmName}/instanceView", func(w http.ResponseWriter, r *http.Request) { + b, err := json.Marshal(&instanceViewItem) + if err != nil { + http.Error(w, "unable to marshal request: "+err.Error(), http.StatusBadRequest) + return + } + if _, err := w.Write(b); err != nil { + http.Error(w, "failed to write", http.StatusBadRequest) + return + } + }) return nil } From 033b6d9815a97688a69a7af365527361b9bd054c Mon Sep 17 00:00:00 2001 From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com> Date: Sat, 24 Dec 2022 10:53:12 +0200 Subject: [PATCH 6/8] update docs --- plugins/source/azure/codegen2/recipes/compute_e.go | 2 +- plugins/source/azure/docs/tables/README.md | 2 +- .../source/azure/docs/tables/azure_compute_virtual_machines.md | 1 + website/pages/docs/plugins/sources/azure/tables.md | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/plugins/source/azure/codegen2/recipes/compute_e.go b/plugins/source/azure/codegen2/recipes/compute_e.go index 0248d5ed1bf0ec..d58927cc68d1e3 100644 --- a/plugins/source/azure/codegen2/recipes/compute_e.go +++ b/plugins/source/azure/codegen2/recipes/compute_e.go @@ -36,7 +36,7 @@ func init() { NewFunc: armcompute.NewVirtualMachinesClient, URL: "/subscriptions/{subscriptionId}/providers/Microsoft.Compute/virtualMachines", Multiplex: `client.SubscriptionMultiplexRegisteredNamespace(client.Namespacemicrosoft_compute)`, - SkipMock: true, + SkipMock: true, ExtraColumns: []codegen.ColumnDefinition{ SubscriptionIdColumn, { diff --git a/plugins/source/azure/docs/tables/README.md b/plugins/source/azure/docs/tables/README.md index 4e51e8d074bc24..fe234db7cedc7f 100644 --- a/plugins/source/azure/docs/tables/README.md +++ b/plugins/source/azure/docs/tables/README.md @@ -53,8 +53,8 @@ - [azure_compute_restore_point_collections](azure_compute_restore_point_collections.md) - [azure_compute_snapshots](azure_compute_snapshots.md) - [azure_compute_virtual_machine_scale_sets](azure_compute_virtual_machine_scale_sets.md) -- [azure_compute_virtual_machines](azure_compute_virtual_machines.md) - [azure_compute_skus](azure_compute_skus.md) +- [azure_compute_virtual_machines](azure_compute_virtual_machines.md) - [azure_confluent_marketplace_agreements](azure_confluent_marketplace_agreements.md) - [azure_connectedvmware_clusters](azure_connectedvmware_clusters.md) - [azure_connectedvmware_datastores](azure_connectedvmware_datastores.md) diff --git a/plugins/source/azure/docs/tables/azure_compute_virtual_machines.md b/plugins/source/azure/docs/tables/azure_compute_virtual_machines.md index aa2e8b8b752095..61ef7bbf4a3f4e 100644 --- a/plugins/source/azure/docs/tables/azure_compute_virtual_machines.md +++ b/plugins/source/azure/docs/tables/azure_compute_virtual_machines.md @@ -11,6 +11,7 @@ The primary key for this table is **id**. |_cq_id|UUID| |_cq_parent_id|UUID| |subscription_id|String| +|instance_view|JSON| |location|String| |extended_location|JSON| |identity|JSON| diff --git a/website/pages/docs/plugins/sources/azure/tables.md b/website/pages/docs/plugins/sources/azure/tables.md index d8bcb7097ff2d5..697c6a32f7223b 100644 --- a/website/pages/docs/plugins/sources/azure/tables.md +++ b/website/pages/docs/plugins/sources/azure/tables.md @@ -53,8 +53,8 @@ - [azure_compute_restore_point_collections](https://github.com/cloudquery/cloudquery/blob/main/plugins/source/azure/docs/tables/azure_compute_restore_point_collections.md) - [azure_compute_snapshots](https://github.com/cloudquery/cloudquery/blob/main/plugins/source/azure/docs/tables/azure_compute_snapshots.md) - [azure_compute_virtual_machine_scale_sets](https://github.com/cloudquery/cloudquery/blob/main/plugins/source/azure/docs/tables/azure_compute_virtual_machine_scale_sets.md) -- [azure_compute_virtual_machines](https://github.com/cloudquery/cloudquery/blob/main/plugins/source/azure/docs/tables/azure_compute_virtual_machines.md) - [azure_compute_skus](https://github.com/cloudquery/cloudquery/blob/main/plugins/source/azure/docs/tables/azure_compute_skus.md) +- [azure_compute_virtual_machines](https://github.com/cloudquery/cloudquery/blob/main/plugins/source/azure/docs/tables/azure_compute_virtual_machines.md) - [azure_confluent_marketplace_agreements](https://github.com/cloudquery/cloudquery/blob/main/plugins/source/azure/docs/tables/azure_confluent_marketplace_agreements.md) - [azure_connectedvmware_clusters](https://github.com/cloudquery/cloudquery/blob/main/plugins/source/azure/docs/tables/azure_connectedvmware_clusters.md) - [azure_connectedvmware_datastores](https://github.com/cloudquery/cloudquery/blob/main/plugins/source/azure/docs/tables/azure_connectedvmware_datastores.md) From fe4531c0d41c116fd12bedd9a88230469a768a3d Mon Sep 17 00:00:00 2001 From: Yevgeny Pats Date: Sun, 25 Dec 2022 12:27:44 +0200 Subject: [PATCH 7/8] Update plugins/source/azure/codegen2/main.go Co-authored-by: Erez Rokah --- plugins/source/azure/codegen2/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/source/azure/codegen2/main.go b/plugins/source/azure/codegen2/main.go index 78e8e20790d234..9b95ea8bb19988 100644 --- a/plugins/source/azure/codegen2/main.go +++ b/plugins/source/azure/codegen2/main.go @@ -165,7 +165,6 @@ func generateTable(parent *recipes.Table, r *recipes.Table) error { var buff bytes.Buffer if err := tpl.Execute(&buff, r); err != nil { return fmt.Errorf("failed to execute template for %w", err) - // return fmt.Errorf("failed to execute template for %s: %w", r.Table.Name, err) } filePath := path.Join(currentDir, "../resources/services", r.PackageName) From ca8429b8e5a8da9fd5d140a2bcc5a410277345c5 Mon Sep 17 00:00:00 2001 From: Yevgeny Pats Date: Sun, 25 Dec 2022 12:27:52 +0200 Subject: [PATCH 8/8] Update plugins/source/azure/codegen2/recipes/compute_e.go Co-authored-by: Erez Rokah --- .../azure/codegen2/recipes/compute_e.go | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/plugins/source/azure/codegen2/recipes/compute_e.go b/plugins/source/azure/codegen2/recipes/compute_e.go index d58927cc68d1e3..176846d969bd2e 100644 --- a/plugins/source/azure/codegen2/recipes/compute_e.go +++ b/plugins/source/azure/codegen2/recipes/compute_e.go @@ -45,28 +45,6 @@ func init() { Resolver: `getInstanceView`, }, }, - // ExtraColumns: DefaultExtraColumns, - // Relations: []*Table{ - // { - // Name: "virtual_machine_instance_views", - // Service: "armcompute", - // Struct: &armcompute.VirtualMachineInstanceView{}, - // // ResponseStruct is just a stub for codegen not to fail - // ResponseStruct: &armcompute.VirtualMachinesClientInstanceViewResponse{}, - // Client: &armcompute.VirtualMachinesClient{}, - // ListFunc: (&armcompute.VirtualMachinesClient{}).InstanceView, - // NewFunc: armcompute.NewVirtualMachinesClient, - // URL: "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Compute/virtualMachines/{vmName}/instanceView", - // SkipFetch: true, - // ExtraColumns: []codegen.ColumnDefinition{ - // { - // Name: "id", - // Type: schema.TypeString, - // Resolver: `schema.PathResolver("ComputerName")`, - // }, - // }, - // }, - // }, }, } Tables = append(Tables, tables...)