From 8e2336da046d72e85abd55f242d792d2134aa517 Mon Sep 17 00:00:00 2001 From: liuyixun Date: Thu, 23 Mar 2023 10:15:08 +0800 Subject: [PATCH 01/17] fix clickhouse migrate_mode: forced error,"table is already exist". add alicloud daily bill table. --- .../destination/clickhouse/queries/tables.go | 2 +- .../alicloud/resources/plugin/plugin.go | 1 + .../resources/services/bss/bill_des.go | 58 +++++++++ .../resources/services/bss/bill_des_fetch.go | 117 ++++++++++++++++++ .../services/bss/bill_des_fetch_mock_test.go | 28 +++++ 5 files changed, 205 insertions(+), 1 deletion(-) create mode 100644 plugins/source/alicloud/resources/services/bss/bill_des.go create mode 100644 plugins/source/alicloud/resources/services/bss/bill_des_fetch.go create mode 100644 plugins/source/alicloud/resources/services/bss/bill_des_fetch_mock_test.go diff --git a/plugins/destination/clickhouse/queries/tables.go b/plugins/destination/clickhouse/queries/tables.go index 1040a6cbba4f38..079dfb6cf7409f 100644 --- a/plugins/destination/clickhouse/queries/tables.go +++ b/plugins/destination/clickhouse/queries/tables.go @@ -19,7 +19,7 @@ func sortKeys(table *schema.Table) []string { func CreateTable(table *schema.Table, cluster string, engine *Engine) string { normalized := normalizeTable(table) strBuilder := strings.Builder{} - strBuilder.WriteString("CREATE TABLE ") + strBuilder.WriteString("CREATE TABLE IF NOT EXISTS") strBuilder.WriteString(tableNamePart(normalized.Name, cluster)) strBuilder.WriteString(" (\n") for i, col := range normalized.Columns { diff --git a/plugins/source/alicloud/resources/plugin/plugin.go b/plugins/source/alicloud/resources/plugin/plugin.go index 23f03d6dc4409d..5bc82e01569e81 100644 --- a/plugins/source/alicloud/resources/plugin/plugin.go +++ b/plugins/source/alicloud/resources/plugin/plugin.go @@ -21,6 +21,7 @@ func Plugin() *source.Plugin { []*schema.Table{ bss.BillOverview(), bss.Bill(), + bss.Bill_Des(), ecs.Instances(), oss.Buckets(), }, diff --git a/plugins/source/alicloud/resources/services/bss/bill_des.go b/plugins/source/alicloud/resources/services/bss/bill_des.go new file mode 100644 index 00000000000000..63464df534e504 --- /dev/null +++ b/plugins/source/alicloud/resources/services/bss/bill_des.go @@ -0,0 +1,58 @@ +package bss + +import ( + "github.com/cloudquery/cloudquery/plugins/source/alicloud/client" + "github.com/cloudquery/plugin-sdk/schema" + "github.com/cloudquery/plugin-sdk/transformers" +) + +func Bill_Des() *schema.Table { + return &schema.Table{ + Name: "alicloud_bss_bill_des", + Resolver: fetchBssDescribeinstanceBill, + Multiplex: client.AccountMultiplex, + Transform: transformers.TransformWithStruct( + &BillDesModel{}, + //transformers.WithPrimaryKeys( + // "BillingCycle", "AccountID", "ProductCode", + // "ProductType", "PipCode", "RecordID", "SubscriptionType", "CommodityCode", + //), + ), + } +} + +type BillDesModel struct { + BillingCycle string `json:"BillingCycle"` + BillingDate string `json:"BillingDate"` + AccountID string `json:"AccountID"` + AccountName string `json:"AccountName"` + ProductName string `json:"ProductName"` + SubOrderId string `json:"SubOrderId"` + DeductedByCashCoupons float64 `json:"DeductedByCashCoupons"` + PaymentTime string `json:"PaymentTime"` + PaymentAmount float64 `json:"PaymentAmount"` + DeductedByPrepaidCard float64 `json:"DeductedByPrepaidCard"` + InvoiceDiscount float64 `json:"InvoiceDiscount"` + UsageEndTime string `json:"UsageEndTime"` + Item string `json:"Item"` + SubscriptionType string `json:"SubscriptionType"` + PretaxGrossAmount float64 `json:"PretaxGrossAmount"` + Currency string `json:"Currency"` + CommodityCode string `json:"CommodityCode"` + UsageStartTime string `json:"UsageStartTime"` + AdjustAmount float64 `json:"AdjustAmount"` + Status string `json:"Status"` + DeductedByCoupons float64 `json:"DeductedByCoupons"` + RoundDownDiscount string `json:"RoundDownDiscount"` + ProductDetail string `json:"ProductDetail"` + ProductCode string `json:"ProductCode"` + ProductType string `json:"ProductType"` + OutstandingAmount float64 `json:"OutstandingAmount"` + PipCode string `json:"PipCode"` + PretaxAmount float64 `json:"PretaxAmount"` + OwnerID string `json:"OwnerID"` + RecordID string `json:"RecordID"` + ResourceGroup string `json:"ResourceGroup"` + InstanceID string `json:"InstanceID"` + CashAmount float64 `json:"CashAmount"` +} diff --git a/plugins/source/alicloud/resources/services/bss/bill_des_fetch.go b/plugins/source/alicloud/resources/services/bss/bill_des_fetch.go new file mode 100644 index 00000000000000..e05195bf89285e --- /dev/null +++ b/plugins/source/alicloud/resources/services/bss/bill_des_fetch.go @@ -0,0 +1,117 @@ +package bss + +import ( + "context" + "fmt" + "github.com/rs/zerolog/log" + "net/http" + "time" + + "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests" + "github.com/aliyun/alibaba-cloud-sdk-go/services/bssopenapi" + "github.com/cloudquery/cloudquery/plugins/source/alicloud/client" + "github.com/cloudquery/plugin-sdk/schema" +) + +/* +* https://help.aliyun.com/document_detail/100392.html + */ +func fetchBssDescribeinstanceBill(_ context.Context, meta schema.ClientMeta, _ *schema.Resource, res chan<- any) error { + c := meta.(*client.Client) + billingCycles := getDesBillingCycles() + if c.Spec.BillHistoryMonths > 0 { + billingCycles = append(getHistoryBillingCycles(c.Spec.BillHistoryMonths), billingCycles...) + } + for _, billingCycle := range billingCycles { + billingDates := getDesBillingDates(billingCycle) + for _, billingDate := range billingDates { + request := bssopenapi.CreateQueryInstanceBillRequest() + log.Info().Str("data", billingDate) + request.BillingCycle = billingCycle + pageNum := 1 + total := 0 + DesmaxLimit := 100 + request.PageNum = requests.NewInteger(pageNum) + request.BillingDate = billingDate + request.PageSize = requests.NewInteger(DesmaxLimit) + request.Granularity = "DAILY" + for { + response, err := c.Services().BSS.QueryInstanceBill(request) + if err != nil { + return err + } + if !response.Success { + code := response.GetHttpStatus() + return fmt.Errorf("got response status code %d (%v)", code, http.StatusText(code)) + } + for _, item := range response.Data.Items.Item { + res <- &BillDesModel{ + BillingCycle: response.Data.BillingCycle, + BillingDate: item.BillingDate, + AccountID: response.Data.AccountID, + AccountName: response.Data.AccountName, + ProductName: item.ProductName, + SubOrderId: item.SubOrderId, + DeductedByCashCoupons: item.DeductedByCashCoupons, + PaymentTime: item.PaymentTime, + PaymentAmount: item.PaymentAmount, + DeductedByPrepaidCard: item.DeductedByPrepaidCard, + InvoiceDiscount: item.InvoiceDiscount, + UsageEndTime: item.UsageEndTime, + Item: item.Item, + SubscriptionType: item.SubscriptionType, + PretaxGrossAmount: item.PretaxGrossAmount, + Currency: item.Currency, + CommodityCode: item.CommodityCode, + UsageStartTime: item.UsageStartTime, + AdjustAmount: item.AdjustAmount, + Status: item.Status, + DeductedByCoupons: item.DeductedByCoupons, + RoundDownDiscount: item.RoundDownDiscount, + ProductDetail: item.ProductDetail, + ProductCode: item.ProductCode, + ProductType: item.ProductType, + OutstandingAmount: item.OutstandingAmount, + PipCode: item.PipCode, + PretaxAmount: item.PretaxAmount, + OwnerID: item.OwnerID, + RecordID: item.RecordID, + ResourceGroup: item.ResourceGroup, + InstanceID: item.InstanceID, + CashAmount: item.CashAmount, + } + + } + total += len(response.Data.Items.Item) + if len(response.Data.Items.Item) == 0 || total >= response.Data.TotalCount { + break + } + pageNum++ + request.PageNum = requests.NewInteger(pageNum) + } + } + } + return nil +} + +func getDesBillingCycles() []string { + var months []string + curMonth := time.Now().Format("2006-01") + months = append(months, curMonth) + lastMonth := time.Now().AddDate(0, 0, -5).Format("2006-01") + if lastMonth != curMonth { + months = append(months, lastMonth) + } + return months +} + +func getDesBillingDates(month string) []string { + var days []string + firstDate := month + "-01" + middle, _ := time.ParseInLocation("2006-01-02", firstDate, time.Local) + totalDate := middle.AddDate(0, 1, -middle.Day()).Day() + for i := 0; i < totalDate; i++ { + days = append(days, middle.AddDate(0, 0, i).Format("2006-01-02")) + } + return days +} diff --git a/plugins/source/alicloud/resources/services/bss/bill_des_fetch_mock_test.go b/plugins/source/alicloud/resources/services/bss/bill_des_fetch_mock_test.go new file mode 100644 index 00000000000000..7c8d6fcaf616ba --- /dev/null +++ b/plugins/source/alicloud/resources/services/bss/bill_des_fetch_mock_test.go @@ -0,0 +1,28 @@ +package bss + +import ( + "testing" + + "github.com/aliyun/alibaba-cloud-sdk-go/services/bssopenapi" + "github.com/cloudquery/cloudquery/plugins/source/alicloud/client" + "github.com/cloudquery/cloudquery/plugins/source/alicloud/client/mocks" + "github.com/cloudquery/plugin-sdk/faker" + "github.com/golang/mock/gomock" +) + +func buildBssDesBill(t *testing.T, ctrl *gomock.Controller) client.Services { + mock := mocks.NewMockBssopenapiClient(ctrl) + + var b *bssopenapi.QueryBillResponse + if err := faker.FakeObject(&b); err != nil { + t.Fatal(err) + } + b.Success = true + b.Data.TotalCount = 2 + mock.EXPECT().QueryBill(gomock.Any()).AnyTimes().Return(b, nil) + return client.Services{BSS: mock} +} + +func TestBssDesBill(t *testing.T) { + client.MockTestHelper(t, Bill(), buildBssDesBill, client.TestOptions{}) +} From 2cfa74a68c16824e99df50368fb0560ec9f61ed9 Mon Sep 17 00:00:00 2001 From: liuyixun Date: Thu, 23 Mar 2023 16:39:08 +0800 Subject: [PATCH 02/17] add alicloud daily bill table. --- plugins/destination/clickhouse/queries/tables.go | 2 +- plugins/source/alicloud/client/spec.go | 1 + plugins/source/alicloud/resources/services/bss/bill_des.go | 7 +++---- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/destination/clickhouse/queries/tables.go b/plugins/destination/clickhouse/queries/tables.go index 079dfb6cf7409f..4e154279f8868a 100644 --- a/plugins/destination/clickhouse/queries/tables.go +++ b/plugins/destination/clickhouse/queries/tables.go @@ -19,7 +19,7 @@ func sortKeys(table *schema.Table) []string { func CreateTable(table *schema.Table, cluster string, engine *Engine) string { normalized := normalizeTable(table) strBuilder := strings.Builder{} - strBuilder.WriteString("CREATE TABLE IF NOT EXISTS") + strBuilder.WriteString("CREATE TABLE") strBuilder.WriteString(tableNamePart(normalized.Name, cluster)) strBuilder.WriteString(" (\n") for i, col := range normalized.Columns { diff --git a/plugins/source/alicloud/client/spec.go b/plugins/source/alicloud/client/spec.go index 738d9238c508c9..78b3d1fdc9c1e2 100644 --- a/plugins/source/alicloud/client/spec.go +++ b/plugins/source/alicloud/client/spec.go @@ -5,6 +5,7 @@ import "fmt" type Spec struct { Accounts []AccountSpec `json:"accounts,omitempty"` BillHistoryMonths int `json:"bill_history_months,omitempty"` + BillHistoryDays int `json:"bill_history_days,omitempty"` } type AccountSpec struct { diff --git a/plugins/source/alicloud/resources/services/bss/bill_des.go b/plugins/source/alicloud/resources/services/bss/bill_des.go index 63464df534e504..e79b491e20cb98 100644 --- a/plugins/source/alicloud/resources/services/bss/bill_des.go +++ b/plugins/source/alicloud/resources/services/bss/bill_des.go @@ -13,10 +13,9 @@ func Bill_Des() *schema.Table { Multiplex: client.AccountMultiplex, Transform: transformers.TransformWithStruct( &BillDesModel{}, - //transformers.WithPrimaryKeys( - // "BillingCycle", "AccountID", "ProductCode", - // "ProductType", "PipCode", "RecordID", "SubscriptionType", "CommodityCode", - //), + transformers.WithPrimaryKeys( + "BillingCycle", + ), ), } } From 1698dff99342f1a2feab2fe6d523852f44cc5b17 Mon Sep 17 00:00:00 2001 From: liuyixun Date: Thu, 23 Mar 2023 17:00:25 +0800 Subject: [PATCH 03/17] fix --- plugins/destination/clickhouse/queries/tables.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/destination/clickhouse/queries/tables.go b/plugins/destination/clickhouse/queries/tables.go index 4e154279f8868a..1040a6cbba4f38 100644 --- a/plugins/destination/clickhouse/queries/tables.go +++ b/plugins/destination/clickhouse/queries/tables.go @@ -19,7 +19,7 @@ func sortKeys(table *schema.Table) []string { func CreateTable(table *schema.Table, cluster string, engine *Engine) string { normalized := normalizeTable(table) strBuilder := strings.Builder{} - strBuilder.WriteString("CREATE TABLE") + strBuilder.WriteString("CREATE TABLE ") strBuilder.WriteString(tableNamePart(normalized.Name, cluster)) strBuilder.WriteString(" (\n") for i, col := range normalized.Columns { From ef97fb15055421f02b3fea7f4c6b6c624615b79d Mon Sep 17 00:00:00 2001 From: easayliu <54569917+easayliu@users.noreply.github.com> Date: Fri, 24 Mar 2023 01:16:25 +0800 Subject: [PATCH 04/17] Update plugins/source/alicloud/resources/services/bss/bill_des_fetch.go Co-authored-by: candiduslynx --- .../source/alicloud/resources/services/bss/bill_des_fetch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/source/alicloud/resources/services/bss/bill_des_fetch.go b/plugins/source/alicloud/resources/services/bss/bill_des_fetch.go index e05195bf89285e..1698fe09dafede 100644 --- a/plugins/source/alicloud/resources/services/bss/bill_des_fetch.go +++ b/plugins/source/alicloud/resources/services/bss/bill_des_fetch.go @@ -30,7 +30,7 @@ func fetchBssDescribeinstanceBill(_ context.Context, meta schema.ClientMeta, _ * request.BillingCycle = billingCycle pageNum := 1 total := 0 - DesmaxLimit := 100 + desmaxLimit := 100 request.PageNum = requests.NewInteger(pageNum) request.BillingDate = billingDate request.PageSize = requests.NewInteger(DesmaxLimit) From 65bfbb35963007c5f1304354462941be837700dd Mon Sep 17 00:00:00 2001 From: easayliu <54569917+easayliu@users.noreply.github.com> Date: Fri, 24 Mar 2023 01:16:38 +0800 Subject: [PATCH 05/17] Update plugins/source/alicloud/resources/services/bss/bill_des_fetch.go Co-authored-by: candiduslynx --- plugins/source/alicloud/resources/services/bss/bill_des_fetch.go | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/source/alicloud/resources/services/bss/bill_des_fetch.go b/plugins/source/alicloud/resources/services/bss/bill_des_fetch.go index 1698fe09dafede..bd37397293ab1c 100644 --- a/plugins/source/alicloud/resources/services/bss/bill_des_fetch.go +++ b/plugins/source/alicloud/resources/services/bss/bill_des_fetch.go @@ -80,7 +80,6 @@ func fetchBssDescribeinstanceBill(_ context.Context, meta schema.ClientMeta, _ * InstanceID: item.InstanceID, CashAmount: item.CashAmount, } - } total += len(response.Data.Items.Item) if len(response.Data.Items.Item) == 0 || total >= response.Data.TotalCount { From 12f4aee949901a1bb7b40953e4a160199cfa4717 Mon Sep 17 00:00:00 2001 From: easayliu <54569917+easayliu@users.noreply.github.com> Date: Thu, 23 Mar 2023 17:29:46 +0000 Subject: [PATCH 06/17] fix: Suggested change --- plugins/source/alicloud/client/spec.go | 1 - plugins/source/alicloud/resources/plugin/plugin.go | 2 +- .../services/bss/{bill_des.go => bill_details.go} | 7 ++++--- .../bss/{bill_des_fetch.go => bill_details_fetch.go} | 9 ++++----- ...etch_mock_test.go => bill_details_fetch_mock_test.go} | 0 5 files changed, 9 insertions(+), 10 deletions(-) rename plugins/source/alicloud/resources/services/bss/{bill_des.go => bill_details.go} (91%) rename plugins/source/alicloud/resources/services/bss/{bill_des_fetch.go => bill_details_fetch.go} (93%) rename plugins/source/alicloud/resources/services/bss/{bill_des_fetch_mock_test.go => bill_details_fetch_mock_test.go} (100%) diff --git a/plugins/source/alicloud/client/spec.go b/plugins/source/alicloud/client/spec.go index 78b3d1fdc9c1e2..738d9238c508c9 100644 --- a/plugins/source/alicloud/client/spec.go +++ b/plugins/source/alicloud/client/spec.go @@ -5,7 +5,6 @@ import "fmt" type Spec struct { Accounts []AccountSpec `json:"accounts,omitempty"` BillHistoryMonths int `json:"bill_history_months,omitempty"` - BillHistoryDays int `json:"bill_history_days,omitempty"` } type AccountSpec struct { diff --git a/plugins/source/alicloud/resources/plugin/plugin.go b/plugins/source/alicloud/resources/plugin/plugin.go index 5bc82e01569e81..2d156d9d61e090 100644 --- a/plugins/source/alicloud/resources/plugin/plugin.go +++ b/plugins/source/alicloud/resources/plugin/plugin.go @@ -21,7 +21,7 @@ func Plugin() *source.Plugin { []*schema.Table{ bss.BillOverview(), bss.Bill(), - bss.Bill_Des(), + bss.BillDetails(), ecs.Instances(), oss.Buckets(), }, diff --git a/plugins/source/alicloud/resources/services/bss/bill_des.go b/plugins/source/alicloud/resources/services/bss/bill_details.go similarity index 91% rename from plugins/source/alicloud/resources/services/bss/bill_des.go rename to plugins/source/alicloud/resources/services/bss/bill_details.go index e79b491e20cb98..aecc3c2becf2f1 100644 --- a/plugins/source/alicloud/resources/services/bss/bill_des.go +++ b/plugins/source/alicloud/resources/services/bss/bill_details.go @@ -6,15 +6,16 @@ import ( "github.com/cloudquery/plugin-sdk/transformers" ) -func Bill_Des() *schema.Table { +func BillDetails() *schema.Table { return &schema.Table{ - Name: "alicloud_bss_bill_des", + Name: "alicloud_bss_bill_details", Resolver: fetchBssDescribeinstanceBill, Multiplex: client.AccountMultiplex, Transform: transformers.TransformWithStruct( &BillDesModel{}, transformers.WithPrimaryKeys( - "BillingCycle", + "BillingCycle", "AccountID", "ProductCode", + "ProductType", "PipCode", "RecordID", "SubscriptionType", "CommodityCode", ), ), } diff --git a/plugins/source/alicloud/resources/services/bss/bill_des_fetch.go b/plugins/source/alicloud/resources/services/bss/bill_details_fetch.go similarity index 93% rename from plugins/source/alicloud/resources/services/bss/bill_des_fetch.go rename to plugins/source/alicloud/resources/services/bss/bill_details_fetch.go index bd37397293ab1c..4a5f702b263b20 100644 --- a/plugins/source/alicloud/resources/services/bss/bill_des_fetch.go +++ b/plugins/source/alicloud/resources/services/bss/bill_details_fetch.go @@ -30,7 +30,7 @@ func fetchBssDescribeinstanceBill(_ context.Context, meta schema.ClientMeta, _ * request.BillingCycle = billingCycle pageNum := 1 total := 0 - desmaxLimit := 100 + DesmaxLimit := 100 request.PageNum = requests.NewInteger(pageNum) request.BillingDate = billingDate request.PageSize = requests.NewInteger(DesmaxLimit) @@ -107,10 +107,9 @@ func getDesBillingCycles() []string { func getDesBillingDates(month string) []string { var days []string firstDate := month + "-01" - middle, _ := time.ParseInLocation("2006-01-02", firstDate, time.Local) - totalDate := middle.AddDate(0, 1, -middle.Day()).Day() - for i := 0; i < totalDate; i++ { - days = append(days, middle.AddDate(0, 0, i).Format("2006-01-02")) + first, _ := time.ParseInLocation("2006-01-02", firstDate, time.Local) + for curr := first; curr.Month() == first.Month(); curr = curr.AddDate(0, 0, 1) { + days = append(days, curr.Format("2006-01-02")) } return days } diff --git a/plugins/source/alicloud/resources/services/bss/bill_des_fetch_mock_test.go b/plugins/source/alicloud/resources/services/bss/bill_details_fetch_mock_test.go similarity index 100% rename from plugins/source/alicloud/resources/services/bss/bill_des_fetch_mock_test.go rename to plugins/source/alicloud/resources/services/bss/bill_details_fetch_mock_test.go From 85bb865254cb225c99d60486140bdbab002a1f42 Mon Sep 17 00:00:00 2001 From: easayliu <54569917+easayliu@users.noreply.github.com> Date: Thu, 23 Mar 2023 17:35:53 +0000 Subject: [PATCH 07/17] change name --- .../resources/services/bss/bill_details.go | 6 +++--- .../resources/services/bss/bill_details_fetch.go | 16 ++++++++-------- .../services/bss/bill_details_fetch_mock_test.go | 6 +++--- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/plugins/source/alicloud/resources/services/bss/bill_details.go b/plugins/source/alicloud/resources/services/bss/bill_details.go index aecc3c2becf2f1..8d473e94f16b4f 100644 --- a/plugins/source/alicloud/resources/services/bss/bill_details.go +++ b/plugins/source/alicloud/resources/services/bss/bill_details.go @@ -9,10 +9,10 @@ import ( func BillDetails() *schema.Table { return &schema.Table{ Name: "alicloud_bss_bill_details", - Resolver: fetchBssDescribeinstanceBill, + Resolver: fetchBillDetails, Multiplex: client.AccountMultiplex, Transform: transformers.TransformWithStruct( - &BillDesModel{}, + &BillDetailsModel{}, transformers.WithPrimaryKeys( "BillingCycle", "AccountID", "ProductCode", "ProductType", "PipCode", "RecordID", "SubscriptionType", "CommodityCode", @@ -21,7 +21,7 @@ func BillDetails() *schema.Table { } } -type BillDesModel struct { +type BillDetailsModel struct { BillingCycle string `json:"BillingCycle"` BillingDate string `json:"BillingDate"` AccountID string `json:"AccountID"` diff --git a/plugins/source/alicloud/resources/services/bss/bill_details_fetch.go b/plugins/source/alicloud/resources/services/bss/bill_details_fetch.go index 4a5f702b263b20..1cf960378f76e4 100644 --- a/plugins/source/alicloud/resources/services/bss/bill_details_fetch.go +++ b/plugins/source/alicloud/resources/services/bss/bill_details_fetch.go @@ -16,24 +16,24 @@ import ( /* * https://help.aliyun.com/document_detail/100392.html */ -func fetchBssDescribeinstanceBill(_ context.Context, meta schema.ClientMeta, _ *schema.Resource, res chan<- any) error { +func fetchBillDetails(_ context.Context, meta schema.ClientMeta, _ *schema.Resource, res chan<- any) error { c := meta.(*client.Client) - billingCycles := getDesBillingCycles() + billingCycles := getDetailsBillingCycles() if c.Spec.BillHistoryMonths > 0 { billingCycles = append(getHistoryBillingCycles(c.Spec.BillHistoryMonths), billingCycles...) } for _, billingCycle := range billingCycles { - billingDates := getDesBillingDates(billingCycle) + billingDates := getDetailsBillingDates(billingCycle) for _, billingDate := range billingDates { request := bssopenapi.CreateQueryInstanceBillRequest() log.Info().Str("data", billingDate) request.BillingCycle = billingCycle pageNum := 1 total := 0 - DesmaxLimit := 100 + DetailsmaxLimit := 100 request.PageNum = requests.NewInteger(pageNum) request.BillingDate = billingDate - request.PageSize = requests.NewInteger(DesmaxLimit) + request.PageSize = requests.NewInteger(DetailsmaxLimit) request.Granularity = "DAILY" for { response, err := c.Services().BSS.QueryInstanceBill(request) @@ -45,7 +45,7 @@ func fetchBssDescribeinstanceBill(_ context.Context, meta schema.ClientMeta, _ * return fmt.Errorf("got response status code %d (%v)", code, http.StatusText(code)) } for _, item := range response.Data.Items.Item { - res <- &BillDesModel{ + res <- &BillDetailsModel{ BillingCycle: response.Data.BillingCycle, BillingDate: item.BillingDate, AccountID: response.Data.AccountID, @@ -93,7 +93,7 @@ func fetchBssDescribeinstanceBill(_ context.Context, meta schema.ClientMeta, _ * return nil } -func getDesBillingCycles() []string { +func getDetailsBillingCycles() []string { var months []string curMonth := time.Now().Format("2006-01") months = append(months, curMonth) @@ -104,7 +104,7 @@ func getDesBillingCycles() []string { return months } -func getDesBillingDates(month string) []string { +func getDetailsBillingDates(month string) []string { var days []string firstDate := month + "-01" first, _ := time.ParseInLocation("2006-01-02", firstDate, time.Local) diff --git a/plugins/source/alicloud/resources/services/bss/bill_details_fetch_mock_test.go b/plugins/source/alicloud/resources/services/bss/bill_details_fetch_mock_test.go index 7c8d6fcaf616ba..926807ecb01a7a 100644 --- a/plugins/source/alicloud/resources/services/bss/bill_details_fetch_mock_test.go +++ b/plugins/source/alicloud/resources/services/bss/bill_details_fetch_mock_test.go @@ -10,7 +10,7 @@ import ( "github.com/golang/mock/gomock" ) -func buildBssDesBill(t *testing.T, ctrl *gomock.Controller) client.Services { +func buildBillDetails(t *testing.T, ctrl *gomock.Controller) client.Services { mock := mocks.NewMockBssopenapiClient(ctrl) var b *bssopenapi.QueryBillResponse @@ -23,6 +23,6 @@ func buildBssDesBill(t *testing.T, ctrl *gomock.Controller) client.Services { return client.Services{BSS: mock} } -func TestBssDesBill(t *testing.T) { - client.MockTestHelper(t, Bill(), buildBssDesBill, client.TestOptions{}) +func TestBillDetails(t *testing.T) { + client.MockTestHelper(t, Bill(), buildBillDetails, client.TestOptions{}) } From 902e08584b84566f535f2489b25518f9305b15ae Mon Sep 17 00:00:00 2001 From: easayliu <54569917+easayliu@users.noreply.github.com> Date: Thu, 23 Mar 2023 17:38:38 +0000 Subject: [PATCH 08/17] rm log --- .../alicloud/resources/services/bss/bill_details_fetch.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/plugins/source/alicloud/resources/services/bss/bill_details_fetch.go b/plugins/source/alicloud/resources/services/bss/bill_details_fetch.go index 1cf960378f76e4..4f4e2ec12f9f0e 100644 --- a/plugins/source/alicloud/resources/services/bss/bill_details_fetch.go +++ b/plugins/source/alicloud/resources/services/bss/bill_details_fetch.go @@ -3,7 +3,6 @@ package bss import ( "context" "fmt" - "github.com/rs/zerolog/log" "net/http" "time" @@ -26,7 +25,6 @@ func fetchBillDetails(_ context.Context, meta schema.ClientMeta, _ *schema.Resou billingDates := getDetailsBillingDates(billingCycle) for _, billingDate := range billingDates { request := bssopenapi.CreateQueryInstanceBillRequest() - log.Info().Str("data", billingDate) request.BillingCycle = billingCycle pageNum := 1 total := 0 From 4e538d8233c2f8c6ddf77a32a6ab5fb2f5a28edd Mon Sep 17 00:00:00 2001 From: easayliu <54569917+easayliu@users.noreply.github.com> Date: Fri, 24 Mar 2023 16:54:34 +0800 Subject: [PATCH 09/17] Update plugins/source/alicloud/resources/services/bss/bill_details_fetch.go Co-authored-by: candiduslynx --- .../alicloud/resources/services/bss/bill_details_fetch.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/source/alicloud/resources/services/bss/bill_details_fetch.go b/plugins/source/alicloud/resources/services/bss/bill_details_fetch.go index 4f4e2ec12f9f0e..93c573b7bfe9a3 100644 --- a/plugins/source/alicloud/resources/services/bss/bill_details_fetch.go +++ b/plugins/source/alicloud/resources/services/bss/bill_details_fetch.go @@ -28,10 +28,10 @@ func fetchBillDetails(_ context.Context, meta schema.ClientMeta, _ *schema.Resou request.BillingCycle = billingCycle pageNum := 1 total := 0 - DetailsmaxLimit := 100 + detailsMaxLimit := 100 request.PageNum = requests.NewInteger(pageNum) request.BillingDate = billingDate - request.PageSize = requests.NewInteger(DetailsmaxLimit) + request.PageSize = requests.NewInteger(detailsMaxLimit) request.Granularity = "DAILY" for { response, err := c.Services().BSS.QueryInstanceBill(request) From b1e5bffe8b3c1b353973c09119ae2b5fe24e46b1 Mon Sep 17 00:00:00 2001 From: easayliu <54569917+easayliu@users.noreply.github.com> Date: Fri, 24 Mar 2023 16:59:13 +0800 Subject: [PATCH 10/17] Update plugins/source/alicloud/resources/services/bss/bill_details_fetch.go copy from bill_fetch.go. u are right Co-authored-by: candiduslynx --- .../alicloud/resources/services/bss/bill_details_fetch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/source/alicloud/resources/services/bss/bill_details_fetch.go b/plugins/source/alicloud/resources/services/bss/bill_details_fetch.go index 93c573b7bfe9a3..23a2e954002ee3 100644 --- a/plugins/source/alicloud/resources/services/bss/bill_details_fetch.go +++ b/plugins/source/alicloud/resources/services/bss/bill_details_fetch.go @@ -95,7 +95,7 @@ func getDetailsBillingCycles() []string { var months []string curMonth := time.Now().Format("2006-01") months = append(months, curMonth) - lastMonth := time.Now().AddDate(0, 0, -5).Format("2006-01") + lastMonth := time.Now().AddDate(0, -1, 0).Format("2006-01") if lastMonth != curMonth { months = append(months, lastMonth) } From 37f8519e61915f77d4c7f61728fc46e6440f3c6c Mon Sep 17 00:00:00 2001 From: liuyixun Date: Fri, 24 Mar 2023 17:11:25 +0800 Subject: [PATCH 11/17] feat(alicloud): add documentation --- plugins/source/alicloud/docs/tables/README.md | 1 + website/pages/docs/plugins/sources/alicloud/tables.md | 1 + 2 files changed, 2 insertions(+) diff --git a/plugins/source/alicloud/docs/tables/README.md b/plugins/source/alicloud/docs/tables/README.md index 79e6018f0e2f3c..9473ed176fc0c5 100644 --- a/plugins/source/alicloud/docs/tables/README.md +++ b/plugins/source/alicloud/docs/tables/README.md @@ -3,6 +3,7 @@ ## Tables - [alicloud_bss_bill](../../../../../website/tables/alicloud/alicloud_bss_bill.md) +- [alicloud_bss_bill_details](../../../../../website/tables/alicloud/alicloud_bss_bill_details.md) - [alicloud_bss_bill_overview](../../../../../website/tables/alicloud/alicloud_bss_bill_overview.md) - [alicloud_ecs_instances](../../../../../website/tables/alicloud/alicloud_ecs_instances.md) - [alicloud_oss_buckets](../../../../../website/tables/alicloud/alicloud_oss_buckets.md) diff --git a/website/pages/docs/plugins/sources/alicloud/tables.md b/website/pages/docs/plugins/sources/alicloud/tables.md index 012c4020f4efe8..854df0ff220a9c 100644 --- a/website/pages/docs/plugins/sources/alicloud/tables.md +++ b/website/pages/docs/plugins/sources/alicloud/tables.md @@ -3,6 +3,7 @@ ## Tables - [alicloud_bss_bill](tables/alicloud_bss_bill) +- [alicloud_bss_bill_details](tables/alicloud_bss_bill_details) - [alicloud_bss_bill_overview](tables/alicloud_bss_bill_overview) - [alicloud_ecs_instances](tables/alicloud_ecs_instances) - [alicloud_oss_buckets](tables/alicloud_oss_buckets) From e4ebac5a7578adea426ff37f200ed66e5547cc1d Mon Sep 17 00:00:00 2001 From: liuyixun Date: Fri, 24 Mar 2023 17:14:06 +0800 Subject: [PATCH 12/17] feat(alicloud): add documentation --- .../alicloud/alicloud_bss_bill_details.md | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 website/tables/alicloud/alicloud_bss_bill_details.md diff --git a/website/tables/alicloud/alicloud_bss_bill_details.md b/website/tables/alicloud/alicloud_bss_bill_details.md new file mode 100644 index 00000000000000..eeb6f458b6df38 --- /dev/null +++ b/website/tables/alicloud/alicloud_bss_bill_details.md @@ -0,0 +1,47 @@ +# Table: alicloud_bss_bill_details + +This table shows data for Alicloud Bss Bill Details. + +The composite primary key for this table is (**billing_cycle**, **account_id**, **subscription_type**, **commodity_code**, **product_code**, **product_type**, **pip_code**, **record_id**). + +## Columns + +| Name | Type | +| ------------- | ------------- | +|_cq_source_name|String| +|_cq_sync_time|Timestamp| +|_cq_id|UUID| +|_cq_parent_id|UUID| +|billing_cycle (PK)|String| +|billing_date|String| +|account_id (PK)|String| +|account_name|String| +|product_name|String| +|sub_order_id|String| +|deducted_by_cash_coupons|Float| +|payment_time|String| +|payment_amount|Float| +|deducted_by_prepaid_card|Float| +|invoice_discount|Float| +|usage_end_time|String| +|item|String| +|subscription_type (PK)|String| +|pretax_gross_amount|Float| +|currency|String| +|commodity_code (PK)|String| +|usage_start_time|String| +|adjust_amount|Float| +|status|String| +|deducted_by_coupons|Float| +|round_down_discount|String| +|product_detail|String| +|product_code (PK)|String| +|product_type (PK)|String| +|outstanding_amount|Float| +|pip_code (PK)|String| +|pretax_amount|Float| +|owner_id|String| +|record_id (PK)|String| +|resource_group|String| +|instance_id|String| +|cash_amount|Float| \ No newline at end of file From e5aa6b38d29b530bc32d8a0ea04f103f5aa81606 Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Fri, 24 Mar 2023 09:38:26 +0000 Subject: [PATCH 13/17] Update title and description --- .../alicloud/resources/services/bss/bill_details.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/plugins/source/alicloud/resources/services/bss/bill_details.go b/plugins/source/alicloud/resources/services/bss/bill_details.go index 8d473e94f16b4f..4b3e7f9988f379 100644 --- a/plugins/source/alicloud/resources/services/bss/bill_details.go +++ b/plugins/source/alicloud/resources/services/bss/bill_details.go @@ -8,9 +8,11 @@ import ( func BillDetails() *schema.Table { return &schema.Table{ - Name: "alicloud_bss_bill_details", - Resolver: fetchBillDetails, - Multiplex: client.AccountMultiplex, + Name: "alicloud_bss_bill_details", + Title: "Alibaba Cloud BSS Bill Details", + Description: "https://help.aliyun.com/document_detail/100392.html", + Resolver: fetchBillDetails, + Multiplex: client.AccountMultiplex, Transform: transformers.TransformWithStruct( &BillDetailsModel{}, transformers.WithPrimaryKeys( From f281a83cf2c624968f9d73b4eb91b529b9e6a8ad Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Fri, 24 Mar 2023 09:39:47 +0000 Subject: [PATCH 14/17] Update docs --- website/tables/alicloud/alicloud_bss_bill_details.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/website/tables/alicloud/alicloud_bss_bill_details.md b/website/tables/alicloud/alicloud_bss_bill_details.md index eeb6f458b6df38..06b3e9f5431f53 100644 --- a/website/tables/alicloud/alicloud_bss_bill_details.md +++ b/website/tables/alicloud/alicloud_bss_bill_details.md @@ -1,6 +1,8 @@ # Table: alicloud_bss_bill_details -This table shows data for Alicloud Bss Bill Details. +This table shows data for Alibaba Cloud BSS Bill Details. + +https://help.aliyun.com/document_detail/100392.html The composite primary key for this table is (**billing_cycle**, **account_id**, **subscription_type**, **commodity_code**, **product_code**, **product_type**, **pip_code**, **record_id**). From 2603033d154be14713a1d2423b509f0eacafea87 Mon Sep 17 00:00:00 2001 From: easayliu <54569917+easayliu@users.noreply.github.com> Date: Mon, 27 Mar 2023 02:24:15 +0000 Subject: [PATCH 15/17] the bill update for 5th of each month. --- .../alicloud/resources/services/bss/bill_details_fetch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/source/alicloud/resources/services/bss/bill_details_fetch.go b/plugins/source/alicloud/resources/services/bss/bill_details_fetch.go index 23a2e954002ee3..93c573b7bfe9a3 100644 --- a/plugins/source/alicloud/resources/services/bss/bill_details_fetch.go +++ b/plugins/source/alicloud/resources/services/bss/bill_details_fetch.go @@ -95,7 +95,7 @@ func getDetailsBillingCycles() []string { var months []string curMonth := time.Now().Format("2006-01") months = append(months, curMonth) - lastMonth := time.Now().AddDate(0, -1, 0).Format("2006-01") + lastMonth := time.Now().AddDate(0, 0, -5).Format("2006-01") if lastMonth != curMonth { months = append(months, lastMonth) } From f14ea5fc0f192b21deb7e1682fcd685900782034 Mon Sep 17 00:00:00 2001 From: easayliu <54569917+easayliu@users.noreply.github.com> Date: Mon, 3 Apr 2023 10:29:38 +0000 Subject: [PATCH 16/17] fix(alicloud): duplicate primary key error --- .../source/alicloud/resources/services/bss/bill_details.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/source/alicloud/resources/services/bss/bill_details.go b/plugins/source/alicloud/resources/services/bss/bill_details.go index 4b3e7f9988f379..d79abd88846a55 100644 --- a/plugins/source/alicloud/resources/services/bss/bill_details.go +++ b/plugins/source/alicloud/resources/services/bss/bill_details.go @@ -16,8 +16,8 @@ func BillDetails() *schema.Table { Transform: transformers.TransformWithStruct( &BillDetailsModel{}, transformers.WithPrimaryKeys( - "BillingCycle", "AccountID", "ProductCode", - "ProductType", "PipCode", "RecordID", "SubscriptionType", "CommodityCode", + "BillingCycle", "BillingDate","AccountID", "ProductCode", + "ProductType", "PipCode", "RecordID", "SubscriptionType", "CommodityCode","InstanceID", ), ), } From ef4a96cc51685d880a30c6cfe8adcf1a3c3757c6 Mon Sep 17 00:00:00 2001 From: liuyixun Date: Tue, 4 Apr 2023 09:21:01 +0800 Subject: [PATCH 17/17] fix(alicloud)!: Fix duplicate primary key in alicloud_bss_bill_details --- .../source/alicloud/resources/services/bss/bill_details.go | 4 ++-- website/tables/alicloud/alicloud_bss_bill_details.md | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/source/alicloud/resources/services/bss/bill_details.go b/plugins/source/alicloud/resources/services/bss/bill_details.go index d79abd88846a55..363be95fc783dc 100644 --- a/plugins/source/alicloud/resources/services/bss/bill_details.go +++ b/plugins/source/alicloud/resources/services/bss/bill_details.go @@ -16,8 +16,8 @@ func BillDetails() *schema.Table { Transform: transformers.TransformWithStruct( &BillDetailsModel{}, transformers.WithPrimaryKeys( - "BillingCycle", "BillingDate","AccountID", "ProductCode", - "ProductType", "PipCode", "RecordID", "SubscriptionType", "CommodityCode","InstanceID", + "BillingCycle", "BillingDate", "AccountID", "ProductCode", + "ProductType", "PipCode", "RecordID", "SubscriptionType", "CommodityCode", "InstanceID", ), ), } diff --git a/website/tables/alicloud/alicloud_bss_bill_details.md b/website/tables/alicloud/alicloud_bss_bill_details.md index 06b3e9f5431f53..2489e8b8e01086 100644 --- a/website/tables/alicloud/alicloud_bss_bill_details.md +++ b/website/tables/alicloud/alicloud_bss_bill_details.md @@ -4,7 +4,7 @@ This table shows data for Alibaba Cloud BSS Bill Details. https://help.aliyun.com/document_detail/100392.html -The composite primary key for this table is (**billing_cycle**, **account_id**, **subscription_type**, **commodity_code**, **product_code**, **product_type**, **pip_code**, **record_id**). +The composite primary key for this table is (**billing_cycle**, **billing_date**, **account_id**, **subscription_type**, **commodity_code**, **product_code**, **product_type**, **pip_code**, **record_id**, **instance_id**). ## Columns @@ -15,7 +15,7 @@ The composite primary key for this table is (**billing_cycle**, **account_id**, |_cq_id|UUID| |_cq_parent_id|UUID| |billing_cycle (PK)|String| -|billing_date|String| +|billing_date (PK)|String| |account_id (PK)|String| |account_name|String| |product_name|String| @@ -45,5 +45,5 @@ The composite primary key for this table is (**billing_cycle**, **account_id**, |owner_id|String| |record_id (PK)|String| |resource_group|String| -|instance_id|String| +|instance_id (PK)|String| |cash_amount|Float| \ No newline at end of file