cloudquery · kodiakhq · Jan 17, 2023 · Jan 16, 2023 · Jan 17, 2023 · Jan 17, 2023
@@ -0,0 +1,85 @@
+name: Source Plugin Alibaba Cloud Workflow
+
+on:
+  pull_request:
+    paths:
+      - "plugins/source/alicloud/**"
+      - ".github/workflows/source_alicloud.yml"
+  push:
+    branches:
+      - main
+    paths:
+      - "plugins/source/alicloud/**"
+      - ".github/workflows/source_alicloud.yml"
+
+jobs:
+  plugins-source-alicloud:
+    timeout-minutes: 30
+    name: "plugins/source/alicloud"
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./plugins/source/alicloud
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v3
+        with:
+          go-version-file: plugins/source/alicloud/go.mod
+          cache: true
+          cache-dependency-path: plugins/source/alicloud/go.sum
+      - name: golangci-lint
+        uses: cloudquery/golangci-lint-action@master
+        with:
+          version: v1.50.1
+          working-directory: plugins/source/alicloud
+          args: "--config ../../.golangci.yml"
+      - name: Get dependencies
+        run: go get -t -d ./...
+      - name: Build
+        run: go build .
+      - name: Test
+        run: make test
+      - name: gen
+        if: github.event_name == 'pull_request'
+        run: make gen
+      - name: Fail if generation updated files
+        if: github.event_name == 'pull_request'
+        run: test "$(git status -s | wc -l)" -eq 0 || (git status -s; exit 1)
+  validate-release:
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    env:
+      CGO_ENABLED: 0
+    steps:
+      - name: Checkout
+        if: startsWith(github.head_ref, 'release-please--branches--main--components') || github.event_name == 'push'
+        uses: actions/checkout@v3
+      - uses: actions/cache@v3
+        if: startsWith(github.head_ref, 'release-please--branches--main--components') || github.event_name == 'push'
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-1.19.4-release-cache-${{ hashFiles('plugins/source/alicloud/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-1.19.4-release-cache-plugins-source-alicloud
+      - name: Set up Go
+        if: startsWith(github.head_ref, 'release-please--branches--main--components') || github.event_name == 'push'
+        uses: actions/setup-go@v3
+        with:
+          go-version-file: plugins/source/alicloud/go.mod
+      - name: Install GoReleaser
+        if: startsWith(github.head_ref, 'release-please--branches--main--components') || github.event_name == 'push'
+        uses: goreleaser/goreleaser-action@v3
+        with:
+          distribution: goreleaser-pro
+          version: latest
+          install-only: true
+      - name: Run GoReleaser Dry-Run
+        if: startsWith(github.head_ref, 'release-please--branches--main--components') || github.event_name == 'push'
+        run: goreleaser release --snapshot --rm-dist --skip-validate --skip-publish --skip-sign -f ./plugins/source/alicloud/.goreleaser.yaml
+        env:
+          GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}
@@ -41,6 +41,7 @@ jobs:
             }
             let actions = ["cli",
                            "scaffold",
+                           "plugins/source/alicloud",
                            "plugins/source/aws",
                            "plugins/source/azure",
                            "plugins/source/azuredevops",

@@ -30,6 +30,7 @@ gen-mocks:
 gen-docs:
 	rm -rf ./docs/tables/*
 	go run main.go doc ./docs/tables
+	sed 's_(\(.*\))_(https://github.com/cloudquery/cloudquery/blob/main/plugins/source/alicloud/docs/tables/\1)_' docs/tables/README.md > ../../../website/pages/docs/plugins/sources/alicloud/tables.md
 
 .PHONY: build
 build:

@@ -2,8 +2,10 @@ package client
 
 import (
 	"context"
+	"time"
 
 	"github.com/cloudquery/plugin-sdk/schema"
+	"github.com/thoas/go-funk"
 )
 
 func ResolveAccount(_ context.Context, meta schema.ClientMeta, r *schema.Resource, _ schema.Column) error {
@@ -15,3 +17,17 @@ func ResolveRegion(_ context.Context, meta schema.ClientMeta, r *schema.Resource
 	client := meta.(*Client)
 	return r.Set("region", client.Region)
 }
+
+func TimestampResolver(layout, path string) schema.ColumnResolver {
+	return func(_ context.Context, meta schema.ClientMeta, r *schema.Resource, c schema.Column) error {
+		s := funk.Get(r.Item, path, funk.WithAllowZero()).(string)
+		if s == "" {
+			return r.Set(c.Name, nil)
+		}
+		t, err := time.Parse(layout, s)
+		if err != nil {
+			return err
+		}
+		return r.Set(c.Name, t)
+	}
+}
diff --git a/plugins/source/alicloud/docs/tables/alicloud_ecs_instances.md b/plugins/source/alicloud/docs/tables/alicloud_ecs_instances.md
@@ -16,12 +16,12 @@ The composite primary key for this table is (**account_id**, **region_id**, **in
 |hostname|String|
 |image_id|String|
 |instance_type|String|
-|auto_release_time|String|
-|last_invoked_time|String|
+|auto_release_time|Timestamp|
+|last_invoked_time|Timestamp|
 |os_type|String|
 |device_available|Bool|
 |instance_network_type|String|
-|registration_time|String|
+|registration_time|Timestamp|
 |local_storage_amount|Int|
 |network_type|String|
 |intranet_ip|String|
@@ -37,7 +37,7 @@ The composite primary key for this table is (**account_id**, **region_id**, **in
 |gpu_amount|Int|
 |connected|Bool|
 |invocation_count|Int|
-|start_time|String|
+|start_time|Timestamp|
 |zone_id|String|
 |internet_max_bandwidth_in|Int|
 |internet_charge_type|String|
@@ -62,10 +62,10 @@ The composite primary key for this table is (**account_id**, **region_id**, **in
 |description|String|
 |recyclable|Bool|
 |sale_cycle|String|
-|expired_time|String|
+|expired_time|Timestamp|
 |internet_ip|String|
 |memory|Int|
-|creation_time|String|
+|creation_time|Timestamp|
 |agent_version|String|
 |key_pair_name|String|
 |hpc_cluster_id|String|

diff --git a/plugins/source/alicloud/docs/tables/alicloud_oss_bucket_stats.md b/plugins/source/alicloud/docs/tables/alicloud_oss_bucket_stats.md
@@ -1,6 +1,6 @@
 # Table: alicloud_oss_bucket_stats
 
-The primary key for this table is **_cq_id**.
+The composite primary key for this table is (**bucket_name**, **account_id**).
 
 ## Relations
 
@@ -12,14 +12,16 @@ This table depends on [alicloud_oss_buckets](alicloud_oss_buckets.md).
 | ------------- | ------------- |
 |_cq_source_name|String|
 |_cq_sync_time|Timestamp|
-|_cq_id (PK)|UUID|
+|_cq_id|UUID|
 |_cq_parent_id|UUID|
+|bucket_name (PK)|String|
+|account_id (PK)|String|
 |xml_name|JSON|
 |storage|Int|
 |object_count|Int|
 |multipart_upload_count|Int|
 |live_channel_count|Int|
-|last_modified_time|Int|
+|last_modified_time|Timestamp|
 |standard_storage|Int|
 |standard_object_count|Int|
 |infrequent_access_storage|Int|

diff --git a/plugins/source/alicloud/docs/tables/alicloud_oss_buckets.md b/plugins/source/alicloud/docs/tables/alicloud_oss_buckets.md
@@ -1,6 +1,6 @@
 # Table: alicloud_oss_buckets
 
-The primary key for this table is **name**.
+The composite primary key for this table is (**account_id**, **name**).
 
 ## Relations
 
@@ -15,6 +15,7 @@ The following tables depend on alicloud_oss_buckets:
 |_cq_sync_time|Timestamp|
 |_cq_id|UUID|
 |_cq_parent_id|UUID|
+|account_id (PK)|String|
 |xml_name|JSON|
 |name (PK)|String|
 |location|String|

@@ -19,7 +19,7 @@ func buildBssBill(t *testing.T, ctrl *gomock.Controller) client.Services {
 	}
 	b.Success = true
 	b.Data.TotalCount = 2
-	mock.EXPECT().QueryBill(gomock.Any()).Times(2).Return(b, nil)
+	mock.EXPECT().QueryBill(gomock.Any()).AnyTimes().Return(b, nil)
 	return client.Services{BSS: mock}
 }
 

@@ -18,7 +18,7 @@ func buildBssBillOverview(t *testing.T, ctrl *gomock.Controller) client.Services
 		t.Fatal(err)
 	}
 	b.Success = true
-	mock.EXPECT().QueryBillOverview(gomock.Any()).Times(1).Return(b, nil)
+	mock.EXPECT().QueryBillOverview(gomock.Any()).AnyTimes().Return(b, nil)
 	return client.Services{BSS: mock}
 }
 

@@ -5,6 +5,9 @@ import (
 	"github.com/cloudquery/cloudquery/plugins/source/alicloud/client"
 	"github.com/cloudquery/plugin-sdk/schema"
 	"github.com/cloudquery/plugin-sdk/transformers"
+
+	"reflect"
+	"strings"
 )
 
 func Instances() *schema.Table {
@@ -18,6 +21,18 @@ func Instances() *schema.Table {
 			transformers.WithPrimaryKeys(
 				"RegionId", "InstanceId",
 			),
+			transformers.WithTypeTransformer(func(f reflect.StructField) (schema.ValueType, error) {
+				if strings.HasSuffix(f.Name, "Time") {
+					return schema.TypeTimestamp, nil
+				}
+				return transformers.DefaultTypeTransformer(f)
+			}),
+			transformers.WithResolverTransformer(func(f reflect.StructField, path string) schema.ColumnResolver {
+				if strings.HasSuffix(f.Name, "Time") {
+					return client.TimestampResolver("2006-01-02T15:04Z", path)
+				}
+				return transformers.DefaultResolverTransformer(f, path)
+			}),
 		),
 		Columns: []schema.Column{
 			{

@@ -22,6 +22,12 @@ func buildEcsInstances(t *testing.T, ctrl *gomock.Controller) client.Services {
 		t.Fatal(err)
 	}
 	b.BaseResponse = fakeSuccessRequest(t)
+	b.Instances.Instance[0].CreationTime = "2020-01-01T01:23Z"
+	b.Instances.Instance[0].StartTime = "2020-01-01T01:23Z"
+	b.Instances.Instance[0].ExpiredTime = "2020-01-01T01:23Z"
+	b.Instances.Instance[0].RegistrationTime = "2020-01-01T01:23Z"
+	b.Instances.Instance[0].AutoReleaseTime = "2020-01-01T01:23Z"
+	b.Instances.Instance[0].LastInvokedTime = "2020-01-01T01:23Z"
 	b.TotalCount = 2
 	mock.EXPECT().DescribeInstances(gomock.Any()).Times(2).Return(b, nil)
 	return client.Services{ECS: mock}

@@ -1,7 +1,10 @@
 package oss
 
 import (
+	"reflect"
+
 	"github.com/aliyun/aliyun-oss-go-sdk/oss"
+	"github.com/cloudquery/cloudquery/plugins/source/alicloud/client"
 	"github.com/cloudquery/plugin-sdk/schema"
 	"github.com/cloudquery/plugin-sdk/transformers"
 )
@@ -12,10 +15,30 @@ func BucketStats() *schema.Table {
 		Resolver: fetchOssBucketStats,
 		Transform: transformers.TransformWithStruct(
 			&oss.BucketStat{},
-			transformers.WithPrimaryKeys(
-				"Name",
-			),
+			transformers.WithTypeTransformer(func(f reflect.StructField) (schema.ValueType, error) {
+				if f.Name == "LastModifiedTime" {
+					return schema.TypeTimestamp, nil
+				}
+				return transformers.DefaultTypeTransformer(f)
+			}),
 		),
-		Columns: []schema.Column{},
+		Columns: []schema.Column{
+			{
+				Name:     "bucket_name",
+				Type:     schema.TypeString,
+				Resolver: schema.ParentColumnResolver("name"),
+				CreationOptions: schema.ColumnCreationOptions{
+					PrimaryKey: true,
+				},
+			},
+			{
+				Name:     "account_id",
+				Type:     schema.TypeString,
+				Resolver: client.ResolveAccount,
+				CreationOptions: schema.ColumnCreationOptions{
+					PrimaryKey: true,
+				},
+			},
+		},
 	}
 }
@@ -18,7 +18,16 @@ func Buckets() *schema.Table {
 				"Name",
 			),
 		),
-		Columns: []schema.Column{},
+		Columns: []schema.Column{
+			{
+				Name:     "account_id",
+				Type:     schema.TypeString,
+				Resolver: client.ResolveAccount,
+				CreationOptions: schema.ColumnCreationOptions{
+					PrimaryKey: true,
+				},
+			},
+		},
 		Relations: []*schema.Table{
 			BucketStats(),
 		},

@@ -17,6 +17,7 @@ func buildOssBuckets(t *testing.T, ctrl *gomock.Controller) client.Services {
 	if err := faker.FakeObject(&b); err != nil {
 		t.Fatal(err)
 	}
+	b.Buckets[0].Location = "cn-hangzhou"
 	mock.EXPECT().ListBuckets().Return(b, nil)
 
 	buildOssBucketStats(t, mock, b.Buckets[0].Name)

@@ -4,6 +4,7 @@
   "packages": {
     "cli": { "component": "cli" },
     "scaffold": { "component": "scaffold" },
+    "plugins/source/alicloud": { "component": "plugins-source-alicloud" },
     "plugins/source/aws": { "component": "plugins-source-aws" },
     "plugins/source/azure": { "component": "plugins-source-azure" },
     "plugins/source/azuredevops": { "component": "plugins-source-azuredevops" },

@@ -1,5 +1,6 @@
 {
   "overview": "Overview",
+  "alicloud": "Alibaba Cloud",
   "aws": "AWS",
   "azure": "Azure",
   "azuredevops": "Azure DevOps",

@@ -0,0 +1,4 @@
+{
+  "overview": "Overview",
+  "tables": "Tables"
+}