From 4b6f2016056f21ff92b47000d96c361df3302145 Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Tue, 24 Jan 2023 17:18:07 +0000 Subject: [PATCH 1/4] Add Makefile and test workflow to scaffold, and flesh out readme --- scaffold/cmd/source.go | 2 + .../source/.github/workflows/test.yaml.tpl | 43 ++++++++++++++++ scaffold/cmd/templates/source/Makefile.tpl | 16 ++++++ scaffold/cmd/templates/source/README.md.tpl | 50 +++++++++++++++++++ 4 files changed, 111 insertions(+) create mode 100644 scaffold/cmd/templates/source/.github/workflows/test.yaml.tpl create mode 100644 scaffold/cmd/templates/source/Makefile.tpl diff --git a/scaffold/cmd/source.go b/scaffold/cmd/source.go index f974fe14c0ffad..c3d905fc013a7f 100644 --- a/scaffold/cmd/source.go +++ b/scaffold/cmd/source.go @@ -35,9 +35,11 @@ func newCmdScaffoldSource() *cobra.Command { var scaffoldTemplates = map[string]string{ "release.yaml.tpl": ".github/workflows/release.yaml", + "test.yaml.tpl": ".github/workflows/test.yaml", ".goreleaser.yaml.tpl": ".goreleaser.yaml", "go.mod.tpl": "go.mod", "main.go.tpl": "main.go", + "Makefile.tpl": "Makefile", "README.md.tpl": "README.md", "client.go.tpl": "client/client.go", "spec.go.tpl": "client/spec.go", diff --git a/scaffold/cmd/templates/source/.github/workflows/test.yaml.tpl b/scaffold/cmd/templates/source/.github/workflows/test.yaml.tpl new file mode 100644 index 00000000000000..4cd03779fb985d --- /dev/null +++ b/scaffold/cmd/templates/source/.github/workflows/test.yaml.tpl @@ -0,0 +1,43 @@ +name: test + +on: + push: + branches: + - main + pull_request: + branches: [ main ] + +jobs: + test: + timeout-minutes: 30 + name: "test" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 2 + - name: Set up Go 1.x + uses: actions/setup-go@v3 + with: + go-version-file: go.mod + cache: true + cache-dependency-path: go.sum + - name: golangci-lint + uses: cloudquery/golangci-lint-action@master + with: + version: v1.50.1 + args: "--config .golangci.yml" + skip-pkg-cache: true + skip-build-cache: true + - name: Get dependencies + run: go get -t -d ./... + - name: Build + run: go build . + - name: Test + run: make test + - name: gen + if: github.event_name == 'pull_request' + run: make gen + - name: Fail if generation updated files + if: github.event_name == 'pull_request' + run: test "$(git status -s | wc -l)" -eq 0 || (git status -s; exit 1) \ No newline at end of file diff --git a/scaffold/cmd/templates/source/Makefile.tpl b/scaffold/cmd/templates/source/Makefile.tpl new file mode 100644 index 00000000000000..cbd3ea0caa1613 --- /dev/null +++ b/scaffold/cmd/templates/source/Makefile.tpl @@ -0,0 +1,16 @@ +.PHONY: test +test: + go test -timeout 3m ./... + +.PHONY: lint +lint: + @golangci-lint run --timeout 10m --verbose + +.PHONY: gen-docs +gen-docs: + rm -rf ./docs/tables/* + go run main.go doc ./docs/tables + +# All gen targets +.PHONY: gen +gen: gen-docs \ No newline at end of file diff --git a/scaffold/cmd/templates/source/README.md.tpl b/scaffold/cmd/templates/source/README.md.tpl index 3af66399a8ff89..888081ac7da4ef 100644 --- a/scaffold/cmd/templates/source/README.md.tpl +++ b/scaffold/cmd/templates/source/README.md.tpl @@ -1 +1,51 @@ # CloudQuery {{.Name}} Source Plugin + +[![test](https://github.com/{{.Org}}/cq-source-{{.Name}}/actions/workflows/test.yaml/badge.svg)](https://github.com/{{.Org}}/cq-source-{{.Name}}/actions/workflows/test.yaml) +[![Go Report Card](https://goreportcard.com/badge/github.com/{{.Org}}/cq-source-{{.Name}})](https://goreportcard.com/report/github.com/{{.Org}}/cq-source-{{.Name}}) + +A {{.Name}} source plugin for CloudQuery that loads data from {{.Name}} to any database, data warehouse or data lake supported by [CloudQuery](https://www.cloudquery.io/), such as PostgreSQL, BigQuery, Athena, and many more. + +## Links + + - [CloudQuery Quickstart Guide](https://www.cloudquery.io/docs/quickstart) + - [Supported Tables](docs/tables/README.md) + + +## Configuration + +The following source configuration file will sync to a PostgreSQL database. See [the CloudQuery Quickstart](https://www.cloudquery.io/docs/quickstart) for more information on how to configure the source and destination. + +```yaml +kind: source +spec: + name: "{{.Name}}" + path: "{{.Org}}/{{.Name}}" + version: "${VERSION}" + backend: "local" # remove this to disable incremental syncing + tables: + ["*"] + destinations: + - "postgresql" + spec: + # plugin spec section +``` + +## Development + +### Run tests + +```bash +make test +``` + +### Run linter + +```bash +make lint +``` + +### Generate docs + +```bash +make gen-docs +``` \ No newline at end of file From 5dad96b1c77192055698a643410c695ff01bd647 Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Wed, 25 Jan 2023 08:56:11 +0000 Subject: [PATCH 2/4] Use official golangci --- scaffold/cmd/templates/source/.github/workflows/test.yaml.tpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scaffold/cmd/templates/source/.github/workflows/test.yaml.tpl b/scaffold/cmd/templates/source/.github/workflows/test.yaml.tpl index 4cd03779fb985d..7106a4e7a03df5 100644 --- a/scaffold/cmd/templates/source/.github/workflows/test.yaml.tpl +++ b/scaffold/cmd/templates/source/.github/workflows/test.yaml.tpl @@ -23,7 +23,7 @@ jobs: cache: true cache-dependency-path: go.sum - name: golangci-lint - uses: cloudquery/golangci-lint-action@master + uses: golangci/golangci-lint-action@v3 with: version: v1.50.1 args: "--config .golangci.yml" @@ -40,4 +40,4 @@ jobs: run: make gen - name: Fail if generation updated files if: github.event_name == 'pull_request' - run: test "$(git status -s | wc -l)" -eq 0 || (git status -s; exit 1) \ No newline at end of file + run: test "$(git status -s | wc -l)" -eq 0 || (git status -s; exit 1) From 399470a59a90a16973a36ce326c5a10bd3b3b11a Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Wed, 25 Jan 2023 08:57:55 +0000 Subject: [PATCH 3/4] Apply suggestions from code review Co-authored-by: Erez Rokah --- scaffold/cmd/templates/source/Makefile.tpl | 2 +- scaffold/cmd/templates/source/README.md.tpl | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/scaffold/cmd/templates/source/Makefile.tpl b/scaffold/cmd/templates/source/Makefile.tpl index cbd3ea0caa1613..963c91fd8f7ecb 100644 --- a/scaffold/cmd/templates/source/Makefile.tpl +++ b/scaffold/cmd/templates/source/Makefile.tpl @@ -4,7 +4,7 @@ test: .PHONY: lint lint: - @golangci-lint run --timeout 10m --verbose + @golangci-lint run --timeout 10m .PHONY: gen-docs gen-docs: diff --git a/scaffold/cmd/templates/source/README.md.tpl b/scaffold/cmd/templates/source/README.md.tpl index 888081ac7da4ef..1975c951dc0c0f 100644 --- a/scaffold/cmd/templates/source/README.md.tpl +++ b/scaffold/cmd/templates/source/README.md.tpl @@ -21,9 +21,6 @@ spec: name: "{{.Name}}" path: "{{.Org}}/{{.Name}}" version: "${VERSION}" - backend: "local" # remove this to disable incremental syncing - tables: - ["*"] destinations: - "postgresql" spec: From 8cc25571c2f9a545ffea82e54ebe0e8d279c62b3 Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Wed, 25 Jan 2023 08:58:14 +0000 Subject: [PATCH 4/4] Apply suggestions from code review Co-authored-by: Erez Rokah --- .../cmd/templates/source/.github/workflows/test.yaml.tpl | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/scaffold/cmd/templates/source/.github/workflows/test.yaml.tpl b/scaffold/cmd/templates/source/.github/workflows/test.yaml.tpl index 7106a4e7a03df5..f0a9a195d593b4 100644 --- a/scaffold/cmd/templates/source/.github/workflows/test.yaml.tpl +++ b/scaffold/cmd/templates/source/.github/workflows/test.yaml.tpl @@ -5,12 +5,11 @@ on: branches: - main pull_request: - branches: [ main ] + branches: [main] jobs: test: timeout-minutes: 30 - name: "test" runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -21,14 +20,10 @@ jobs: with: go-version-file: go.mod cache: true - cache-dependency-path: go.sum - name: golangci-lint uses: golangci/golangci-lint-action@v3 with: version: v1.50.1 - args: "--config .golangci.yml" - skip-pkg-cache: true - skip-build-cache: true - name: Get dependencies run: go get -t -d ./... - name: Build