diff --git a/pkg/client/client.go b/pkg/client/client.go index 7cc53659e844ee..b07cc1f22c3b99 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -259,9 +259,9 @@ func New(ctx context.Context, options ...Option) (*Client, error) { SkipBuildTables: false, HubProgressUpdater: nil, HistoryCfg: nil, - RegistryURL: registry.CloudQueryRegistryURl, + RegistryURL: registry.CloudQueryRegistryURL, Logger: logging.NewZHcLog(&zerolog.Logger, ""), - Hub: *registry.NewRegistryHub(registry.CloudQueryRegistryURl), + Hub: *registry.NewRegistryHub(registry.CloudQueryRegistryURL), } for _, o := range options { o(c) diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 417f9141ee554a..08a4e2181f310d 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -16,13 +16,10 @@ import ( "github.com/cloudquery/cloudquery/internal/test/provider" "github.com/cloudquery/cloudquery/pkg/config" - "github.com/cloudquery/cloudquery/pkg/plugin/registry" "github.com/cloudquery/cq-provider-sdk/provider/schema" "github.com/cloudquery/cq-provider-sdk/serve" "github.com/fsnotify/fsnotify" "github.com/golang-migrate/migrate/v4" - "github.com/google/go-github/v35/github" - "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-version" "github.com/hashicorp/hcl/v2/hclparse" "github.com/jackc/pgx/v4" @@ -710,100 +707,3 @@ func Test_collectProviderVersions(t *testing.T) { }) } } - -func TestCheckForProviderUpdates(t *testing.T) { - type githubResult struct { - release *github.RepositoryRelease - err error - } - version1 := "1.0.0" - version2 := "2.0.0" - tests := []struct { - name string - providers []*config.RequiredProvider - githubResults []githubResult - want []ProviderUpdateSummary - }{ - { - "empty list of providers", - nil, - nil, - []ProviderUpdateSummary{}, - }, - { - "one provider, github error", - []*config.RequiredProvider{{Name: "test", Version: version1}}, - []githubResult{{nil, errors.New("fake")}}, - []ProviderUpdateSummary{}, - }, - { - "one provider, no update", - []*config.RequiredProvider{{Name: "test", Version: version1}}, - []githubResult{{&github.RepositoryRelease{TagName: &version1}, nil}}, - []ProviderUpdateSummary{}, - }, - { - "latest provider, no update", - []*config.RequiredProvider{{Name: "test", Version: "latest"}}, - []githubResult{{&github.RepositoryRelease{TagName: &version1}, nil}}, - []ProviderUpdateSummary{}, - }, - { - "two providers, one update", - []*config.RequiredProvider{ - {Name: "test", Version: version1}, - {Name: "other", Version: version1}, - }, - []githubResult{ - {&github.RepositoryRelease{TagName: &version1}, nil}, - {&github.RepositoryRelease{TagName: &version2}, nil}, - }, - []ProviderUpdateSummary{ - {Name: "other", Version: version1, LatestVersion: version2}, - }, - }, - { - "three providers, github error, one update", - []*config.RequiredProvider{{Name: "test", Version: version1}, {Name: "other", Version: version1}, {Name: "third", Version: version1}}, - []githubResult{ - {&github.RepositoryRelease{TagName: &version1}, nil}, - {nil, errors.New("fake")}, - {&github.RepositoryRelease{TagName: &version2}, nil}, - }, - []ProviderUpdateSummary{ - {Name: "third", Version: version1, LatestVersion: version2}, - }, - }, - { - "three providers, three updates", - []*config.RequiredProvider{{Name: "test", Version: version1}, {Name: "other", Version: version1}, {Name: "third", Version: version1}}, - []githubResult{ - {&github.RepositoryRelease{TagName: &version2}, nil}, - {&github.RepositoryRelease{TagName: &version2}, nil}, - {&github.RepositoryRelease{TagName: &version2}, nil}, - }, - []ProviderUpdateSummary{ - {Name: "test", Version: version1, LatestVersion: version2}, - {Name: "other", Version: version1, LatestVersion: version2}, - {Name: "third", Version: version1, LatestVersion: version2}, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ctx := context.Background() - getReleaseCall := 0 - c := Client{ - Providers: tt.providers, - Logger: hclog.Default(), - Hub: *registry.NewRegistryHub("", registry.WithLatestReleaseGetter(func(ctx context.Context, owner, repo string) (*github.RepositoryRelease, error) { - r := tt.githubResults[getReleaseCall] - getReleaseCall++ - return r.release, r.err - })), - } - got := c.CheckForProviderUpdates(ctx) - assert.Equal(t, tt.want, got) - }) - } -} diff --git a/pkg/plugin/registry/hub.go b/pkg/plugin/registry/hub.go index 35dad71a00f208..71f2bf0fff530c 100644 --- a/pkg/plugin/registry/hub.go +++ b/pkg/plugin/registry/hub.go @@ -2,8 +2,10 @@ package registry import ( "context" + "encoding/json" "fmt" "net/http" + "net/url" "os" "path/filepath" "runtime" @@ -14,14 +16,13 @@ import ( "github.com/cloudquery/cloudquery/internal/logging" "github.com/cloudquery/cloudquery/pkg/config" "github.com/cloudquery/cloudquery/pkg/ui" - "github.com/google/go-github/v35/github" "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-version" zerolog "github.com/rs/zerolog/log" ) const ( - CloudQueryRegistryURl = "https://firestore.googleapis.com/v1/projects/hub-cloudquery/databases/(default)/documents/orgs/%s/providers/%s" + CloudQueryRegistryURL = "https://firestore.googleapis.com/v1/projects/hub-cloudquery/databases/(default)/documents/orgs/%s/providers/%s" // Timeout for http requests related to CloudQuery providers version check. versionCheckHTTPTimeout = time.Second * 10 @@ -45,26 +46,16 @@ type Hub struct { url string // map of downloaded providers providers map[string]ProviderDetails - - // getLatestRelease is a function that will return latest release from Github repo given organization and repo name. - getLatestRelease LatestReleaseGetter } type Option func(h *Hub) -func WithLatestReleaseGetter(g LatestReleaseGetter) Option { - return func(h *Hub) { - h.getLatestRelease = g - } -} - func NewRegistryHub(url string, opts ...Option) *Hub { h := &Hub{ - PluginDirectory: filepath.Join(".", ".cq", "providers"), - Logger: logging.NewZHcLog(&zerolog.Logger, ""), - url: url, - providers: make(map[string]ProviderDetails), - getLatestRelease: getLatestRelease, + PluginDirectory: filepath.Join(".", ".cq", "providers"), + Logger: logging.NewZHcLog(&zerolog.Logger, ""), + url: url, + providers: make(map[string]ProviderDetails), } // apply the list of options to hub for _, opt := range opts { @@ -170,11 +161,10 @@ func (h Hub) CheckProviderUpdate(ctx context.Context, requestedProvider *config. ctx, cancel := context.WithTimeout(ctx, versionCheckHTTPTimeout) defer cancel() - release, err := h.getLatestRelease(ctx, organization, ProviderRepoName(providerName)) + latestVersion, err := h.getLatestRelease(ctx, organization, ProviderRepoName(providerName)) if err != nil { return "", err } - latestVersion := release.GetTagName() v, err := version.NewVersion(latestVersion) if err != nil { return "", fmt.Errorf("bad version received: provider %s, version %s", providerName, latestVersion) @@ -193,11 +183,10 @@ func (h Hub) DownloadProvider(ctx context.Context, requestedProvider *config.Req } if providerVersion == "latest" { - release, err := h.getRelease(ctx, organization, providerName, providerVersion) + providerVersion, err = h.getLatestRelease(ctx, organization, providerName) if err != nil { return ProviderDetails{}, err } - providerVersion = release.GetTagName() } p, ok := h.providers[fmt.Sprintf("%s-%s", providerName, providerVersion)] if !ok { @@ -269,14 +258,46 @@ func (h Hub) downloadProvider(ctx context.Context, organization, providerName, p return details, nil } -func (h Hub) getRelease(ctx context.Context, organization, providerName, version string) (*github.RepositoryRelease, error) { - client := github.NewClient(nil) - if version != "latest" { - release, _, err := client.Repositories.GetReleaseByTag(ctx, organization, ProviderRepoName(providerName), version) - return release, err +func (h Hub) getLatestRelease(ctx context.Context, organization, providerName string) (string, error) { + versions, err := url.Parse(fmt.Sprintf(h.url+"/versions", organization, providerName)) + if err != nil { + return "", err + } + qv := versions.Query() + qv.Set("pageSize", "1") + qv.Set("orderBy", "v_major desc, v_minor desc, v_patch desc, published_at desc") + qv.Set("mask.fieldPaths", "tag") + versions.RawQuery = qv.Encode() + + hc := &http.Client{Timeout: 15 * time.Second} + req, _ := http.NewRequestWithContext(ctx, http.MethodGet, versions.String(), nil) + res, err := hc.Do(req) + if err != nil { + return "", err } - release, _, err := client.Repositories.GetLatestRelease(ctx, organization, ProviderRepoName(providerName)) - return release, err + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return "", fmt.Errorf("unexpected status code %d", res.StatusCode) + } + + var doc struct { + Documents []struct { + Name string `json:"name"` + Fields struct { + Tag struct { + Val string `json:"stringValue"` + } `json:"tag"` + } `json:"fields"` + } `json:"documents"` + } + if err := json.NewDecoder(res.Body).Decode(&doc); err != nil { + return "", err + } + + if len(doc.Documents) == 0 || doc.Documents[0].Fields.Tag.Val == "" { + return "", fmt.Errorf("failed to find provider %s latest version", providerName) + } + return doc.Documents[0].Fields.Tag.Val, nil } func (h Hub) verifyRegistered(organization, providerName, version string, noVerify bool) bool { @@ -377,11 +398,3 @@ func parseProviderSource(requestedProvider *config.RequiredProvider) (string, st } return ParseProviderName(requestedSource) } - -type LatestReleaseGetter func(ctx context.Context, owner, repo string) (*github.RepositoryRelease, error) - -func getLatestRelease(ctx context.Context, owner, repo string) (*github.RepositoryRelease, error) { - gh := github.NewClient(nil) - r, _, err := gh.Repositories.GetLatestRelease(ctx, owner, repo) - return r, err -} diff --git a/pkg/plugin/registry/hub_test.go b/pkg/plugin/registry/hub_test.go deleted file mode 100644 index d18e4d90437f7c..00000000000000 --- a/pkg/plugin/registry/hub_test.go +++ /dev/null @@ -1,96 +0,0 @@ -package registry - -import ( - "context" - "errors" - "testing" - - "github.com/cloudquery/cloudquery/pkg/config" - "github.com/google/go-github/v35/github" -) - -func TestCheckProviderUpdate(t *testing.T) { - type githubResult struct { - version string - err error - } - tests := []struct { - name string - requestedProvider *config.RequiredProvider - github githubResult - want string - wantErr bool - }{ - { - "bad provider name", - &config.RequiredProvider{Name: "very/strange/name"}, - githubResult{}, - "", - true, - }, - { - "bad local provider version", - &config.RequiredProvider{Name: "test", Version: "bad"}, - githubResult{}, - "", - true, - }, - { - "github returns an error", - &config.RequiredProvider{Name: "test", Version: "1.0.0"}, - githubResult{"", errors.New("fake")}, - "", - true, - }, - { - "bad remote provider version", - &config.RequiredProvider{Name: "test", Version: "1.0.0"}, - githubResult{"bad", nil}, - "", - true, - }, - { - "remote version is newer", - &config.RequiredProvider{Name: "test", Version: "1.0.0"}, - githubResult{"1.0.1", nil}, - "1.0.1", - false, - }, - { - "versions are equal", - &config.RequiredProvider{Name: "test", Version: "1.0.0"}, - githubResult{"1.0.0", nil}, - "", - false, - }, - { - "local version is newer", - &config.RequiredProvider{Name: "test", Version: "1.0.1"}, - githubResult{"1.0.0", nil}, - "", - false, - }, - { - "latest version", - &config.RequiredProvider{Name: "test", Version: "latest"}, - githubResult{"1.0.0", nil}, - "", - true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - h := NewRegistryHub("", WithLatestReleaseGetter(func(ctx context.Context, owner, repo string) (*github.RepositoryRelease, error) { - return &github.RepositoryRelease{TagName: &tt.github.version}, tt.github.err - })) - got, err := h.CheckProviderUpdate(context.Background(), tt.requestedProvider) - if (err != nil) != tt.wantErr { - t.Errorf("Hub.CheckProviderUpdate() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != tt.want { - t.Errorf("Hub.CheckProviderUpdate() = %v, want %v", got, tt.want) - } - }) - } -}