cloudquery · roneli · Feb 10, 2022 · Feb 7, 2022 · Feb 9, 2022 · Feb 9, 2022
@@ -259,9 +259,9 @@ func New(ctx context.Context, options ...Option) (*Client, error) {
 		SkipBuildTables:    false,
 		HubProgressUpdater: nil,
 		HistoryCfg:         nil,
-		RegistryURL:        registry.CloudQueryRegistryURl,
+		RegistryURL:        registry.CloudQueryRegistryURL,
 		Logger:             logging.NewZHcLog(&zerolog.Logger, ""),
-		Hub:                *registry.NewRegistryHub(registry.CloudQueryRegistryURl),
+		Hub:                *registry.NewRegistryHub(registry.CloudQueryRegistryURL),
 	}
 	for _, o := range options {
 		o(c)

@@ -16,13 +16,10 @@ import (
 
 	"github.com/cloudquery/cloudquery/internal/test/provider"
 	"github.com/cloudquery/cloudquery/pkg/config"
-	"github.com/cloudquery/cloudquery/pkg/plugin/registry"
 	"github.com/cloudquery/cq-provider-sdk/provider/schema"
 	"github.com/cloudquery/cq-provider-sdk/serve"
 	"github.com/fsnotify/fsnotify"
 	"github.com/golang-migrate/migrate/v4"
-	"github.com/google/go-github/v35/github"
-	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/go-version"
 	"github.com/hashicorp/hcl/v2/hclparse"
 	"github.com/jackc/pgx/v4"
@@ -710,100 +707,3 @@ func Test_collectProviderVersions(t *testing.T) {
 		})
 	}
 }
-
-func TestCheckForProviderUpdates(t *testing.T) {
-	type githubResult struct {
-		release *github.RepositoryRelease
-		err     error
-	}
-	version1 := "1.0.0"
-	version2 := "2.0.0"
-	tests := []struct {
-		name          string
-		providers     []*config.RequiredProvider
-		githubResults []githubResult
-		want          []ProviderUpdateSummary
-	}{
-		{
-			"empty list of providers",
-			nil,
-			nil,
-			[]ProviderUpdateSummary{},
-		},
-		{
-			"one provider, github error",
-			[]*config.RequiredProvider{{Name: "test", Version: version1}},
-			[]githubResult{{nil, errors.New("fake")}},
-			[]ProviderUpdateSummary{},
-		},
-		{
-			"one provider, no update",
-			[]*config.RequiredProvider{{Name: "test", Version: version1}},
-			[]githubResult{{&github.RepositoryRelease{TagName: &version1}, nil}},
-			[]ProviderUpdateSummary{},
-		},
-		{
-			"latest provider, no update",
-			[]*config.RequiredProvider{{Name: "test", Version: "latest"}},
-			[]githubResult{{&github.RepositoryRelease{TagName: &version1}, nil}},
-			[]ProviderUpdateSummary{},
-		},
-		{
-			"two providers, one update",
-			[]*config.RequiredProvider{
-				{Name: "test", Version: version1},
-				{Name: "other", Version: version1},
-			},
-			[]githubResult{
-				{&github.RepositoryRelease{TagName: &version1}, nil},
-				{&github.RepositoryRelease{TagName: &version2}, nil},
-			},
-			[]ProviderUpdateSummary{
-				{Name: "other", Version: version1, LatestVersion: version2},
-			},
-		},
-		{
-			"three providers, github error, one update",
-			[]*config.RequiredProvider{{Name: "test", Version: version1}, {Name: "other", Version: version1}, {Name: "third", Version: version1}},
-			[]githubResult{
-				{&github.RepositoryRelease{TagName: &version1}, nil},
-				{nil, errors.New("fake")},
-				{&github.RepositoryRelease{TagName: &version2}, nil},
-			},
-			[]ProviderUpdateSummary{
-				{Name: "third", Version: version1, LatestVersion: version2},
-			},
-		},
-		{
-			"three providers, three updates",
-			[]*config.RequiredProvider{{Name: "test", Version: version1}, {Name: "other", Version: version1}, {Name: "third", Version: version1}},
-			[]githubResult{
-				{&github.RepositoryRelease{TagName: &version2}, nil},
-				{&github.RepositoryRelease{TagName: &version2}, nil},
-				{&github.RepositoryRelease{TagName: &version2}, nil},
-			},
-			[]ProviderUpdateSummary{
-				{Name: "test", Version: version1, LatestVersion: version2},
-				{Name: "other", Version: version1, LatestVersion: version2},
-				{Name: "third", Version: version1, LatestVersion: version2},
-			},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			ctx := context.Background()
-			getReleaseCall := 0
-			c := Client{
-				Providers: tt.providers,
-				Logger:    hclog.Default(),
-				Hub: *registry.NewRegistryHub("", registry.WithLatestReleaseGetter(func(ctx context.Context, owner, repo string) (*github.RepositoryRelease, error) {
-					r := tt.githubResults[getReleaseCall]
-					getReleaseCall++
-					return r.release, r.err
-				})),
-			}
-			got := c.CheckForProviderUpdates(ctx)
-			assert.Equal(t, tt.want, got)
-		})
-	}
-}
@@ -2,8 +2,10 @@ package registry
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net/http"
+	"net/url"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -14,14 +16,13 @@ import (
 	"github.com/cloudquery/cloudquery/internal/logging"
 	"github.com/cloudquery/cloudquery/pkg/config"
 	"github.com/cloudquery/cloudquery/pkg/ui"
-	"github.com/google/go-github/v35/github"
 	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/go-version"
 	zerolog "github.com/rs/zerolog/log"
 )
 
 const (
-	CloudQueryRegistryURl = "https://firestore.googleapis.com/v1/projects/hub-cloudquery/databases/(default)/documents/orgs/%s/providers/%s"
+	CloudQueryRegistryURL = "https://firestore.googleapis.com/v1/projects/hub-cloudquery/databases/(default)/documents/orgs/%s/providers/%s"
 
 	// Timeout for http requests related to CloudQuery providers version check.
 	versionCheckHTTPTimeout = time.Second * 10
@@ -45,26 +46,16 @@ type Hub struct {
 	url string
 	// map of downloaded providers
 	providers map[string]ProviderDetails
-
-	// getLatestRelease is a function that will return latest release from Github repo given organization and repo name.
-	getLatestRelease LatestReleaseGetter
 }
 
 type Option func(h *Hub)
 
-func WithLatestReleaseGetter(g LatestReleaseGetter) Option {
-	return func(h *Hub) {
-		h.getLatestRelease = g
-	}
-}
-
 func NewRegistryHub(url string, opts ...Option) *Hub {
 	h := &Hub{
-		PluginDirectory:  filepath.Join(".", ".cq", "providers"),
-		Logger:           logging.NewZHcLog(&zerolog.Logger, ""),
-		url:              url,
-		providers:        make(map[string]ProviderDetails),
-		getLatestRelease: getLatestRelease,
+		PluginDirectory: filepath.Join(".", ".cq", "providers"),
+		Logger:          logging.NewZHcLog(&zerolog.Logger, ""),
+		url:             url,
+		providers:       make(map[string]ProviderDetails),
 	}
 	// apply the list of options to hub
 	for _, opt := range opts {
@@ -170,11 +161,10 @@ func (h Hub) CheckProviderUpdate(ctx context.Context, requestedProvider *config.
 
 	ctx, cancel := context.WithTimeout(ctx, versionCheckHTTPTimeout)
 	defer cancel()
-	release, err := h.getLatestRelease(ctx, organization, ProviderRepoName(providerName))
+	latestVersion, err := h.getLatestRelease(ctx, organization, ProviderRepoName(providerName))
 	if err != nil {
 		return "", err
 	}
-	latestVersion := release.GetTagName()
 	v, err := version.NewVersion(latestVersion)
 	if err != nil {
 		return "", fmt.Errorf("bad version received: provider %s, version %s", providerName, latestVersion)
@@ -193,11 +183,10 @@ func (h Hub) DownloadProvider(ctx context.Context, requestedProvider *config.Req
 	}
 
 	if providerVersion == "latest" {
-		release, err := h.getRelease(ctx, organization, providerName, providerVersion)
+		providerVersion, err = h.getLatestRelease(ctx, organization, providerName)
 		if err != nil {
 			return ProviderDetails{}, err
 		}
-		providerVersion = release.GetTagName()
 	}
 	p, ok := h.providers[fmt.Sprintf("%s-%s", providerName, providerVersion)]
 	if !ok {
@@ -269,14 +258,46 @@ func (h Hub) downloadProvider(ctx context.Context, organization, providerName, p
 	return details, nil
 }
 
-func (h Hub) getRelease(ctx context.Context, organization, providerName, version string) (*github.RepositoryRelease, error) {
-	client := github.NewClient(nil)
-	if version != "latest" {
-		release, _, err := client.Repositories.GetReleaseByTag(ctx, organization, ProviderRepoName(providerName), version)
-		return release, err
+func (h Hub) getLatestRelease(ctx context.Context, organization, providerName string) (string, error) {
+	versions, err := url.Parse(fmt.Sprintf(h.url+"/versions", organization, providerName))
+	if err != nil {
+		return "", err
+	}
+	qv := versions.Query()
+	qv.Set("pageSize", "1")
+	qv.Set("orderBy", "v_major desc, v_minor desc, v_patch desc, published_at desc")
+	qv.Set("mask.fieldPaths", "tag")
+	versions.RawQuery = qv.Encode()
+
+	hc := &http.Client{Timeout: 15 * time.Second}
+	req, _ := http.NewRequestWithContext(ctx, http.MethodGet, versions.String(), nil)
+	res, err := hc.Do(req)
+	if err != nil {
+		return "", err
 	}
-	release, _, err := client.Repositories.GetLatestRelease(ctx, organization, ProviderRepoName(providerName))
-	return release, err
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("unexpected status code %d", res.StatusCode)
+	}
+
+	var doc struct {
+		Documents []struct {
+			Name   string `json:"name"`
+			Fields struct {
+				Tag struct {
+					Val string `json:"stringValue"`
+				} `json:"tag"`
+			} `json:"fields"`
+		} `json:"documents"`
+	}
+	if err := json.NewDecoder(res.Body).Decode(&doc); err != nil {
+		return "", err
+	}
+
+	if len(doc.Documents) == 0 || doc.Documents[0].Fields.Tag.Val == "" {
+		return "", fmt.Errorf("failed to find provider %s latest version", providerName)
+	}
+	return doc.Documents[0].Fields.Tag.Val, nil
 }
 
 func (h Hub) verifyRegistered(organization, providerName, version string, noVerify bool) bool {
@@ -377,11 +398,3 @@ func parseProviderSource(requestedProvider *config.RequiredProvider) (string, st
 	}
 	return ParseProviderName(requestedSource)
 }
-
-type LatestReleaseGetter func(ctx context.Context, owner, repo string) (*github.RepositoryRelease, error)
-
-func getLatestRelease(ctx context.Context, owner, repo string) (*github.RepositoryRelease, error) {
-	gh := github.NewClient(nil)
-	r, _, err := gh.Repositories.GetLatestRelease(ctx, owner, repo)
-	return r, err
-}