BUG/MEDIUM: cluster: wait until all resources and routines are ready

oktalz · oktalz · commit b71553824a7a · 2021-10-13T12:04:41.000Z
wait until all resources and routines are ready before starting connection process. previously connection process could be started earlier that other resources expected
diff --git a/cmd/dataplaneapi/main.go b/cmd/dataplaneapi/main.go
@@ -16,10 +16,13 @@
 package main
 
 import (
+	"crypto/tls"
 	"fmt"
+	"net/http"
 	"os"
 	"path"
 	"syscall"
+	"time"
 
 	loads "github.com/go-openapi/loads"
 	"github.com/go-openapi/runtime"
@@ -223,6 +226,28 @@ func startServer(cfg *configuration.Configuration) (reload configuration.AtomicB
 	}()
 
 	server.ConfigureAPI()
+	schema := "http"
+	if len(server.EnabledListeners) > 0 {
+		schema = server.EnabledListeners[0]
+	}
+	path := fmt.Sprintf("%s://%s:%d%s", schema, server.Host, server.Port, cfg.RuntimeData.APIBasePath)
+	go func(path string) {
+		tr := &http.Transport{
+			// we just need to check if we have a response
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec
+		}
+		client := &http.Client{Transport: tr}
+		numTrys := 0
+		for ; numTrys < 10; numTrys++ {
+			_, err := client.Get(path)
+			if err == nil {
+				cfg.Notify.ServerStarted.Notify()
+				return
+			}
+			time.Sleep(500 * time.Millisecond)
+		}
+		log.Fatalf("check if dataplane is running failed on %s", path)
+	}(path)
 	if err := server.Serve(); err != nil {
 		log.Fatalf("Error running HAProxy Data Plane API: %s", err.Error())
 	}
diff --git a/configuration/cluster_sync.go b/configuration/cluster_sync.go
@@ -31,7 +31,6 @@ import (
 	"net/http"
 	"os"
 	"path"
-	"runtime"
 	"strconv"
 	"strings"
 	"time"
@@ -88,11 +87,12 @@ func (c *ClusterSync) Monitor(cfg *Configuration, cli *client_native.HAProxyClie
 	c.certFetch = make(chan struct{}, 2)
 	go c.fetchCert()
 
+	<-c.cfg.Notify.ServerStarted.Subscribe("clusterMonitor")
+
 	key := c.cfg.Cluster.BootstrapKey.Load()
 	certFetched := cfg.Cluster.CertificateFetched.Load()
 
 	if key != "" && !certFetched {
-		runtime.Gosched()
 		c.cfg.Notify.BootstrapKeyChanged.Notify()
 	}
 }
diff --git a/configuration/configuration.go b/configuration/configuration.go
@@ -113,6 +113,7 @@ type RuntimeData struct {
 
 type NotifyConfiguration struct {
 	BootstrapKeyChanged *ChanNotify `yaml:"-"`
+	ServerStarted       *ChanNotify `yaml:"-"`
 	CertificateRefresh  *ChanNotify `yaml:"-"`
 	Reload              *ChanNotify `yaml:"-"`
 	Shutdown            *ChanNotify `yaml:"-"`
@@ -152,6 +153,7 @@ func Get() *Configuration {
 		cfg = &Configuration{}
 		cfg.initSignalHandler()
 		cfg.Notify.BootstrapKeyChanged = NewChanNotify()
+		cfg.Notify.ServerStarted = NewChanNotify()
 		cfg.Notify.CertificateRefresh = NewChanNotify()
 		cfg.Notify.Reload = NewChanNotify()
 		cfg.Notify.Shutdown = NewChanNotify()
@@ -177,6 +179,7 @@ func (c *Configuration) GetStorageData() *StorageDataplaneAPIConfiguration {
 
 func (c *Configuration) UnSubscribeAll() {
 	c.Notify.BootstrapKeyChanged.UnSubscribeAll()
+	c.Notify.ServerStarted.UnSubscribeAll()
 	c.Notify.CertificateRefresh.UnSubscribeAll()
 	c.Notify.Reload.UnSubscribeAll()
 	c.Notify.Shutdown.UnSubscribeAll()
