health check

main.go

@@ -125,7 +125,7 @@ func main() {
 	// if local
 	//syncedPool.AddRemote("localhost")
 
-	_, err = NewGrainHandler(app.pool, ":1337")
+	hg, err := NewGrainHandler(app.pool, ":1337")
 	if err != nil {
 		log.Fatalf("Error creating handler: %v\n", err)
 	}
@@ -155,6 +155,29 @@ func main() {
 	mux.HandleFunc("/pprof/symbol", pprof.Symbol)
 	mux.HandleFunc("/pprof/trace", pprof.Trace)
 	mux.Handle("/metrics", promhttp.Handler())
+	mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+		if !hg.IsHealthy() {
+			w.WriteHeader(http.StatusInternalServerError)
+			w.Write([]byte("handler not healthy"))
+			return
+		}
+		if !syncedPool.IsHealthy() {
+			w.WriteHeader(http.StatusInternalServerError)
+			w.Write([]byte("pool not healthy"))
+			return
+		}
+
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte("ok"))
+	})
+	mux.HandleFunc("/readyz", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte("ok"))
+	})
+	mux.HandleFunc("/livez", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte("ok"))
+	})
 
 	sigs := make(chan os.Signal, 1)
 	done := make(chan bool, 1)

rpc-server.go

@@ -30,6 +30,10 @@ func NewGrainHandler(pool *GrainLocalPool, listen string) (*GrainHandler, error)
 	return handler, err
 }
 
+func (h *GrainHandler) IsHealthy() bool {
+	return len(h.pool.grains) < h.pool.PoolSize
+}
+
 func (h *GrainHandler) RemoteHandleMessageHandler(id CartId, data []byte) (uint32, []byte, error) {
 	var msg Message
 	err := ReadMessage(bytes.NewReader(data), &msg)

synced-pool.go

@@ -17,6 +17,10 @@ type Quorum interface {
 	OwnerChanged(CartId, host string) error
 }
 
+type HealthHandler interface {
+	IsHealthy() bool
+}
+
 type RemoteHost struct {
 	*Client
 	Host        string
@@ -200,6 +204,15 @@ func NewSyncedPool(local *GrainLocalPool, hostname string, discovery Discovery)
 	return pool, nil
 }
 
+func (p *SyncedPool) IsHealthy() bool {
+	for _, r := range p.remotes {
+		if r.MissedPings > 3 {
+			return false
+		}
+	}
+	return true
+}
+
 func (p *SyncedPool) IsKnown(host string) bool {
 	for _, r := range p.remotes {
 		if r.Host == host {