check nodes

deployment/deployment.yaml

@@ -84,10 +84,10 @@ metadata:
   name: cart-ingress
   annotations:
     cert-manager.io/cluster-issuer: letsencrypt-prod
-    nginx.ingress.kubernetes.io/affinity: "cookie"
-    nginx.ingress.kubernetes.io/session-cookie-name: "cart-session"
-    nginx.ingress.kubernetes.io/session-cookie-expires: "172800"
-    nginx.ingress.kubernetes.io/session-cookie-max-age: "172800"
+    # nginx.ingress.kubernetes.io/affinity: "cookie"
+    # nginx.ingress.kubernetes.io/session-cookie-name: "cart-session"
+    # nginx.ingress.kubernetes.io/session-cookie-expires: "172800"
+    # nginx.ingress.kubernetes.io/session-cookie-max-age: "172800"
     nginx.ingress.kubernetes.io/proxy-body-size: 4m
 spec:
   ingressClassName: nginx

discovery.go

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"context"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+type Discovery interface {
+	Discover() ([]string, error)
+}
+
+type K8sDiscovery struct {
+	ctx    context.Context
+	client *kubernetes.Clientset
+}
+
+func (k *K8sDiscovery) Discover() ([]string, error) {
+	return k.DiscoverInNamespace("")
+}
+func (k *K8sDiscovery) DiscoverInNamespace(namespace string) ([]string, error) {
+	pods, err := k.client.CoreV1().Pods(namespace).List(k.ctx, metav1.ListOptions{
+		LabelSelector: "actor-pool=cart",
+	})
+	if err != nil {
+		return nil, err
+	}
+	hosts := make([]string, 0, len(pods.Items))
+	for _, pod := range pods.Items {
+		hosts = append(hosts, pod.Status.PodIP)
+	}
+	return hosts, nil
+}
+
+func NewK8sDiscovery(client *kubernetes.Clientset) *K8sDiscovery {
+	return &K8sDiscovery{
+		ctx:    context.Background(),
+		client: client,
+	}
+}

synced-pool.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"context"
 	"encoding/binary"
 	"fmt"
 	"io"
@@ -13,43 +12,8 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes"
 )
 
-type Discovery interface {
-	Discover() ([]string, error)
-}
-
-type K8sDiscovery struct {
-	ctx    context.Context
-	client *kubernetes.Clientset
-}
-
-func (k *K8sDiscovery) Discover() ([]string, error) {
-	return k.DiscoverInNamespace("")
-}
-func (k *K8sDiscovery) DiscoverInNamespace(namespace string) ([]string, error) {
-	pods, err := k.client.CoreV1().Pods(namespace).List(k.ctx, metav1.ListOptions{
-		LabelSelector: "actor-pool=cart",
-	})
-	if err != nil {
-		return nil, err
-	}
-	hosts := make([]string, 0, len(pods.Items))
-	for _, pod := range pods.Items {
-		hosts = append(hosts, pod.Status.PodIP)
-	}
-	return hosts, nil
-}
-
-func NewK8sDiscovery(client *kubernetes.Clientset) *K8sDiscovery {
-	return &K8sDiscovery{
-		ctx:    context.Background(),
-		client: client,
-	}
-}
-
 type Quorum interface {
 	Negotiate(knownHosts []string) ([]string, error)
 	OwnerChanged(CartId, host string) error
@@ -106,12 +70,14 @@ func NewSyncedPool(local *GrainLocalPool, hostname string, d Discovery) (*Synced
 					r.MissedPings = 0
 				}
 			}
+			connectedRemotes.Set(float64(len(pool.remotes)))
 		}
 	}()
 	if d != nil {
 		discoveryTimer := time.NewTicker(time.Second * 5)
 		go func() {
 			<-discoveryTimer.C
+			log.Printf("Looking for new nodes\n")
 			hosts, err := d.Discover()
 			if err != nil {
 				log.Printf("Error discovering hosts: %v\n", err)
@@ -296,7 +262,7 @@ func (p *SyncedPool) handleConnection(conn net.Conn) {
 			// remote grain changed
 			grainSyncCount.Inc()
 			log.Printf("Remote grain changed\n")
-			for err == nil {
+
 			idAndHost := make([]byte, packet.DataLength)
 			_, err = conn.Read(idAndHost)
 			log.Printf("Remote grain %s changed\n", idAndHost)
@@ -321,12 +287,12 @@ func (p *SyncedPool) handleConnection(conn net.Conn) {
 				log.Printf("Remote host %s not found\n", idAndHostParts[1])
 				log.Printf("Remotes %v\n", p.remotes)
 			} else {
-					err = SendPacket(conn, AckChange, func(w io.Writer) error {
+				SendPacket(conn, AckChange, func(w io.Writer) error {
 					_, err := w.Write([]byte("ok"))
 					return err
 				})
 			}
-			}
+
 		}
 	}
 }
@@ -364,11 +330,14 @@ func (p *SyncedPool) Negotiate(knownHosts []string) ([]string, error) {
 }
 
 func (r *RemoteHost) ConfirmChange(id CartId, host string) error {
-	SendPacket(r.connection, RemoteGrainChanged, func(w io.Writer) error {
+	err := SendPacket(r.connection, RemoteGrainChanged, func(w io.Writer) error {
 		_, err := w.Write([]byte(fmt.Sprintf("%s;%s", id, host)))
 		return err
 	})
-	_, err := r.queue.Expect(AckChange, time.Second)
+	if err != nil {
+		return err
+	}
+	_, err = r.queue.Expect(AckChange, time.Second)
 
 	if err != nil {
 		return err
@@ -382,7 +351,7 @@ func (p *SyncedPool) OwnerChanged(id CartId, host string) error {
 		err := r.ConfirmChange(id, host)
 
 		if err != nil {
-			log.Printf("Error confirming change: %v\n", err)
+			log.Printf("Error confirming change: %v from %s\n", err, host)
 			return err
 		}
 	}