watch pods instead of polling

synced-pool.go

@@ -78,20 +78,23 @@ func NewSyncedPool(local *GrainLocalPool, hostname string, d Discovery) (*Synced
 
 	if d != nil {
 		go func() {
-			for range time.Tick(time.Second * 5) {
-				hosts, err := d.Discover()
-				if err != nil {
-					log.Printf("Error discovering hosts: %v", err)
+			ch, err := d.Watch()
+			if err != nil {
+				log.Printf("Error discovering hosts: %v", err)
+				return
+			}
+			for host := range ch {
+				if pool.IsKnown(host) {
+					continue
 				}
-				for _, h := range pool.ExcludeKnown(hosts) {
-
-					log.Printf("Discovered host %s", h)
-
+				go func(h string) {
+					log.Printf("Discovered host %s, waiting for startup", h)
+					time.Sleep(time.Second)
 					err := pool.AddRemote(h)
 					if err != nil {
 						log.Printf("Error adding remote %s: %v", h, err)
 					}
-				}
+				}(host)
 			}
 		}()
 	} else {
@@ -112,6 +115,15 @@ func NewSyncedPool(local *GrainLocalPool, hostname string, d Discovery) (*Synced
 	return pool, nil
 }
 
+func (p *SyncedPool) IsKnown(host string) bool {
+	for _, r := range p.remotes {
+		if r.Host == host {
+			return true
+		}
+	}
+	return host != p.Hostname
+}
+
 func (p *SyncedPool) ExcludeKnown(hosts []string) []string {
 	ret := make([]string, 0, len(hosts))
 	for _, h := range hosts {
@@ -172,6 +184,14 @@ var (
 		Name: "cart_packet_queue_size",
 		Help: "The total number of packets in the queue",
 	})
+	packetsSent = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "cart_pool_packets_sent_total",
+		Help: "The total number of packets sent",
+	})
+	packetsReceived = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "cart_pool_packets_received_total",
+		Help: "The total number of packets received",
+	})
 )
 
 const (
@@ -216,7 +236,7 @@ func NewPacketQueue(connection net.Conn) *PacketQueue {
 
 				//return
 			}
-			packetQueue.Inc()
+
 			queue.mu.Lock()
 			for i, packet := range queue.Packets {
 				if time.Since(packet.Added) < time.Second*5 {
@@ -231,6 +251,8 @@ func NewPacketQueue(connection net.Conn) *PacketQueue {
 				Data:        data,
 			})
 			queue.mu.Unlock()
+			packetsReceived.Inc()
+			packetQueue.Inc()
 		}
 	}()
 	return queue