better retry

pool-server.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"encoding/json"
-	"log"
 	"net/http"
 	"strconv"
 
@@ -23,7 +22,7 @@ func NewPoolServer(pool GrainPool, pod_name string) *PoolServer {
 
 func (s *PoolServer) HandleGet(w http.ResponseWriter, r *http.Request) error {
 	id := r.PathValue("id")
-	log.Printf("Getting cart %s\n", id)
+
 	data, err := s.pool.Get(ToCartId(id))
 	if err != nil {
 		return err

synced-pool.go

@@ -399,6 +399,10 @@ func (p *SyncedPool) AddRemote(host string) error {
 	}()
 	go func() {
 		for range time.Tick(time.Second * 3) {
+			if !remote.IsHealthy() {
+				return
+			}
+			log.Printf("Pinging remote %s\n", host)
 			err := remote.Ping()
 
 			for err != nil {

tcp-client.go

@@ -54,16 +54,25 @@ func NewPersistentConnection(address string) (*PersistentConnection, error) {
 }
 
 func (m *PersistentConnection) Connect() error {
-	if !m.Dead {
+	fails := 0
+	for {
 		connection, err := net.Dial("tcp", m.address)
 		if err != nil {
 			log.Printf("Error connecting to %s: %v\n", m.address, err)
-			m.Died <- true
+			fails++
-			m.Dead = true
+			if fails > 15 {
-			return err
+				m.Died <- true
+				m.Dead = true
+				return err
+			}
+		} else {
+			m.Conn = connection
+			break
 		}
-		m.Conn = connection
+		time.Sleep(time.Millisecond * 300)
+
 	}
+
 	return nil
 }