better node removal

synced-pool.go

@@ -56,16 +56,15 @@ func NewSyncedPool(local *GrainLocalPool, hostname string, d Discovery) (*Synced
 	go func(pingTimer *time.Ticker) {
 		for {
 			<-pingTimer.C
-			log.Printf("Pinging remotes %d\n", len(pool.remotes))
-			for i, r := range pool.remotes {
+			for _, r := range pool.remotes {
 				err := DoPing(r)
 				if err != nil {
 					r.MissedPings++
 					log.Printf("Error pinging remote %s: %v\n, missed pings: %d", r.Host, err, r.MissedPings)
 					if r.MissedPings > 3 {
 						log.Printf("Removing remote %s\n", r.Host)
-
-						pool.remotes = append(pool.remotes[:i], pool.remotes[i+1:]...)
+						go pool.RemoveHost(r)
+						//pool.remotes = append(pool.remotes[:i], pool.remotes[i+1:]...)
 
 					}
 				} else {
@@ -133,6 +132,28 @@ func NewSyncedPool(local *GrainLocalPool, hostname string, d Discovery) (*Synced
 	return pool, nil
 }
 
+func (p *SyncedPool) RemoveHost(host *RemoteHost) {
+
+	for i, r := range p.remotes {
+		if r == host {
+			p.RemoveHostMappedCarts(r)
+			p.remotes = append(p.remotes[:i], p.remotes[i+1:]...)
+			connectedRemotes.Set(float64(len(p.remotes)))
+			return
+		}
+	}
+}
+
+func (p *SyncedPool) RemoveHostMappedCarts(host *RemoteHost) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	for id, r := range p.remoteIndex {
+		if r == host.Pool {
+			delete(p.remoteIndex, id)
+		}
+	}
+}
+
 var (
 	negotiationCount = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "cart_remote_negotiation_total",