@@ -513,7 +513,9 @@ struct lnet_ni *lnet_get_next_ni_locked(struct lnet_net *mynet,
int lnet_get_peer_list(u32 *countp, u32 *sizep,
struct lnet_process_id __user *ids);
extern void lnet_peer_ni_set_healthv(lnet_nid_t nid, int value, bool all);
-extern void lnet_peer_ni_add_to_recoveryq_locked(struct lnet_peer_ni *lpni);
+extern void lnet_peer_ni_add_to_recoveryq_locked(struct lnet_peer_ni *lpni,
+ struct list_head *queue,
+ time64_t now);
extern int lnet_peer_add_pref_nid(struct lnet_peer_ni *lpni, lnet_nid_t nid);
extern void lnet_peer_clr_pref_nids(struct lnet_peer_ni *lpni);
extern int lnet_peer_del_pref_nid(struct lnet_peer_ni *lpni, lnet_nid_t nid);
@@ -3356,6 +3356,7 @@ struct lnet_mt_event_info {
struct lnet_peer_ni *lpni;
struct lnet_peer_ni *tmp;
lnet_nid_t nid;
+ time64_t now;
int healthv;
int rc;
@@ -3367,6 +3368,8 @@ struct lnet_mt_event_info {
&local_queue);
lnet_net_unlock(0);
+ now = ktime_get_seconds();
+
list_for_each_entry_safe(lpni, tmp, &local_queue,
lpni_recovery) {
/* The same protection strategy is used here as is in the
@@ -3444,30 +3447,22 @@ struct lnet_mt_event_info {
}
lpni->lpni_recovery_ping_mdh = mdh;
- /* While we're unlocked the lpni could've been
- * readded on the recovery queue. In this case we
- * don't need to add it to the local queue, since
- * it's already on there and the thread that added
- * it would've incremented the refcount on the
- * peer, which means we need to decref the refcount
- * that was implicitly grabbed by find_peer_ni_locked.
- * Otherwise, if the lpni is still not on
- * the recovery queue, then we'll add it to the
- * processed list.
- */
- if (list_empty(&lpni->lpni_recovery))
- list_add_tail(&lpni->lpni_recovery,
- &processed_list);
- else
- lnet_peer_ni_decref_locked(lpni);
- lnet_net_unlock(0);
-
- spin_lock(&lpni->lpni_lock);
- if (rc)
+ lnet_peer_ni_add_to_recoveryq_locked(lpni,
+ &processed_list,
+ now);
+ if (rc) {
+ spin_lock(&lpni->lpni_lock);
lpni->lpni_state &=
~LNET_PEER_NI_RECOVERY_PENDING;
+ spin_unlock(&lpni->lpni_lock);
+ }
+
+ /* Drop the ref taken by lnet_find_peer_ni_locked() */
+ lnet_peer_ni_decref_locked(lpni);
+ lnet_net_unlock(0);
+ } else {
+ spin_unlock(&lpni->lpni_lock);
}
- spin_unlock(&lpni->lpni_lock);
}
list_splice_init(&processed_list, &local_queue);
@@ -4384,8 +4379,7 @@ void lnet_monitor_thr_stop(void)
}
}
- if (the_lnet.ln_routing)
- lpni->lpni_last_alive = ktime_get_seconds();
+ lpni->lpni_last_alive = ktime_get_seconds();
msg->msg_rxpeer = lpni;
msg->msg_rxni = ni;
@@ -488,19 +488,13 @@
lnet_net_unlock(0);
}
+/* must hold net_lock/0 */
void
lnet_handle_remote_failure_locked(struct lnet_peer_ni *lpni)
{
u32 sensitivity = lnet_health_sensitivity;
u32 lp_sensitivity;
- /* NO-OP if:
- * 1. lpni could be NULL if we're in the LOLND case
- * 2. this is a recovery message
- */
- if (!lpni)
- return;
-
/* If there is a health sensitivity in the peer then use that
* instead of the globally set one.
*/
@@ -519,7 +513,9 @@
* value will not be reduced. In this case, there is no reason to
* invoke recovery
*/
- lnet_peer_ni_add_to_recoveryq_locked(lpni);
+ lnet_peer_ni_add_to_recoveryq_locked(lpni,
+ &the_lnet.ln_mt_peerNIRecovq,
+ ktime_get_seconds());
}
static void
@@ -892,8 +888,19 @@
u32 sensitivity;
lpn_peer = lpni->lpni_peer_net->lpn_peer;
- sensitivity = lpn_peer->lp_health_sensitivity;
+ sensitivity = lpn_peer->lp_health_sensitivity ?
+ lpn_peer->lp_health_sensitivity :
+ lnet_health_sensitivity;
lnet_inc_lpni_healthv_locked(lpni, sensitivity);
+ /* This peer NI may have previously aged out
+ * of recovery. Now that we've received a
+ * message from it, we can continue recovery
+ * if its health value is still below the
+ * maximum.
+ */
+ lnet_peer_ni_add_to_recoveryq_locked(lpni,
+ &the_lnet.ln_mt_peerNIRecovq,
+ ktime_get_seconds());
}
lnet_net_unlock(0);
}
@@ -3978,22 +3978,38 @@ int lnet_get_peer_info(struct lnet_ioctl_peer_cfg *cfg, void __user *bulk)
return rc;
}
+/* must hold net_lock/0 */
void
-lnet_peer_ni_add_to_recoveryq_locked(struct lnet_peer_ni *lpni)
+lnet_peer_ni_add_to_recoveryq_locked(struct lnet_peer_ni *lpni,
+ struct list_head *recovery_queue,
+ time64_t now)
{
/* the mt could've shutdown and cleaned up the queues */
if (the_lnet.ln_mt_state != LNET_MT_STATE_RUNNING)
return;
- if (list_empty(&lpni->lpni_recovery) &&
- atomic_read(&lpni->lpni_healthv) < LNET_MAX_HEALTH_VALUE) {
- CDEBUG(D_NET, "lpni %s added to recovery queue. Health = %d\n",
+ if (!list_empty(&lpni->lpni_recovery))
+ return;
+
+ if (atomic_read(&lpni->lpni_healthv) == LNET_MAX_HEALTH_VALUE)
+ return;
+
+ if (now > lpni->lpni_last_alive + lnet_recovery_limit) {
+ CDEBUG(D_NET, "lpni %s aged out last alive %lld\n",
libcfs_nid2str(lpni->lpni_nid),
- atomic_read(&lpni->lpni_healthv));
- list_add_tail(&lpni->lpni_recovery,
- &the_lnet.ln_mt_peerNIRecovq);
- lnet_peer_ni_addref_locked(lpni);
+ lpni->lpni_last_alive);
+ return;
}
+
+ /* This peer NI is going on the recovery queue, so take a ref on it */
+ lnet_peer_ni_addref_locked(lpni);
+
+ CDEBUG(D_NET, "%s added to recovery queue. last alive: %lld health: %d\n",
+ libcfs_nid2str(lpni->lpni_nid),
+ lpni->lpni_last_alive,
+ atomic_read(&lpni->lpni_healthv));
+
+ list_add_tail(&lpni->lpni_recovery, recovery_queue);
}
/* Call with the ln_api_mutex held */
@@ -4006,10 +4022,13 @@ int lnet_get_peer_info(struct lnet_ioctl_peer_cfg *cfg, void __user *bulk)
struct lnet_peer_ni *lpni;
int lncpt;
int cpt;
+ time64_t now;
if (the_lnet.ln_state != LNET_STATE_RUNNING)
return;
+ now = ktime_get_seconds();
+
if (!all) {
lnet_net_lock(LNET_LOCK_EX);
lpni = lnet_find_peer_ni_locked(nid);
@@ -4018,7 +4037,8 @@ int lnet_get_peer_info(struct lnet_ioctl_peer_cfg *cfg, void __user *bulk)
return;
}
atomic_set(&lpni->lpni_healthv, value);
- lnet_peer_ni_add_to_recoveryq_locked(lpni);
+ lnet_peer_ni_add_to_recoveryq_locked(lpni,
+ &the_lnet.ln_mt_peerNIRecovq, now);
lnet_peer_ni_decref_locked(lpni);
lnet_net_unlock(LNET_LOCK_EX);
return;
@@ -4026,8 +4046,8 @@ int lnet_get_peer_info(struct lnet_ioctl_peer_cfg *cfg, void __user *bulk)
lncpt = cfs_percpt_number(the_lnet.ln_peer_tables);
- /* Walk all the peers and reset the healhv for each one to the
- * maximum value.
+ /* Walk all the peers and reset the healh value for each one to the
+ * specified value.
*/
lnet_net_lock(LNET_LOCK_EX);
for (cpt = 0; cpt < lncpt; cpt++) {
@@ -4038,7 +4058,9 @@ int lnet_get_peer_info(struct lnet_ioctl_peer_cfg *cfg, void __user *bulk)
list_for_each_entry(lpni, &lpn->lpn_peer_nis,
lpni_peer_nis) {
atomic_set(&lpni->lpni_healthv, value);
- lnet_peer_ni_add_to_recoveryq_locked(lpni);
+ lnet_peer_ni_add_to_recoveryq_locked(lpni,
+ &the_lnet.ln_mt_peerNIRecovq,
+ now);
}
}
}