From patchwork Fri Feb 16 18:41:44 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Oliver Upton X-Patchwork-Id: 13560457 Received: from out-181.mta0.migadu.com (out-181.mta0.migadu.com [91.218.175.181]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 297441353EA for ; Fri, 16 Feb 2024 18:42:09 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=91.218.175.181 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1708108931; cv=none; b=oKrRaoqTYHUiLlylTobnnBMVNXyJE2rcNjnJQGmtHheXDp8QjmMVoFoIK0UD2uD8yzpupQOILcRhNQKoGq6/Lyzv2g0GgkCxRb585XQpn2XFuezFASnw6PjBOagBVfNlTHIeGO0QwsKvVodXMpTOEk4CFYVQjpWfvlfTkDAZ49w= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1708108931; c=relaxed/simple; bh=dH0k6IN0MHlaNID2kd6SHBELB8QrefZu66kvwU5n4cs=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=XbP+XNs35cAGNolGVSj+M1Ll0/EGFPZdRHP9s8lMe4hb2VdSx/T2oCmKoxEfBdsL4ijRHqLcW9pF0BCXeBH0fwXqg2/oMdapiteVJo9uhf2BJeO4+4C+qaO3EbVZR60D/vKdhczJCuH7WQnADzjjIpznFHQyaZvHUYSF/uWI1/M= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev; spf=pass smtp.mailfrom=linux.dev; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b=H/JVGd4/; arc=none smtp.client-ip=91.218.175.181 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.dev Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b="H/JVGd4/" X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.dev; s=key1; t=1708108927; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=qQ663bMVQErfiC1nnRDiwhyxZXGKefti52HAGa4Tyg4=; b=H/JVGd4/7Qq9RFnGveFyggh0J3Hiz/SYlGE4oLCu0pRw3RL+NYbIT9UL+2eGUevSFydFmQ PChqNQZt1fy48ANP9BQdYkMZZY2UaaMHjgyxXAX09BXBszi6N6AdOjz0n4yFyxssgBGdGf +wOUy0mYdWR1ebkABIDqrOxY2J1j9AU= From: Oliver Upton To: kvmarm@lists.linux.dev Cc: kvm@vger.kernel.org, Marc Zyngier , James Morse , Suzuki K Poulose , Zenghui Yu , linux-kernel@vger.kernel.org, Oliver Upton Subject: [PATCH v3 01/10] KVM: arm64: vgic: Store LPIs in an xarray Date: Fri, 16 Feb 2024 18:41:44 +0000 Message-ID: <20240216184153.2714504-2-oliver.upton@linux.dev> In-Reply-To: <20240216184153.2714504-1-oliver.upton@linux.dev> References: <20240216184153.2714504-1-oliver.upton@linux.dev> Precedence: bulk X-Mailing-List: kvm@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 X-Migadu-Flow: FLOW_OUT Using a linked-list for LPIs is less than ideal as it of course requires iterative searches to find a particular entry. An xarray is a better data structure for this use case, as it provides faster searches and can still handle a potentially sparse range of INTID allocations. Start by storing LPIs in an xarray, punting usage of the xarray to a subsequent change. Signed-off-by: Oliver Upton --- arch/arm64/kvm/vgic/vgic-init.c | 3 +++ arch/arm64/kvm/vgic/vgic-its.c | 16 ++++++++++++++++ arch/arm64/kvm/vgic/vgic.c | 1 + include/kvm/arm_vgic.h | 2 ++ 4 files changed, 22 insertions(+) diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index e949e1d0fd9f..411719053107 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -56,6 +56,7 @@ void kvm_vgic_early_init(struct kvm *kvm) INIT_LIST_HEAD(&dist->lpi_list_head); INIT_LIST_HEAD(&dist->lpi_translation_cache); raw_spin_lock_init(&dist->lpi_list_lock); + xa_init_flags(&dist->lpi_xa, XA_FLAGS_LOCK_IRQ); } /* CREATION */ @@ -366,6 +367,8 @@ static void kvm_vgic_dist_destroy(struct kvm *kvm) if (vgic_supports_direct_msis(kvm)) vgic_v4_teardown(kvm); + + xa_destroy(&dist->lpi_xa); } static void __kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu) diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index e2764d0ffa9f..fb2d3c356984 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -52,6 +52,12 @@ static struct vgic_irq *vgic_add_lpi(struct kvm *kvm, u32 intid, if (!irq) return ERR_PTR(-ENOMEM); + ret = xa_reserve_irq(&dist->lpi_xa, intid, GFP_KERNEL_ACCOUNT); + if (ret) { + kfree(irq); + return ERR_PTR(ret); + } + INIT_LIST_HEAD(&irq->lpi_list); INIT_LIST_HEAD(&irq->ap_list); raw_spin_lock_init(&irq->irq_lock); @@ -86,12 +92,22 @@ static struct vgic_irq *vgic_add_lpi(struct kvm *kvm, u32 intid, goto out_unlock; } + ret = xa_err(xa_store(&dist->lpi_xa, intid, irq, 0)); + if (ret) { + xa_release(&dist->lpi_xa, intid); + kfree(irq); + goto out_unlock; + } + list_add_tail(&irq->lpi_list, &dist->lpi_list_head); dist->lpi_list_count++; out_unlock: raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags); + if (ret) + return ERR_PTR(ret); + /* * We "cache" the configuration table entries in our struct vgic_irq's. * However we only have those structs for mapped IRQs, so we read in diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index db2a95762b1b..c126014f8395 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -131,6 +131,7 @@ void __vgic_put_lpi_locked(struct kvm *kvm, struct vgic_irq *irq) return; list_del(&irq->lpi_list); + xa_erase(&dist->lpi_xa, irq->intid); dist->lpi_list_count--; kfree(irq); diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index 8cc38e836f54..795b35656b54 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -275,6 +276,7 @@ struct vgic_dist { /* Protects the lpi_list and the count value below. */ raw_spinlock_t lpi_list_lock; + struct xarray lpi_xa; struct list_head lpi_list_head; int lpi_list_count; From patchwork Fri Feb 16 18:41:45 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Oliver Upton X-Patchwork-Id: 13560458 Received: from out-183.mta0.migadu.com (out-183.mta0.migadu.com [91.218.175.183]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 6F57313664E for ; Fri, 16 Feb 2024 18:42:11 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=91.218.175.183 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1708108933; cv=none; b=TOCTCe4WVSsBjIo8adS6PxfDY/Rnu5HPI8EuQb79nwEhAnl6R8muRXAIBkuK57Yt9or1cTGhpzT36vXM6ShDc6ze+vEeA16Lx5PEaeBJ3oEVV1vUbxx6qLOLKowGEf1KsZv8Nm1A1aM0yljPldUH+IelBfFQxPyMLksOjps6qaw= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1708108933; c=relaxed/simple; bh=Vcvs9scbETBbx3K0EOYQAZ7h+k9GiCx1VVK7pwIOKmc=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=msWcqxTfSGDWf2h7320PKec6bl6lRtpWrcQjRwRUDdif+WHSADi9CS/358aM+b9dftrCU88vL3wD4kMk0VvXYMxJlcfDGUI5k7i5P2BuDjXsn+rxFY9gvGflF61g/c1Omj2mARgQMNxEuvO43hFqX+g+p9zhxupqjGEDOU9cayY= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev; spf=pass smtp.mailfrom=linux.dev; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b=fHkOMxWW; arc=none smtp.client-ip=91.218.175.183 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.dev Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b="fHkOMxWW" X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.dev; s=key1; t=1708108929; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=J1JR94TUCmjSuW67hdoXXOUBEMTu1g9M0PtI4CuvL0o=; b=fHkOMxWWKme5F7VJkNfW+SwbWdAz9GsVCnFpRhADCScBrrGU7MQrfxUVI2JE/0KBwZ94j5 dbg5VQW4Mm7l8ZMpBNR94yRYY9lIwtkG/t0ffTRAJ+FMAiB58P1pqTQPAjneWt/VoxOxb4 rbgzSMMM9/TtYZYQN8d6MeanbSV03Po= From: Oliver Upton To: kvmarm@lists.linux.dev Cc: kvm@vger.kernel.org, Marc Zyngier , James Morse , Suzuki K Poulose , Zenghui Yu , linux-kernel@vger.kernel.org, Oliver Upton Subject: [PATCH v3 02/10] KVM: arm64: vgic: Use xarray to find LPI in vgic_get_lpi() Date: Fri, 16 Feb 2024 18:41:45 +0000 Message-ID: <20240216184153.2714504-3-oliver.upton@linux.dev> In-Reply-To: <20240216184153.2714504-1-oliver.upton@linux.dev> References: <20240216184153.2714504-1-oliver.upton@linux.dev> Precedence: bulk X-Mailing-List: kvm@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 X-Migadu-Flow: FLOW_OUT Iterating over the LPI linked-list is less than ideal when the desired index is already known. Use the INTID to index the LPI xarray instead. Signed-off-by: Oliver Upton --- arch/arm64/kvm/vgic/vgic.c | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index c126014f8395..d90c42ff051d 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -54,8 +54,9 @@ struct vgic_global kvm_vgic_global_state __ro_after_init = { */ /* - * Iterate over the VM's list of mapped LPIs to find the one with a - * matching interrupt ID and return a reference to the IRQ structure. + * Index the VM's xarray of mapped LPIs and return a reference to the IRQ + * structure. The caller is expected to call vgic_put_irq() later once it's + * finished with the IRQ. */ static struct vgic_irq *vgic_get_lpi(struct kvm *kvm, u32 intid) { @@ -65,20 +66,10 @@ static struct vgic_irq *vgic_get_lpi(struct kvm *kvm, u32 intid) raw_spin_lock_irqsave(&dist->lpi_list_lock, flags); - list_for_each_entry(irq, &dist->lpi_list_head, lpi_list) { - if (irq->intid != intid) - continue; - - /* - * This increases the refcount, the caller is expected to - * call vgic_put_irq() later once it's finished with the IRQ. - */ + irq = xa_load(&dist->lpi_xa, intid); + if (irq) vgic_get_irq_kref(irq); - goto out_unlock; - } - irq = NULL; -out_unlock: raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags); return irq; From patchwork Fri Feb 16 18:41:46 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Oliver Upton X-Patchwork-Id: 13560459 Received: from out-183.mta0.migadu.com (out-183.mta0.migadu.com [91.218.175.183]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 73A7A137C47 for ; Fri, 16 Feb 2024 18:42:13 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=91.218.175.183 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1708108935; cv=none; b=p9t1CQtA5Mwg/UzaE2V3tm+tT+C9zRG7bzfrZIjI9WA0eXhcytqVagGWB/4482oGw9uWMnLLuDEEANMF+CBnsOaJdshcPvYT7ABOsK4oYssfy4GmXhcZWduX79Hntl6PQnL1Q3reB94blCdhgBcClI9Msba75wncvVHqKIbwX9I= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1708108935; c=relaxed/simple; bh=RxF9w33LgnXLTuXV5O9RLdzK9dXUqE5MSbsOmdxxwVA=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=QcBw1rNymI2FfIa3Uww+yi7JPrNzMDqpWtuWVFiF9aVWlAXoVfFz7o+2SwSyRrHO0lrZ8JRo8EJvccDgtaTL1V+L9hu5mlbLv6qMlccMnKmDa1h4l6jrxF1i3MRaCGxQjK9a9S8dsD4vAk4jZOjRFNdcRjrkoCueRUpVT+yNNCk= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev; spf=pass smtp.mailfrom=linux.dev; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b=ZLbfNeSQ; arc=none smtp.client-ip=91.218.175.183 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.dev Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b="ZLbfNeSQ" X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.dev; s=key1; t=1708108931; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=p7/gBwWxFbIZwFC+F2czZFaZoLMKZvcePyaMRonjua4=; b=ZLbfNeSQ+vvMysguJereiuL6HX9y7yTkwAqfZDi01alhaBPsgpmVjE5rDkwu1Yn8Htvw9v fzVoJNNRJHANhXohgMOEmDPfTxdaCfZD8ONhfVqCE7ZJVf2ocgo0jDD1DRg4a2cUdGSBHp 8wDn/MKghWUzK463T149MUkCPXm0D6Y= From: Oliver Upton To: kvmarm@lists.linux.dev Cc: kvm@vger.kernel.org, Marc Zyngier , James Morse , Suzuki K Poulose , Zenghui Yu , linux-kernel@vger.kernel.org, Oliver Upton Subject: [PATCH v3 03/10] KVM: arm64: vgic-v3: Iterate the xarray to find pending LPIs Date: Fri, 16 Feb 2024 18:41:46 +0000 Message-ID: <20240216184153.2714504-4-oliver.upton@linux.dev> In-Reply-To: <20240216184153.2714504-1-oliver.upton@linux.dev> References: <20240216184153.2714504-1-oliver.upton@linux.dev> Precedence: bulk X-Mailing-List: kvm@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 X-Migadu-Flow: FLOW_OUT Start walking the LPI xarray to find pending LPIs in preparation for the removal of the LPI linked-list. Note that the 'basic' iterator is chosen here as each iteration needs to drop the xarray read lock (RCU) as reads/writes to guest memory can potentially block. Signed-off-by: Oliver Upton --- arch/arm64/kvm/vgic/vgic-v3.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c index 9465d3706ab9..4ea3340786b9 100644 --- a/arch/arm64/kvm/vgic/vgic-v3.c +++ b/arch/arm64/kvm/vgic/vgic-v3.c @@ -380,6 +380,7 @@ int vgic_v3_save_pending_tables(struct kvm *kvm) struct vgic_irq *irq; gpa_t last_ptr = ~(gpa_t)0; bool vlpi_avail = false; + unsigned long index; int ret = 0; u8 val; @@ -396,7 +397,7 @@ int vgic_v3_save_pending_tables(struct kvm *kvm) vlpi_avail = true; } - list_for_each_entry(irq, &dist->lpi_list_head, lpi_list) { + xa_for_each(&dist->lpi_xa, index, irq) { int byte_offset, bit_nr; struct kvm_vcpu *vcpu; gpa_t pendbase, ptr; From patchwork Fri Feb 16 18:41:47 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Oliver Upton X-Patchwork-Id: 13560460 Received: from out-172.mta0.migadu.com (out-172.mta0.migadu.com [91.218.175.172]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 18B5F139573; Fri, 16 Feb 2024 18:42:14 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=91.218.175.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1708108936; cv=none; b=sb+lKRfkaGhNwhD6Ou8wfjWe2aZVMGr4LY+qk9Z3a6ZiOC2Mg/TBb3qFbyB2FN3lZKPl20JXJS7ZMZ3+dOgr7rbVS7T6V4zRN2ysBapYzWyNAEGFvwQtDaD0WbVBKYPupHeKFUEC8sK/3SnJWmbApkPOzC0T3SrO3Hms0FL/4oQ= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1708108936; c=relaxed/simple; bh=kWOvmd79TSBa85yZMhQxLashdvnOQitPJm7fzCmDhgw=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=DNtqmsL5ZEwoKFeBZB5I2rhDGOhnucNWd/Z3W2ugC4D/4eyGOfbOuBQdzXCYmCmhVZ++Tkkr0tFhgNLi9VP25ELnjgjURtu0aJvsjYCKoc0YudtXMfOhKW+Ss0bHPWPeMYekcrfptr4pvKdIel/ojUeL1AZkunaptMaxJ1Oiuhc= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev; spf=pass smtp.mailfrom=linux.dev; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b=n1FUOVlE; arc=none smtp.client-ip=91.218.175.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.dev Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b="n1FUOVlE" X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.dev; s=key1; t=1708108933; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=ZQGaJWYKQGMmTnU81XUm3LbLxD/nkGmlez2wBi9FY5g=; b=n1FUOVlE36G2VVZcvP+dqcFaQP/pVkYnqUcaLYGdtm9/lxWqLZYHUlFvWZz/JpQPNCRJP6 1wTx7GgFkHB6w/xSy6Ht/4/qxBloxyYg8mhMIejHWFi/xDFcNVBqjzuUqKKH/EJ2liryZJ wRYPIDA1qMmkaeagAY7w1JT56E90o40= From: Oliver Upton To: kvmarm@lists.linux.dev Cc: kvm@vger.kernel.org, Marc Zyngier , James Morse , Suzuki K Poulose , Zenghui Yu , linux-kernel@vger.kernel.org, Oliver Upton Subject: [PATCH v3 04/10] KVM: arm64: vgic-its: Walk the LPI xarray in vgic_copy_lpi_list() Date: Fri, 16 Feb 2024 18:41:47 +0000 Message-ID: <20240216184153.2714504-5-oliver.upton@linux.dev> In-Reply-To: <20240216184153.2714504-1-oliver.upton@linux.dev> References: <20240216184153.2714504-1-oliver.upton@linux.dev> Precedence: bulk X-Mailing-List: kvm@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 X-Migadu-Flow: FLOW_OUT Start iterating the LPI xarray in anticipation of removing the LPI linked-list. Signed-off-by: Oliver Upton --- arch/arm64/kvm/vgic/vgic-its.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index fb2d3c356984..9ce2edfadd11 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -335,6 +335,7 @@ static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq, int vgic_copy_lpi_list(struct kvm *kvm, struct kvm_vcpu *vcpu, u32 **intid_ptr) { struct vgic_dist *dist = &kvm->arch.vgic; + XA_STATE(xas, &dist->lpi_xa, GIC_LPI_OFFSET); struct vgic_irq *irq; unsigned long flags; u32 *intids; @@ -353,7 +354,9 @@ int vgic_copy_lpi_list(struct kvm *kvm, struct kvm_vcpu *vcpu, u32 **intid_ptr) return -ENOMEM; raw_spin_lock_irqsave(&dist->lpi_list_lock, flags); - list_for_each_entry(irq, &dist->lpi_list_head, lpi_list) { + rcu_read_lock(); + + xas_for_each(&xas, irq, INTERRUPT_ID_BITS_ITS) { if (i == irq_count) break; /* We don't need to "get" the IRQ, as we hold the list lock. */ @@ -361,6 +364,8 @@ int vgic_copy_lpi_list(struct kvm *kvm, struct kvm_vcpu *vcpu, u32 **intid_ptr) continue; intids[i++] = irq->intid; } + + rcu_read_unlock(); raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags); *intid_ptr = intids; From patchwork Fri Feb 16 18:41:48 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Oliver Upton X-Patchwork-Id: 13560461 Received: from out-177.mta0.migadu.com (out-177.mta0.migadu.com [91.218.175.177]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id DD3AD13B2B0 for ; Fri, 16 Feb 2024 18:42:16 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=91.218.175.177 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1708108938; cv=none; b=hH05fuf52irITPxQjJgc1DfZ8xpXHD5NRPQf4mUoRkE/eF26BCAS2trmtAFkUVKQCZuvlB4mpGI3bJX/nT7E4Ufwc+WxPYBTTnfOPLZZDNL9bV0SnGFdZ8Yc3pBeFSOn687cXtZRcUAsJUG8kACtrIOrFoHGl5vyC1llZoFZ558= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1708108938; c=relaxed/simple; bh=D3m+FHFIaj+jhZ3odonI17F4V0pkUqn0ADel7SjBEng=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=CNzcvm4Hd0ObsVT6+u10FPsIKMbNuvb++GU34WGNOlx3Cs/SDFvkk7Eqwyom89GPFPUWsW/SwX6H7CbziSB6Ea67VXGHa7eeC6BP15tMBkeZ+Ul5axTiSxXs6qAtuf4dDx4u5jOdvQhAhe8Q/Prw7xSiTX/yBPBMCIVEaDRjl2I= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev; spf=pass smtp.mailfrom=linux.dev; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b=MRuFiYgz; arc=none smtp.client-ip=91.218.175.177 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.dev Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b="MRuFiYgz" X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.dev; s=key1; t=1708108935; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=EffQNLJRTkroBiMKmKtexwMDjVwsGrj4DMlqwBeCnuY=; b=MRuFiYgzsOhaKh4Vv3717zkk2cwLeWkro5zgb5TFMYgHKAWwC/alTwHQuKZGS6PrsNNqhn yPzf5SIf86NlHT4vBoMmYoUHJPXFmatU9PQSIfZSJ154XjRNSNziyWYyhtideaooTXTpIW 2gW16e1g78OLTxCsGrjj7DWuHLd9KJA= From: Oliver Upton To: kvmarm@lists.linux.dev Cc: kvm@vger.kernel.org, Marc Zyngier , James Morse , Suzuki K Poulose , Zenghui Yu , linux-kernel@vger.kernel.org, Oliver Upton Subject: [PATCH v3 05/10] KVM: arm64: vgic: Get rid of the LPI linked-list Date: Fri, 16 Feb 2024 18:41:48 +0000 Message-ID: <20240216184153.2714504-6-oliver.upton@linux.dev> In-Reply-To: <20240216184153.2714504-1-oliver.upton@linux.dev> References: <20240216184153.2714504-1-oliver.upton@linux.dev> Precedence: bulk X-Mailing-List: kvm@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 X-Migadu-Flow: FLOW_OUT All readers of LPI configuration have been transitioned to use the LPI xarray. Get rid of the linked-list altogether. Signed-off-by: Oliver Upton --- arch/arm64/kvm/vgic/vgic-init.c | 1 - arch/arm64/kvm/vgic/vgic-its.c | 8 ++------ arch/arm64/kvm/vgic/vgic.c | 1 - include/kvm/arm_vgic.h | 2 -- 4 files changed, 2 insertions(+), 10 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index 411719053107..e25672d6e846 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -53,7 +53,6 @@ void kvm_vgic_early_init(struct kvm *kvm) { struct vgic_dist *dist = &kvm->arch.vgic; - INIT_LIST_HEAD(&dist->lpi_list_head); INIT_LIST_HEAD(&dist->lpi_translation_cache); raw_spin_lock_init(&dist->lpi_list_lock); xa_init_flags(&dist->lpi_xa, XA_FLAGS_LOCK_IRQ); diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index 9ce2edfadd11..0d1cfc776f47 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -58,7 +58,6 @@ static struct vgic_irq *vgic_add_lpi(struct kvm *kvm, u32 intid, return ERR_PTR(ret); } - INIT_LIST_HEAD(&irq->lpi_list); INIT_LIST_HEAD(&irq->ap_list); raw_spin_lock_init(&irq->irq_lock); @@ -74,10 +73,8 @@ static struct vgic_irq *vgic_add_lpi(struct kvm *kvm, u32 intid, * There could be a race with another vgic_add_lpi(), so we need to * check that we don't add a second list entry with the same LPI. */ - list_for_each_entry(oldirq, &dist->lpi_list_head, lpi_list) { - if (oldirq->intid != intid) - continue; - + oldirq = xa_load(&dist->lpi_xa, intid); + if (oldirq) { /* Someone was faster with adding this LPI, lets use that. */ kfree(irq); irq = oldirq; @@ -99,7 +96,6 @@ static struct vgic_irq *vgic_add_lpi(struct kvm *kvm, u32 intid, goto out_unlock; } - list_add_tail(&irq->lpi_list, &dist->lpi_list_head); dist->lpi_list_count++; out_unlock: diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index d90c42ff051d..e58ce68e325c 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -121,7 +121,6 @@ void __vgic_put_lpi_locked(struct kvm *kvm, struct vgic_irq *irq) if (!kref_put(&irq->refcount, vgic_irq_release)) return; - list_del(&irq->lpi_list); xa_erase(&dist->lpi_xa, irq->intid); dist->lpi_list_count--; diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index 795b35656b54..aeff363e3ba6 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -117,7 +117,6 @@ struct irq_ops { struct vgic_irq { raw_spinlock_t irq_lock; /* Protects the content of the struct */ - struct list_head lpi_list; /* Used to link all LPIs together */ struct list_head ap_list; struct kvm_vcpu *vcpu; /* SGIs and PPIs: The VCPU @@ -277,7 +276,6 @@ struct vgic_dist { /* Protects the lpi_list and the count value below. */ raw_spinlock_t lpi_list_lock; struct xarray lpi_xa; - struct list_head lpi_list_head; int lpi_list_count; /* LPI translation cache */ From patchwork Fri Feb 16 18:41:49 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Oliver Upton X-Patchwork-Id: 13560462 Received: from out-174.mta0.migadu.com (out-174.mta0.migadu.com [91.218.175.174]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 33B15145FF6 for ; Fri, 16 Feb 2024 18:42:18 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=91.218.175.174 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1708108940; cv=none; b=Mn34znNWGHQeOiLxveVqOA6RExzex9tplXa6/BXpr37licTmAol0eJuqSPzBy8HRs0X+fuQ0AAPzsnb680ElkrVRSKG7M3AZsQbjBqo3u0SyM17ALvpSUytf8FgkziX+hqVQL3siqCD+GiD9oGl8zfRKIwQFkTN+/z0uIn7zRXM= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1708108940; c=relaxed/simple; bh=YQt7HllDOIOc6JmEVYjsohh1WJr5UrgxtFnKA5+RyhU=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=H8JLl31ENZu0YWnen/vMmovUA1VKDpMM7lApfsTStLIJ4tpZYdPSAdddK4qRMkQnR95eJHx+mxZwW7rXNZSShWaaZKiRxIHpmMObzpd69Sy+G7jzirMj1mct2THOLyjZhoiYkWYRYHz2Yd3IprWn7xJxdXoPQVgWPwdtDYXs3PQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev; spf=pass smtp.mailfrom=linux.dev; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b=sptWkG5D; arc=none smtp.client-ip=91.218.175.174 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.dev Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b="sptWkG5D" X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.dev; s=key1; t=1708108937; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=Ovim50P2MGl2zeU/Tv+W8v3zNQiqHM6FYpVR4brBrho=; b=sptWkG5DobDUHQ1ct3NjhArOg4jO4ZffFqJZudlIxMmKLdK/yO+1qIuMtOYePl1Qz6o37c NQNa9lydPyh5r9aTznErWkktC6Is36TB1b1beKShaSLIRjeJOt6+lDVTyfddnPKNPTBQM6 fYnKx6JZmOA7Q9c8uYNyrVev/uSypt4= From: Oliver Upton To: kvmarm@lists.linux.dev Cc: kvm@vger.kernel.org, Marc Zyngier , James Morse , Suzuki K Poulose , Zenghui Yu , linux-kernel@vger.kernel.org, Oliver Upton Subject: [PATCH v3 06/10] KVM: arm64: vgic: Use atomics to count LPIs Date: Fri, 16 Feb 2024 18:41:49 +0000 Message-ID: <20240216184153.2714504-7-oliver.upton@linux.dev> In-Reply-To: <20240216184153.2714504-1-oliver.upton@linux.dev> References: <20240216184153.2714504-1-oliver.upton@linux.dev> Precedence: bulk X-Mailing-List: kvm@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 X-Migadu-Flow: FLOW_OUT Switch to using atomics for LPI accounting, allowing vgic_irq references to be dropped in parallel. Signed-off-by: Oliver Upton --- arch/arm64/kvm/vgic/vgic-debug.c | 2 +- arch/arm64/kvm/vgic/vgic-its.c | 4 ++-- arch/arm64/kvm/vgic/vgic.c | 2 +- include/kvm/arm_vgic.h | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-debug.c b/arch/arm64/kvm/vgic/vgic-debug.c index 85606a531dc3..389025ce7749 100644 --- a/arch/arm64/kvm/vgic/vgic-debug.c +++ b/arch/arm64/kvm/vgic/vgic-debug.c @@ -149,7 +149,7 @@ static void print_dist_state(struct seq_file *s, struct vgic_dist *dist) seq_printf(s, "vgic_model:\t%s\n", v3 ? "GICv3" : "GICv2"); seq_printf(s, "nr_spis:\t%d\n", dist->nr_spis); if (v3) - seq_printf(s, "nr_lpis:\t%d\n", dist->lpi_list_count); + seq_printf(s, "nr_lpis:\t%d\n", atomic_read(&dist->lpi_count)); seq_printf(s, "enabled:\t%d\n", dist->enabled); seq_printf(s, "\n"); diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index 0d1cfc776f47..27a7451ad8b9 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -96,7 +96,7 @@ static struct vgic_irq *vgic_add_lpi(struct kvm *kvm, u32 intid, goto out_unlock; } - dist->lpi_list_count++; + atomic_inc(&dist->lpi_count); out_unlock: raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags); @@ -344,7 +344,7 @@ int vgic_copy_lpi_list(struct kvm *kvm, struct kvm_vcpu *vcpu, u32 **intid_ptr) * command). If coming from another path (such as enabling LPIs), * we must be careful not to overrun the array. */ - irq_count = READ_ONCE(dist->lpi_list_count); + irq_count = atomic_read(&dist->lpi_count); intids = kmalloc_array(irq_count, sizeof(intids[0]), GFP_KERNEL_ACCOUNT); if (!intids) return -ENOMEM; diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index e58ce68e325c..5988d162b765 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -122,7 +122,7 @@ void __vgic_put_lpi_locked(struct kvm *kvm, struct vgic_irq *irq) return; xa_erase(&dist->lpi_xa, irq->intid); - dist->lpi_list_count--; + atomic_dec(&dist->lpi_count); kfree(irq); } diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index aeff363e3ba6..71e9d719533b 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -273,10 +273,10 @@ struct vgic_dist { */ u64 propbaser; - /* Protects the lpi_list and the count value below. */ + /* Protects the lpi_list. */ raw_spinlock_t lpi_list_lock; struct xarray lpi_xa; - int lpi_list_count; + atomic_t lpi_count; /* LPI translation cache */ struct list_head lpi_translation_cache; From patchwork Fri Feb 16 18:41:50 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Oliver Upton X-Patchwork-Id: 13560463 Received: from out-178.mta0.migadu.com (out-178.mta0.migadu.com [91.218.175.178]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id E1E9814601C for ; Fri, 16 Feb 2024 18:42:20 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=91.218.175.178 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1708108942; cv=none; b=PRLLl1SjLh5No1Irr1Y9nNP7jaUZe6mYwW2E7Tr+YeJfTevPTu7V+6WpBd2uLm5i6zxDOW6oUwTBEI1t6U/XTD3l3KfEHjhbnSIqbFkfxDzIc59z8NjKgjWG360rfN1nwKYb3TtaKa027kSSdIJZ2AdDW0pi8jMQ+Uu4WBME1Vg= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1708108942; c=relaxed/simple; bh=vtekqhGgtjmqisr8NulSQvgtjNS1TOilwFJMKeBnMg4=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=j0y10LBZNc4Zyz2hhFJzJiB3nfzz/HX7IbRYI3+RC2DUfvjIyEtYu5MrZRxpqzpbjl9vZcFvE3LZyBvdifl60l83Z+vDDlnl5phPfe52EtzvEJArUOzhsBzApJxp60ni1FINx9W/AYIumyOmn+PUgQd5vuDmiKpqmd9SEAMtxlU= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev; spf=pass smtp.mailfrom=linux.dev; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b=B8h55U4v; arc=none smtp.client-ip=91.218.175.178 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.dev Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b="B8h55U4v" X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.dev; s=key1; t=1708108939; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=z9GN6WeBnSe+DXw8hn3rK0c8U+qc433lF4ad1LOLZiY=; b=B8h55U4vyFlSMWnLw3WokyuHHqf8SDrZYLk+F5/4/VmqI7gxyrbjsPcP5qiY84KN4ccDQL F+qnE61n5c9PbbeXv8H0A5xjvpRrfV9ItrPXnR+conGlKE00t8cEPfgeS7GGbp9ME7If5Q 8kpOuHuA8ql4HXQg4tlW5W9asI4smsE= From: Oliver Upton To: kvmarm@lists.linux.dev Cc: kvm@vger.kernel.org, Marc Zyngier , James Morse , Suzuki K Poulose , Zenghui Yu , linux-kernel@vger.kernel.org, Oliver Upton Subject: [PATCH v3 07/10] KVM: arm64: vgic: Free LPI vgic_irq structs in an RCU-safe manner Date: Fri, 16 Feb 2024 18:41:50 +0000 Message-ID: <20240216184153.2714504-8-oliver.upton@linux.dev> In-Reply-To: <20240216184153.2714504-1-oliver.upton@linux.dev> References: <20240216184153.2714504-1-oliver.upton@linux.dev> Precedence: bulk X-Mailing-List: kvm@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 X-Migadu-Flow: FLOW_OUT Free the vgic_irq structs in an RCU-safe manner to allow reads of the LPI configuration data to happen in parallel with the release of LPIs. Signed-off-by: Oliver Upton --- arch/arm64/kvm/vgic/vgic.c | 2 +- include/kvm/arm_vgic.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index 5988d162b765..be3ed4c5e1fa 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -124,7 +124,7 @@ void __vgic_put_lpi_locked(struct kvm *kvm, struct vgic_irq *irq) xa_erase(&dist->lpi_xa, irq->intid); atomic_dec(&dist->lpi_count); - kfree(irq); + kfree_rcu(irq, rcu); } void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq) diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index 71e9d719533b..47035946648e 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -117,6 +117,7 @@ struct irq_ops { struct vgic_irq { raw_spinlock_t irq_lock; /* Protects the content of the struct */ + struct rcu_head rcu; struct list_head ap_list; struct kvm_vcpu *vcpu; /* SGIs and PPIs: The VCPU From patchwork Fri Feb 16 18:41:51 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Oliver Upton X-Patchwork-Id: 13560464 Received: from out-180.mta0.migadu.com (out-180.mta0.migadu.com [91.218.175.180]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 9C4C0146912 for ; Fri, 16 Feb 2024 18:42:22 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=91.218.175.180 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1708108944; cv=none; b=ni0C+g4vVStQAI5tSHi4ocbva/DQtoC56eZuHC12lqfnKhoVIQ/TBozxeLsd4OPPxPswC5EENFpnDxO8zUVjgIpf9OlocrSoJ+T6dAV6Epqj1WwnbY/euekgWp8X7rOLPd7eL/1PByLHEj/m2OuRguVia/WSZ0ml+LAPaErXUAA= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1708108944; c=relaxed/simple; bh=SeNrU4ebThHySloZ/TR+kB1GC+oWuzUM+5Ajy3ISOQY=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=uIJJA2tSUsE4yRnAmUnCWGJgET08+hqZe6UJLJjm26yUErajbW3EDt4AeuBpeknwi6Fhx6wt0ayZnP4LWpBQtCJKSzkMG316ZKKRf0T5VebTUgSHg2FqjXzR19CYt8SUfhnKpanF1vDXiPIftRNYzlSYa2sLk3zMW6fu/neLYEs= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev; spf=pass smtp.mailfrom=linux.dev; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b=oTBnQU5q; arc=none smtp.client-ip=91.218.175.180 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.dev Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b="oTBnQU5q" X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.dev; s=key1; t=1708108941; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=cQF8jmMEEUM2DwWlHNkDBl2apuHpFelpH7G2Kk29lho=; b=oTBnQU5qPGkYHCizl8J+qjyAehCVu38cmLrtpEEygEROkm0VoerNZhquN0IIRIcY5RnvjN 6ANsbJ6rbKZCeF8F1yno7ZklnqkDwLYGHWg/qWlVXBtXCfQno+URKfRUqvNpbp9Afxhyqc 7TOCUUKhqw1uKfu2420fNRxQiMnKuvc= From: Oliver Upton To: kvmarm@lists.linux.dev Cc: kvm@vger.kernel.org, Marc Zyngier , James Morse , Suzuki K Poulose , Zenghui Yu , linux-kernel@vger.kernel.org, Oliver Upton Subject: [PATCH v3 08/10] KVM: arm64: vgic: Rely on RCU protection in vgic_get_lpi() Date: Fri, 16 Feb 2024 18:41:51 +0000 Message-ID: <20240216184153.2714504-9-oliver.upton@linux.dev> In-Reply-To: <20240216184153.2714504-1-oliver.upton@linux.dev> References: <20240216184153.2714504-1-oliver.upton@linux.dev> Precedence: bulk X-Mailing-List: kvm@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 X-Migadu-Flow: FLOW_OUT Stop acquiring the lpi_list_lock in favor of RCU for protecting the read-side critical section in vgic_get_lpi(). In order for this to be safe, we also need to be careful not to take a reference on an irq with a refcount of 0, as it is about to be freed. Signed-off-by: Oliver Upton --- arch/arm64/kvm/vgic/vgic.c | 9 ++++----- arch/arm64/kvm/vgic/vgic.h | 14 +++++++++++--- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index be3ed4c5e1fa..128ae53a0a55 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -62,15 +62,14 @@ static struct vgic_irq *vgic_get_lpi(struct kvm *kvm, u32 intid) { struct vgic_dist *dist = &kvm->arch.vgic; struct vgic_irq *irq = NULL; - unsigned long flags; - raw_spin_lock_irqsave(&dist->lpi_list_lock, flags); + rcu_read_lock(); irq = xa_load(&dist->lpi_xa, intid); - if (irq) - vgic_get_irq_kref(irq); + if (!vgic_try_get_irq_kref(irq)) + irq = NULL; - raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags); + rcu_read_unlock(); return irq; } diff --git a/arch/arm64/kvm/vgic/vgic.h b/arch/arm64/kvm/vgic/vgic.h index 8d134569d0a1..f874b9932c5a 100644 --- a/arch/arm64/kvm/vgic/vgic.h +++ b/arch/arm64/kvm/vgic/vgic.h @@ -220,12 +220,20 @@ void vgic_v2_vmcr_sync(struct kvm_vcpu *vcpu); void vgic_v2_save_state(struct kvm_vcpu *vcpu); void vgic_v2_restore_state(struct kvm_vcpu *vcpu); -static inline void vgic_get_irq_kref(struct vgic_irq *irq) +static inline bool vgic_try_get_irq_kref(struct vgic_irq *irq) { + if (!irq) + return false; + if (irq->intid < VGIC_MIN_LPI) - return; + return true; - kref_get(&irq->refcount); + return kref_get_unless_zero(&irq->refcount); +} + +static inline void vgic_get_irq_kref(struct vgic_irq *irq) +{ + WARN_ON_ONCE(!vgic_try_get_irq_kref(irq)); } void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu); From patchwork Fri Feb 16 18:41:52 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Oliver Upton X-Patchwork-Id: 13560465 Received: from out-186.mta0.migadu.com (out-186.mta0.migadu.com [91.218.175.186]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 5349C1474BB for ; Fri, 16 Feb 2024 18:42:24 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=91.218.175.186 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1708108946; cv=none; b=ey/AGbOhWychN2BP+It/GgOoqzdH58y8PA162Eazdu5mGxU138GZx0PTlzZ/gm4M8W4bZ9LIj7x1uXfsfLuQmA0Rop0a/hQ/5G939/ttpPE2tgSBT5h7vwyb+JKLpZFeB3JWAp+6sitxU01SacHC0rpZVv62V5KTETfyaROioy4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1708108946; c=relaxed/simple; bh=0+9hvpIMVuumxZZRLzdSlgvQCBBmw4uMTPz59W0JCTQ=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=X+YPydUkAerk7qP8m/CJGvyTEztjJrZ0PejEvisHqqdjkAaP+LvEL13D7UnikWrC5bP64hTR6HE3JELLjyo9Eempn8LULANC3ut3lOVVcUyu7XCD8MWAeX/Q9MRJLT/jywO3OQ4FG7PH658Ip/g9s0RpeRCrCIJZUL3z8PR8loE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev; spf=pass smtp.mailfrom=linux.dev; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b=LlwyUY4z; arc=none smtp.client-ip=91.218.175.186 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.dev Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b="LlwyUY4z" X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.dev; s=key1; t=1708108942; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=9exRx46WJAI7PPjinT5FzyCoHPJk8OWrIMcDrv8MJtA=; b=LlwyUY4zVt9oeDsGjl2cESfGbvcBUqxmU9X0jYGPlk4hY4xItlonwv/uzIe7LzX3lnVt68 vFjohrhpxUXrXI6UrVnZG+79PnUpoPLOvYkHiYPA0HGpUNmt0TaFN81eGP5B8MddedUgPs NG/hqEjrnCXoUTbmcxfw3U94Vul/1e4= From: Oliver Upton To: kvmarm@lists.linux.dev Cc: kvm@vger.kernel.org, Marc Zyngier , James Morse , Suzuki K Poulose , Zenghui Yu , linux-kernel@vger.kernel.org, Oliver Upton Subject: [PATCH v3 09/10] KVM: arm64: vgic: Ensure the irq refcount is nonzero when taking a ref Date: Fri, 16 Feb 2024 18:41:52 +0000 Message-ID: <20240216184153.2714504-10-oliver.upton@linux.dev> In-Reply-To: <20240216184153.2714504-1-oliver.upton@linux.dev> References: <20240216184153.2714504-1-oliver.upton@linux.dev> Precedence: bulk X-Mailing-List: kvm@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 X-Migadu-Flow: FLOW_OUT It will soon be possible for get() and put() calls to happen in parallel, which means in most cases we must ensure the refcount is nonzero when taking a new reference. Switch to using vgic_try_get_irq_kref() where necessary, and document the few conditions where an IRQ's refcount is guaranteed to be nonzero. Signed-off-by: Oliver Upton --- arch/arm64/kvm/vgic/vgic-its.c | 18 ++++++++---------- arch/arm64/kvm/vgic/vgic.c | 3 ++- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index 27a7451ad8b9..c3d1bca0b458 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -74,18 +74,11 @@ static struct vgic_irq *vgic_add_lpi(struct kvm *kvm, u32 intid, * check that we don't add a second list entry with the same LPI. */ oldirq = xa_load(&dist->lpi_xa, intid); - if (oldirq) { + if (vgic_try_get_irq_kref(oldirq)) { /* Someone was faster with adding this LPI, lets use that. */ kfree(irq); irq = oldirq; - /* - * This increases the refcount, the caller is expected to - * call vgic_put_irq() on the returned pointer once it's - * finished with the IRQ. - */ - vgic_get_irq_kref(irq); - goto out_unlock; } @@ -609,8 +602,8 @@ static struct vgic_irq *vgic_its_check_cache(struct kvm *kvm, phys_addr_t db, raw_spin_lock_irqsave(&dist->lpi_list_lock, flags); irq = __vgic_its_check_cache(dist, db, devid, eventid); - if (irq) - vgic_get_irq_kref(irq); + if (!vgic_try_get_irq_kref(irq)) + irq = NULL; raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags); @@ -656,6 +649,11 @@ static void vgic_its_cache_translation(struct kvm *kvm, struct vgic_its *its, if (cte->irq) __vgic_put_lpi_locked(kvm, cte->irq); + /* + * The irq refcount is guaranteed to be nonzero while holding the + * its_lock, as the ITE (and the reference it holds) cannot be freed. + */ + lockdep_assert_held(&its->its_lock); vgic_get_irq_kref(irq); cte->db = db; diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index 128ae53a0a55..2a288d6c0be7 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -394,7 +394,8 @@ bool vgic_queue_irq_unlock(struct kvm *kvm, struct vgic_irq *irq, /* * Grab a reference to the irq to reflect the fact that it is - * now in the ap_list. + * now in the ap_list. This is safe as the caller must already hold a + * reference on the irq. */ vgic_get_irq_kref(irq); list_add_tail(&irq->ap_list, &vcpu->arch.vgic_cpu.ap_list_head); From patchwork Fri Feb 16 18:41:53 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Oliver Upton X-Patchwork-Id: 13560466 Received: from out-189.mta0.migadu.com (out-189.mta0.migadu.com [91.218.175.189]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 042E514830C for ; Fri, 16 Feb 2024 18:42:25 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=91.218.175.189 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1708108948; cv=none; b=Pf1y1N74mQZ5iOMnDcDI1FZXYY1wLcIsvRlAxKuhy8aDOMMYPOFddBGU1xM7EtJbisaHmHKi2kVgL3259t5gOpP3cf1fjIgBmj5bq521YnW4CTE4qOIpGRVQ76hdH5NSgaAW5mMP64QBKbrkHculg47JWd7f6+9nFioOGULaQf0= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1708108948; c=relaxed/simple; bh=YOt99fgiOeGafpp1nEhSIUqA34CgCiK6op8GicPElws=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=gqEmEcP0YzR9BYDhzsJFIA34Jd3Ry6nhRF0MspISQadu82KTlPwh5ZYeJp7vKGBQnvQRR6LQLCFwjBjXl4rs/jPfgyrnc+Q9rB64blZ52KadkaBvO7wf5lmmJXXdXZEjFVNjhb8O8xaHcfhv5FSBEQ4w9DCo6pqwBp3viDezd2U= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev; spf=pass smtp.mailfrom=linux.dev; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b=DLxdkdtb; arc=none smtp.client-ip=91.218.175.189 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.dev Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b="DLxdkdtb" X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.dev; s=key1; t=1708108944; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=+6/2b6cdY93cfhwg+wgl1iJpv6T7icr8cPzff5WRiiM=; b=DLxdkdtbNwcuA4gLo4jd1yD05qIOD+omQNvakwfPpgTo6UMoTeIbl9XzhWZhJZwtRffA+j S6COGILK/oPKkNX85TmWz314nYUf1N7NFk3Rob9ONKeUXrHUuFrI9W4PpXWgZer+wkhrb/ NjTdniGek0pEoQhqcV0ARz6GeS0cHrQ= From: Oliver Upton To: kvmarm@lists.linux.dev Cc: kvm@vger.kernel.org, Marc Zyngier , James Morse , Suzuki K Poulose , Zenghui Yu , linux-kernel@vger.kernel.org, Oliver Upton Subject: [PATCH v3 10/10] KVM: arm64: vgic: Don't acquire the lpi_list_lock in vgic_put_irq() Date: Fri, 16 Feb 2024 18:41:53 +0000 Message-ID: <20240216184153.2714504-11-oliver.upton@linux.dev> In-Reply-To: <20240216184153.2714504-1-oliver.upton@linux.dev> References: <20240216184153.2714504-1-oliver.upton@linux.dev> Precedence: bulk X-Mailing-List: kvm@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 X-Migadu-Flow: FLOW_OUT The LPI xarray's xa_lock is sufficient for synchronizing writers when freeing a given LPI. Furthermore, readers can only take a new reference on an IRQ if it was already nonzero. Stop taking the lpi_list_lock unnecessarily and get rid of __vgic_put_lpi_locked(). Signed-off-by: Oliver Upton --- arch/arm64/kvm/vgic/vgic-its.c | 4 ++-- arch/arm64/kvm/vgic/vgic.c | 21 ++++----------------- arch/arm64/kvm/vgic/vgic.h | 1 - 3 files changed, 6 insertions(+), 20 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index c3d1bca0b458..d84cb7618c59 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -647,7 +647,7 @@ static void vgic_its_cache_translation(struct kvm *kvm, struct vgic_its *its, * was in the cache, and increment it on the new interrupt. */ if (cte->irq) - __vgic_put_lpi_locked(kvm, cte->irq); + vgic_put_irq(kvm, cte->irq); /* * The irq refcount is guaranteed to be nonzero while holding the @@ -684,7 +684,7 @@ void vgic_its_invalidate_cache(struct kvm *kvm) if (!cte->irq) break; - __vgic_put_lpi_locked(kvm, cte->irq); + vgic_put_irq(kvm, cte->irq); cte->irq = NULL; } diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index 2a288d6c0be7..c8208f81fdc8 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -110,13 +110,13 @@ static void vgic_irq_release(struct kref *ref) { } -/* - * Drop the refcount on the LPI. Must be called with lpi_list_lock held. - */ -void __vgic_put_lpi_locked(struct kvm *kvm, struct vgic_irq *irq) +void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq) { struct vgic_dist *dist = &kvm->arch.vgic; + if (irq->intid < VGIC_MIN_LPI) + return; + if (!kref_put(&irq->refcount, vgic_irq_release)) return; @@ -126,19 +126,6 @@ void __vgic_put_lpi_locked(struct kvm *kvm, struct vgic_irq *irq) kfree_rcu(irq, rcu); } -void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq) -{ - struct vgic_dist *dist = &kvm->arch.vgic; - unsigned long flags; - - if (irq->intid < VGIC_MIN_LPI) - return; - - raw_spin_lock_irqsave(&dist->lpi_list_lock, flags); - __vgic_put_lpi_locked(kvm, irq); - raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags); -} - void vgic_flush_pending_lpis(struct kvm_vcpu *vcpu) { struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; diff --git a/arch/arm64/kvm/vgic/vgic.h b/arch/arm64/kvm/vgic/vgic.h index f874b9932c5a..0c2b82de8fa3 100644 --- a/arch/arm64/kvm/vgic/vgic.h +++ b/arch/arm64/kvm/vgic/vgic.h @@ -180,7 +180,6 @@ vgic_get_mmio_region(struct kvm_vcpu *vcpu, struct vgic_io_device *iodev, gpa_t addr, int len); struct vgic_irq *vgic_get_irq(struct kvm *kvm, struct kvm_vcpu *vcpu, u32 intid); -void __vgic_put_lpi_locked(struct kvm *kvm, struct vgic_irq *irq); void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq); bool vgic_get_phys_line_level(struct vgic_irq *irq); void vgic_irq_set_phys_pending(struct vgic_irq *irq, bool pending);