| Message ID | 20210629160104.2893-2-ameynarkhede03@gmail.com (mailing list archive) |
|---|---|
| State | Superseded |
| Delegated to: | Bjorn Helgaas |
| Headers | show |
| Series | Expose and manage PCI device reset | expand |
On Tue, 29 Jun 2021 21:30:57 +0530 Amey Narkhede <ameynarkhede03@gmail.com> wrote: > Add has_flr bitfield in struct pci_dev to indicate support for pcie flr > to avoid reading PCI_EXP_DEVCAP multiple times and get rid of > PCI_DEV_FLAGS_NO_FLR_RESET in quirk_no_flr(). > > Currently there is separate function pcie_has_flr() to probe if pcie flr is > supported by the device which does not match the calling convention > followed by reset methods which use second function argument to decide > whether to probe or not. Add new function pcie_reset_flr() that follows > the calling convention of reset methods. > > Co-developed-by: Alex Williamson <alex.williamson@redhat.com> > Signed-off-by: Alex Williamson <alex.williamson@redhat.com> There are some non-trivial changes here vs previous, should probably drop this. > Signed-off-by: Amey Narkhede <ameynarkhede03@gmail.com> > --- > drivers/crypto/cavium/nitrox/nitrox_main.c | 4 +- > drivers/pci/pci.c | 62 +++++++++++----------- > drivers/pci/pcie/aer.c | 12 ++--- > drivers/pci/probe.c | 6 ++- > drivers/pci/quirks.c | 11 ++-- > include/linux/pci.h | 7 ++- > 6 files changed, 47 insertions(+), 55 deletions(-) > > diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c > index facc8e6bc..15d6c8452 100644 > --- a/drivers/crypto/cavium/nitrox/nitrox_main.c > +++ b/drivers/crypto/cavium/nitrox/nitrox_main.c > @@ -306,9 +306,7 @@ static int nitrox_device_flr(struct pci_dev *pdev) > return -ENOMEM; > } > > - /* check flr support */ > - if (pcie_has_flr(pdev)) > - pcie_flr(pdev); > + pcie_reset_flr(pdev, 0); > > pci_restore_state(pdev); > > diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c > index 452351025..28f099a4f 100644 > --- a/drivers/pci/pci.c > +++ b/drivers/pci/pci.c > @@ -4611,32 +4611,12 @@ int pci_wait_for_pending_transaction(struct pci_dev *dev) > } > EXPORT_SYMBOL(pci_wait_for_pending_transaction); > > -/** > - * pcie_has_flr - check if a device supports function level resets > - * @dev: device to check > - * > - * Returns true if the device advertises support for PCIe function level > - * resets. > - */ > -bool pcie_has_flr(struct pci_dev *dev) > -{ > - u32 cap; > - > - if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET) > - return false; > - > - pcie_capability_read_dword(dev, PCI_EXP_DEVCAP, &cap); > - return cap & PCI_EXP_DEVCAP_FLR; > -} > -EXPORT_SYMBOL_GPL(pcie_has_flr); > - > /** > * pcie_flr - initiate a PCIe function level reset > * @dev: device to reset > * > - * Initiate a function level reset on @dev. The caller should ensure the > - * device supports FLR before calling this function, e.g. by using the > - * pcie_has_flr() helper. > + * Initiate a function level reset unconditionally on @dev without > + * checking any flags and DEVCAP > */ > int pcie_flr(struct pci_dev *dev) > { > @@ -4659,16 +4639,35 @@ int pcie_flr(struct pci_dev *dev) > } > EXPORT_SYMBOL_GPL(pcie_flr); > > +/** > + * pcie_reset_flr - initiate a PCIe function level reset > + * @dev: device to reset > + * @probe: If set, only check if the device can be reset this way. > + * > + * Initiate a function level reset on @dev. > + */ > +int pcie_reset_flr(struct pci_dev *dev, int probe) > +{ > + if (!dev->has_flr) > + return -ENOTTY; > + > + if (probe) > + return 0; > + > + return pcie_flr(dev); > +} > +EXPORT_SYMBOL_GPL(pcie_reset_flr); > + > static int pci_af_flr(struct pci_dev *dev, int probe) > { > int pos; > u8 cap; > > - pos = pci_find_capability(dev, PCI_CAP_ID_AF); > - if (!pos) > + if (!dev->has_flr) > return -ENOTTY; > > - if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET) > + pos = pci_find_capability(dev, PCI_CAP_ID_AF); > + if (!pos) > return -ENOTTY; > > pci_read_config_byte(dev, pos + PCI_AF_CAP, &cap); How can has_flr encompass both methods of invoking FLR? PCIe FLR is not a prerequisite to AF FLR. > @@ -5139,11 +5138,9 @@ int __pci_reset_function_locked(struct pci_dev *dev) > rc = pci_dev_specific_reset(dev, 0); > if (rc != -ENOTTY) > return rc; > - if (pcie_has_flr(dev)) { > - rc = pcie_flr(dev); > - if (rc != -ENOTTY) > - return rc; > - } > + rc = pcie_reset_flr(dev, 0); > + if (rc != -ENOTTY) > + return rc; > rc = pci_af_flr(dev, 0); > if (rc != -ENOTTY) > return rc; > @@ -5174,8 +5171,9 @@ int pci_probe_reset_function(struct pci_dev *dev) > rc = pci_dev_specific_reset(dev, 1); > if (rc != -ENOTTY) > return rc; > - if (pcie_has_flr(dev)) > - return 0; > + rc = pcie_reset_flr(dev, 1); > + if (rc != -ENOTTY) > + return rc; > rc = pci_af_flr(dev, 1); > if (rc != -ENOTTY) > return rc; > diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c > index ec943cee5..98077595a 100644 > --- a/drivers/pci/pcie/aer.c > +++ b/drivers/pci/pcie/aer.c > @@ -1405,13 +1405,11 @@ static pci_ers_result_t aer_root_reset(struct pci_dev *dev) > } > > if (type == PCI_EXP_TYPE_RC_EC || type == PCI_EXP_TYPE_RC_END) { > - if (pcie_has_flr(dev)) { > - rc = pcie_flr(dev); > - pci_info(dev, "has been reset (%d)\n", rc); > - } else { > - pci_info(dev, "not reset (no FLR support)\n"); > - rc = -ENOTTY; > - } > + rc = pcie_reset_flr(dev, 0); > + if (!rc) > + pci_info(dev, "has been reset\n"); > + else > + pci_info(dev, "not reset (no FLR support: %d)\n", rc); > } else { > rc = pci_bus_error_reset(dev); > pci_info(dev, "%s Port link has been reset (%d)\n", > diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c > index 3a62d09b8..862d91615 100644 > --- a/drivers/pci/probe.c > +++ b/drivers/pci/probe.c > @@ -1487,6 +1487,7 @@ void set_pcie_port_type(struct pci_dev *pdev) > { > int pos; > u16 reg16; > + u32 reg32; > int type; > struct pci_dev *parent; > > @@ -1497,8 +1498,9 @@ void set_pcie_port_type(struct pci_dev *pdev) > pdev->pcie_cap = pos; > pci_read_config_word(pdev, pos + PCI_EXP_FLAGS, ®16); > pdev->pcie_flags_reg = reg16; > - pci_read_config_word(pdev, pos + PCI_EXP_DEVCAP, ®16); > - pdev->pcie_mpss = reg16 & PCI_EXP_DEVCAP_PAYLOAD; > + pci_read_config_dword(pdev, pos + PCI_EXP_DEVCAP, ®32); > + pdev->pcie_mpss = reg32 & PCI_EXP_DEVCAP_PAYLOAD; > + pdev->has_flr = reg32 & PCI_EXP_DEVCAP_FLR ? 1 : 0; We only set has_flr for PCIe FLR, so I think this effectively kills AF FLR. The ternary should be unnecessary here. > > parent = pci_upstream_bridge(pdev); > if (!parent) > diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c > index d85914afe..1efdc4e5a 100644 > --- a/drivers/pci/quirks.c > +++ b/drivers/pci/quirks.c > @@ -3819,7 +3819,7 @@ static int nvme_disable_and_flr(struct pci_dev *dev, int probe) > u32 cfg; > > if (dev->class != PCI_CLASS_STORAGE_EXPRESS || > - !pcie_has_flr(dev) || !pci_resource_start(dev, 0)) > + pcie_reset_flr(dev, 1) || !pci_resource_start(dev, 0)) > return -ENOTTY; > > if (probe) > @@ -3888,13 +3888,10 @@ static int nvme_disable_and_flr(struct pci_dev *dev, int probe) > */ > static int delay_250ms_after_flr(struct pci_dev *dev, int probe) > { > - if (!pcie_has_flr(dev)) > - return -ENOTTY; > + int ret = pcie_reset_flr(dev, probe); > > if (probe) > - return 0; > - > - pcie_flr(dev); > + return ret; > > msleep(250); > > @@ -5182,7 +5179,7 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x443, quirk_intel_qat_vf_cap); > */ > static void quirk_no_flr(struct pci_dev *dev) > { > - dev->dev_flags |= PCI_DEV_FLAGS_NO_FLR_RESET; > + dev->has_flr = 0; > } > DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_AMD, 0x1487, quirk_no_flr); > DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_AMD, 0x148c, quirk_no_flr); > diff --git a/include/linux/pci.h b/include/linux/pci.h > index c20211e59..74f42a2cd 100644 > --- a/include/linux/pci.h > +++ b/include/linux/pci.h > @@ -223,10 +223,8 @@ enum pci_dev_flags { > PCI_DEV_FLAGS_VPD_REF_F0 = (__force pci_dev_flags_t) (1 << 8), > /* A non-root bridge where translation occurs, stop alias search here */ > PCI_DEV_FLAGS_BRIDGE_XLATE_ROOT = (__force pci_dev_flags_t) (1 << 9), > - /* Do not use FLR even if device advertises PCI_AF_CAP */ > - PCI_DEV_FLAGS_NO_FLR_RESET = (__force pci_dev_flags_t) (1 << 10), > /* Don't use Relaxed Ordering for TLPs directed at this device */ > - PCI_DEV_FLAGS_NO_RELAXED_ORDERING = (__force pci_dev_flags_t) (1 << 11), > + PCI_DEV_FLAGS_NO_RELAXED_ORDERING = (__force pci_dev_flags_t) (1 << 10), > }; > > enum pci_irq_reroute_variant { > @@ -381,6 +379,7 @@ struct pci_dev { > bit manually */ > unsigned int d3hot_delay; /* D3hot->D0 transition time in ms */ > unsigned int d3cold_delay; /* D3cold->D0 transition time in ms */ > + unsigned int has_flr:1; There are more space efficient places to define this. Thanks, Alex > > #ifdef CONFIG_PCIEASPM > struct pcie_link_state *link_state; /* ASPM link state */ > @@ -1225,7 +1224,7 @@ u32 pcie_bandwidth_available(struct pci_dev *dev, struct pci_dev **limiting_dev, > enum pci_bus_speed *speed, > enum pcie_link_width *width); > void pcie_print_link_status(struct pci_dev *dev); > -bool pcie_has_flr(struct pci_dev *dev); > +int pcie_reset_flr(struct pci_dev *dev, int probe); > int pcie_flr(struct pci_dev *dev); > int __pci_reset_function_locked(struct pci_dev *dev); > int pci_reset_function(struct pci_dev *dev);
On 21/06/30 11:56AM, Alex Williamson wrote: > On Tue, 29 Jun 2021 21:30:57 +0530 > Amey Narkhede <ameynarkhede03@gmail.com> wrote: > > > Add has_flr bitfield in struct pci_dev to indicate support for pcie flr > > to avoid reading PCI_EXP_DEVCAP multiple times and get rid of > > PCI_DEV_FLAGS_NO_FLR_RESET in quirk_no_flr(). > > > > Currently there is separate function pcie_has_flr() to probe if pcie flr is > > supported by the device which does not match the calling convention > > followed by reset methods which use second function argument to decide > > whether to probe or not. Add new function pcie_reset_flr() that follows > > the calling convention of reset methods. > > > > Co-developed-by: Alex Williamson <alex.williamson@redhat.com> > > Signed-off-by: Alex Williamson <alex.williamson@redhat.com> > > There are some non-trivial changes here vs previous, should probably > drop this. > > > Signed-off-by: Amey Narkhede <ameynarkhede03@gmail.com> > > --- > > drivers/crypto/cavium/nitrox/nitrox_main.c | 4 +- > > drivers/pci/pci.c | 62 +++++++++++----------- > > drivers/pci/pcie/aer.c | 12 ++--- > > drivers/pci/probe.c | 6 ++- > > drivers/pci/quirks.c | 11 ++-- > > include/linux/pci.h | 7 ++- > > 6 files changed, 47 insertions(+), 55 deletions(-) > > > > diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c > > index facc8e6bc..15d6c8452 100644 > > --- a/drivers/crypto/cavium/nitrox/nitrox_main.c > > +++ b/drivers/crypto/cavium/nitrox/nitrox_main.c > > @@ -306,9 +306,7 @@ static int nitrox_device_flr(struct pci_dev *pdev) > > return -ENOMEM; > > } > > > > - /* check flr support */ > > - if (pcie_has_flr(pdev)) > > - pcie_flr(pdev); > > + pcie_reset_flr(pdev, 0); > > > > pci_restore_state(pdev); > > > > diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c > > index 452351025..28f099a4f 100644 > > --- a/drivers/pci/pci.c > > +++ b/drivers/pci/pci.c > > @@ -4611,32 +4611,12 @@ int pci_wait_for_pending_transaction(struct pci_dev *dev) > > } > > EXPORT_SYMBOL(pci_wait_for_pending_transaction); > > > > -/** > > - * pcie_has_flr - check if a device supports function level resets > > - * @dev: device to check > > - * > > - * Returns true if the device advertises support for PCIe function level > > - * resets. > > - */ > > -bool pcie_has_flr(struct pci_dev *dev) > > -{ > > - u32 cap; > > - > > - if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET) > > - return false; > > - > > - pcie_capability_read_dword(dev, PCI_EXP_DEVCAP, &cap); > > - return cap & PCI_EXP_DEVCAP_FLR; > > -} > > -EXPORT_SYMBOL_GPL(pcie_has_flr); > > - > > /** > > * pcie_flr - initiate a PCIe function level reset > > * @dev: device to reset > > * > > - * Initiate a function level reset on @dev. The caller should ensure the > > - * device supports FLR before calling this function, e.g. by using the > > - * pcie_has_flr() helper. > > + * Initiate a function level reset unconditionally on @dev without > > + * checking any flags and DEVCAP > > */ > > int pcie_flr(struct pci_dev *dev) > > { > > @@ -4659,16 +4639,35 @@ int pcie_flr(struct pci_dev *dev) > > } > > EXPORT_SYMBOL_GPL(pcie_flr); > > > > +/** > > + * pcie_reset_flr - initiate a PCIe function level reset > > + * @dev: device to reset > > + * @probe: If set, only check if the device can be reset this way. > > + * > > + * Initiate a function level reset on @dev. > > + */ > > +int pcie_reset_flr(struct pci_dev *dev, int probe) > > +{ > > + if (!dev->has_flr) > > + return -ENOTTY; > > + > > + if (probe) > > + return 0; > > + > > + return pcie_flr(dev); > > +} > > +EXPORT_SYMBOL_GPL(pcie_reset_flr); > > + > > static int pci_af_flr(struct pci_dev *dev, int probe) > > { > > int pos; > > u8 cap; > > > > - pos = pci_find_capability(dev, PCI_CAP_ID_AF); > > - if (!pos) > > + if (!dev->has_flr) > > return -ENOTTY; > > > > - if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET) > > + pos = pci_find_capability(dev, PCI_CAP_ID_AF); > > + if (!pos) > > return -ENOTTY; > > > > pci_read_config_byte(dev, pos + PCI_AF_CAP, &cap); > > > How can has_flr encompass both methods of invoking FLR? PCIe FLR is > not a prerequisite to AF FLR. > I see. Does this mean that there should be a separate flag for disabling AF FLR? > > > @@ -5139,11 +5138,9 @@ int __pci_reset_function_locked(struct pci_dev *dev) > > rc = pci_dev_specific_reset(dev, 0); > > if (rc != -ENOTTY) > > return rc; > > - if (pcie_has_flr(dev)) { > > - rc = pcie_flr(dev); > > - if (rc != -ENOTTY) > > - return rc; > > - } > > + rc = pcie_reset_flr(dev, 0); > > + if (rc != -ENOTTY) > > + return rc; > > rc = pci_af_flr(dev, 0); > > if (rc != -ENOTTY) > > return rc; > > @@ -5174,8 +5171,9 @@ int pci_probe_reset_function(struct pci_dev *dev) > > rc = pci_dev_specific_reset(dev, 1); > > if (rc != -ENOTTY) > > return rc; > > - if (pcie_has_flr(dev)) > > - return 0; > > + rc = pcie_reset_flr(dev, 1); > > + if (rc != -ENOTTY) > > + return rc; > > rc = pci_af_flr(dev, 1); > > if (rc != -ENOTTY) > > return rc; > > diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c > > index ec943cee5..98077595a 100644 > > --- a/drivers/pci/pcie/aer.c > > +++ b/drivers/pci/pcie/aer.c > > @@ -1405,13 +1405,11 @@ static pci_ers_result_t aer_root_reset(struct pci_dev *dev) > > } > > > > if (type == PCI_EXP_TYPE_RC_EC || type == PCI_EXP_TYPE_RC_END) { > > - if (pcie_has_flr(dev)) { > > - rc = pcie_flr(dev); > > - pci_info(dev, "has been reset (%d)\n", rc); > > - } else { > > - pci_info(dev, "not reset (no FLR support)\n"); > > - rc = -ENOTTY; > > - } > > + rc = pcie_reset_flr(dev, 0); > > + if (!rc) > > + pci_info(dev, "has been reset\n"); > > + else > > + pci_info(dev, "not reset (no FLR support: %d)\n", rc); > > } else { > > rc = pci_bus_error_reset(dev); > > pci_info(dev, "%s Port link has been reset (%d)\n", > > diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c > > index 3a62d09b8..862d91615 100644 > > --- a/drivers/pci/probe.c > > +++ b/drivers/pci/probe.c > > @@ -1487,6 +1487,7 @@ void set_pcie_port_type(struct pci_dev *pdev) > > { > > int pos; > > u16 reg16; > > + u32 reg32; > > int type; > > struct pci_dev *parent; > > > > @@ -1497,8 +1498,9 @@ void set_pcie_port_type(struct pci_dev *pdev) > > pdev->pcie_cap = pos; > > pci_read_config_word(pdev, pos + PCI_EXP_FLAGS, ®16); > > pdev->pcie_flags_reg = reg16; > > - pci_read_config_word(pdev, pos + PCI_EXP_DEVCAP, ®16); > > - pdev->pcie_mpss = reg16 & PCI_EXP_DEVCAP_PAYLOAD; > > + pci_read_config_dword(pdev, pos + PCI_EXP_DEVCAP, ®32); > > + pdev->pcie_mpss = reg32 & PCI_EXP_DEVCAP_PAYLOAD; > > + pdev->has_flr = reg32 & PCI_EXP_DEVCAP_FLR ? 1 : 0; > > We only set has_flr for PCIe FLR, so I think this effectively kills AF > FLR. The ternary should be unnecessary here. > I believe only one of the PCIe FLR and PCI AF FLR will be available at a time. So I think it should be okay to check for PCI_AF_CAP_FLR and set pdev->has_flr accordingly. One question though, what is PCI_AF_CAP_TP and how is it related to PCI AF FLR? > > > > parent = pci_upstream_bridge(pdev); > > if (!parent) > > diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c > > index d85914afe..1efdc4e5a 100644 > > --- a/drivers/pci/quirks.c > > +++ b/drivers/pci/quirks.c > > @@ -3819,7 +3819,7 @@ static int nvme_disable_and_flr(struct pci_dev *dev, int probe) > > u32 cfg; > > > > if (dev->class != PCI_CLASS_STORAGE_EXPRESS || > > - !pcie_has_flr(dev) || !pci_resource_start(dev, 0)) > > + pcie_reset_flr(dev, 1) || !pci_resource_start(dev, 0)) > > return -ENOTTY; > > > > if (probe) > > @@ -3888,13 +3888,10 @@ static int nvme_disable_and_flr(struct pci_dev *dev, int probe) > > */ > > static int delay_250ms_after_flr(struct pci_dev *dev, int probe) > > { > > - if (!pcie_has_flr(dev)) > > - return -ENOTTY; > > + int ret = pcie_reset_flr(dev, probe); > > > > if (probe) > > - return 0; > > - > > - pcie_flr(dev); > > + return ret; > > > > msleep(250); > > > > @@ -5182,7 +5179,7 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x443, quirk_intel_qat_vf_cap); > > */ > > static void quirk_no_flr(struct pci_dev *dev) > > { > > - dev->dev_flags |= PCI_DEV_FLAGS_NO_FLR_RESET; > > + dev->has_flr = 0; > > } > > DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_AMD, 0x1487, quirk_no_flr); > > DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_AMD, 0x148c, quirk_no_flr); > > diff --git a/include/linux/pci.h b/include/linux/pci.h > > index c20211e59..74f42a2cd 100644 > > --- a/include/linux/pci.h > > +++ b/include/linux/pci.h > > @@ -223,10 +223,8 @@ enum pci_dev_flags { > > PCI_DEV_FLAGS_VPD_REF_F0 = (__force pci_dev_flags_t) (1 << 8), > > /* A non-root bridge where translation occurs, stop alias search here */ > > PCI_DEV_FLAGS_BRIDGE_XLATE_ROOT = (__force pci_dev_flags_t) (1 << 9), > > - /* Do not use FLR even if device advertises PCI_AF_CAP */ > > - PCI_DEV_FLAGS_NO_FLR_RESET = (__force pci_dev_flags_t) (1 << 10), > > /* Don't use Relaxed Ordering for TLPs directed at this device */ > > - PCI_DEV_FLAGS_NO_RELAXED_ORDERING = (__force pci_dev_flags_t) (1 << 11), > > + PCI_DEV_FLAGS_NO_RELAXED_ORDERING = (__force pci_dev_flags_t) (1 << 10), > > }; > > > > enum pci_irq_reroute_variant { > > @@ -381,6 +379,7 @@ struct pci_dev { > > bit manually */ > > unsigned int d3hot_delay; /* D3hot->D0 transition time in ms */ > > unsigned int d3cold_delay; /* D3cold->D0 transition time in ms */ > > + unsigned int has_flr:1; > > There are more space efficient places to define this. Thanks, > > Alex > Oops. Will update this in v9. Thanks, Amey > > > > #ifdef CONFIG_PCIEASPM > > struct pcie_link_state *link_state; /* ASPM link state */ > > @@ -1225,7 +1224,7 @@ u32 pcie_bandwidth_available(struct pci_dev *dev, struct pci_dev **limiting_dev, > > enum pci_bus_speed *speed, > > enum pcie_link_width *width); > > void pcie_print_link_status(struct pci_dev *dev); > > -bool pcie_has_flr(struct pci_dev *dev); > > +int pcie_reset_flr(struct pci_dev *dev, int probe); > > int pcie_flr(struct pci_dev *dev); > > int __pci_reset_function_locked(struct pci_dev *dev); > > int pci_reset_function(struct pci_dev *dev); >
On Thu, 1 Jul 2021 01:34:15 +0530 Amey Narkhede <ameynarkhede03@gmail.com> wrote: > On 21/06/30 11:56AM, Alex Williamson wrote: > > On Tue, 29 Jun 2021 21:30:57 +0530 > > Amey Narkhede <ameynarkhede03@gmail.com> wrote: > > > > > Add has_flr bitfield in struct pci_dev to indicate support for pcie flr > > > to avoid reading PCI_EXP_DEVCAP multiple times and get rid of > > > PCI_DEV_FLAGS_NO_FLR_RESET in quirk_no_flr(). > > > > > > Currently there is separate function pcie_has_flr() to probe if pcie flr is > > > supported by the device which does not match the calling convention > > > followed by reset methods which use second function argument to decide > > > whether to probe or not. Add new function pcie_reset_flr() that follows > > > the calling convention of reset methods. > > > > > > Co-developed-by: Alex Williamson <alex.williamson@redhat.com> > > > Signed-off-by: Alex Williamson <alex.williamson@redhat.com> > > > > There are some non-trivial changes here vs previous, should probably > > drop this. > > > > > Signed-off-by: Amey Narkhede <ameynarkhede03@gmail.com> > > > --- > > > drivers/crypto/cavium/nitrox/nitrox_main.c | 4 +- > > > drivers/pci/pci.c | 62 +++++++++++----------- > > > drivers/pci/pcie/aer.c | 12 ++--- > > > drivers/pci/probe.c | 6 ++- > > > drivers/pci/quirks.c | 11 ++-- > > > include/linux/pci.h | 7 ++- > > > 6 files changed, 47 insertions(+), 55 deletions(-) > > > > > > diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c > > > index facc8e6bc..15d6c8452 100644 > > > --- a/drivers/crypto/cavium/nitrox/nitrox_main.c > > > +++ b/drivers/crypto/cavium/nitrox/nitrox_main.c > > > @@ -306,9 +306,7 @@ static int nitrox_device_flr(struct pci_dev *pdev) > > > return -ENOMEM; > > > } > > > > > > - /* check flr support */ > > > - if (pcie_has_flr(pdev)) > > > - pcie_flr(pdev); > > > + pcie_reset_flr(pdev, 0); > > > > > > pci_restore_state(pdev); > > > > > > diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c > > > index 452351025..28f099a4f 100644 > > > --- a/drivers/pci/pci.c > > > +++ b/drivers/pci/pci.c > > > @@ -4611,32 +4611,12 @@ int pci_wait_for_pending_transaction(struct pci_dev *dev) > > > } > > > EXPORT_SYMBOL(pci_wait_for_pending_transaction); > > > > > > -/** > > > - * pcie_has_flr - check if a device supports function level resets > > > - * @dev: device to check > > > - * > > > - * Returns true if the device advertises support for PCIe function level > > > - * resets. > > > - */ > > > -bool pcie_has_flr(struct pci_dev *dev) > > > -{ > > > - u32 cap; > > > - > > > - if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET) > > > - return false; > > > - > > > - pcie_capability_read_dword(dev, PCI_EXP_DEVCAP, &cap); > > > - return cap & PCI_EXP_DEVCAP_FLR; > > > -} > > > -EXPORT_SYMBOL_GPL(pcie_has_flr); > > > - > > > /** > > > * pcie_flr - initiate a PCIe function level reset > > > * @dev: device to reset > > > * > > > - * Initiate a function level reset on @dev. The caller should ensure the > > > - * device supports FLR before calling this function, e.g. by using the > > > - * pcie_has_flr() helper. > > > + * Initiate a function level reset unconditionally on @dev without > > > + * checking any flags and DEVCAP > > > */ > > > int pcie_flr(struct pci_dev *dev) > > > { > > > @@ -4659,16 +4639,35 @@ int pcie_flr(struct pci_dev *dev) > > > } > > > EXPORT_SYMBOL_GPL(pcie_flr); > > > > > > +/** > > > + * pcie_reset_flr - initiate a PCIe function level reset > > > + * @dev: device to reset > > > + * @probe: If set, only check if the device can be reset this way. > > > + * > > > + * Initiate a function level reset on @dev. > > > + */ > > > +int pcie_reset_flr(struct pci_dev *dev, int probe) > > > +{ > > > + if (!dev->has_flr) > > > + return -ENOTTY; > > > + > > > + if (probe) > > > + return 0; > > > + > > > + return pcie_flr(dev); > > > +} > > > +EXPORT_SYMBOL_GPL(pcie_reset_flr); > > > + > > > static int pci_af_flr(struct pci_dev *dev, int probe) > > > { > > > int pos; > > > u8 cap; > > > > > > - pos = pci_find_capability(dev, PCI_CAP_ID_AF); > > > - if (!pos) > > > + if (!dev->has_flr) > > > return -ENOTTY; > > > > > > - if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET) > > > + pos = pci_find_capability(dev, PCI_CAP_ID_AF); > > > + if (!pos) > > > return -ENOTTY; > > > > > > pci_read_config_byte(dev, pos + PCI_AF_CAP, &cap); > > > > > > How can has_flr encompass both methods of invoking FLR? PCIe FLR is > > not a prerequisite to AF FLR. > > > I see. Does this mean that there should be a separate flag for disabling > AF FLR? There hasn't been a need so far. Per the ECN, the AF capability is meant to make select PCIe features available on conventional PCI devices. It seems like it would be against the spirit of the AF capability to implement both an AF capability and a PCIe capability, but I don't see that it's definitively addressed by the spec. AF FLR is sufficiently rare that it's probably reasonable to make a has_pcie_flr bit on the device and leave AF FLR alone. I can't really say that I'm in favor of assigning a has_flr bit the double duty of also quirking broken FLR, if nothing else it's inconsistent with our other means of quirking resets. > > > @@ -5139,11 +5138,9 @@ int __pci_reset_function_locked(struct pci_dev *dev) > > > rc = pci_dev_specific_reset(dev, 0); > > > if (rc != -ENOTTY) > > > return rc; > > > - if (pcie_has_flr(dev)) { > > > - rc = pcie_flr(dev); > > > - if (rc != -ENOTTY) > > > - return rc; > > > - } > > > + rc = pcie_reset_flr(dev, 0); > > > + if (rc != -ENOTTY) > > > + return rc; > > > rc = pci_af_flr(dev, 0); > > > if (rc != -ENOTTY) > > > return rc; > > > @@ -5174,8 +5171,9 @@ int pci_probe_reset_function(struct pci_dev *dev) > > > rc = pci_dev_specific_reset(dev, 1); > > > if (rc != -ENOTTY) > > > return rc; > > > - if (pcie_has_flr(dev)) > > > - return 0; > > > + rc = pcie_reset_flr(dev, 1); > > > + if (rc != -ENOTTY) > > > + return rc; > > > rc = pci_af_flr(dev, 1); > > > if (rc != -ENOTTY) > > > return rc; > > > diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c > > > index ec943cee5..98077595a 100644 > > > --- a/drivers/pci/pcie/aer.c > > > +++ b/drivers/pci/pcie/aer.c > > > @@ -1405,13 +1405,11 @@ static pci_ers_result_t aer_root_reset(struct pci_dev *dev) > > > } > > > > > > if (type == PCI_EXP_TYPE_RC_EC || type == PCI_EXP_TYPE_RC_END) { > > > - if (pcie_has_flr(dev)) { > > > - rc = pcie_flr(dev); > > > - pci_info(dev, "has been reset (%d)\n", rc); > > > - } else { > > > - pci_info(dev, "not reset (no FLR support)\n"); > > > - rc = -ENOTTY; > > > - } > > > + rc = pcie_reset_flr(dev, 0); > > > + if (!rc) > > > + pci_info(dev, "has been reset\n"); > > > + else > > > + pci_info(dev, "not reset (no FLR support: %d)\n", rc); > > > } else { > > > rc = pci_bus_error_reset(dev); > > > pci_info(dev, "%s Port link has been reset (%d)\n", > > > diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c > > > index 3a62d09b8..862d91615 100644 > > > --- a/drivers/pci/probe.c > > > +++ b/drivers/pci/probe.c > > > @@ -1487,6 +1487,7 @@ void set_pcie_port_type(struct pci_dev *pdev) > > > { > > > int pos; > > > u16 reg16; > > > + u32 reg32; > > > int type; > > > struct pci_dev *parent; > > > > > > @@ -1497,8 +1498,9 @@ void set_pcie_port_type(struct pci_dev *pdev) > > > pdev->pcie_cap = pos; > > > pci_read_config_word(pdev, pos + PCI_EXP_FLAGS, ®16); > > > pdev->pcie_flags_reg = reg16; > > > - pci_read_config_word(pdev, pos + PCI_EXP_DEVCAP, ®16); > > > - pdev->pcie_mpss = reg16 & PCI_EXP_DEVCAP_PAYLOAD; > > > + pci_read_config_dword(pdev, pos + PCI_EXP_DEVCAP, ®32); > > > + pdev->pcie_mpss = reg32 & PCI_EXP_DEVCAP_PAYLOAD; > > > + pdev->has_flr = reg32 & PCI_EXP_DEVCAP_FLR ? 1 : 0; > > > > We only set has_flr for PCIe FLR, so I think this effectively kills AF > > FLR. The ternary should be unnecessary here. > > > I believe only one of the PCIe FLR and PCI AF FLR will be available at a time. Likely, possibly required depending on spec interpretation. > So I think it should be okay to check for PCI_AF_CAP_FLR and set > pdev->has_flr accordingly. AF FLR is rare, are the reasons we're wanting to cache its availability as relevant as they are for PCIe FLR? > One question though, what is PCI_AF_CAP_TP Transaction Pending > and how is it related to PCI AF FLR? It's required to be implemented if AF FLR is implemented. We're supposed to wait for TP to clear before issuing an FLR to avoid an issue with stale completions of outstanding transactions. Thanks, Alex
Please add "()" after function names in subject lines. On Wed, Jun 30, 2021 at 02:49:55PM -0600, Alex Williamson wrote: > On Thu, 1 Jul 2021 01:34:15 +0530 > Amey Narkhede <ameynarkhede03@gmail.com> wrote: > > On 21/06/30 11:56AM, Alex Williamson wrote: > > > On Tue, 29 Jun 2021 21:30:57 +0530 > > > Amey Narkhede <ameynarkhede03@gmail.com> wrote: > > > > > > > Add has_flr bitfield in struct pci_dev to indicate support for pcie flr > > > > to avoid reading PCI_EXP_DEVCAP multiple times and get rid of > > > > PCI_DEV_FLAGS_NO_FLR_RESET in quirk_no_flr(). > > > > > > > > Currently there is separate function pcie_has_flr() to probe if pcie flr is > > > > supported by the device which does not match the calling convention > > > > followed by reset methods which use second function argument to decide > > > > whether to probe or not. Add new function pcie_reset_flr() that follows > > > > the calling convention of reset methods. s/pcie/PCIe/ (except for variables, function names, etc) > > > > static int pci_af_flr(struct pci_dev *dev, int probe) > > > > { > > > > int pos; > > > > u8 cap; > > > > > > > > - pos = pci_find_capability(dev, PCI_CAP_ID_AF); > > > > - if (!pos) > > > > + if (!dev->has_flr) > > > > return -ENOTTY; > > > > > > > > - if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET) > > > > + pos = pci_find_capability(dev, PCI_CAP_ID_AF); > > > > + if (!pos) > > > > return -ENOTTY; > > > > > > > > pci_read_config_byte(dev, pos + PCI_AF_CAP, &cap); > > > > > > > > > How can has_flr encompass both methods of invoking FLR? PCIe FLR is > > > not a prerequisite to AF FLR. > > > > > I see. Does this mean that there should be a separate flag for disabling > > AF FLR? > > There hasn't been a need so far. Per the ECN, the AF capability is > meant to make select PCIe features available on conventional PCI > devices. It seems like it would be against the spirit of the AF > capability to implement both an AF capability and a PCIe capability, > but I don't see that it's definitively addressed by the spec. > > AF FLR is sufficiently rare that it's probably reasonable to make a > has_pcie_flr bit on the device and leave AF FLR alone. This sounds good to me. I agree that I'd prefer not to have a single bit that applies to both AF FLR and PCIe FLR since they are distinct mechanisms, discovered and initiated differently. > I can't really say that I'm in favor of assigning a has_flr bit the > double duty of also quirking broken FLR, if nothing else it's > inconsistent with our other means of quirking resets. By "other means of quirking resets," do you mean PCI_DEV_FLAGS_NO_BUS_RESET, PCI_DEV_FLAGS_NO_PM_RESET, and PCI_DEV_FLAGS_NO_FLR_RESET? I agree that a pdev->has_pcie_flr bit would be different from those, and maybe it's better to stick with PCI_DEV_FLAGS_NO_FLR_RESET for now. In general, I don't like the dual approach of some things being in the pci_dev_flags enum and others being "unsigned int foo:1" in the struct pci_dev because I can't tell if there's a reason to choose one over the other. If there's not a reason to have both, I'd like to migrate them all to the ":1" approach because it seems a little more readable to me. Bjorn
diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c index facc8e6bc..15d6c8452 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_main.c +++ b/drivers/crypto/cavium/nitrox/nitrox_main.c @@ -306,9 +306,7 @@ static int nitrox_device_flr(struct pci_dev *pdev) return -ENOMEM; } - /* check flr support */ - if (pcie_has_flr(pdev)) - pcie_flr(pdev); + pcie_reset_flr(pdev, 0); pci_restore_state(pdev); diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 452351025..28f099a4f 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -4611,32 +4611,12 @@ int pci_wait_for_pending_transaction(struct pci_dev *dev) } EXPORT_SYMBOL(pci_wait_for_pending_transaction); -/** - * pcie_has_flr - check if a device supports function level resets - * @dev: device to check - * - * Returns true if the device advertises support for PCIe function level - * resets. - */ -bool pcie_has_flr(struct pci_dev *dev) -{ - u32 cap; - - if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET) - return false; - - pcie_capability_read_dword(dev, PCI_EXP_DEVCAP, &cap); - return cap & PCI_EXP_DEVCAP_FLR; -} -EXPORT_SYMBOL_GPL(pcie_has_flr); - /** * pcie_flr - initiate a PCIe function level reset * @dev: device to reset * - * Initiate a function level reset on @dev. The caller should ensure the - * device supports FLR before calling this function, e.g. by using the - * pcie_has_flr() helper. + * Initiate a function level reset unconditionally on @dev without + * checking any flags and DEVCAP */ int pcie_flr(struct pci_dev *dev) { @@ -4659,16 +4639,35 @@ int pcie_flr(struct pci_dev *dev) } EXPORT_SYMBOL_GPL(pcie_flr); +/** + * pcie_reset_flr - initiate a PCIe function level reset + * @dev: device to reset + * @probe: If set, only check if the device can be reset this way. + * + * Initiate a function level reset on @dev. + */ +int pcie_reset_flr(struct pci_dev *dev, int probe) +{ + if (!dev->has_flr) + return -ENOTTY; + + if (probe) + return 0; + + return pcie_flr(dev); +} +EXPORT_SYMBOL_GPL(pcie_reset_flr); + static int pci_af_flr(struct pci_dev *dev, int probe) { int pos; u8 cap; - pos = pci_find_capability(dev, PCI_CAP_ID_AF); - if (!pos) + if (!dev->has_flr) return -ENOTTY; - if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET) + pos = pci_find_capability(dev, PCI_CAP_ID_AF); + if (!pos) return -ENOTTY; pci_read_config_byte(dev, pos + PCI_AF_CAP, &cap); @@ -5139,11 +5138,9 @@ int __pci_reset_function_locked(struct pci_dev *dev) rc = pci_dev_specific_reset(dev, 0); if (rc != -ENOTTY) return rc; - if (pcie_has_flr(dev)) { - rc = pcie_flr(dev); - if (rc != -ENOTTY) - return rc; - } + rc = pcie_reset_flr(dev, 0); + if (rc != -ENOTTY) + return rc; rc = pci_af_flr(dev, 0); if (rc != -ENOTTY) return rc; @@ -5174,8 +5171,9 @@ int pci_probe_reset_function(struct pci_dev *dev) rc = pci_dev_specific_reset(dev, 1); if (rc != -ENOTTY) return rc; - if (pcie_has_flr(dev)) - return 0; + rc = pcie_reset_flr(dev, 1); + if (rc != -ENOTTY) + return rc; rc = pci_af_flr(dev, 1); if (rc != -ENOTTY) return rc; diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c index ec943cee5..98077595a 100644 --- a/drivers/pci/pcie/aer.c +++ b/drivers/pci/pcie/aer.c @@ -1405,13 +1405,11 @@ static pci_ers_result_t aer_root_reset(struct pci_dev *dev) } if (type == PCI_EXP_TYPE_RC_EC || type == PCI_EXP_TYPE_RC_END) { - if (pcie_has_flr(dev)) { - rc = pcie_flr(dev); - pci_info(dev, "has been reset (%d)\n", rc); - } else { - pci_info(dev, "not reset (no FLR support)\n"); - rc = -ENOTTY; - } + rc = pcie_reset_flr(dev, 0); + if (!rc) + pci_info(dev, "has been reset\n"); + else + pci_info(dev, "not reset (no FLR support: %d)\n", rc); } else { rc = pci_bus_error_reset(dev); pci_info(dev, "%s Port link has been reset (%d)\n", diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index 3a62d09b8..862d91615 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -1487,6 +1487,7 @@ void set_pcie_port_type(struct pci_dev *pdev) { int pos; u16 reg16; + u32 reg32; int type; struct pci_dev *parent; @@ -1497,8 +1498,9 @@ void set_pcie_port_type(struct pci_dev *pdev) pdev->pcie_cap = pos; pci_read_config_word(pdev, pos + PCI_EXP_FLAGS, ®16); pdev->pcie_flags_reg = reg16; - pci_read_config_word(pdev, pos + PCI_EXP_DEVCAP, ®16); - pdev->pcie_mpss = reg16 & PCI_EXP_DEVCAP_PAYLOAD; + pci_read_config_dword(pdev, pos + PCI_EXP_DEVCAP, ®32); + pdev->pcie_mpss = reg32 & PCI_EXP_DEVCAP_PAYLOAD; + pdev->has_flr = reg32 & PCI_EXP_DEVCAP_FLR ? 1 : 0; parent = pci_upstream_bridge(pdev); if (!parent) diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index d85914afe..1efdc4e5a 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -3819,7 +3819,7 @@ static int nvme_disable_and_flr(struct pci_dev *dev, int probe) u32 cfg; if (dev->class != PCI_CLASS_STORAGE_EXPRESS || - !pcie_has_flr(dev) || !pci_resource_start(dev, 0)) + pcie_reset_flr(dev, 1) || !pci_resource_start(dev, 0)) return -ENOTTY; if (probe) @@ -3888,13 +3888,10 @@ static int nvme_disable_and_flr(struct pci_dev *dev, int probe) */ static int delay_250ms_after_flr(struct pci_dev *dev, int probe) { - if (!pcie_has_flr(dev)) - return -ENOTTY; + int ret = pcie_reset_flr(dev, probe); if (probe) - return 0; - - pcie_flr(dev); + return ret; msleep(250); @@ -5182,7 +5179,7 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x443, quirk_intel_qat_vf_cap); */ static void quirk_no_flr(struct pci_dev *dev) { - dev->dev_flags |= PCI_DEV_FLAGS_NO_FLR_RESET; + dev->has_flr = 0; } DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_AMD, 0x1487, quirk_no_flr); DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_AMD, 0x148c, quirk_no_flr); diff --git a/include/linux/pci.h b/include/linux/pci.h index c20211e59..74f42a2cd 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -223,10 +223,8 @@ enum pci_dev_flags { PCI_DEV_FLAGS_VPD_REF_F0 = (__force pci_dev_flags_t) (1 << 8), /* A non-root bridge where translation occurs, stop alias search here */ PCI_DEV_FLAGS_BRIDGE_XLATE_ROOT = (__force pci_dev_flags_t) (1 << 9), - /* Do not use FLR even if device advertises PCI_AF_CAP */ - PCI_DEV_FLAGS_NO_FLR_RESET = (__force pci_dev_flags_t) (1 << 10), /* Don't use Relaxed Ordering for TLPs directed at this device */ - PCI_DEV_FLAGS_NO_RELAXED_ORDERING = (__force pci_dev_flags_t) (1 << 11), + PCI_DEV_FLAGS_NO_RELAXED_ORDERING = (__force pci_dev_flags_t) (1 << 10), }; enum pci_irq_reroute_variant { @@ -381,6 +379,7 @@ struct pci_dev { bit manually */ unsigned int d3hot_delay; /* D3hot->D0 transition time in ms */ unsigned int d3cold_delay; /* D3cold->D0 transition time in ms */ + unsigned int has_flr:1; #ifdef CONFIG_PCIEASPM struct pcie_link_state *link_state; /* ASPM link state */ @@ -1225,7 +1224,7 @@ u32 pcie_bandwidth_available(struct pci_dev *dev, struct pci_dev **limiting_dev, enum pci_bus_speed *speed, enum pcie_link_width *width); void pcie_print_link_status(struct pci_dev *dev); -bool pcie_has_flr(struct pci_dev *dev); +int pcie_reset_flr(struct pci_dev *dev, int probe); int pcie_flr(struct pci_dev *dev); int __pci_reset_function_locked(struct pci_dev *dev); int pci_reset_function(struct pci_dev *dev);