diff --git a/Documentation/ABI/testing/sysfs-class-devlink b/Documentation/ABI/testing/sysfs-class-devlink new file mode 100644 index 0000000000000000000000000000000000000000..3a24973abb83509cfc6aba46ddb7e7ba482cacc8 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-class-devlink @@ -0,0 +1,126 @@ +What: /sys/class/devlink/.../ +Date: May 2020 +Contact: Saravana Kannan +Description: + Provide a place in sysfs for the device link objects in the + kernel at any given time. The name of a device link directory, + denoted as ... above, is of the form : + where is the supplier device name and is + the consumer device name. + +What: /sys/class/devlink/.../auto_remove_on +Date: May 2020 +Contact: Saravana Kannan +Description: + This file indicates if the device link will ever be + automatically removed by the driver core when the consumer and + supplier devices themselves are still present. + + This will be one of the following strings: + + 'consumer unbind' + 'supplier unbind' + 'never' + + 'consumer unbind' means the device link will be removed when + the consumer's driver is unbound from the consumer device. + + 'supplier unbind' means the device link will be removed when + the supplier's driver is unbound from the supplier device. + + 'never' means the device link will not be automatically removed + when as long as the supplier and consumer devices themselves + are still present. + +What: /sys/class/devlink/.../consumer +Date: May 2020 +Contact: Saravana Kannan +Description: + This file is a symlink to the consumer device's sysfs directory. + +What: /sys/class/devlink/.../runtime_pm +Date: May 2020 +Contact: Saravana Kannan +Description: + This file indicates if the device link has any impact on the + runtime power management behavior of the consumer and supplier + devices. For example: Making sure the supplier doesn't enter + runtime suspend while the consumer is active. + + This will be one of the following strings: + + '0' - Does not affect runtime power management + '1' - Affects runtime power management + +What: /sys/class/devlink/.../status +Date: May 2020 +Contact: Saravana Kannan +Description: + This file indicates the status of the device link. The status + of a device link is affected by whether the supplier and + consumer devices have been bound to their corresponding + drivers. The status of a device link also affects the binding + and unbinding of the supplier and consumer devices with their + drivers and also affects whether the software state of the + supplier device is synced with the hardware state of the + supplier device after boot up. + See also: sysfs-devices-state_synced. + + This will be one of the following strings: + + 'not tracked' + 'dormant' + 'available' + 'consumer probing' + 'active' + 'supplier unbinding' + 'unknown' + + 'not tracked' means this device link does not track the status + and has no impact on the binding, unbinding and syncing the + hardware and software device state. + + 'dormant' means the supplier and the consumer devices have not + bound to their driver. + + 'available' means the supplier has bound to its driver and is + available to supply resources to the consumer device. + + 'consumer probing' means the consumer device is currently + trying to bind to its driver. + + 'active' means the supplier and consumer devices have both + bound successfully to their drivers. + + 'supplier unbinding' means the supplier devices is currently in + the process of unbinding from its driver. + + 'unknown' means the state of the device link is not any of the + above. If this is ever the value, there's a bug in the kernel. + +What: /sys/class/devlink/.../supplier +Date: May 2020 +Contact: Saravana Kannan +Description: + This file is a symlink to the supplier device's sysfs directory. + +What: /sys/class/devlink/.../sync_state_only +Date: May 2020 +Contact: Saravana Kannan +Description: + This file indicates if the device link is limited to only + affecting the syncing of the hardware and software state of the + supplier device. + + This will be one of the following strings: + + '0' + '1' - Affects runtime power management + + '0' means the device link can affect other device behaviors + like binding/unbinding, suspend/resume, runtime power + management, etc. + + '1' means the device link will only affect the syncing of + hardware and software state of the supplier device after boot + up and doesn't not affect other behaviors of the devices. diff --git a/Documentation/ABI/testing/sysfs-devices-consumer b/Documentation/ABI/testing/sysfs-devices-consumer new file mode 100644 index 0000000000000000000000000000000000000000..1f06d74d1c3ccca76cb67e48c456d269ca3b2435 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-devices-consumer @@ -0,0 +1,8 @@ +What: /sys/devices/.../consumer: +Date: May 2020 +Contact: Saravana Kannan +Description: + The /sys/devices/.../consumer: are symlinks to device + links where this device is the supplier. denotes the + name of the consumer in that device link. There can be zero or + more of these symlinks for a given device. diff --git a/Documentation/ABI/testing/sysfs-devices-hisi_ptt b/Documentation/ABI/testing/sysfs-devices-hisi_ptt new file mode 100644 index 0000000000000000000000000000000000000000..d7e206b4901c092babc3ca87e2b13df6c51207b3 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-devices-hisi_ptt @@ -0,0 +1,113 @@ +What: /sys/devices/hisi_ptt_/tune +Date: October 2022 +KernelVersion: 6.1 +Contact: Yicong Yang +Description: This directory contains files for tuning the PCIe link + parameters(events). Each file is named after the event + of the PCIe link. + + See Documentation/trace/hisi-ptt.rst for more information. + +What: /sys/devices/hisi_ptt_/tune/qos_tx_cpl +Date: October 2022 +KernelVersion: 6.1 +Contact: Yicong Yang +Description: (RW) Controls the weight of Tx completion TLPs, which influence + the proportion of outbound completion TLPs on the PCIe link. + The available tune data is [0, 1, 2]. Writing a negative value + will return an error, and out of range values will be converted + to 2. The value indicates a probable level of the event. + +What: /sys/devices/hisi_ptt_/tune/qos_tx_np +Date: October 2022 +KernelVersion: 6.1 +Contact: Yicong Yang +Description: (RW) Controls the weight of Tx non-posted TLPs, which influence + the proportion of outbound non-posted TLPs on the PCIe link. + The available tune data is [0, 1, 2]. Writing a negative value + will return an error, and out of range values will be converted + to 2. The value indicates a probable level of the event. + +What: /sys/devices/hisi_ptt_/tune/qos_tx_p +Date: October 2022 +KernelVersion: 6.1 +Contact: Yicong Yang +Description: (RW) Controls the weight of Tx posted TLPs, which influence the + proportion of outbound posted TLPs on the PCIe link. + The available tune data is [0, 1, 2]. Writing a negative value + will return an error, and out of range values will be converted + to 2. The value indicates a probable level of the event. + +What: /sys/devices/hisi_ptt_/tune/rx_alloc_buf_level +Date: October 2022 +KernelVersion: 6.1 +Contact: Yicong Yang +Description: (RW) Control the allocated buffer watermark for inbound packets. + The packets will be stored in the buffer first and then transmitted + either when the watermark reached or when timed out. + The available tune data is [0, 1, 2]. Writing a negative value + will return an error, and out of range values will be converted + to 2. The value indicates a probable level of the event. + +What: /sys/devices/hisi_ptt_/tune/tx_alloc_buf_level +Date: October 2022 +KernelVersion: 6.1 +Contact: Yicong Yang +Description: (RW) Control the allocated buffer watermark of outbound packets. + The packets will be stored in the buffer first and then transmitted + either when the watermark reached or when timed out. + The available tune data is [0, 1, 2]. Writing a negative value + will return an error, and out of range values will be converted + to 2. The value indicates a probable level of the event. + +What: /sys/devices/hisi_ptt_/root_port_filters +Date: May 2023 +KernelVersion: 6.5 +Contact: Yicong Yang +Description: This directory contains the files providing the PCIe Root Port filters + information used for PTT trace. Each file is named after the supported + Root Port device name ::.. + + See the description of the "filter" in Documentation/trace/hisi-ptt.rst + for more information. + +What: /sys/devices/hisi_ptt_/root_port_filters/multiselect +Date: May 2023 +KernelVersion: 6.5 +Contact: Yicong Yang +Description: (Read) Indicates if this kind of filter can be selected at the same + time as others filters, or must be used on it's own. 1 indicates + the former case and 0 indicates the latter. + +What: /sys/devices/hisi_ptt_/root_port_filters/ +Date: May 2023 +KernelVersion: 6.5 +Contact: Yicong Yang +Description: (Read) Indicates the filter value of this Root Port filter, which + can be used to control the TLP headers to trace by the PTT trace. + +What: /sys/devices/hisi_ptt_/requester_filters +Date: May 2023 +KernelVersion: 6.5 +Contact: Yicong Yang +Description: This directory contains the files providing the PCIe Requester filters + information used for PTT trace. Each file is named after the supported + Endpoint device name ::.. + + See the description of the "filter" in Documentation/trace/hisi-ptt.rst + for more information. + +What: /sys/devices/hisi_ptt_/requester_filters/multiselect +Date: May 2023 +KernelVersion: 6.5 +Contact: Yicong Yang +Description: (Read) Indicates if this kind of filter can be selected at the same + time as others filters, or must be used on it's own. 1 indicates + the former case and 0 indicates the latter. + +What: /sys/devices/hisi_ptt_/requester_filters/ +Date: May 2023 +KernelVersion: 6.5 +Contact: Yicong Yang +Description: (Read) Indicates the filter value of this Requester filter, which + can be used to control the TLP headers to trace by the PTT trace. diff --git a/Documentation/ABI/testing/sysfs-devices-platform-kunpeng_hccs b/Documentation/ABI/testing/sysfs-devices-platform-kunpeng_hccs new file mode 100644 index 0000000000000000000000000000000000000000..1ddfdb790efe13bed97fc9d4e38674574d7fb0e2 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-devices-platform-kunpeng_hccs @@ -0,0 +1,58 @@ +What: /sys/devices/platform/HISI04Bx:00/chipX/dieY/all_linked +What: /sys/devices/platform/HISI04Bx:00/chipX/dieY/linked_full_lane +What: /sys/devices/platform/HISI04Bx:00/chipX/dieY/crc_err_cnt +Date: November 2023 +KernelVersion: 6.6 +Contact: Huisong Li +Description: + The /sys/devices/platform/HISI04Bx:00/chipX/dieY/ directory + contains read-only attributes exposing some summarization + information of all HCCS ports under a specified die. + The Y in 'dieY' indicates the hardware id of the die on chip who + has chip id X. + + There are following attributes in this directory: + + ================= ==== ========================================= + all_linked: (RO) if all enabled ports on this die are + linked (bool). + linked_full_lane: (RO) if all linked ports on this die are full + lane (bool). + crc_err_cnt: (RO) total CRC err count for all ports on this + chip. + ================= ==== ========================================= + +What: /sys/devices/platform/HISI04Bx:00/chipX/dieY/hccsN/type +What: /sys/devices/platform/HISI04Bx:00/chipX/dieY/hccsN/lane_mode +What: /sys/devices/platform/HISI04Bx:00/chipX/dieY/hccsN/enable +What: /sys/devices/platform/HISI04Bx:00/chipX/dieY/hccsN/cur_lane_num +What: /sys/devices/platform/HISI04Bx:00/chipX/dieY/hccsN/link_fsm +What: /sys/devices/platform/HISI04Bx:00/chipX/dieY/hccsN/lane_mask +What: /sys/devices/platform/HISI04Bx:00/chipX/dieY/hccsN/crc_err_cnt +Date: November 2023 +KernelVersion: 6.6 +Contact: Huisong Li +Description: + The /sys/devices/platform/HISI04Bx/chipX/dieX/hccsN/ directory + contains read-only attributes exposing information about + a HCCS port. The N value in 'hccsN' indicates this port id. + The X in 'chipX' indicates the ID of the chip to which the + HCCS port belongs. For example, X ranges from to 'n - 1' if the + chip number on platform is n. + The Y in 'dieY' indicates the hardware id of the die to which + the hccs port belongs. + Note: type, lane_mode and enable are fixed attributes on running + platform. + + The HCCS port have the following attributes: + + ============= ==== ============================================= + type: (RO) port type (string), e.g. HCCS-v1 -> H32 + lane_mode: (RO) the lane mode of this port (string), e.g. x8 + enable: (RO) indicate if this port is enabled (bool). + cur_lane_num: (RO) current lane number of this port. + link_fsm: (RO) link finite state machine of this port. + lane_mask: (RO) current lane mask of this port, every bit + indicates a lane. + crc_err_cnt: (RO) CRC err count on this port. + ============= ==== ============================================= diff --git a/Documentation/ABI/testing/sysfs-devices-supplier b/Documentation/ABI/testing/sysfs-devices-supplier new file mode 100644 index 0000000000000000000000000000000000000000..a919e0db5e902ca572ddde6560663a2ef05ad15d --- /dev/null +++ b/Documentation/ABI/testing/sysfs-devices-supplier @@ -0,0 +1,8 @@ +What: /sys/devices/.../supplier: +Date: May 2020 +Contact: Saravana Kannan +Description: + The /sys/devices/.../supplier: are symlinks to device + links where this device is the consumer. denotes the + name of the supplier in that device link. There can be zero or + more of these symlinks for a given device. diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index adee43c2cf4976880cb5158fb3daecfbe3b02fc2..895d5089090a0d32957ff6c30c66dbf1ea3cf95f 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -1965,9 +1965,31 @@ begins at 0 and the maximum value is "number of CPUs in system - 1". - The format of is described above. - + managed_irq + + Isolate from being targeted by managed interrupts + which have an interrupt mask containing isolated + CPUs. The affinity of managed interrupts is + handled by the kernel and cannot be changed via + the /proc/irq/* interfaces. + + This isolation is best effort and only effective + if the automatically assigned interrupt mask of a + device queue contains isolated and housekeeping + CPUs. If housekeeping CPUs are online then such + interrupts are directed to the housekeeping CPU + so that IO submitted on the housekeeping CPU + cannot disturb the isolated CPU. + + If a queue's affinity mask contains only isolated + CPUs then this parameter has no effect on the + interrupt routing decision, though interrupts are + only delivered when tasks running on those + isolated CPUs submit IO. IO submitted on + housekeeping CPUs has no influence on those + queues. + The format of is described above. iucv= [HW,NET] diff --git a/Documentation/admin-guide/perf/hisi-pcie-pmu.rst b/Documentation/admin-guide/perf/hisi-pcie-pmu.rst new file mode 100644 index 0000000000000000000000000000000000000000..f93869db85a8e9ef76b1fc4ff8d6407774328a69 --- /dev/null +++ b/Documentation/admin-guide/perf/hisi-pcie-pmu.rst @@ -0,0 +1,124 @@ +================================================ +HiSilicon PCIe Performance Monitoring Unit (PMU) +================================================ + +On Hip09, HiSilicon PCIe Performance Monitoring Unit (PMU) could monitor +bandwidth, latency, bus utilization and buffer occupancy data of PCIe. + +Each PCIe Core has a PMU to monitor multi Root Ports of this PCIe Core and +all Endpoints downstream these Root Ports. + + +HiSilicon PCIe PMU driver +========================= + +The PCIe PMU driver registers a perf PMU with the name of its sicl-id and PCIe +Core id.:: + + /sys/bus/event_source/hisi_pcie_ + +PMU driver provides description of available events and filter options in sysfs, +see /sys/bus/event_source/devices/hisi_pcie_. + +The "format" directory describes all formats of the config (events) and config1 +(filter options) fields of the perf_event_attr structure. The "events" directory +describes all documented events shown in perf list. + +The "identifier" sysfs file allows users to identify the version of the +PMU hardware device. + +The "bus" sysfs file allows users to get the bus number of Root Ports +monitored by PMU. + +Example usage of perf:: + + $# perf list + hisi_pcie0_0/rx_mwr_latency/ [kernel PMU event] + hisi_pcie0_0/rx_mwr_cnt/ [kernel PMU event] + ------------------------------------------ + + $# perf stat -e hisi_pcie0_0/rx_mwr_latency/ + $# perf stat -e hisi_pcie0_0/rx_mwr_cnt/ + $# perf stat -g -e hisi_pcie0_0/rx_mwr_latency/ -e hisi_pcie0_0/rx_mwr_cnt/ + +The current driver does not support sampling. So "perf record" is unsupported. +Also attach to a task is unsupported for PCIe PMU. + +Filter options +-------------- + +1. Target filter +PMU could only monitor the performance of traffic downstream target Root Ports +or downstream target Endpoint. PCIe PMU driver support "port" and "bdf" +interfaces for users, and these two interfaces aren't supported at the same +time. + +-port +"port" filter can be used in all PCIe PMU events, target Root Port can be +selected by configuring the 16-bits-bitmap "port". Multi ports can be selected +for AP-layer-events, and only one port can be selected for TL/DL-layer-events. + +For example, if target Root Port is 0000:00:00.0 (x8 lanes), bit0 of bitmap +should be set, port=0x1; if target Root Port is 0000:00:04.0 (x4 lanes), +bit8 is set, port=0x100; if these two Root Ports are both monitored, port=0x101. + +Example usage of perf:: + + $# perf stat -e hisi_pcie0_0/rx_mwr_latency,port=0x1/ sleep 5 + +-bdf + +"bdf" filter can only be used in bandwidth events, target Endpoint is selected +by configuring BDF to "bdf". Counter only counts the bandwidth of message +requested by target Endpoint. + +For example, "bdf=0x3900" means BDF of target Endpoint is 0000:39:00.0. + +Example usage of perf:: + + $# perf stat -e hisi_pcie0_0/rx_mrd_flux,bdf=0x3900/ sleep 5 + +2. Trigger filter +Event statistics start when the first time TLP length is greater/smaller +than trigger condition. You can set the trigger condition by writing "trig_len", +and set the trigger mode by writing "trig_mode". This filter can only be used +in bandwidth events. + +For example, "trig_len=4" means trigger condition is 2^4 DW, "trig_mode=0" +means statistics start when TLP length > trigger condition, "trig_mode=1" +means start when TLP length < condition. + +Example usage of perf:: + + $# perf stat -e hisi_pcie0_0/rx_mrd_flux,trig_len=0x4,trig_mode=1/ sleep 5 + +3. Threshold filter +Counter counts when TLP length within the specified range. You can set the +threshold by writing "thr_len", and set the threshold mode by writing +"thr_mode". This filter can only be used in bandwidth events. + +For example, "thr_len=4" means threshold is 2^4 DW, "thr_mode=0" means +counter counts when TLP length >= threshold, and "thr_mode=1" means counts +when TLP length < threshold. + +Example usage of perf:: + + $# perf stat -e hisi_pcie0_core0/rx_mrd_flux,thr_len=0x4,thr_mode=1/ sleep 5 + +4. TLP Length filter + + When counting bandwidth, the data can be composed of certain parts of TLP + packets. You can specify it through "len_mode": + + - 2'b00: Reserved (Do not use this since the behaviour is undefined) + - 2'b01: Bandwidth of TLP payloads + - 2'b10: Bandwidth of TLP headers + - 2'b11: Bandwidth of both TLP payloads and headers + + For example, "len_mode=2" means only counting the bandwidth of TLP headers + and "len_mode=3" means the final bandwidth data is composed of both TLP + headers and payloads. Default value if not specified is 2'b11. + + Example usage of perf:: + + $# perf stat -e hisi_pcie0_core0/rx_mrd_flux,len_mode=0x1/ sleep 5 diff --git a/Documentation/arm64/silicon-errata.rst b/Documentation/arm64/silicon-errata.rst index 6bafd420a69475314eb60af8c611b0930e0d905d..0ef894fe7a25cabb7071b678749e7c6924a6ab6e 100644 --- a/Documentation/arm64/silicon-errata.rst +++ b/Documentation/arm64/silicon-errata.rst @@ -140,6 +140,13 @@ stable kernels. | Hisilicon | Hip08 SMMU PMCG | #162001900 | N/A | | | Hip09 SMMU PMCG | | | +----------------+-----------------+-----------------+-----------------------------+ +| Hisilicon | TSV{110,200} | #1980005 | HISILICON_ERRATUM_1980005 | ++----------------+-----------------+-----------------+-----------------------------+ +| Hisilicon | Hip09 | #162100801 | HISILICON_ERRATUM_162100801 | ++----------------+-----------------+-----------------+-----------------------------+ +| Hisilicon | LINXICORE9100 | #162100125 | HISILICON_ERRATUM_162100125 | ++----------------+-----------------+-----------------+-----------------------------+ +| Hisilicon | HIP09 | #162102203 | HISILICON_ERRATUM_162102203 | +----------------+-----------------+-----------------+-----------------------------+ | Qualcomm Tech. | Kryo/Falkor v1 | E1003 | QCOM_FALKOR_ERRATUM_1003 | +----------------+-----------------+-----------------+-----------------------------+ diff --git a/Documentation/driver-api/driver-model/devres.rst b/Documentation/driver-api/driver-model/devres.rst index a100bef5495284d93ee487a7b2ad44985869497d..959d6f3941747fc5922fb19738951952b68161cf 100644 --- a/Documentation/driver-api/driver-model/devres.rst +++ b/Documentation/driver-api/driver-model/devres.rst @@ -412,6 +412,8 @@ SLAVE DMA ENGINE devm_acpi_dma_controller_register() SPI + devm_spi_alloc_master() + devm_spi_alloc_slave() devm_spi_register_master() WATCHDOG diff --git a/Documentation/scsi/libsas.txt b/Documentation/scsi/libsas.txt index 8cac6492aade7c41cc1cffafc4b4d0500ff9d624..7a1e9e06174d54adf912c093d25859328df6ed8d 100644 --- a/Documentation/scsi/libsas.txt +++ b/Documentation/scsi/libsas.txt @@ -154,12 +154,8 @@ The event interface: /* LLDD calls these to notify the class of an event. */ void (*notify_ha_event)(struct sas_ha_struct *, enum ha_event); - void (*notify_port_event)(struct sas_phy *, enum port_event); - void (*notify_phy_event)(struct sas_phy *, enum phy_event); - -When sas_register_ha() returns, those are set and can be -called by the LLDD to notify the SAS layer of such events -the SAS layer. + void sas_notify_port_event(struct sas_phy *, enum port_event, gfp_t); + void sas_notify_phy_event(struct sas_phy *, enum phy_event, gfp_t); The port notification: diff --git a/Documentation/trace/hisi-ptt.rst b/Documentation/trace/hisi-ptt.rst new file mode 100644 index 0000000000000000000000000000000000000000..989255eb56221e7fe56d990f03cc7e2970c9a4c5 --- /dev/null +++ b/Documentation/trace/hisi-ptt.rst @@ -0,0 +1,304 @@ +.. SPDX-License-Identifier: GPL-2.0 + +====================================== +HiSilicon PCIe Tune and Trace device +====================================== + +Introduction +============ + +HiSilicon PCIe tune and trace device (PTT) is a PCIe Root Complex +integrated Endpoint (RCiEP) device, providing the capability +to dynamically monitor and tune the PCIe link's events (tune), +and trace the TLP headers (trace). The two functions are independent, +but is recommended to use them together to analyze and enhance the +PCIe link's performance. + +On Kunpeng 930 SoC, the PCIe Root Complex is composed of several +PCIe cores. Each PCIe core includes several Root Ports and a PTT +RCiEP, like below. The PTT device is capable of tuning and +tracing the links of the PCIe core. +:: + + +--------------Core 0-------+ + | | [ PTT ] | + | | [Root Port]---[Endpoint] + | | [Root Port]---[Endpoint] + | | [Root Port]---[Endpoint] + Root Complex |------Core 1-------+ + | | [ PTT ] | + | | [Root Port]---[ Switch ]---[Endpoint] + | | [Root Port]---[Endpoint] `-[Endpoint] + | | [Root Port]---[Endpoint] + +---------------------------+ + +The PTT device driver registers one PMU device for each PTT device. +The name of each PTT device is composed of 'hisi_ptt' prefix with +the id of the SICL and the Core where it locates. The Kunpeng 930 +SoC encapsulates multiple CPU dies (SCCL, Super CPU Cluster) and +IO dies (SICL, Super I/O Cluster), where there's one PCIe Root +Complex for each SICL. +:: + + /sys/devices/hisi_ptt_ + +Tune +==== + +PTT tune is designed for monitoring and adjusting PCIe link parameters (events). +Currently we support events in 2 classes. The scope of the events +covers the PCIe core to which the PTT device belongs. + +Each event is presented as a file under $(PTT PMU dir)/tune, and +a simple open/read/write/close cycle will be used to tune the event. +:: + + $ cd /sys/devices/hisi_ptt_/tune + $ ls + qos_tx_cpl qos_tx_np qos_tx_p + tx_path_rx_req_alloc_buf_level + tx_path_tx_req_alloc_buf_level + $ cat qos_tx_dp + 1 + $ echo 2 > qos_tx_dp + $ cat qos_tx_dp + 2 + +Current value (numerical value) of the event can be simply read +from the file, and the desired value written to the file to tune. + +1. Tx Path QoS Control +------------------------ + +The following files are provided to tune the QoS of the tx path of +the PCIe core. + +- qos_tx_cpl: weight of Tx completion TLPs +- qos_tx_np: weight of Tx non-posted TLPs +- qos_tx_p: weight of Tx posted TLPs + +The weight influences the proportion of certain packets on the PCIe link. +For example, for the storage scenario, increase the proportion +of the completion packets on the link to enhance the performance as +more completions are consumed. + +The available tune data of these events is [0, 1, 2]. +Writing a negative value will return an error, and out of range +values will be converted to 2. Note that the event value just +indicates a probable level, but is not precise. + +2. Tx Path Buffer Control +------------------------- + +Following files are provided to tune the buffer of tx path of the PCIe core. + +- rx_alloc_buf_level: watermark of Rx requested +- tx_alloc_buf_level: watermark of Tx requested + +These events influence the watermark of the buffer allocated for each +type. Rx means the inbound while Tx means outbound. The packets will +be stored in the buffer first and then transmitted either when the +watermark reached or when timed out. For a busy direction, you should +increase the related buffer watermark to avoid frequently posting and +thus enhance the performance. In most cases just keep the default value. + +The available tune data of above events is [0, 1, 2]. +Writing a negative value will return an error, and out of range +values will be converted to 2. Note that the event value just +indicates a probable level, but is not precise. + +Trace +===== + +PTT trace is designed for dumping the TLP headers to the memory, which +can be used to analyze the transactions and usage condition of the PCIe +Link. You can choose to filter the traced headers by either Requester ID, +or those downstream of a set of Root Ports on the same core of the PTT +device. It's also supported to trace the headers of certain type and of +certain direction. + +You can use the perf command `perf record` to set the parameters, start +trace and get the data. It's also supported to decode the trace +data with `perf report`. The control parameters for trace is inputted +as event code for each events, which will be further illustrated later. +An example usage is like +:: + + $ perf record -e hisi_ptt0_2/filter=0x80001,type=1,direction=1, + format=1/ -- sleep 5 + +This will trace the TLP headers downstream root port 0000:00:10.1 (event +code for event 'filter' is 0x80001) with type of posted TLP requests, +direction of inbound and traced data format of 8DW. + +1. Filter +--------- + +The TLP headers to trace can be filtered by the Root Ports or the Requester ID +of the Endpoint, which are located on the same core of the PTT device. You can +set the filter by specifying the `filter` parameter which is required to start +the trace. The parameter value is 20 bit. Bit 19 indicates the filter type. +1 for Root Port filter and 0 for Requester filter. Bit[15:0] indicates the +filter value. The value for a Root Port is a mask of the core port id which is +calculated from its PCI Slot ID as (slotid & 7) * 2. The value for a Requester +is the Requester ID (Device ID of the PCIe function). Bit[18:16] is currently +reserved for extension. + +For example, if the desired filter is Endpoint function 0000:01:00.1 the filter +value will be 0x00101. If the desired filter is Root Port 0000:00:10.0 then +then filter value is calculated as 0x80001. + +The driver also presents every supported Root Port and Requester filter through +sysfs. Each filter will be an individual file with name of its related PCIe +device name (domain:bus:device.function). The files of Root Port filters are +under $(PTT PMU dir)/root_port_filters and files of Requester filters +are under $(PTT PMU dir)/requester_filters. + +Note that multiple Root Ports can be specified at one time, but only one +Endpoint function can be specified in one trace. Specifying both Root Port +and function at the same time is not supported. Driver maintains a list of +available filters and will check the invalid inputs. + +The available filters will be dynamically updated, which means you will always +get correct filter information when hotplug events happen, or when you manually +remove/rescan the devices. + +2. Type +------- + +You can trace the TLP headers of certain types by specifying the `type` +parameter, which is required to start the trace. The parameter value is +8 bit. Current supported types and related values are shown below: + +- 8'b00000001: posted requests (P) +- 8'b00000010: non-posted requests (NP) +- 8'b00000100: completions (CPL) + +You can specify multiple types when tracing inbound TLP headers, but can only +specify one when tracing outbound TLP headers. + +3. Direction +------------ + +You can trace the TLP headers from certain direction, which is relative +to the Root Port or the PCIe core, by specifying the `direction` parameter. +This is optional and the default parameter is inbound. The parameter value +is 4 bit. When the desired format is 4DW, directions and related values +supported are shown below: + +- 4'b0000: inbound TLPs (P, NP, CPL) +- 4'b0001: outbound TLPs (P, NP, CPL) +- 4'b0010: outbound TLPs (P, NP, CPL) and inbound TLPs (P, NP, CPL B) +- 4'b0011: outbound TLPs (P, NP, CPL) and inbound TLPs (CPL A) + +When the desired format is 8DW, directions and related values supported are +shown below: + +- 4'b0000: reserved +- 4'b0001: outbound TLPs (P, NP, CPL) +- 4'b0010: inbound TLPs (P, NP, CPL B) +- 4'b0011: inbound TLPs (CPL A) + +Inbound completions are classified into two types: + +- completion A (CPL A): completion of CHI/DMA/Native non-posted requests, except for CPL B +- completion B (CPL B): completion of DMA remote2local and P2P non-posted requests + +4. Format +-------------- + +You can change the format of the traced TLP headers by specifying the +`format` parameter. The default format is 4DW. The parameter value is 4 bit. +Current supported formats and related values are shown below: + +- 4'b0000: 4DW length per TLP header +- 4'b0001: 8DW length per TLP header + +The traced TLP header format is different from the PCIe standard. + +When using the 8DW data format, the entire TLP header is logged +(Header DW0-3 shown below). For example, the TLP header for Memory +Reads with 64-bit addresses is shown in PCIe r5.0, Figure 2-17; +the header for Configuration Requests is shown in Figure 2.20, etc. + +In addition, 8DW trace buffer entries contain a timestamp and +possibly a prefix for a PASID TLP prefix (see Figure 6-20, PCIe r5.0). +Otherwise this field will be all 0. + +The bit[31:11] of DW0 is always 0x1fffff, which can be +used to distinguish the data format. 8DW format is like +:: + + bits [ 31:11 ][ 10:0 ] + |---------------------------------------|-------------------| + DW0 [ 0x1fffff ][ Reserved (0x7ff) ] + DW1 [ Prefix ] + DW2 [ Header DW0 ] + DW3 [ Header DW1 ] + DW4 [ Header DW2 ] + DW5 [ Header DW3 ] + DW6 [ Reserved (0x0) ] + DW7 [ Time ] + +When using the 4DW data format, DW0 of the trace buffer entry +contains selected fields of DW0 of the TLP, together with a +timestamp. DW1-DW3 of the trace buffer entry contain DW1-DW3 +directly from the TLP header. + +4DW format is like +:: + + bits [31:30] [ 29:25 ][24][23][22][21][ 20:11 ][ 10:0 ] + |-----|---------|---|---|---|---|-------------|-------------| + DW0 [ Fmt ][ Type ][T9][T8][TH][SO][ Length ][ Time ] + DW1 [ Header DW1 ] + DW2 [ Header DW2 ] + DW3 [ Header DW3 ] + +5. Memory Management +-------------------- + +The traced TLP headers will be written to the memory allocated +by the driver. The hardware accepts 4 DMA address with same size, +and writes the buffer sequentially like below. If DMA addr 3 is +finished and the trace is still on, it will return to addr 0. +:: + + +->[DMA addr 0]->[DMA addr 1]->[DMA addr 2]->[DMA addr 3]-+ + +---------------------------------------------------------+ + +Driver will allocate each DMA buffer of 4MiB. The finished buffer +will be copied to the perf AUX buffer allocated by the perf core. +Once the AUX buffer is full while the trace is still on, driver +will commit the AUX buffer first and then apply for a new one with +the same size. The size of AUX buffer is default to 16MiB. User can +adjust the size by specifying the `-m` parameter of the perf command. + +6. Decoding +----------- + +You can decode the traced data with `perf report -D` command (currently +only support to dump the raw trace data). The traced data will be decoded +according to the format described previously (take 8DW as an example): +:: + + [...perf headers and other information] + . ... HISI PTT data: size 4194304 bytes + . 00000000: 00 00 00 00 Prefix + . 00000004: 01 00 00 60 Header DW0 + . 00000008: 0f 1e 00 01 Header DW1 + . 0000000c: 04 00 00 00 Header DW2 + . 00000010: 40 00 81 02 Header DW3 + . 00000014: 33 c0 04 00 Time + . 00000020: 00 00 00 00 Prefix + . 00000024: 01 00 00 60 Header DW0 + . 00000028: 0f 1e 00 01 Header DW1 + . 0000002c: 04 00 00 00 Header DW2 + . 00000030: 40 00 81 02 Header DW3 + . 00000034: 02 00 00 00 Time + . 00000040: 00 00 00 00 Prefix + . 00000044: 01 00 00 60 Header DW0 + . 00000048: 0f 1e 00 01 Header DW1 + . 0000004c: 04 00 00 00 Header DW2 + . 00000050: 40 00 81 02 Header DW3 + [...] diff --git a/Documentation/trace/index.rst b/Documentation/trace/index.rst index b7891cb1ab4dd9fa79c9b9f1ac52af2c74c8fd2c..2ca74fccffa7614c25965b0ecd5d115997b29a82 100644 --- a/Documentation/trace/index.rst +++ b/Documentation/trace/index.rst @@ -25,3 +25,7 @@ Linux Tracing Technologies sys-t coresight coresight-cpu-debug + coresight/index + user_events + rv/index + hisi-ptt diff --git a/MAINTAINERS b/MAINTAINERS index b153ef0ca00096fed22130d2f977a75c592bf775..126650c59db7cc3b5e9e91586e0e793eadb7b237 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -7370,6 +7370,13 @@ F: include/uapi/linux/if_hippi.h F: net/802/hippi.c F: drivers/net/hippi/ +HISILICON KUNPENG SOC HCCS DRIVER +M: Huisong Li +S: Maintained +F: Documentation/ABI/testing/sysfs-devices-platform-kunpeng_hccs +F: drivers/soc/hisilicon/kunpeng_hccs.c +F: drivers/soc/hisilicon/kunpeng_hccs.h + HISILICON NETWORK SUBSYSTEM 3 DRIVER (HNS3) M: Yisen Zhuang M: Salil Mehta @@ -7378,6 +7385,13 @@ W: http://www.hisilicon.com S: Maintained F: drivers/net/ethernet/hisilicon/hns3/ +HISILICON I2C CONTROLLER DRIVER +M: Yicong Yang +L: linux-i2c@vger.kernel.org +S: Maintained +W: https://www.hisilicon.com +F: drivers/i2c/busses/i2c-hisi.c + HISILICON LPC BUS DRIVER M: john.garry@huawei.com W: http://www.hisilicon.com @@ -7400,6 +7414,35 @@ W: http://www.hisilicon.com S: Supported F: drivers/perf/hisilicon F: Documentation/admin-guide/perf/hisi-pmu.rst +F: Documentation/admin-guide/perf/hns3-pmu.rst +F: drivers/perf/hisilicon/hns3_pmu.c + +HISILICON PTT DRIVER +M: Yicong Yang +L: linux-kernel@vger.kernel.org +S: Maintained +F: Documentation/ABI/testing/sysfs-devices-hisi_ptt +F: Documentation/trace/hisi-ptt.rst +F: drivers/hwtracing/ptt/ + +HISILICON QM DRIVER +M: Weili Qian +M: Zhou Wang +L: linux-crypto@vger.kernel.org +S: Maintained +F: drivers/crypto/hisilicon/Kconfig +F: drivers/crypto/hisilicon/Makefile +F: drivers/crypto/hisilicon/qm.c +F: drivers/crypto/hisilicon/sgl.c +F: include/linux/hisi_acc_qm.h + +HISILICON ZIP Controller DRIVER +M: Yang Shen +M: Zhou Wang +L: linux-crypto@vger.kernel.org +S: Maintained +F: Documentation/ABI/testing/debugfs-hisi-zip +F: drivers/crypto/hisilicon/zip/ HISILICON ROCE DRIVER M: Lijun Ou @@ -7416,6 +7459,12 @@ S: Supported F: drivers/scsi/hisi_sas/ F: Documentation/devicetree/bindings/scsi/hisilicon-sas.txt +HISILICON V3XX SPI NOR FLASH Controller Driver +M: John Garry +W: http://www.hisilicon.com +S: Maintained +F: drivers/spi/spi-hisi-sfc-v3xx.c + HISILICON QM AND ZIP Controller DRIVER M: Zhou Wang L: linux-crypto@vger.kernel.org @@ -7427,6 +7476,13 @@ F: drivers/crypto/hisilicon/sgl.h F: drivers/crypto/hisilicon/zip/ F: Documentation/ABI/testing/debugfs-hisi-zip +HISILICON SPI Controller DRIVER FOR KUNPENG SOCS +M: Jay Fang +L: linux-spi@vger.kernel.org +S: Maintained +W: http://www.hisilicon.com +F: drivers/spi/spi-hisi-kunpeng.c + HMM - Heterogeneous Memory Management M: Jérôme Glisse L: linux-mm@kvack.org diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index ae9b53b7835e397b78af075475459d896eaea2e7..d7ab699470e726fe316c7a4b6e72a499e7b0ebe0 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -691,6 +691,76 @@ config CAVIUM_TX2_ERRATUM_219 If unsure, say Y. +config FUJITSU_ERRATUM_010001 + bool "Fujitsu-A64FX erratum E#010001: Undefined fault may occur wrongly" + default y + help + This option adds a workaround for Fujitsu-A64FX erratum E#010001. + On some variants of the Fujitsu-A64FX cores ver(1.0, 1.1), memory + accesses may cause undefined fault (Data abort, DFSC=0b111111). + This fault occurs under a specific hardware condition when a + load/store instruction performs an address translation using: + case-1 TTBR0_EL1 with TCR_EL1.NFD0 == 1. + case-2 TTBR0_EL2 with TCR_EL2.NFD0 == 1. + case-3 TTBR1_EL1 with TCR_EL1.NFD1 == 1. + case-4 TTBR1_EL2 with TCR_EL2.NFD1 == 1. + + The workaround is to ensure these bits are clear in TCR_ELx. + The workaround only affects the Fujitsu-A64FX. + + If unsure, say Y. + +config HISILICON_ERRATUM_161600802 + bool "Hip07 161600802: Erroneous redistributor VLPI base" + default y + help + The HiSilicon Hip07 SoC uses the wrong redistributor base + when issued ITS commands such as VMOVP and VMAPP, and requires + a 128kB offset to be applied to the target address in this commands. + + If unsure, say Y. + +config HISILICON_ERRATUM_1980005 + bool "Hisilicon erratum IDC support" + default n + help + The HiSilicon TSV100/200 SoC support idc but report wrong value to + kernel. + + If unsure, say N. + +config HISILICON_ERRATUM_162100801 + bool "Hip09 162100801 erratum support" + default y + help + When enabled GICv4.1 in hip09, there are some invalid vPE config + in configuration tables for some situation, which will cause vSGI + interrupts lost. So fix it by sending vinvall commands after vmovp. + + If unsure, say Y. + +config HISILICON_ERRATUM_162100125 + bool "Hisilicon erratum 162100125" + default y + help + On Hisilicon LINXICORE9100 cores, sharing tlb entries on two cores when + TTBRx.CNP=1 differs from the standard ARM core. This causes issues when + tlb entries sharing between CPU cores. Avoid these issues by disabling + CNP support for Hisilicon LINXICORE9100 cores. + + If unsure, say Y. + +config HISILICON_ERRATUM_162102203 + bool "Hisilicon erratum 162102203" + default n + help + On HIP09, SMS_ATN bit may be cleared by hardware before it is driven + to process. This will causes bridge commands timeout. Avoid these issues + by modifying the IPMI driver to use variables to store SMS_ATNs that may + be lost. + + If unsure, say N. + config QCOM_FALKOR_ERRATUM_1003 bool "Falkor E1003: Incorrect translation due to ASID change" default y @@ -1512,6 +1582,24 @@ config ARM64_PTR_AUTH endmenu +menu "ARMv8.4 architectural features" + +config AS_HAS_ARMV8_4 + def_bool $(cc-option,-Wa$(comma)-march=armv8.4-a) + +config ARM64_TLB_RANGE + bool "Enable support for tlbi range feature" + default y + depends on AS_HAS_ARMV8_4 + help + ARMv8.4-TLBI provides TLBI invalidation instruction that apply to a + range of input addresses. + + The feature introduces new assembly instructions, and they were + support when binutils >= 2.30. + +endmenu + config ARM64_SVE bool "ARM Scalable Vector Extension support" default y diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile index f089323d823aec4c24c8994b3223a01e09d989dc..beed7058f5737d655dbcfdea81f86c79bcf251ca 100644 --- a/arch/arm64/Makefile +++ b/arch/arm64/Makefile @@ -76,6 +76,11 @@ endif # off. KBUILD_CFLAGS += $(call cc-option,-mbranch-protection=none) +ifeq ($(CONFIG_AS_HAS_ARMV8_4), y) +# make sure to pass the newest target architecture to -march. +KBUILD_CFLAGS += -Wa,-march=armv8.4-a +endif + ifeq ($(CONFIG_CPU_BIG_ENDIAN), y) KBUILD_CPPFLAGS += -mbig-endian CHECKFLAGS += -D__AARCH64EB__ diff --git a/arch/arm64/configs/tencent.config b/arch/arm64/configs/tencent.config index 20ae9af7987abdbe7074133b2faaa31a7c0c97d1..a03f562a51726b867c4e13013f988f3af1218f8b 100644 --- a/arch/arm64/configs/tencent.config +++ b/arch/arm64/configs/tencent.config @@ -12,7 +12,6 @@ CONFIG_TASK_DELAY_ACCT=y CONFIG_TASK_XACCT=y CONFIG_TASK_IO_ACCOUNTING=y CONFIG_PSI=y -# CONFIG_IKCONFIG is not set CONFIG_LOG_BUF_SHIFT=18 CONFIG_LOG_CPU_MAX_BUF_SHIFT=17 CONFIG_NUMA_BALANCING=y @@ -29,6 +28,7 @@ CONFIG_CGROUP_DEVICE=y CONFIG_CGROUP_CPUACCT=y CONFIG_CGROUP_PERF=y CONFIG_CGROUP_BPF=y +CONFIG_CGROUP_SLI=y CONFIG_NAMESPACES=y CONFIG_USER_NS=y CONFIG_CHECKPOINT_RESTORE=y @@ -121,7 +121,6 @@ CONFIG_MODULE_UNLOAD=y CONFIG_MODVERSIONS=y CONFIG_MODULE_SRCVERSION_ALL=y CONFIG_MODULE_SIG=y -CONFIG_MODULE_SIG_TENCENT=y CONFIG_BLK_DEV_THROTTLING=y CONFIG_BLK_WBT=y CONFIG_BLK_CGROUP_IOLATENCY=y @@ -134,6 +133,7 @@ CONFIG_CMDLINE_PARTITION=y CONFIG_IOSCHED_BFQ=y CONFIG_BFQ_GROUP_IOSCHED=y CONFIG_TKERNEL=y +CONFIG_MODULE_SIG_TENCENT=y CONFIG_BINFMT_MISC=m CONFIG_MEMORY_HOTPLUG=y CONFIG_KSM=y @@ -585,6 +585,7 @@ CONFIG_PCIE_KIRIN=y CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y CONFIG_CONNECTOR=y +CONFIG_MTD=y CONFIG_OF_OVERLAY=y CONFIG_BLK_DEV_NULL_BLK=m CONFIG_ZRAM=m @@ -620,6 +621,7 @@ CONFIG_BE2ISCSI=m CONFIG_SCSI_HPSA=m CONFIG_SCSI_HISI_SAS=y CONFIG_SCSI_HISI_SAS_PCI=y +CONFIG_SCSI_HISI_DEBUGFS_DEFAULT_ENABLE=y CONFIG_SCSI_MVSAS=m # CONFIG_SCSI_MVSAS_DEBUG is not set CONFIG_SCSI_MVSAS_TASKLET=y @@ -878,12 +880,16 @@ CONFIG_TCG_TIS=y CONFIG_I2C_CHARDEV=y CONFIG_I2C_MUX=m CONFIG_I2C_DESIGNWARE_PLATFORM=y +CONFIG_I2C_HISI=m CONFIG_I2C_XGENE_SLIMPRO=m CONFIG_I2C_SLAVE=y CONFIG_SPI=y +CONFIG_SPI_HISI_KUNPENG=m +CONFIG_SPI_HISI_SFC_V3XX=m CONFIG_GPIOLIB=y CONFIG_GPIO_SYSFS=y CONFIG_GPIO_DWAPB=y +CONFIG_GPIO_HISI=m CONFIG_POWER_RESET_HISI=y CONFIG_SENSORS_ALTRA=m CONFIG_SENSORS_W83795=y @@ -1031,6 +1037,7 @@ CONFIG_ARM_SMMU_V3_PMU=y CONFIG_HISI_PMU=y CONFIG_ARM_SPE_PMU=m CONFIG_DEV_DAX=m +CONFIG_HISI_PTT=m CONFIG_THIRDPARTY_IXGBE=m CONFIG_THIRDPARTY_IXGBEVF=m CONFIG_THIRDPARTY_I40E=m @@ -1203,8 +1210,8 @@ CONFIG_STRIP_ASM_SYMS=y CONFIG_DEBUG_SECTION_MISMATCH=y CONFIG_DEBUG_MEMORY_INIT=y CONFIG_DEBUG_SHIRQ=y -CONFIG_KILL_BLOCK=y CONFIG_SOFTLOCKUP_DETECTOR=y +CONFIG_KILL_BLOCK=y CONFIG_PANIC_ON_OOPS=y CONFIG_SCHEDSTATS=y CONFIG_SCHED_STACK_END_CHECK=y @@ -1236,4 +1243,3 @@ CONFIG_BUG_ON_DATA_CORRUPTION=y CONFIG_KGDB=y CONFIG_KGDB_KDB=y CONFIG_KDB_KEYBOARD=y -CONFIG_CGROUP_SLI=y diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h index 3b16cbc945cfa0c80256532c84e6745dd638a30b..9713a99920d5aa870a8658f49eec5069b22c6b26 100644 --- a/arch/arm64/include/asm/cpucaps.h +++ b/arch/arm64/include/asm/cpucaps.h @@ -57,7 +57,9 @@ #define ARM64_WORKAROUND_1542419 47 #define ARM64_SPECTRE_BHB 48 #define ARM64_WORKAROUND_1742098 49 +#define ARM64_HAS_ARMv8_4_TTL 50 +#define ARM64_HAS_TLB_RANGE 51 -#define ARM64_NCAPS 50 +#define ARM64_NCAPS 52 #endif /* __ASM_CPUCAPS_H */ diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 83bb2712dcc6f5669da492f6cfd566a7eff6fd9c..d4fd3b29e31b5d1b7aa0c95fda2920a6c5502b01 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -643,6 +643,12 @@ static inline bool system_has_prio_mask_debugging(void) system_uses_irq_prio_masking(); } +static inline bool system_supports_tlb_range(void) +{ + return IS_ENABLED(CONFIG_ARM64_TLB_RANGE) && + cpus_have_const_cap(ARM64_HAS_TLB_RANGE); +} + #define ARM64_BP_HARDEN_UNKNOWN -1 #define ARM64_BP_HARDEN_WA_NEEDED 0 #define ARM64_BP_HARDEN_NOT_REQUIRED 1 diff --git a/arch/arm64/include/asm/pgtable-hwdef.h b/arch/arm64/include/asm/pgtable-hwdef.h index a0f789fa25f395c237dd161e133a86057373b51c..efb88ec02e884eb0c525f6a1201c1b63faaff021 100644 --- a/arch/arm64/include/asm/pgtable-hwdef.h +++ b/arch/arm64/include/asm/pgtable-hwdef.h @@ -176,10 +176,12 @@ #define PTE_S2_RDONLY (_AT(pteval_t, 1) << 6) /* HAP[2:1] */ #define PTE_S2_RDWR (_AT(pteval_t, 3) << 6) /* HAP[2:1] */ #define PTE_S2_XN (_AT(pteval_t, 2) << 53) /* XN[1:0] */ +#define PTE_S2_SW_RESVD (_AT(pteval_t, 15) << 55) /* Reserved for SW */ #define PMD_S2_RDONLY (_AT(pmdval_t, 1) << 6) /* HAP[2:1] */ #define PMD_S2_RDWR (_AT(pmdval_t, 3) << 6) /* HAP[2:1] */ #define PMD_S2_XN (_AT(pmdval_t, 2) << 53) /* XN[1:0] */ +#define PMD_S2_SW_RESVD (_AT(pmdval_t, 15) << 55) /* Reserved for SW */ #define PUD_S2_RDONLY (_AT(pudval_t, 1) << 6) /* HAP[2:1] */ #define PUD_S2_RDWR (_AT(pudval_t, 3) << 6) /* HAP[2:1] */ diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 9c02741a30a2eaaac87e7e9b96ce99702b87653f..5e87be42563632ddee832d0a635d323885dc2918 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -40,6 +40,16 @@ extern void __pmd_error(const char *file, int line, unsigned long val); extern void __pud_error(const char *file, int line, unsigned long val); extern void __pgd_error(const char *file, int line, unsigned long val); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE + +/* Set stride and tlb_level in flush_*_tlb_range */ +#define flush_pmd_tlb_range(vma, addr, end) \ + __flush_tlb_range(vma, addr, end, PMD_SIZE, false, 2) +#define flush_pud_tlb_range(vma, addr, end) \ + __flush_tlb_range(vma, addr, end, PUD_SIZE, false, 1) +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + /* * ZERO_PAGE is a global shared page that is always zero: used * for zero-mapped memory areas etc.. diff --git a/arch/arm64/include/asm/stage2_pgtable.h b/arch/arm64/include/asm/stage2_pgtable.h index 326aac658b9da86fd4be5a98f18eb6781b9653eb..7ed5c1a769a9b33b71e956f302a2433be3f2b4b4 100644 --- a/arch/arm64/include/asm/stage2_pgtable.h +++ b/arch/arm64/include/asm/stage2_pgtable.h @@ -230,4 +230,13 @@ stage2_pgd_addr_end(struct kvm *kvm, phys_addr_t addr, phys_addr_t end) return (boundary - 1 < end - 1) ? boundary : end; } +/* + * Level values for the ARMv8.4-TTL extension, mapping PUD/PMD/PTE and + * the architectural page-table level. + */ +#define S2_NO_LEVEL_HINT 0 +#define S2_PUD_LEVEL 1 +#define S2_PMD_LEVEL 2 +#define S2_PTE_LEVEL 3 + #endif /* __ARM64_S2_PGTABLE_H_ */ diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 1ff93878c8132b7b1a27060688bfb02f11c37536..717e3f8e630633466a7b723fe8f41bc2b0043420 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -280,7 +280,11 @@ #define SYS_PMSFCR_EL1_ST_SHIFT 18 #define SYS_PMSEVFR_EL1 sys_reg(3, 0, 9, 9, 5) -#define SYS_PMSEVFR_EL1_RES0 0x0000ffff00ff0f55UL +#define SYS_PMSEVFR_EL1_RES0_8_2 \ + (GENMASK_ULL(47, 32) | GENMASK_ULL(23, 16) | GENMASK_ULL(11, 8) |\ + BIT_ULL(6) | BIT_ULL(4) | BIT_ULL(2) | BIT_ULL(0)) +#define SYS_PMSEVFR_EL1_RES0_8_3 \ + (SYS_PMSEVFR_EL1_RES0_8_2 & ~(BIT_ULL(18) | BIT_ULL(17) | BIT_ULL(11))) #define SYS_PMSLATFR_EL1 sys_reg(3, 0, 9, 9, 6) #define SYS_PMSLATFR_EL1_MINLAT_SHIFT 0 @@ -548,6 +552,7 @@ ENDIAN_SET_EL1 | SCTLR_EL1_UCI | SCTLR_EL1_RES1) /* id_aa64isar0 */ +#define ID_AA64ISAR0_TLB_SHIFT 56 #define ID_AA64ISAR0_TS_SHIFT 52 #define ID_AA64ISAR0_FHM_SHIFT 48 #define ID_AA64ISAR0_DP_SHIFT 44 @@ -561,6 +566,9 @@ #define ID_AA64ISAR0_SHA1_SHIFT 8 #define ID_AA64ISAR0_AES_SHIFT 4 +#define ID_AA64ISAR0_TLB_RANGE_NI 0x0 +#define ID_AA64ISAR0_TLB_RANGE 0x2 + /* id_aa64isar1 */ #define ID_AA64ISAR1_SB_SHIFT 36 #define ID_AA64ISAR1_FRINTTS_SHIFT 32 @@ -680,6 +688,7 @@ #define ID_AA64MMFR1_VMIDBITS_16 2 /* id_aa64mmfr2 */ +#define ID_AA64MMFR2_TTL_SHIFT 48 #define ID_AA64MMFR2_FWB_SHIFT 40 #define ID_AA64MMFR2_AT_SHIFT 32 #define ID_AA64MMFR2_LVA_SHIFT 16 @@ -697,6 +706,9 @@ #define ID_AA64DFR0_TRACEVER_SHIFT 4 #define ID_AA64DFR0_DEBUGVER_SHIFT 0 +#define ID_AA64DFR0_PMSVER_8_2 0x1 +#define ID_AA64DFR0_PMSVER_8_3 0x2 + #define ID_ISAR5_RDM_SHIFT 24 #define ID_ISAR5_CRC32_SHIFT 16 #define ID_ISAR5_SHA2_SHIFT 12 diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h index b76df828e6b73d1faadc15b1cf921dffd14c7f67..61c97d3b58c7080a8b20ac7bd12accf25a9f81ab 100644 --- a/arch/arm64/include/asm/tlb.h +++ b/arch/arm64/include/asm/tlb.h @@ -21,11 +21,37 @@ static void tlb_flush(struct mmu_gather *tlb); #include +/* + * get the tlbi levels in arm64. Default value is 0 if more than one + * of cleared_* is set or neither is set. + * Arm64 doesn't support p4ds now. + */ +static inline int tlb_get_level(struct mmu_gather *tlb) +{ + if (tlb->cleared_ptes && !(tlb->cleared_pmds || + tlb->cleared_puds || + tlb->cleared_p4ds)) + return 3; + + if (tlb->cleared_pmds && !(tlb->cleared_ptes || + tlb->cleared_puds || + tlb->cleared_p4ds)) + return 2; + + if (tlb->cleared_puds && !(tlb->cleared_ptes || + tlb->cleared_pmds || + tlb->cleared_p4ds)) + return 1; + + return 0; +} + static inline void tlb_flush(struct mmu_gather *tlb) { struct vm_area_struct vma = TLB_FLUSH_VMA(tlb->mm, 0); bool last_level = !tlb->freed_tables; unsigned long stride = tlb_get_unmap_size(tlb); + int tlb_level = tlb_get_level(tlb); /* * If we're tearing down the address space then we only care about @@ -38,7 +64,8 @@ static inline void tlb_flush(struct mmu_gather *tlb) return; } - __flush_tlb_range(&vma, tlb->start, tlb->end, stride, last_level); + __flush_tlb_range(&vma, tlb->start, tlb->end, stride, + last_level, tlb_level); } static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index bc39490647259aeaf54c6bc9645dbaea3d07bd4f..d493174415dbf55a4ade7d13b7379988e990829d 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -10,6 +10,7 @@ #ifndef __ASSEMBLY__ +#include #include #include #include @@ -59,6 +60,102 @@ __ta; \ }) +/* + * Get translation granule of the system, which is decided by + * PAGE_SIZE. Used by TTL. + * - 4KB : 1 + * - 16KB : 2 + * - 64KB : 3 + */ +#define TLBI_TTL_TG_4K 1 +#define TLBI_TTL_TG_16K 2 +#define TLBI_TTL_TG_64K 3 + +static inline unsigned long get_trans_granule(void) +{ + switch (PAGE_SIZE) { + case SZ_4K: + return TLBI_TTL_TG_4K; + case SZ_16K: + return TLBI_TTL_TG_16K; + case SZ_64K: + return TLBI_TTL_TG_64K; + default: + return 0; + } +} + +/* + * Level-based TLBI operations. + * + * When ARMv8.4-TTL exists, TLBI operations take an additional hint for + * the level at which the invalidation must take place. If the level is + * wrong, no invalidation may take place. In the case where the level + * cannot be easily determined, a 0 value for the level parameter will + * perform a non-hinted invalidation. + * + * For Stage-2 invalidation, use the level values provided to that effect + * in asm/stage2_pgtable.h. + */ +#define TLBI_TTL_MASK GENMASK_ULL(47, 44) + +#define __tlbi_level(op, addr, level) do { \ + u64 arg = addr; \ + \ + if (cpus_have_const_cap(ARM64_HAS_ARMv8_4_TTL) && \ + level) { \ + u64 ttl = level & 3; \ + ttl |= get_trans_granule() << 2; \ + arg &= ~TLBI_TTL_MASK; \ + arg |= FIELD_PREP(TLBI_TTL_MASK, ttl); \ + } \ + \ + __tlbi(op, arg); \ +} while(0) + +#define __tlbi_user_level(op, arg, level) do { \ + if (arm64_kernel_unmapped_at_el0()) \ + __tlbi_level(op, (arg | USER_ASID_FLAG), level); \ +} while (0) + +/* + * This macro creates a properly formatted VA operand for the TLB RANGE. + * The value bit assignments are: + * + * +----------+------+-------+-------+-------+----------------------+ + * | ASID | TG | SCALE | NUM | TTL | BADDR | + * +-----------------+-------+-------+-------+----------------------+ + * |63 48|47 46|45 44|43 39|38 37|36 0| + * + * The address range is determined by below formula: + * [BADDR, BADDR + (NUM + 1) * 2^(5*SCALE + 1) * PAGESIZE) + * + */ +#define __TLBI_VADDR_RANGE(addr, asid, scale, num, ttl) \ + ({ \ + unsigned long __ta = (addr) >> PAGE_SHIFT; \ + __ta &= GENMASK_ULL(36, 0); \ + __ta |= (unsigned long)(ttl) << 37; \ + __ta |= (unsigned long)(num) << 39; \ + __ta |= (unsigned long)(scale) << 44; \ + __ta |= get_trans_granule() << 46; \ + __ta |= (unsigned long)(asid) << 48; \ + __ta; \ + }) + +/* These macros are used by the TLBI RANGE feature. */ +#define __TLBI_RANGE_PAGES(num, scale) \ + ((unsigned long)((num) + 1) << (5 * (scale) + 1)) +#define MAX_TLBI_RANGE_PAGES __TLBI_RANGE_PAGES(31, 3) + +/* + * Generate 'num' values from -1 to 30 with -1 rejected by the + * __flush_tlb_range() loop below. + */ +#define TLBI_RANGE_MASK GENMASK_ULL(4, 0) +#define __TLBI_RANGE_NUM(pages, scale) \ + ((((pages) >> (5 * (scale) + 1)) & TLBI_RANGE_MASK) - 1) + /* * TLB Invalidation * ================ @@ -179,34 +276,83 @@ static inline void flush_tlb_page(struct vm_area_struct *vma, static inline void __flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end, - unsigned long stride, bool last_level) + unsigned long stride, bool last_level, + int tlb_level) { + int num = 0; + int scale = 0; unsigned long asid = ASID(vma->vm_mm); unsigned long addr; + unsigned long pages; start = round_down(start, stride); end = round_up(end, stride); + pages = (end - start) >> PAGE_SHIFT; - if ((end - start) >= (MAX_TLBI_OPS * stride)) { + /* + * When not uses TLB range ops, we can handle up to + * (MAX_TLBI_OPS - 1) pages; + * When uses TLB range ops, we can handle up to + * (MAX_TLBI_RANGE_PAGES - 1) pages. + */ + if ((!system_supports_tlb_range() && + (end - start) >= (MAX_TLBI_OPS * stride)) || + pages >= MAX_TLBI_RANGE_PAGES) { flush_tlb_mm(vma->vm_mm); return; } - /* Convert the stride into units of 4k */ - stride >>= 12; + dsb(ishst); - start = __TLBI_VADDR(start, asid); - end = __TLBI_VADDR(end, asid); + /* + * When the CPU does not support TLB range operations, flush the TLB + * entries one by one at the granularity of 'stride'. If the the TLB + * range ops are supported, then: + * + * 1. If 'pages' is odd, flush the first page through non-range + * operations; + * + * 2. For remaining pages: the minimum range granularity is decided + * by 'scale', so multiple range TLBI operations may be required. + * Start from scale = 0, flush the corresponding number of pages + * ((num+1)*2^(5*scale+1) starting from 'addr'), then increase it + * until no pages left. + * + * Note that certain ranges can be represented by either num = 31 and + * scale or num = 0 and scale + 1. The loop below favours the latter + * since num is limited to 30 by the __TLBI_RANGE_NUM() macro. + */ + while (pages > 0) { + if (!system_supports_tlb_range() || + pages % 2 == 1) { + addr = __TLBI_VADDR(start, asid); + if (last_level) { + __tlbi_level(vale1is, addr, tlb_level); + __tlbi_user_level(vale1is, addr, tlb_level); + } else { + __tlbi_level(vae1is, addr, tlb_level); + __tlbi_user_level(vae1is, addr, tlb_level); + } + start += stride; + pages -= stride >> PAGE_SHIFT; + continue; + } - dsb(ishst); - for (addr = start; addr < end; addr += stride) { - if (last_level) { - __tlbi(vale1is, addr); - __tlbi_user(vale1is, addr); - } else { - __tlbi(vae1is, addr); - __tlbi_user(vae1is, addr); + num = __TLBI_RANGE_NUM(pages, scale); + if (num >= 0) { + addr = __TLBI_VADDR_RANGE(start, asid, scale, + num, tlb_level); + if (last_level) { + __tlbi(rvale1is, addr); + __tlbi_user(rvale1is, addr); + } else { + __tlbi(rvae1is, addr); + __tlbi_user(rvae1is, addr); + } + start += __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; + pages -= __TLBI_RANGE_PAGES(num, scale); } + scale++; } dsb(ish); } @@ -217,8 +363,9 @@ static inline void flush_tlb_range(struct vm_area_struct *vma, /* * We cannot use leaf-only invalidation here, since we may be invalidating * table entries as part of collapsing hugepages or moving page tables. + * Set the tlb_level to 0 because we can not get enough information here. */ - __flush_tlb_range(vma, start, end, PAGE_SIZE, false); + __flush_tlb_range(vma, start, end, PAGE_SIZE, false, 0); } static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end) diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 396d96224b48bb6862fb9f3d5e84d71bbd5bbde9..9b6e437f031e8948a341863aae7166e00835a120 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -123,6 +123,7 @@ static void cpu_enable_cnp(struct arm64_cpu_capabilities const *cap); * sync with the documentation of the CPU feature register ABI. */ static const struct arm64_ftr_bits ftr_id_aa64isar0[] = { + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_TLB_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_TS_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_FHM_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_DP_SHIFT, 4, 0), @@ -233,6 +234,7 @@ static const struct arm64_ftr_bits ftr_id_aa64mmfr1[] = { }; static const struct arm64_ftr_bits ftr_id_aa64mmfr2[] = { + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_TTL_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_FWB_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_AT_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_LVA_SHIFT, 4, 0), @@ -1465,6 +1467,26 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .matches = has_cpuid_feature, .cpu_enable = cpu_has_fwb, }, + { + .desc = "ARMv8.4 Translation Table Level", + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .capability = ARM64_HAS_ARMv8_4_TTL, + .sys_reg = SYS_ID_AA64MMFR2_EL1, + .sign = FTR_UNSIGNED, + .field_pos = ID_AA64MMFR2_TTL_SHIFT, + .min_field_value = 1, + .matches = has_cpuid_feature, + }, + { + .desc = "TLB range maintenance instructions", + .capability = ARM64_HAS_TLB_RANGE, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = has_cpuid_feature, + .sys_reg = SYS_ID_AA64ISAR0_EL1, + .field_pos = ID_AA64ISAR0_TLB_SHIFT, + .sign = FTR_UNSIGNED, + .min_field_value = ID_AA64ISAR0_TLB_RANGE, + }, #ifdef CONFIG_ARM64_HW_AFDBM { /* diff --git a/drivers/Makefile b/drivers/Makefile index 16abd3b8252796dbbfa74c102bb24b8d7b800156..3f2e3731538aa2144fc2850b2752976a73d2ed2f 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -176,6 +176,7 @@ obj-$(CONFIG_CORESIGHT) += hwtracing/coresight/ obj-y += hwtracing/intel_th/ obj-$(CONFIG_STM) += hwtracing/stm/ obj-$(CONFIG_ANDROID) += android/ +obj-$(CONFIG_HISI_PTT) += hwtracing/ptt/ obj-$(CONFIG_NVMEM) += nvmem/ obj-$(CONFIG_FPGA) += fpga/ obj-$(CONFIG_FSI) += fsi/ diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index 6a4216f99e56e3967833d8a42537d0531b3f2e4a..d294dc18575ba74f55298071dcccfa8e5f5439a2 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -42,7 +42,7 @@ #include struct cppc_pcc_data { - struct mbox_chan *pcc_channel; + struct pcc_mbox_chan *pcc_channel; void __iomem *pcc_comm_addr; bool pcc_channel_acquired; unsigned int deadline_us; @@ -294,7 +294,7 @@ static int send_pcc_cmd(int pcc_ss_id, u16 cmd) pcc_ss_data->platform_owns_pcc = true; /* Ring doorbell */ - ret = mbox_send_message(pcc_ss_data->pcc_channel, &cmd); + ret = mbox_send_message(pcc_ss_data->pcc_channel->mchan, &cmd); if (ret < 0) { pr_err("Err sending PCC mbox message. ss: %d cmd:%d, ret:%d\n", pcc_ss_id, cmd, ret); @@ -307,10 +307,10 @@ static int send_pcc_cmd(int pcc_ss_id, u16 cmd) if (pcc_ss_data->pcc_mrtt) pcc_ss_data->last_cmd_cmpl_time = ktime_get(); - if (pcc_ss_data->pcc_channel->mbox->txdone_irq) - mbox_chan_txdone(pcc_ss_data->pcc_channel, ret); + if (pcc_ss_data->pcc_channel->mchan->mbox->txdone_irq) + mbox_chan_txdone(pcc_ss_data->pcc_channel->mchan, ret); else - mbox_client_txdone(pcc_ss_data->pcc_channel, ret); + mbox_client_txdone(pcc_ss_data->pcc_channel->mchan, ret); end: if (cmd == CMD_WRITE) { @@ -531,46 +531,33 @@ EXPORT_SYMBOL_GPL(acpi_get_psd_map); static int register_pcc_channel(int pcc_ss_idx) { - struct acpi_pcct_hw_reduced *cppc_ss; + struct pcc_mbox_chan *pcc_chan; u64 usecs_lat; if (pcc_ss_idx >= 0) { - pcc_data[pcc_ss_idx]->pcc_channel = - pcc_mbox_request_channel(&cppc_mbox_cl, pcc_ss_idx); + pcc_chan = pcc_mbox_request_channel(&cppc_mbox_cl, pcc_ss_idx); - if (IS_ERR(pcc_data[pcc_ss_idx]->pcc_channel)) { + if (IS_ERR(pcc_chan)) { pr_err("Failed to find PCC channel for subspace %d\n", pcc_ss_idx); return -ENODEV; } - /* - * The PCC mailbox controller driver should - * have parsed the PCCT (global table of all - * PCC channels) and stored pointers to the - * subspace communication region in con_priv. - */ - cppc_ss = (pcc_data[pcc_ss_idx]->pcc_channel)->con_priv; - - if (!cppc_ss) { - pr_err("No PCC subspace found for %d CPPC\n", - pcc_ss_idx); - return -ENODEV; - } - + pcc_data[pcc_ss_idx]->pcc_channel = pcc_chan; /* * cppc_ss->latency is just a Nominal value. In reality * the remote processor could be much slower to reply. * So add an arbitrary amount of wait on top of Nominal. */ - usecs_lat = NUM_RETRIES * cppc_ss->latency; + usecs_lat = NUM_RETRIES * pcc_chan->latency; pcc_data[pcc_ss_idx]->deadline_us = usecs_lat; - pcc_data[pcc_ss_idx]->pcc_mrtt = cppc_ss->min_turnaround_time; - pcc_data[pcc_ss_idx]->pcc_mpar = cppc_ss->max_access_rate; - pcc_data[pcc_ss_idx]->pcc_nominal = cppc_ss->latency; + pcc_data[pcc_ss_idx]->pcc_mrtt = pcc_chan->min_turnaround_time; + pcc_data[pcc_ss_idx]->pcc_mpar = pcc_chan->max_access_rate; + pcc_data[pcc_ss_idx]->pcc_nominal = pcc_chan->latency; pcc_data[pcc_ss_idx]->pcc_comm_addr = - acpi_os_ioremap(cppc_ss->base_address, cppc_ss->length); + acpi_os_ioremap(pcc_chan->shmem_base_addr, + pcc_chan->shmem_size); if (!pcc_data[pcc_ss_idx]->pcc_comm_addr) { pr_err("Failed to ioremap PCC comm region mem for %d\n", pcc_ss_idx); diff --git a/drivers/base/core.c b/drivers/base/core.c index 36d2bdcea5d149faeb91f04377a0c1be0f407445..2ae7bc8fe69598ebb054254154ebcaaf8c206f51 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -74,6 +74,11 @@ int device_links_read_lock_held(void) { return srcu_read_lock_held(&device_links_srcu); } + +static void device_link_synchronize_removal(void) +{ + synchronize_srcu(&device_links_srcu); +} #else /* !CONFIG_SRCU */ static DECLARE_RWSEM(device_links_lock); @@ -104,6 +109,10 @@ int device_links_read_lock_held(void) return lockdep_is_held(&device_links_lock); } #endif + +static inline void device_link_synchronize_removal(void) +{ +} #endif /* !CONFIG_SRCU */ static bool device_is_ancestor(struct device *dev, struct device *target) @@ -142,6 +151,9 @@ static int device_is_dependent(struct device *dev, void *target) return ret; list_for_each_entry(link, &dev->links.consumers, s_node) { + if (link->flags == (DL_FLAG_SYNC_STATE_ONLY | DL_FLAG_MANAGED)) + continue; + if (link->consumer == target) return 1; @@ -211,8 +223,11 @@ static int device_reorder_to_tail(struct device *dev, void *not_used) device_pm_move_last(dev); device_for_each_child(dev, NULL, device_reorder_to_tail); - list_for_each_entry(link, &dev->links.consumers, s_node) + list_for_each_entry(link, &dev->links.consumers, s_node) { + if (link->flags == (DL_FLAG_SYNC_STATE_ONLY | DL_FLAG_MANAGED)) + continue; device_reorder_to_tail(link->consumer, NULL); + } return 0; } @@ -237,9 +252,214 @@ void device_pm_move_to_tail(struct device *dev) device_links_read_unlock(idx); } +#define to_devlink(dev) container_of((dev), struct device_link, link_dev) + +static ssize_t status_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + char *status; + + switch (to_devlink(dev)->status) { + case DL_STATE_NONE: + status = "not tracked"; break; + case DL_STATE_DORMANT: + status = "dormant"; break; + case DL_STATE_AVAILABLE: + status = "available"; break; + case DL_STATE_CONSUMER_PROBE: + status = "consumer probing"; break; + case DL_STATE_ACTIVE: + status = "active"; break; + case DL_STATE_SUPPLIER_UNBIND: + status = "supplier unbinding"; break; + default: + status = "unknown"; break; + } + return sprintf(buf, "%s\n", status); +} +static DEVICE_ATTR_RO(status); + +static ssize_t auto_remove_on_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct device_link *link = to_devlink(dev); + char *str; + + if (link->flags & DL_FLAG_AUTOREMOVE_SUPPLIER) + str = "supplier unbind"; + else if (link->flags & DL_FLAG_AUTOREMOVE_CONSUMER) + str = "consumer unbind"; + else + str = "never"; + + return sprintf(buf, "%s\n", str); +} +static DEVICE_ATTR_RO(auto_remove_on); + +static ssize_t runtime_pm_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct device_link *link = to_devlink(dev); + + return sprintf(buf, "%d\n", !!(link->flags & DL_FLAG_PM_RUNTIME)); +} +static DEVICE_ATTR_RO(runtime_pm); + +static ssize_t sync_state_only_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct device_link *link = to_devlink(dev); + + return sprintf(buf, "%d\n", !!(link->flags & DL_FLAG_SYNC_STATE_ONLY)); +} +static DEVICE_ATTR_RO(sync_state_only); + +static struct attribute *devlink_attrs[] = { + &dev_attr_status.attr, + &dev_attr_auto_remove_on.attr, + &dev_attr_runtime_pm.attr, + &dev_attr_sync_state_only.attr, + NULL, +}; +ATTRIBUTE_GROUPS(devlink); + +static void device_link_release_fn(struct work_struct *work) +{ + struct device_link *link = container_of(work, struct device_link, rm_work); + + /* Ensure that all references to the link object have been dropped. */ + device_link_synchronize_removal(); + + while (refcount_dec_not_one(&link->rpm_active)) + pm_runtime_put(link->supplier); + + put_device(link->consumer); + put_device(link->supplier); + kfree(link); +} + +static void devlink_dev_release(struct device *dev) +{ + struct device_link *link = to_devlink(dev); + + INIT_WORK(&link->rm_work, device_link_release_fn); + /* + * It may take a while to complete this work because of the SRCU + * synchronization in device_link_release_fn() and if the consumer or + * supplier devices get deleted when it runs, so put it into the "long" + * workqueue. + */ + queue_work(system_long_wq, &link->rm_work); +} + +static struct class devlink_class = { + .name = "devlink", + .owner = THIS_MODULE, + .dev_groups = devlink_groups, + .dev_release = devlink_dev_release, +}; + +static int devlink_add_symlinks(struct device *dev, + struct class_interface *class_intf) +{ + int ret; + size_t len; + struct device_link *link = to_devlink(dev); + struct device *sup = link->supplier; + struct device *con = link->consumer; + char *buf; + + len = max(strlen(dev_name(sup)), strlen(dev_name(con))); + len += strlen("supplier:") + 1; + buf = kzalloc(len, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + ret = sysfs_create_link(&link->link_dev.kobj, &sup->kobj, "supplier"); + if (ret) + goto out; + + ret = sysfs_create_link(&link->link_dev.kobj, &con->kobj, "consumer"); + if (ret) + goto err_con; + + snprintf(buf, len, "consumer:%s", dev_name(con)); + ret = sysfs_create_link(&sup->kobj, &link->link_dev.kobj, buf); + if (ret) + goto err_con_dev; + + snprintf(buf, len, "supplier:%s", dev_name(sup)); + ret = sysfs_create_link(&con->kobj, &link->link_dev.kobj, buf); + if (ret) + goto err_sup_dev; + + goto out; + +err_sup_dev: + snprintf(buf, len, "consumer:%s", dev_name(con)); + sysfs_remove_link(&sup->kobj, buf); +err_con_dev: + sysfs_remove_link(&link->link_dev.kobj, "consumer"); +err_con: + sysfs_remove_link(&link->link_dev.kobj, "supplier"); +out: + kfree(buf); + return ret; +} + +static void devlink_remove_symlinks(struct device *dev, + struct class_interface *class_intf) +{ + struct device_link *link = to_devlink(dev); + size_t len; + struct device *sup = link->supplier; + struct device *con = link->consumer; + char *buf; + + sysfs_remove_link(&link->link_dev.kobj, "consumer"); + sysfs_remove_link(&link->link_dev.kobj, "supplier"); + + len = max(strlen(dev_name(sup)), strlen(dev_name(con))); + len += strlen("supplier:") + 1; + buf = kzalloc(len, GFP_KERNEL); + if (!buf) { + WARN(1, "Unable to properly free device link symlinks!\n"); + return; + } + + snprintf(buf, len, "supplier:%s", dev_name(sup)); + sysfs_remove_link(&con->kobj, buf); + snprintf(buf, len, "consumer:%s", dev_name(con)); + sysfs_remove_link(&sup->kobj, buf); + kfree(buf); +} + +static struct class_interface devlink_class_intf = { + .class = &devlink_class, + .add_dev = devlink_add_symlinks, + .remove_dev = devlink_remove_symlinks, +}; + +static int __init devlink_class_init(void) +{ + int ret; + + ret = class_register(&devlink_class); + if (ret) + return ret; + + ret = class_interface_register(&devlink_class_intf); + if (ret) + class_unregister(&devlink_class); + + return ret; +} +postcore_initcall(devlink_class_init); + #define DL_MANAGED_LINK_FLAGS (DL_FLAG_AUTOREMOVE_CONSUMER | \ DL_FLAG_AUTOREMOVE_SUPPLIER | \ - DL_FLAG_AUTOPROBE_CONSUMER) + DL_FLAG_AUTOPROBE_CONSUMER | \ + DL_FLAG_SYNC_STATE_ONLY) #define DL_ADD_VALID_FLAGS (DL_MANAGED_LINK_FLAGS | DL_FLAG_STATELESS | \ DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE) @@ -307,6 +527,8 @@ struct device_link *device_link_add(struct device *consumer, if (!consumer || !supplier || flags & ~DL_ADD_VALID_FLAGS || (flags & DL_FLAG_STATELESS && flags & DL_MANAGED_LINK_FLAGS) || + (flags & DL_FLAG_SYNC_STATE_ONLY && + flags != DL_FLAG_SYNC_STATE_ONLY) || (flags & DL_FLAG_AUTOPROBE_CONSUMER && flags & (DL_FLAG_AUTOREMOVE_CONSUMER | DL_FLAG_AUTOREMOVE_SUPPLIER))) @@ -327,11 +549,14 @@ struct device_link *device_link_add(struct device *consumer, /* * If the supplier has not been fully registered yet or there is a - * reverse dependency between the consumer and the supplier already in - * the graph, return NULL. + * reverse (non-SYNC_STATE_ONLY) dependency between the consumer and + * the supplier already in the graph, return NULL. If the link is a + * SYNC_STATE_ONLY link, we don't check for reverse dependencies + * because it only affects sync_state() callbacks. */ if (!device_pm_initialized(supplier) - || device_is_dependent(consumer, supplier)) { + || (!(flags & DL_FLAG_SYNC_STATE_ONLY) && + device_is_dependent(consumer, supplier))) { link = NULL; goto out; } @@ -358,9 +583,14 @@ struct device_link *device_link_add(struct device *consumer, } if (flags & DL_FLAG_STATELESS) { - link->flags |= DL_FLAG_STATELESS; kref_get(&link->kref); - goto out; + if (link->flags & DL_FLAG_SYNC_STATE_ONLY && + !(link->flags & DL_FLAG_STATELESS)) { + link->flags |= DL_FLAG_STATELESS; + goto reorder; + } else { + goto out; + } } /* @@ -382,6 +612,12 @@ struct device_link *device_link_add(struct device *consumer, link->flags |= DL_FLAG_MANAGED; device_link_init_status(link, consumer, supplier); } + if (link->flags & DL_FLAG_SYNC_STATE_ONLY && + !(flags & DL_FLAG_SYNC_STATE_ONLY)) { + link->flags &= ~DL_FLAG_SYNC_STATE_ONLY; + goto reorder; + } + goto out; } @@ -391,13 +627,6 @@ struct device_link *device_link_add(struct device *consumer, refcount_set(&link->rpm_active, 1); - if (flags & DL_FLAG_PM_RUNTIME) { - if (flags & DL_FLAG_RPM_ACTIVE) - refcount_inc(&link->rpm_active); - - pm_runtime_new_link(consumer); - } - get_device(supplier); link->supplier = supplier; INIT_LIST_HEAD(&link->s_node); @@ -407,6 +636,25 @@ struct device_link *device_link_add(struct device *consumer, link->flags = flags; kref_init(&link->kref); + link->link_dev.class = &devlink_class; + device_set_pm_not_required(&link->link_dev); + dev_set_name(&link->link_dev, "%s:%s", + dev_name(supplier), dev_name(consumer)); + if (device_register(&link->link_dev)) { + put_device(consumer); + put_device(supplier); + kfree(link); + link = NULL; + goto out; + } + + if (flags & DL_FLAG_PM_RUNTIME) { + if (flags & DL_FLAG_RPM_ACTIVE) + refcount_inc(&link->rpm_active); + + pm_runtime_new_link(consumer); + } + /* Determine the initial link state. */ if (flags & DL_FLAG_STATELESS) link->status = DL_STATE_NONE; @@ -421,6 +669,13 @@ struct device_link *device_link_add(struct device *consumer, flags & DL_FLAG_PM_RUNTIME) pm_runtime_resume(supplier); + if (flags & DL_FLAG_SYNC_STATE_ONLY) { + dev_dbg(consumer, + "Linked as a sync state only consumer to %s\n", + dev_name(supplier)); + goto out; + } +reorder: /* * Move the consumer and all of the devices depending on it to the end * of dpm_list and the devices_kset list. @@ -446,22 +701,7 @@ struct device_link *device_link_add(struct device *consumer, } EXPORT_SYMBOL_GPL(device_link_add); -static void device_link_free(struct device_link *link) -{ - while (refcount_dec_not_one(&link->rpm_active)) - pm_runtime_put(link->supplier); - - put_device(link->consumer); - put_device(link->supplier); - kfree(link); -} - #ifdef CONFIG_SRCU -static void __device_link_free_srcu(struct rcu_head *rhead) -{ - device_link_free(container_of(rhead, struct device_link, rcu_head)); -} - static void __device_link_del(struct kref *kref) { struct device_link *link = container_of(kref, struct device_link, kref); @@ -473,7 +713,7 @@ static void __device_link_del(struct kref *kref) list_del_rcu(&link->s_node); list_del_rcu(&link->c_node); - call_srcu(&device_links_srcu, &link->rcu_head, __device_link_free_srcu); + device_unregister(&link->link_dev); } #else /* !CONFIG_SRCU */ static void __device_link_del(struct kref *kref) @@ -487,7 +727,7 @@ static void __device_link_del(struct kref *kref) list_del(&link->s_node); list_del(&link->c_node); - device_link_free(link); + device_unregister(&link->link_dev); } #endif /* !CONFIG_SRCU */ @@ -581,7 +821,8 @@ int device_links_check_suppliers(struct device *dev) device_links_write_lock(); list_for_each_entry(link, &dev->links.suppliers, c_node) { - if (!(link->flags & DL_FLAG_MANAGED)) + if (!(link->flags & DL_FLAG_MANAGED) || + link->flags & DL_FLAG_SYNC_STATE_ONLY) continue; if (link->status != DL_STATE_AVAILABLE) { @@ -826,7 +1067,8 @@ void device_links_unbind_consumers(struct device *dev) list_for_each_entry(link, &dev->links.consumers, s_node) { enum device_link_state status; - if (!(link->flags & DL_FLAG_MANAGED)) + if (!(link->flags & DL_FLAG_MANAGED) || + link->flags & DL_FLAG_SYNC_STATE_ONLY) continue; status = link->status; @@ -862,6 +1104,9 @@ static void device_links_purge(struct device *dev) { struct device_link *link, *ln; + if (dev->class == &devlink_class) + return; + /* * Delete all of the remaining links from this device to any other * devices (either consumers or suppliers). diff --git a/drivers/char/ipmi/ipmi_bt_sm.c b/drivers/char/ipmi/ipmi_bt_sm.c index f3f216cdf686d7083a895120f37fcccb56c6cda9..7ecf7e235b813abd812fcf9e11c562a43c3ed5dd 100644 --- a/drivers/char/ipmi/ipmi_bt_sm.c +++ b/drivers/char/ipmi/ipmi_bt_sm.c @@ -88,6 +88,13 @@ struct si_sm_data { enum bt_states complete; /* to divert the state machine */ long BT_CAP_req2rsp; int BT_CAP_retries; /* Recommended retries */ + +#ifdef CONFIG_HISILICON_ERRATUM_162102203 + /* Record sms_atn when sms_atn set && bt_state not in idle */ + int sms_atn_flag; + /* If true, need to store SMS_ATN bit */ + bool sms_atn_quirk; +#endif }; #define BT_CLR_WR_PTR 0x01 /* See IPMI 1.5 table 11.6.4 */ @@ -168,6 +175,32 @@ static char *status2txt(unsigned char status) } #define STATUS2TXT status2txt(status) +#ifdef CONFIG_HISILICON_ERRATUM_162102203 +/* + * To confirm whether the SMS_ATN flag needs to be stored and get + * quirk through the method reported by the BIOS. Because in special + * cases SMS_ATN flag bits may be lost before being processed. + */ +static bool get_sms_atn_quirk(struct si_sm_io *io) +{ + acpi_handle handle; + acpi_status status; + unsigned long long tmp; + + handle = ACPI_HANDLE(io->dev); + if (!handle) + return false; + + status = acpi_evaluate_integer(handle, "SATN", NULL, &tmp); + if (ACPI_FAILURE(status)) + return false; + else if (tmp != 1) + return false; + + return true; +} +#endif + /* called externally at insmod time, and internally on cleanup */ static unsigned int bt_init_data(struct si_sm_data *bt, struct si_sm_io *io) @@ -182,6 +215,12 @@ static unsigned int bt_init_data(struct si_sm_data *bt, struct si_sm_io *io) bt->complete = BT_STATE_IDLE; /* end here */ bt->BT_CAP_req2rsp = BT_NORMAL_TIMEOUT * USEC_PER_SEC; bt->BT_CAP_retries = BT_NORMAL_RETRY_LIMIT; + +#ifdef CONFIG_HISILICON_ERRATUM_162102203 + bt->sms_atn_quirk = get_sms_atn_quirk(io); + bt->sms_atn_flag = 0; +#endif + return 3; /* We claim 3 bytes of space; ought to check SPMI table */ } @@ -281,6 +320,11 @@ static void reset_flags(struct si_sm_data *bt) BT_CONTROL(BT_H_BUSY); /* force clear */ BT_CONTROL(BT_CLR_WR_PTR); /* always reset */ BT_CONTROL(BT_SMS_ATN); /* always clear */ + +#ifdef CONFIG_HISILICON_ERRATUM_162102203 + bt->sms_atn_flag = 0; /* Reset sms_atn_flag */ +#endif + BT_INTMASK_W(BT_BMC_HWRST); } @@ -452,6 +496,36 @@ static enum si_sm_result bt_event(struct si_sm_data *bt, long time) int i; status = BT_STATUS; +#ifdef CONFIG_HISILICON_ERRATUM_162102203 + if (bt->sms_atn_quirk) { + /* + * Identify whether the current BT_B2H_ATN is possibly due to + * receiving an SMS message from BMC. If so, only clear the + * incorrectly set BT_B2H_ATN without returning, and continue + * to execute downwards. + */ + if ((bt->state < BT_STATE_WRITE_BYTES) && (status & BT_B2H_ATN) && + (status & BT_SMS_ATN)) { + BT_CONTROL(BT_B2H_ATN); /* Clear it */ + status &= ~BT_B2H_ATN; /* Refresh status */ + if (bt_debug) + dev_dbg(bt->io->dev, "clear wrong B2H_ATN, BT: %s\n", + status2txt(BT_STATUS)); + } + + /* + * Record the SMS_ATN by sms_atn_flag, because SMS_ATN would be clear + * incorrectly by hardware when received a SMS message from BMC if + * bt->state is not in IDLE state. And the BT_SMS_ATN will be lost. + */ + if ((bt->state >= BT_STATE_XACTION_START) && (status & BT_SMS_ATN)) + bt->sms_atn_flag = 1; + + /* Restore SMS_ATN */ + if (bt->sms_atn_flag) + status |= BT_SMS_ATN; + } +#endif bt->nonzero_status |= status; if ((bt_debug & BT_DEBUG_STATES) && (bt->state != last_printed)) { dev_dbg(bt->io->dev, "BT: %s %s TO=%ld - %ld\n", @@ -494,6 +568,10 @@ static enum si_sm_result bt_event(struct si_sm_data *bt, long time) case BT_STATE_IDLE: if (status & BT_SMS_ATN) { BT_CONTROL(BT_SMS_ATN); /* clear it */ + +#ifdef CONFIG_HISILICON_ERRATUM_162102203 + bt->sms_atn_flag = 0; /* Reset sms_atn_flag */ +#endif return SI_SM_ATTN; } diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index ec6f28ed21e27fb023752a33763bd23478cf1b06..e0b2da5ef2b4d772eaf308bd08823e11ac21ed34 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -800,7 +800,7 @@ static void __arch_timer_setup(unsigned type, clk->set_state_shutdown(clk); - clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff); + clockevents_config_and_register(clk, arch_timer_rate, 0xf, CLOCKSOURCE_MASK(56)); } static void arch_timer_evtstrm_enable(int divider) diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c index cf7ebe3bd1ad2d89365dd0f4ef83591ea5a8d877..e11e54da37e761f3411d98d7d09227803374fd8f 100644 --- a/drivers/cpufreq/acpi-cpufreq.c +++ b/drivers/cpufreq/acpi-cpufreq.c @@ -126,12 +126,12 @@ static void boost_set_msr_each(void *p_en) boost_set_msr(enable); } -static int set_boost(int val) +static int set_boost(struct cpufreq_policy *policy, int val) { - get_online_cpus(); - on_each_cpu(boost_set_msr_each, (void *)(long)val, 1); - put_online_cpus(); - pr_debug("Core Boosting %sabled.\n", val ? "en" : "dis"); + on_each_cpu_mask(policy->cpus, boost_set_msr_each, + (void *)(long)val, 1); + pr_debug("CPU %*pbl: Core Boosting %sabled.\n", + cpumask_pr_args(policy->cpus), val ? "en" : "dis"); return 0; } @@ -162,7 +162,9 @@ static ssize_t store_cpb(struct cpufreq_policy *policy, const char *buf, if (ret || val > 1) return -EINVAL; - set_boost(val); + get_online_cpus(); + set_boost(policy, val); + put_online_cpus(); return count; } diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c index 8910fd1ae3c6b13ffd1cca4f8bef1450a1637414..900a65156af9303828d88838ce17f051a282f937 100644 --- a/drivers/cpufreq/cppc_cpufreq.c +++ b/drivers/cpufreq/cppc_cpufreq.c @@ -37,6 +37,7 @@ * requested etc. */ static struct cppc_cpudata **all_cpu_data; +static bool boost_supported; struct cppc_workaround_oem_info { char oem_id[ACPI_OEM_ID_SIZE +1]; @@ -306,7 +307,7 @@ static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy) * Section 8.4.7.1.1.5 of ACPI 6.1 spec) */ policy->min = cppc_cpufreq_perf_to_khz(cpu, cpu->perf_caps.lowest_nonlinear_perf); - policy->max = cppc_cpufreq_perf_to_khz(cpu, cpu->perf_caps.highest_perf); + policy->max = cppc_cpufreq_perf_to_khz(cpu, cpu->perf_caps.nominal_perf); /* * Set cpuinfo.min_freq to Lowest to make the full range of performance @@ -314,7 +315,7 @@ static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy) * nonlinear perf */ policy->cpuinfo.min_freq = cppc_cpufreq_perf_to_khz(cpu, cpu->perf_caps.lowest_perf); - policy->cpuinfo.max_freq = cppc_cpufreq_perf_to_khz(cpu, cpu->perf_caps.highest_perf); + policy->cpuinfo.max_freq = cppc_cpufreq_perf_to_khz(cpu, cpu->perf_caps.nominal_perf); policy->transition_delay_us = cppc_cpufreq_get_transition_delay_us(cpu_num); policy->shared_type = cpu->shared_type; @@ -339,6 +340,13 @@ static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy) cpu->cur_policy = policy; + /* + * If 'highest_perf' is greater than 'nominal_perf', we assume CPU Boost + * is supported. + */ + if (cpu->perf_caps.highest_perf > cpu->perf_caps.nominal_perf) + boost_supported = true; + /* Set policy->cur to max now. The governors will adjust later. */ policy->cur = cppc_cpufreq_perf_to_khz(cpu, cpu->perf_caps.highest_perf); @@ -406,6 +414,32 @@ static unsigned int cppc_cpufreq_get_rate(unsigned int cpunum) return cppc_get_rate_from_fbctrs(cpu, fb_ctrs_t0, fb_ctrs_t1); } +static int cppc_cpufreq_set_boost(struct cpufreq_policy *policy, int state) +{ + struct cppc_cpudata *cpudata; + int ret; + + if (!boost_supported) { + pr_err("BOOST not supported by CPU or firmware\n"); + return -EINVAL; + } + + cpudata = all_cpu_data[policy->cpu]; + if (state) + policy->max = cppc_cpufreq_perf_to_khz(cpudata, + cpudata->perf_caps.highest_perf); + else + policy->max = cppc_cpufreq_perf_to_khz(cpudata, + cpudata->perf_caps.nominal_perf); + policy->cpuinfo.max_freq = policy->max; + + ret = freq_qos_update_request(policy->max_freq_req, policy->max); + if (ret < 0) + return ret; + + return 0; +} + static struct cpufreq_driver cppc_cpufreq_driver = { .flags = CPUFREQ_CONST_LOOPS, .verify = cppc_verify_policy, @@ -413,6 +447,7 @@ static struct cpufreq_driver cppc_cpufreq_driver = { .get = cppc_cpufreq_get_rate, .init = cppc_cpufreq_cpu_init, .stop_cpu = cppc_cpufreq_stop_cpu, + .set_boost = cppc_cpufreq_set_boost, .name = "cppc_cpufreq", }; diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 4a8be0e67b00550399ea6471b14f17a4e83312dd..05556dd1217093f44e6fa11e2b6fd8c82af98fd5 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -2528,34 +2528,29 @@ EXPORT_SYMBOL_GPL(cpufreq_update_limits); /********************************************************************* * BOOST * *********************************************************************/ -static int cpufreq_boost_set_sw(int state) +static int cpufreq_boost_set_sw(struct cpufreq_policy *policy, int state) { - struct cpufreq_policy *policy; - - for_each_active_policy(policy) { - int ret; - - if (!policy->freq_table) - return -ENXIO; + int ret; - ret = cpufreq_frequency_table_cpuinfo(policy, - policy->freq_table); - if (ret) { - pr_err("%s: Policy frequency update failed\n", - __func__); - return ret; - } + if (!policy->freq_table) + return -ENXIO; - ret = freq_qos_update_request(policy->max_freq_req, policy->max); - if (ret < 0) - return ret; + ret = cpufreq_frequency_table_cpuinfo(policy, policy->freq_table); + if (ret) { + pr_err("%s: Policy frequency update failed\n", __func__); + return ret; } + ret = freq_qos_update_request(policy->max_freq_req, policy->max); + if (ret < 0) + return ret; + return 0; } int cpufreq_boost_trigger_state(int state) { + struct cpufreq_policy *policy; unsigned long flags; int ret = 0; @@ -2566,15 +2561,25 @@ int cpufreq_boost_trigger_state(int state) cpufreq_driver->boost_enabled = state; write_unlock_irqrestore(&cpufreq_driver_lock, flags); - ret = cpufreq_driver->set_boost(state); - if (ret) { - write_lock_irqsave(&cpufreq_driver_lock, flags); - cpufreq_driver->boost_enabled = !state; - write_unlock_irqrestore(&cpufreq_driver_lock, flags); - - pr_err("%s: Cannot %s BOOST\n", - __func__, state ? "enable" : "disable"); + get_online_cpus(); + for_each_active_policy(policy) { + ret = cpufreq_driver->set_boost(policy, state); + if (ret) + goto err_reset_state; } + put_online_cpus(); + + return 0; + +err_reset_state: + put_online_cpus(); + + write_lock_irqsave(&cpufreq_driver_lock, flags); + cpufreq_driver->boost_enabled = !state; + write_unlock_irqrestore(&cpufreq_driver_lock, flags); + + pr_err("%s: Cannot %s BOOST\n", + __func__, state ? "enable" : "disable"); return ret; } diff --git a/drivers/cpuidle/sysfs.c b/drivers/cpuidle/sysfs.c index 86e89eba8b449a76756676aa5dcd752deb0c9759..afa5f0e2aa3f543579bab4d56b0b0d3b7dbbb871 100644 --- a/drivers/cpuidle/sysfs.c +++ b/drivers/cpuidle/sysfs.c @@ -115,19 +115,20 @@ static ssize_t store_current_governor(struct device *dev, return count; } +static DEVICE_ATTR(available_governors, 0444, show_available_governors, NULL); static DEVICE_ATTR(current_driver, 0444, show_current_driver, NULL); +static DEVICE_ATTR(current_governor, 0644, show_current_governor, + store_current_governor); static DEVICE_ATTR(current_governor_ro, 0444, show_current_governor, NULL); static struct attribute *cpuidle_default_attrs[] = { + &dev_attr_available_governors.attr, &dev_attr_current_driver.attr, + &dev_attr_current_governor.attr, &dev_attr_current_governor_ro.attr, NULL }; -static DEVICE_ATTR(available_governors, 0444, show_available_governors, NULL); -static DEVICE_ATTR(current_governor, 0644, show_current_governor, - store_current_governor); - static struct attribute *cpuidle_switch_attrs[] = { &dev_attr_available_governors.attr, &dev_attr_current_driver.attr, diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig index ae414045a7506ebf78e3e0a9a6ad738ee7f83be5..33d3bd6b55afc09c6ba875cc1b63acb612cb1e05 100644 --- a/drivers/gpio/Kconfig +++ b/drivers/gpio/Kconfig @@ -250,6 +250,17 @@ config GPIO_GRGPIO Select this to support Aeroflex Gaisler GRGPIO cores from the GRLIB VHDL IP core library. +config GPIO_HISI + tristate "HiSilicon GPIO controller driver" + depends on (ARM64 || COMPILE_TEST) && ACPI + select GPIO_GENERIC + select GPIOLIB_IRQCHIP + help + Say Y or M here to build support for the HiSilicon GPIO controller + driver GPIO block. + This GPIO controller support double-edge interrupt and multi-core + concurrent access. + config GPIO_HLWD tristate "Nintendo Wii (Hollywood) GPIO" depends on OF_GPIO diff --git a/drivers/gpio/Makefile b/drivers/gpio/Makefile index d2fd19c15bae3fbad7b7f7ce2fbf2ca9f1429efb..05f97adf1059c7b08c75a6414e55ca3ea1d6a97f 100644 --- a/drivers/gpio/Makefile +++ b/drivers/gpio/Makefile @@ -58,6 +58,7 @@ obj-$(CONFIG_GPIO_GE_FPGA) += gpio-ge.o obj-$(CONFIG_GPIO_GPIO_MM) += gpio-gpio-mm.o obj-$(CONFIG_GPIO_GRGPIO) += gpio-grgpio.o obj-$(CONFIG_GPIO_GW_PLD) += gpio-gw-pld.o +obj-$(CONFIG_GPIO_HISI) += gpio-hisi.o obj-$(CONFIG_GPIO_HLWD) += gpio-hlwd.o obj-$(CONFIG_HTC_EGPIO) += gpio-htc-egpio.o obj-$(CONFIG_GPIO_ICH) += gpio-ich.o diff --git a/drivers/gpio/gpio-hisi.c b/drivers/gpio/gpio-hisi.c new file mode 100644 index 0000000000000000000000000000000000000000..5ece887d5a59f41b9a9d76f75f56e2036fe354a8 --- /dev/null +++ b/drivers/gpio/gpio-hisi.c @@ -0,0 +1,328 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2020 HiSilicon Limited. */ +#include +#include +#include +#include +#include + +#define HISI_GPIO_SWPORT_DR_SET_WX 0x000 +#define HISI_GPIO_SWPORT_DR_CLR_WX 0x004 +#define HISI_GPIO_SWPORT_DDR_SET_WX 0x010 +#define HISI_GPIO_SWPORT_DDR_CLR_WX 0x014 +#define HISI_GPIO_SWPORT_DDR_ST_WX 0x018 +#define HISI_GPIO_INTEN_SET_WX 0x020 +#define HISI_GPIO_INTEN_CLR_WX 0x024 +#define HISI_GPIO_INTMASK_SET_WX 0x030 +#define HISI_GPIO_INTMASK_CLR_WX 0x034 +#define HISI_GPIO_INTTYPE_EDGE_SET_WX 0x040 +#define HISI_GPIO_INTTYPE_EDGE_CLR_WX 0x044 +#define HISI_GPIO_INT_POLARITY_SET_WX 0x050 +#define HISI_GPIO_INT_POLARITY_CLR_WX 0x054 +#define HISI_GPIO_DEBOUNCE_SET_WX 0x060 +#define HISI_GPIO_DEBOUNCE_CLR_WX 0x064 +#define HISI_GPIO_INTSTATUS_WX 0x070 +#define HISI_GPIO_PORTA_EOI_WX 0x078 +#define HISI_GPIO_EXT_PORT_WX 0x080 +#define HISI_GPIO_INTCOMB_MASK_WX 0x0a0 +#define HISI_GPIO_INT_DEDGE_SET 0x0b0 +#define HISI_GPIO_INT_DEDGE_CLR 0x0b4 +#define HISI_GPIO_INT_DEDGE_ST 0x0b8 + +#define HISI_GPIO_LINE_NUM_MAX 32 +#define HISI_GPIO_DRIVER_NAME "gpio-hisi" + +struct hisi_gpio { + struct gpio_chip chip; + struct device *dev; + void __iomem *reg_base; + unsigned int line_num; + struct irq_chip irq_chip; + int irq; +}; + +static inline u32 hisi_gpio_read_reg(struct gpio_chip *chip, + unsigned int off) +{ + struct hisi_gpio *hisi_gpio = + container_of(chip, struct hisi_gpio, chip); + void __iomem *reg = hisi_gpio->reg_base + off; + + return readl(reg); +} + +static inline void hisi_gpio_write_reg(struct gpio_chip *chip, + unsigned int off, u32 val) +{ + struct hisi_gpio *hisi_gpio = + container_of(chip, struct hisi_gpio, chip); + void __iomem *reg = hisi_gpio->reg_base + off; + + writel(val, reg); +} + +static void hisi_gpio_set_debounce(struct gpio_chip *chip, unsigned int off, + u32 debounce) +{ + if (debounce) + hisi_gpio_write_reg(chip, HISI_GPIO_DEBOUNCE_SET_WX, BIT(off)); + else + hisi_gpio_write_reg(chip, HISI_GPIO_DEBOUNCE_CLR_WX, BIT(off)); +} + +static int hisi_gpio_set_config(struct gpio_chip *chip, unsigned int offset, + unsigned long config) +{ + u32 config_para = pinconf_to_config_param(config); + u32 config_arg; + + switch (config_para) { + case PIN_CONFIG_INPUT_DEBOUNCE: + config_arg = pinconf_to_config_argument(config); + hisi_gpio_set_debounce(chip, offset, config_arg); + break; + default: + return -ENOTSUPP; + } + + return 0; +} + +static void hisi_gpio_set_ack(struct irq_data *d) +{ + struct gpio_chip *chip = irq_data_get_irq_chip_data(d); + + hisi_gpio_write_reg(chip, HISI_GPIO_PORTA_EOI_WX, BIT(irqd_to_hwirq(d))); +} + +static void hisi_gpio_irq_set_mask(struct irq_data *d) +{ + struct gpio_chip *chip = irq_data_get_irq_chip_data(d); + + hisi_gpio_write_reg(chip, HISI_GPIO_INTMASK_SET_WX, BIT(irqd_to_hwirq(d))); +} + +static void hisi_gpio_irq_clr_mask(struct irq_data *d) +{ + struct gpio_chip *chip = irq_data_get_irq_chip_data(d); + + hisi_gpio_write_reg(chip, HISI_GPIO_INTMASK_CLR_WX, BIT(irqd_to_hwirq(d))); +} + +static int hisi_gpio_irq_set_type(struct irq_data *d, u32 type) +{ + struct gpio_chip *chip = irq_data_get_irq_chip_data(d); + unsigned int mask = BIT(irqd_to_hwirq(d)); + + switch (type) { + case IRQ_TYPE_EDGE_BOTH: + hisi_gpio_write_reg(chip, HISI_GPIO_INT_DEDGE_SET, mask); + break; + case IRQ_TYPE_EDGE_RISING: + hisi_gpio_write_reg(chip, HISI_GPIO_INTTYPE_EDGE_SET_WX, mask); + hisi_gpio_write_reg(chip, HISI_GPIO_INT_POLARITY_SET_WX, mask); + break; + case IRQ_TYPE_EDGE_FALLING: + hisi_gpio_write_reg(chip, HISI_GPIO_INTTYPE_EDGE_SET_WX, mask); + hisi_gpio_write_reg(chip, HISI_GPIO_INT_POLARITY_CLR_WX, mask); + break; + case IRQ_TYPE_LEVEL_HIGH: + hisi_gpio_write_reg(chip, HISI_GPIO_INTTYPE_EDGE_CLR_WX, mask); + hisi_gpio_write_reg(chip, HISI_GPIO_INT_POLARITY_SET_WX, mask); + break; + case IRQ_TYPE_LEVEL_LOW: + hisi_gpio_write_reg(chip, HISI_GPIO_INTTYPE_EDGE_CLR_WX, mask); + hisi_gpio_write_reg(chip, HISI_GPIO_INT_POLARITY_CLR_WX, mask); + break; + default: + return -EINVAL; + } + + /* + * The dual-edge interrupt and other interrupt's registers do not + * take effect at the same time. The registers of the two-edge + * interrupts have higher priorities, the configuration of + * the dual-edge interrupts must be disabled before the configuration + * of other kind of interrupts. + */ + if (type != IRQ_TYPE_EDGE_BOTH) { + unsigned int both = hisi_gpio_read_reg(chip, HISI_GPIO_INT_DEDGE_ST); + + if (both & mask) + hisi_gpio_write_reg(chip, HISI_GPIO_INT_DEDGE_CLR, mask); + } + + if (type & IRQ_TYPE_LEVEL_MASK) + irq_set_handler_locked(d, handle_level_irq); + else if (type & IRQ_TYPE_EDGE_BOTH) + irq_set_handler_locked(d, handle_edge_irq); + + return 0; +} + +static void hisi_gpio_irq_enable(struct irq_data *d) +{ + struct gpio_chip *chip = irq_data_get_irq_chip_data(d); + + hisi_gpio_irq_clr_mask(d); + hisi_gpio_write_reg(chip, HISI_GPIO_INTEN_SET_WX, BIT(irqd_to_hwirq(d))); +} + +static void hisi_gpio_irq_disable(struct irq_data *d) +{ + struct gpio_chip *chip = irq_data_get_irq_chip_data(d); + + hisi_gpio_irq_set_mask(d); + hisi_gpio_write_reg(chip, HISI_GPIO_INTEN_CLR_WX, BIT(irqd_to_hwirq(d))); +} + +static void hisi_gpio_irq_handler(struct irq_desc *desc) +{ + struct hisi_gpio *hisi_gpio = irq_desc_get_handler_data(desc); + unsigned long irq_msk = hisi_gpio_read_reg(&hisi_gpio->chip, + HISI_GPIO_INTSTATUS_WX); + struct irq_chip *irq_c = irq_desc_get_chip(desc); + int hwirq; + + chained_irq_enter(irq_c, desc); + for_each_set_bit(hwirq, &irq_msk, HISI_GPIO_LINE_NUM_MAX) + generic_handle_irq(irq_find_mapping(hisi_gpio->chip.irq.domain, + hwirq)); + chained_irq_exit(irq_c, desc); +} + +static void hisi_gpio_init_irq(struct hisi_gpio *hisi_gpio) +{ + struct gpio_chip *chip = &hisi_gpio->chip; + struct gpio_irq_chip *girq_chip = &chip->irq; + + /* Set hooks for irq_chip */ + hisi_gpio->irq_chip.irq_ack = hisi_gpio_set_ack; + hisi_gpio->irq_chip.irq_mask = hisi_gpio_irq_set_mask; + hisi_gpio->irq_chip.irq_unmask = hisi_gpio_irq_clr_mask; + hisi_gpio->irq_chip.irq_set_type = hisi_gpio_irq_set_type; + hisi_gpio->irq_chip.irq_enable = hisi_gpio_irq_enable; + hisi_gpio->irq_chip.irq_disable = hisi_gpio_irq_disable; + + girq_chip->chip = &hisi_gpio->irq_chip; + girq_chip->default_type = IRQ_TYPE_NONE; + girq_chip->num_parents = 1; + girq_chip->parents = &hisi_gpio->irq; + girq_chip->parent_handler = hisi_gpio_irq_handler; + girq_chip->parent_handler_data = hisi_gpio; + + /* Clear Mask of GPIO controller combine IRQ */ + hisi_gpio_write_reg(chip, HISI_GPIO_INTCOMB_MASK_WX, 1); +} + +static const struct acpi_device_id hisi_gpio_acpi_match[] = { + {"HISI0184", 0}, + {} +}; +MODULE_DEVICE_TABLE(acpi, hisi_gpio_acpi_match); + +static void hisi_gpio_get_pdata(struct device *dev, + struct hisi_gpio *hisi_gpio) +{ + struct platform_device *pdev = to_platform_device(dev); + struct fwnode_handle *fwnode; + int idx = 0; + + device_for_each_child_node(dev, fwnode) { + /* Cycle for once, no need for an array to save line_num */ + if (fwnode_property_read_u32(fwnode, "ngpios", + &hisi_gpio->line_num)) { + dev_err(dev, + "failed to get number of lines for port%d and use default value instead\n", + idx); + hisi_gpio->line_num = HISI_GPIO_LINE_NUM_MAX; + } + + if (WARN_ON(hisi_gpio->line_num > HISI_GPIO_LINE_NUM_MAX)) + hisi_gpio->line_num = HISI_GPIO_LINE_NUM_MAX; + + hisi_gpio->irq = platform_get_irq(pdev, idx); + + dev_info(dev, + "get hisi_gpio[%d] with %u lines\n", idx, + hisi_gpio->line_num); + + idx++; + } +} + +static int hisi_gpio_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + void __iomem *dat, *set, *clr; + struct hisi_gpio *hisi_gpio; + int port_num; + int ret; + + /* + * One GPIO controller own one port currently, + * if we get more from ACPI table, return error. + */ + port_num = device_get_child_node_count(dev); + if (WARN_ON(port_num != 1)) + return -ENODEV; + + hisi_gpio = devm_kzalloc(dev, sizeof(*hisi_gpio), GFP_KERNEL); + if (!hisi_gpio) + return -ENOMEM; + + hisi_gpio->reg_base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(hisi_gpio->reg_base)) + return PTR_ERR(hisi_gpio->reg_base); + + hisi_gpio_get_pdata(dev, hisi_gpio); + + hisi_gpio->dev = dev; + + dat = hisi_gpio->reg_base + HISI_GPIO_EXT_PORT_WX; + set = hisi_gpio->reg_base + HISI_GPIO_SWPORT_DR_SET_WX; + clr = hisi_gpio->reg_base + HISI_GPIO_SWPORT_DR_CLR_WX; + + ret = bgpio_init(&hisi_gpio->chip, hisi_gpio->dev, 0x4, + hisi_gpio->reg_base + HISI_GPIO_EXT_PORT_WX, + hisi_gpio->reg_base + HISI_GPIO_SWPORT_DR_SET_WX, + hisi_gpio->reg_base + HISI_GPIO_SWPORT_DR_CLR_WX, + hisi_gpio->reg_base + HISI_GPIO_SWPORT_DDR_SET_WX, + hisi_gpio->reg_base + HISI_GPIO_SWPORT_DDR_CLR_WX, + BGPIOF_NO_SET_ON_INPUT); + if (ret) { + dev_err(dev, "failed to init, ret = %d\n", ret); + return ret; + } + + hisi_gpio->chip.set_config = hisi_gpio_set_config; + hisi_gpio->chip.ngpio = hisi_gpio->line_num; + hisi_gpio->chip.bgpio_dir_unreadable = 1; + hisi_gpio->chip.base = -1; + + if (hisi_gpio->irq > 0) + hisi_gpio_init_irq(hisi_gpio); + + ret = devm_gpiochip_add_data(dev, &hisi_gpio->chip, hisi_gpio); + if (ret) { + dev_err(dev, "failed to register gpiochip, ret = %d\n", ret); + return ret; + } + + return 0; +} + +static struct platform_driver hisi_gpio_driver = { + .driver = { + .name = HISI_GPIO_DRIVER_NAME, + .acpi_match_table = hisi_gpio_acpi_match, + }, + .probe = hisi_gpio_probe, +}; + +module_platform_driver(hisi_gpio_driver); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Luo Jiaxing "); +MODULE_DESCRIPTION("HiSilicon GPIO controller driver"); +MODULE_ALIAS("platform:" HISI_GPIO_DRIVER_NAME); diff --git a/drivers/gpio/gpio-mmio.c b/drivers/gpio/gpio-mmio.c index 6f904c87467826c5e48a2e710c4929488f07ceec..4c5e507192ae335115f23e0442dc8d224bad9956 100644 --- a/drivers/gpio/gpio-mmio.c +++ b/drivers/gpio/gpio-mmio.c @@ -381,12 +381,10 @@ static int bgpio_get_dir(struct gpio_chip *gc, unsigned int gpio) return 1; } -static int bgpio_dir_out(struct gpio_chip *gc, unsigned int gpio, int val) +static void bgpio_dir_out(struct gpio_chip *gc, unsigned int gpio, int val) { unsigned long flags; - gc->set(gc, gpio, val); - spin_lock_irqsave(&gc->bgpio_lock, flags); gc->bgpio_dir |= bgpio_line2mask(gc, gpio); @@ -397,7 +395,21 @@ static int bgpio_dir_out(struct gpio_chip *gc, unsigned int gpio, int val) gc->write_reg(gc->reg_dir_out, gc->bgpio_dir); spin_unlock_irqrestore(&gc->bgpio_lock, flags); +} +static int bgpio_dir_out_dir_first(struct gpio_chip *gc, unsigned int gpio, + int val) +{ + bgpio_dir_out(gc, gpio, val); + gc->set(gc, gpio, val); + return 0; +} + +static int bgpio_dir_out_val_first(struct gpio_chip *gc, unsigned int gpio, + int val) +{ + gc->set(gc, gpio, val); + bgpio_dir_out(gc, gpio, val); return 0; } @@ -530,7 +542,10 @@ static int bgpio_setup_direction(struct gpio_chip *gc, if (dirout || dirin) { gc->reg_dir_out = dirout; gc->reg_dir_in = dirin; - gc->direction_output = bgpio_dir_out; + if (flags & BGPIOF_NO_SET_ON_INPUT) + gc->direction_output = bgpio_dir_out_dir_first; + else + gc->direction_output = bgpio_dir_out_val_first; gc->direction_input = bgpio_dir_in; gc->get_direction = bgpio_get_dir; } else { diff --git a/drivers/hwmon/xgene-hwmon.c b/drivers/hwmon/xgene-hwmon.c index f5d3cf86753f76cc683895d5521b25af3264fc86..9c37e2afc5575ab010c6e33d031d0d3ed75189a5 100644 --- a/drivers/hwmon/xgene-hwmon.c +++ b/drivers/hwmon/xgene-hwmon.c @@ -93,6 +93,7 @@ struct slimpro_resp_msg { struct xgene_hwmon_dev { struct device *dev; struct mbox_chan *mbox_chan; + struct pcc_mbox_chan *pcc_chan; struct mbox_client mbox_client; int mbox_idx; @@ -652,7 +653,7 @@ static int xgene_hwmon_probe(struct platform_device *pdev) goto out_mbox_free; } } else { - struct acpi_pcct_hw_reduced *cppc_ss; + struct pcc_mbox_chan *pcc_chan; const struct acpi_device_id *acpi_id; int version; @@ -671,26 +672,16 @@ static int xgene_hwmon_probe(struct platform_device *pdev) } cl->rx_callback = xgene_hwmon_pcc_rx_cb; - ctx->mbox_chan = pcc_mbox_request_channel(cl, ctx->mbox_idx); - if (IS_ERR(ctx->mbox_chan)) { + pcc_chan = pcc_mbox_request_channel(cl, ctx->mbox_idx); + if (IS_ERR(pcc_chan)) { dev_err(&pdev->dev, "PPC channel request failed\n"); rc = -ENODEV; goto out_mbox_free; } - /* - * The PCC mailbox controller driver should - * have parsed the PCCT (global table of all - * PCC channels) and stored pointers to the - * subspace communication region in con_priv. - */ - cppc_ss = ctx->mbox_chan->con_priv; - if (!cppc_ss) { - dev_err(&pdev->dev, "PPC subspace not found\n"); - rc = -ENODEV; - goto out; - } + ctx->pcc_chan = pcc_chan; + ctx->mbox_chan = pcc_chan->mchan; if (!ctx->mbox_chan->mbox->txdone_irq) { dev_err(&pdev->dev, "PCC IRQ not supported\n"); @@ -702,16 +693,16 @@ static int xgene_hwmon_probe(struct platform_device *pdev) * This is the shared communication region * for the OS and Platform to communicate over. */ - ctx->comm_base_addr = cppc_ss->base_address; + ctx->comm_base_addr = pcc_chan->shmem_base_addr; if (ctx->comm_base_addr) { if (version == XGENE_HWMON_V2) ctx->pcc_comm_addr = (void __force *)ioremap( ctx->comm_base_addr, - cppc_ss->length); + pcc_chan->shmem_size); else ctx->pcc_comm_addr = memremap( ctx->comm_base_addr, - cppc_ss->length, + pcc_chan->shmem_size, MEMREMAP_WB); } else { dev_err(&pdev->dev, "Failed to get PCC comm region\n"); @@ -727,11 +718,11 @@ static int xgene_hwmon_probe(struct platform_device *pdev) } /* - * cppc_ss->latency is just a Nominal value. In reality + * pcc_chan->latency is just a Nominal value. In reality * the remote processor could be much slower to reply. * So add an arbitrary amount of wait on top of Nominal. */ - ctx->usecs_lat = PCC_NUM_RETRIES * cppc_ss->latency; + ctx->usecs_lat = PCC_NUM_RETRIES * pcc_chan->latency; } ctx->hwmon_dev = hwmon_device_register_with_groups(ctx->dev, @@ -757,7 +748,7 @@ static int xgene_hwmon_probe(struct platform_device *pdev) if (acpi_disabled) mbox_free_channel(ctx->mbox_chan); else - pcc_mbox_free_channel(ctx->mbox_chan); + pcc_mbox_free_channel(ctx->pcc_chan); out_mbox_free: kfifo_free(&ctx->async_msg_fifo); @@ -774,7 +765,7 @@ static int xgene_hwmon_remove(struct platform_device *pdev) if (acpi_disabled) mbox_free_channel(ctx->mbox_chan); else - pcc_mbox_free_channel(ctx->mbox_chan); + pcc_mbox_free_channel(ctx->pcc_chan); return 0; } diff --git a/drivers/hwtracing/Kconfig b/drivers/hwtracing/Kconfig index 13085835a63672cbe0918cccf9f181358a2c823e..911ee977103c0119f9e9aa0aa373d595d37b0485 100644 --- a/drivers/hwtracing/Kconfig +++ b/drivers/hwtracing/Kconfig @@ -5,4 +5,6 @@ source "drivers/hwtracing/stm/Kconfig" source "drivers/hwtracing/intel_th/Kconfig" +source "drivers/hwtracing/ptt/Kconfig" + endmenu diff --git a/drivers/hwtracing/ptt/Kconfig b/drivers/hwtracing/ptt/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..6d46a09ffeb91992c6e943fa1a8fe7421dd88a99 --- /dev/null +++ b/drivers/hwtracing/ptt/Kconfig @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0-only +config HISI_PTT + tristate "HiSilicon PCIe Tune and Trace Device" + depends on ARM64 || (COMPILE_TEST && 64BIT) + depends on PCI && HAS_DMA && HAS_IOMEM && PERF_EVENTS + help + HiSilicon PCIe Tune and Trace device exists as a PCIe RCiEP + device, and it provides support for PCIe traffic tuning and + tracing TLP headers to the memory. + + This driver can also be built as a module. If so, the module + will be called hisi_ptt. diff --git a/drivers/hwtracing/ptt/Makefile b/drivers/hwtracing/ptt/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..908c09a981619c99a5a5a222188d41cf816b8896 --- /dev/null +++ b/drivers/hwtracing/ptt/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_HISI_PTT) += hisi_ptt.o diff --git a/drivers/hwtracing/ptt/hisi_ptt.c b/drivers/hwtracing/ptt/hisi_ptt.c new file mode 100644 index 0000000000000000000000000000000000000000..198cdc03153a9824ad87f8e84e88e43edac0f0b0 --- /dev/null +++ b/drivers/hwtracing/ptt/hisi_ptt.c @@ -0,0 +1,1436 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Driver for HiSilicon PCIe tune and trace device + * + * Copyright (c) 2022 HiSilicon Technologies Co., Ltd. + * Author: Yicong Yang + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hisi_ptt.h" + +/* Dynamic CPU hotplug state used by PTT */ +static enum cpuhp_state hisi_ptt_pmu_online; + +static bool hisi_ptt_wait_tuning_finish(struct hisi_ptt *hisi_ptt) +{ + u32 val; + + return !readl_poll_timeout(hisi_ptt->iobase + HISI_PTT_TUNING_INT_STAT, + val, !(val & HISI_PTT_TUNING_INT_STAT_MASK), + HISI_PTT_WAIT_POLL_INTERVAL_US, + HISI_PTT_WAIT_TUNE_TIMEOUT_US); +} + +static ssize_t hisi_ptt_tune_attr_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct hisi_ptt *hisi_ptt = to_hisi_ptt(dev_get_drvdata(dev)); + struct dev_ext_attribute *ext_attr; + struct hisi_ptt_tune_desc *desc; + u32 reg; + u16 val; + + ext_attr = container_of(attr, struct dev_ext_attribute, attr); + desc = ext_attr->var; + + mutex_lock(&hisi_ptt->tune_lock); + + reg = readl(hisi_ptt->iobase + HISI_PTT_TUNING_CTRL); + reg &= ~(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB); + reg |= FIELD_PREP(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB, + desc->event_code); + writel(reg, hisi_ptt->iobase + HISI_PTT_TUNING_CTRL); + + /* Write all 1 to indicates it's the read process */ + writel(~0U, hisi_ptt->iobase + HISI_PTT_TUNING_DATA); + + if (!hisi_ptt_wait_tuning_finish(hisi_ptt)) { + mutex_unlock(&hisi_ptt->tune_lock); + return -ETIMEDOUT; + } + + reg = readl(hisi_ptt->iobase + HISI_PTT_TUNING_DATA); + reg &= HISI_PTT_TUNING_DATA_VAL_MASK; + val = FIELD_GET(HISI_PTT_TUNING_DATA_VAL_MASK, reg); + + mutex_unlock(&hisi_ptt->tune_lock); + return sysfs_emit(buf, "%u\n", val); +} + +static ssize_t hisi_ptt_tune_attr_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct hisi_ptt *hisi_ptt = to_hisi_ptt(dev_get_drvdata(dev)); + struct dev_ext_attribute *ext_attr; + struct hisi_ptt_tune_desc *desc; + u32 reg; + u16 val; + + ext_attr = container_of(attr, struct dev_ext_attribute, attr); + desc = ext_attr->var; + + if (kstrtou16(buf, 10, &val)) + return -EINVAL; + + mutex_lock(&hisi_ptt->tune_lock); + + reg = readl(hisi_ptt->iobase + HISI_PTT_TUNING_CTRL); + reg &= ~(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB); + reg |= FIELD_PREP(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB, + desc->event_code); + writel(reg, hisi_ptt->iobase + HISI_PTT_TUNING_CTRL); + writel(FIELD_PREP(HISI_PTT_TUNING_DATA_VAL_MASK, val), + hisi_ptt->iobase + HISI_PTT_TUNING_DATA); + + if (!hisi_ptt_wait_tuning_finish(hisi_ptt)) { + mutex_unlock(&hisi_ptt->tune_lock); + return -ETIMEDOUT; + } + + mutex_unlock(&hisi_ptt->tune_lock); + return count; +} + +#define HISI_PTT_TUNE_ATTR(_name, _val, _show, _store) \ + static struct hisi_ptt_tune_desc _name##_desc = { \ + .name = #_name, \ + .event_code = (_val), \ + }; \ + static struct dev_ext_attribute hisi_ptt_##_name##_attr = { \ + .attr = __ATTR(_name, 0600, _show, _store), \ + .var = &_name##_desc, \ + } + +#define HISI_PTT_TUNE_ATTR_COMMON(_name, _val) \ + HISI_PTT_TUNE_ATTR(_name, _val, \ + hisi_ptt_tune_attr_show, \ + hisi_ptt_tune_attr_store) + +/* + * The value of the tuning event are composed of two parts: main event code + * in BIT[0,15] and subevent code in BIT[16,23]. For example, qox_tx_cpl is + * a subevent of 'Tx path QoS control' which for tuning the weight of Tx + * completion TLPs. See hisi_ptt.rst documentation for more information. + */ +#define HISI_PTT_TUNE_QOS_TX_CPL (0x4 | (3 << 16)) +#define HISI_PTT_TUNE_QOS_TX_NP (0x4 | (4 << 16)) +#define HISI_PTT_TUNE_QOS_TX_P (0x4 | (5 << 16)) +#define HISI_PTT_TUNE_RX_ALLOC_BUF_LEVEL (0x5 | (6 << 16)) +#define HISI_PTT_TUNE_TX_ALLOC_BUF_LEVEL (0x5 | (7 << 16)) + +HISI_PTT_TUNE_ATTR_COMMON(qos_tx_cpl, HISI_PTT_TUNE_QOS_TX_CPL); +HISI_PTT_TUNE_ATTR_COMMON(qos_tx_np, HISI_PTT_TUNE_QOS_TX_NP); +HISI_PTT_TUNE_ATTR_COMMON(qos_tx_p, HISI_PTT_TUNE_QOS_TX_P); +HISI_PTT_TUNE_ATTR_COMMON(rx_alloc_buf_level, HISI_PTT_TUNE_RX_ALLOC_BUF_LEVEL); +HISI_PTT_TUNE_ATTR_COMMON(tx_alloc_buf_level, HISI_PTT_TUNE_TX_ALLOC_BUF_LEVEL); + +static struct attribute *hisi_ptt_tune_attrs[] = { + &hisi_ptt_qos_tx_cpl_attr.attr.attr, + &hisi_ptt_qos_tx_np_attr.attr.attr, + &hisi_ptt_qos_tx_p_attr.attr.attr, + &hisi_ptt_rx_alloc_buf_level_attr.attr.attr, + &hisi_ptt_tx_alloc_buf_level_attr.attr.attr, + NULL, +}; + +static struct attribute_group hisi_ptt_tune_group = { + .name = "tune", + .attrs = hisi_ptt_tune_attrs, +}; + +static u16 hisi_ptt_get_filter_val(u16 devid, bool is_port) +{ + if (is_port) + return BIT(HISI_PCIE_CORE_PORT_ID(devid & 0xff)); + + return devid; +} + +static bool hisi_ptt_wait_trace_hw_idle(struct hisi_ptt *hisi_ptt) +{ + u32 val; + + return !readl_poll_timeout_atomic(hisi_ptt->iobase + HISI_PTT_TRACE_STS, + val, val & HISI_PTT_TRACE_IDLE, + HISI_PTT_WAIT_POLL_INTERVAL_US, + HISI_PTT_WAIT_TRACE_TIMEOUT_US); +} + +static void hisi_ptt_wait_dma_reset_done(struct hisi_ptt *hisi_ptt) +{ + u32 val; + + readl_poll_timeout_atomic(hisi_ptt->iobase + HISI_PTT_TRACE_WR_STS, + val, !val, HISI_PTT_RESET_POLL_INTERVAL_US, + HISI_PTT_RESET_TIMEOUT_US); +} + +static void hisi_ptt_trace_end(struct hisi_ptt *hisi_ptt) +{ + writel(0, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); + hisi_ptt->trace_ctrl.started = false; +} + +static int hisi_ptt_trace_start(struct hisi_ptt *hisi_ptt) +{ + struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl; + u32 val; + int i; + + /* Check device idle before start trace */ + if (!hisi_ptt_wait_trace_hw_idle(hisi_ptt)) { + pci_err(hisi_ptt->pdev, "Failed to start trace, the device is still busy\n"); + return -EBUSY; + } + + ctrl->started = true; + + /* Reset the DMA before start tracing */ + val = readl(hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); + val |= HISI_PTT_TRACE_CTRL_RST; + writel(val, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); + + hisi_ptt_wait_dma_reset_done(hisi_ptt); + + val = readl(hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); + val &= ~HISI_PTT_TRACE_CTRL_RST; + writel(val, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); + + /* Reset the index of current buffer */ + hisi_ptt->trace_ctrl.buf_index = 0; + + /* Zero the trace buffers */ + for (i = 0; i < HISI_PTT_TRACE_BUF_CNT; i++) + memset(ctrl->trace_buf[i].addr, 0, HISI_PTT_TRACE_BUF_SIZE); + + /* Clear the interrupt status */ + writel(HISI_PTT_TRACE_INT_STAT_MASK, hisi_ptt->iobase + HISI_PTT_TRACE_INT_STAT); + writel(0, hisi_ptt->iobase + HISI_PTT_TRACE_INT_MASK); + + /* Set the trace control register */ + val = FIELD_PREP(HISI_PTT_TRACE_CTRL_TYPE_SEL, ctrl->type); + val |= FIELD_PREP(HISI_PTT_TRACE_CTRL_RXTX_SEL, ctrl->direction); + val |= FIELD_PREP(HISI_PTT_TRACE_CTRL_DATA_FORMAT, ctrl->format); + val |= FIELD_PREP(HISI_PTT_TRACE_CTRL_TARGET_SEL, hisi_ptt->trace_ctrl.filter); + if (!hisi_ptt->trace_ctrl.is_port) + val |= HISI_PTT_TRACE_CTRL_FILTER_MODE; + + /* Start the Trace */ + val |= HISI_PTT_TRACE_CTRL_EN; + writel(val, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); + + return 0; +} + +static int hisi_ptt_update_aux(struct hisi_ptt *hisi_ptt, int index, bool stop) +{ + struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl; + struct perf_output_handle *handle = &ctrl->handle; + struct perf_event *event = handle->event; + struct hisi_ptt_pmu_buf *buf; + size_t size; + void *addr; + + buf = perf_get_aux(handle); + if (!buf || !handle->size) + return -EINVAL; + + addr = ctrl->trace_buf[ctrl->buf_index].addr; + + /* + * If we're going to stop, read the size of already traced data from + * HISI_PTT_TRACE_WR_STS. Otherwise we're coming from the interrupt, + * the data size is always HISI_PTT_TRACE_BUF_SIZE. + */ + if (stop) { + u32 reg; + + reg = readl(hisi_ptt->iobase + HISI_PTT_TRACE_WR_STS); + size = FIELD_GET(HISI_PTT_TRACE_WR_STS_WRITE, reg); + } else { + size = HISI_PTT_TRACE_BUF_SIZE; + } + + memcpy(buf->base + buf->pos, addr, size); + buf->pos += size; + + /* + * Just commit the traced data if we're going to stop. Otherwise if the + * resident AUX buffer cannot contain the data of next trace buffer, + * apply a new one. + */ + if (stop) { + perf_aux_output_end(handle, buf->pos); + } else if (buf->length - buf->pos < HISI_PTT_TRACE_BUF_SIZE) { + perf_aux_output_end(handle, buf->pos); + + buf = perf_aux_output_begin(handle, event); + if (!buf) + return -EINVAL; + + buf->pos = handle->head % buf->length; + if (buf->length - buf->pos < HISI_PTT_TRACE_BUF_SIZE) { + perf_aux_output_end(handle, 0); + return -EINVAL; + } + } + + return 0; +} + +static irqreturn_t hisi_ptt_isr(int irq, void *context) +{ + struct hisi_ptt *hisi_ptt = context; + u32 status, buf_idx; + + status = readl(hisi_ptt->iobase + HISI_PTT_TRACE_INT_STAT); + if (!(status & HISI_PTT_TRACE_INT_STAT_MASK)) + return IRQ_NONE; + + buf_idx = ffs(status) - 1; + + /* Clear the interrupt status of buffer @buf_idx */ + writel(status, hisi_ptt->iobase + HISI_PTT_TRACE_INT_STAT); + + /* + * Update the AUX buffer and cache the current buffer index, + * as we need to know this and save the data when the trace + * is ended out of the interrupt handler. End the trace + * if the updating fails. + */ + if (hisi_ptt_update_aux(hisi_ptt, buf_idx, false)) + hisi_ptt_trace_end(hisi_ptt); + else + hisi_ptt->trace_ctrl.buf_index = (buf_idx + 1) % HISI_PTT_TRACE_BUF_CNT; + + return IRQ_HANDLED; +} + +static void hisi_ptt_irq_free_vectors(void *pdev) +{ + pci_free_irq_vectors(pdev); +} + +static int hisi_ptt_register_irq(struct hisi_ptt *hisi_ptt) +{ + struct pci_dev *pdev = hisi_ptt->pdev; + int ret; + + ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI); + if (ret < 0) { + pci_err(pdev, "failed to allocate irq vector, ret = %d\n", ret); + return ret; + } + + ret = devm_add_action_or_reset(&pdev->dev, hisi_ptt_irq_free_vectors, pdev); + if (ret < 0) + return ret; + + hisi_ptt->trace_irq = pci_irq_vector(pdev, HISI_PTT_TRACE_DMA_IRQ); + ret = devm_request_threaded_irq(&pdev->dev, hisi_ptt->trace_irq, + NULL, hisi_ptt_isr, 0, + DRV_NAME, hisi_ptt); + if (ret) { + pci_err(pdev, "failed to request irq %d, ret = %d\n", + hisi_ptt->trace_irq, ret); + return ret; + } + + return 0; +} + +static void hisi_ptt_del_free_filter(struct hisi_ptt *hisi_ptt, + struct hisi_ptt_filter_desc *filter) +{ + if (filter->is_port) + hisi_ptt->port_mask &= ~hisi_ptt_get_filter_val(filter->devid, true); + + list_del(&filter->list); + kfree(filter->name); + kfree(filter); +} + +static struct hisi_ptt_filter_desc * +hisi_ptt_alloc_add_filter(struct hisi_ptt *hisi_ptt, u16 devid, bool is_port) +{ + struct hisi_ptt_filter_desc *filter; + u8 devfn = devid & 0xff; + char *filter_name; + + filter_name = kasprintf(GFP_KERNEL, "%04x:%02x:%02x.%d", pci_domain_nr(hisi_ptt->pdev->bus), + PCI_BUS_NUM(devid), PCI_SLOT(devfn), PCI_FUNC(devfn)); + if (!filter_name) { + pci_err(hisi_ptt->pdev, "failed to allocate name for filter %04x:%02x:%02x.%d\n", + pci_domain_nr(hisi_ptt->pdev->bus), PCI_BUS_NUM(devid), + PCI_SLOT(devfn), PCI_FUNC(devfn)); + return NULL; + } + + /* + * We won't fail the probe if filter allocation failed here. The filters + * should be partial initialized and users would know which filter fails + * through the log. Other functions of PTT device are still available. + */ + filter = kzalloc(sizeof(*filter), GFP_KERNEL); + if (!filter) { + pci_err(hisi_ptt->pdev, "failed to add filter for %s\n", + filter_name); + kfree(filter_name); + return NULL; + } + + filter->name = filter_name; + filter->is_port = is_port; + filter->devid = devid; + + if (filter->is_port) { + list_add_tail(&filter->list, &hisi_ptt->port_filters); + + /* Update the available port mask */ + hisi_ptt->port_mask |= hisi_ptt_get_filter_val(filter->devid, true); + } else { + list_add_tail(&filter->list, &hisi_ptt->req_filters); + } + + return filter; +} + +static ssize_t hisi_ptt_filter_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct hisi_ptt_filter_desc *filter; + unsigned long filter_val; + + filter = container_of(attr, struct hisi_ptt_filter_desc, attr); + filter_val = hisi_ptt_get_filter_val(filter->devid, filter->is_port) | + (filter->is_port ? HISI_PTT_PMU_FILTER_IS_PORT : 0); + + return sysfs_emit(buf, "0x%05lx\n", filter_val); +} + +static int hisi_ptt_create_rp_filter_attr(struct hisi_ptt *hisi_ptt, + struct hisi_ptt_filter_desc *filter) +{ + struct kobject *kobj = &hisi_ptt->hisi_ptt_pmu.dev->kobj; + + sysfs_attr_init(&filter->attr.attr); + filter->attr.attr.name = filter->name; + filter->attr.attr.mode = 0400; /* DEVICE_ATTR_ADMIN_RO */ + filter->attr.show = hisi_ptt_filter_show; + + return sysfs_add_file_to_group(kobj, &filter->attr.attr, + HISI_PTT_RP_FILTERS_GRP_NAME); +} + +static void hisi_ptt_remove_rp_filter_attr(struct hisi_ptt *hisi_ptt, + struct hisi_ptt_filter_desc *filter) +{ + struct kobject *kobj = &hisi_ptt->hisi_ptt_pmu.dev->kobj; + + sysfs_remove_file_from_group(kobj, &filter->attr.attr, + HISI_PTT_RP_FILTERS_GRP_NAME); +} + +static int hisi_ptt_create_req_filter_attr(struct hisi_ptt *hisi_ptt, + struct hisi_ptt_filter_desc *filter) +{ + struct kobject *kobj = &hisi_ptt->hisi_ptt_pmu.dev->kobj; + + sysfs_attr_init(&filter->attr.attr); + filter->attr.attr.name = filter->name; + filter->attr.attr.mode = 0400; /* DEVICE_ATTR_ADMIN_RO */ + filter->attr.show = hisi_ptt_filter_show; + + return sysfs_add_file_to_group(kobj, &filter->attr.attr, + HISI_PTT_REQ_FILTERS_GRP_NAME); +} + +static void hisi_ptt_remove_req_filter_attr(struct hisi_ptt *hisi_ptt, + struct hisi_ptt_filter_desc *filter) +{ + struct kobject *kobj = &hisi_ptt->hisi_ptt_pmu.dev->kobj; + + sysfs_remove_file_from_group(kobj, &filter->attr.attr, + HISI_PTT_REQ_FILTERS_GRP_NAME); +} + +static int hisi_ptt_create_filter_attr(struct hisi_ptt *hisi_ptt, + struct hisi_ptt_filter_desc *filter) +{ + int ret; + + if (filter->is_port) + ret = hisi_ptt_create_rp_filter_attr(hisi_ptt, filter); + else + ret = hisi_ptt_create_req_filter_attr(hisi_ptt, filter); + + if (ret) + pci_err(hisi_ptt->pdev, "failed to create sysfs attribute for filter %s\n", + filter->name); + + return ret; +} + +static void hisi_ptt_remove_filter_attr(struct hisi_ptt *hisi_ptt, + struct hisi_ptt_filter_desc *filter) +{ + if (filter->is_port) + hisi_ptt_remove_rp_filter_attr(hisi_ptt, filter); + else + hisi_ptt_remove_req_filter_attr(hisi_ptt, filter); +} + +static void hisi_ptt_remove_all_filter_attributes(void *data) +{ + struct hisi_ptt_filter_desc *filter; + struct hisi_ptt *hisi_ptt = data; + + mutex_lock(&hisi_ptt->filter_lock); + + list_for_each_entry(filter, &hisi_ptt->req_filters, list) + hisi_ptt_remove_filter_attr(hisi_ptt, filter); + + list_for_each_entry(filter, &hisi_ptt->port_filters, list) + hisi_ptt_remove_filter_attr(hisi_ptt, filter); + + hisi_ptt->sysfs_inited = false; + mutex_unlock(&hisi_ptt->filter_lock); +} + +static int hisi_ptt_init_filter_attributes(struct hisi_ptt *hisi_ptt) +{ + struct hisi_ptt_filter_desc *filter; + int ret; + + mutex_lock(&hisi_ptt->filter_lock); + + /* + * Register the reset callback in the first stage. In reset we traverse + * the filters list to remove the sysfs attributes so the callback can + * be called safely even without below filter attributes creation. + */ + ret = devm_add_action(&hisi_ptt->pdev->dev, + hisi_ptt_remove_all_filter_attributes, + hisi_ptt); + if (ret) + goto out; + + list_for_each_entry(filter, &hisi_ptt->port_filters, list) { + ret = hisi_ptt_create_filter_attr(hisi_ptt, filter); + if (ret) + goto out; + } + + list_for_each_entry(filter, &hisi_ptt->req_filters, list) { + ret = hisi_ptt_create_filter_attr(hisi_ptt, filter); + if (ret) + goto out; + } + + hisi_ptt->sysfs_inited = true; +out: + mutex_unlock(&hisi_ptt->filter_lock); + return ret; +} + +static void hisi_ptt_update_filters(struct work_struct *work) +{ + struct delayed_work *delayed_work = to_delayed_work(work); + struct hisi_ptt_filter_update_info info; + struct hisi_ptt_filter_desc *filter; + struct hisi_ptt *hisi_ptt; + + hisi_ptt = container_of(delayed_work, struct hisi_ptt, work); + + if (!mutex_trylock(&hisi_ptt->filter_lock)) { + schedule_delayed_work(&hisi_ptt->work, HISI_PTT_WORK_DELAY_MS); + return; + } + + while (kfifo_get(&hisi_ptt->filter_update_kfifo, &info)) { + if (info.is_add) { + /* + * Notify the users if failed to add this filter, others + * still work and available. See the comments in + * hisi_ptt_init_filters(). + */ + filter = hisi_ptt_alloc_add_filter(hisi_ptt, info.devid, info.is_port); + if (!filter) + continue; + + /* + * If filters' sysfs entries hasn't been initialized, + * then we're still at probe stage. Add the filters to + * the list and later hisi_ptt_init_filter_attributes() + * will create sysfs attributes for all the filters. + */ + if (hisi_ptt->sysfs_inited && + hisi_ptt_create_filter_attr(hisi_ptt, filter)) { + hisi_ptt_del_free_filter(hisi_ptt, filter); + continue; + } + } else { + struct hisi_ptt_filter_desc *tmp; + struct list_head *target_list; + + target_list = info.is_port ? &hisi_ptt->port_filters : + &hisi_ptt->req_filters; + + list_for_each_entry_safe(filter, tmp, target_list, list) + if (filter->devid == info.devid) { + if (hisi_ptt->sysfs_inited) + hisi_ptt_remove_filter_attr(hisi_ptt, filter); + + hisi_ptt_del_free_filter(hisi_ptt, filter); + break; + } + } + } + + mutex_unlock(&hisi_ptt->filter_lock); +} + +/* + * A PCI bus notifier is used here for dynamically updating the filter + * list. + */ +static int hisi_ptt_notifier_call(struct notifier_block *nb, unsigned long action, + void *data) +{ + struct hisi_ptt *hisi_ptt = container_of(nb, struct hisi_ptt, hisi_ptt_nb); + struct hisi_ptt_filter_update_info info; + struct pci_dev *pdev, *root_port; + struct device *dev = data; + u32 port_devid; + + pdev = to_pci_dev(dev); + root_port = pcie_find_root_port(pdev); + if (!root_port) + return 0; + + port_devid = PCI_DEVID(root_port->bus->number, root_port->devfn); + if (port_devid < hisi_ptt->lower_bdf || + port_devid > hisi_ptt->upper_bdf) + return 0; + + info.is_port = pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT; + info.devid = PCI_DEVID(pdev->bus->number, pdev->devfn); + + switch (action) { + case BUS_NOTIFY_ADD_DEVICE: + info.is_add = true; + break; + case BUS_NOTIFY_DEL_DEVICE: + info.is_add = false; + break; + default: + return 0; + } + + /* + * The FIFO size is 16 which is sufficient for almost all the cases, + * since each PCIe core will have most 8 Root Ports (typically only + * 1~4 Root Ports). On failure log the failed filter and let user + * handle it. + */ + if (kfifo_in_spinlocked(&hisi_ptt->filter_update_kfifo, &info, 1, + &hisi_ptt->filter_update_lock)) + schedule_delayed_work(&hisi_ptt->work, 0); + else + pci_warn(hisi_ptt->pdev, + "filter update fifo overflow for target %s\n", + pci_name(pdev)); + + return 0; +} + +static int hisi_ptt_init_filters(struct pci_dev *pdev, void *data) +{ + struct pci_dev *root_port = pcie_find_root_port(pdev); + struct hisi_ptt_filter_desc *filter; + struct hisi_ptt *hisi_ptt = data; + u32 port_devid; + + if (!root_port) + return 0; + + port_devid = PCI_DEVID(root_port->bus->number, root_port->devfn); + if (port_devid < hisi_ptt->lower_bdf || + port_devid > hisi_ptt->upper_bdf) + return 0; + + /* + * We won't fail the probe if filter allocation failed here. The filters + * should be partial initialized and users would know which filter fails + * through the log. Other functions of PTT device are still available. + */ + filter = hisi_ptt_alloc_add_filter(hisi_ptt, PCI_DEVID(pdev->bus->number, pdev->devfn), + pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT); + if (!filter) + return -ENOMEM; + + return 0; +} + +static void hisi_ptt_release_filters(void *data) +{ + struct hisi_ptt_filter_desc *filter, *tmp; + struct hisi_ptt *hisi_ptt = data; + + list_for_each_entry_safe(filter, tmp, &hisi_ptt->req_filters, list) + hisi_ptt_del_free_filter(hisi_ptt, filter); + + list_for_each_entry_safe(filter, tmp, &hisi_ptt->port_filters, list) + hisi_ptt_del_free_filter(hisi_ptt, filter); +} + +static int hisi_ptt_config_trace_buf(struct hisi_ptt *hisi_ptt) +{ + struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl; + struct device *dev = &hisi_ptt->pdev->dev; + int i; + + ctrl->trace_buf = devm_kcalloc(dev, HISI_PTT_TRACE_BUF_CNT, + sizeof(*ctrl->trace_buf), GFP_KERNEL); + if (!ctrl->trace_buf) + return -ENOMEM; + + for (i = 0; i < HISI_PTT_TRACE_BUF_CNT; ++i) { + ctrl->trace_buf[i].addr = dmam_alloc_coherent(dev, HISI_PTT_TRACE_BUF_SIZE, + &ctrl->trace_buf[i].dma, + GFP_KERNEL); + if (!ctrl->trace_buf[i].addr) + return -ENOMEM; + } + + /* Configure the trace DMA buffer */ + for (i = 0; i < HISI_PTT_TRACE_BUF_CNT; i++) { + writel(lower_32_bits(ctrl->trace_buf[i].dma), + hisi_ptt->iobase + HISI_PTT_TRACE_ADDR_BASE_LO_0 + + i * HISI_PTT_TRACE_ADDR_STRIDE); + writel(upper_32_bits(ctrl->trace_buf[i].dma), + hisi_ptt->iobase + HISI_PTT_TRACE_ADDR_BASE_HI_0 + + i * HISI_PTT_TRACE_ADDR_STRIDE); + } + writel(HISI_PTT_TRACE_BUF_SIZE, hisi_ptt->iobase + HISI_PTT_TRACE_ADDR_SIZE); + + return 0; +} + +static int hisi_ptt_init_ctrls(struct hisi_ptt *hisi_ptt) +{ + struct pci_dev *pdev = hisi_ptt->pdev; + struct pci_bus *bus; + int ret; + u32 reg; + + INIT_DELAYED_WORK(&hisi_ptt->work, hisi_ptt_update_filters); + INIT_KFIFO(hisi_ptt->filter_update_kfifo); + spin_lock_init(&hisi_ptt->filter_update_lock); + + INIT_LIST_HEAD(&hisi_ptt->port_filters); + INIT_LIST_HEAD(&hisi_ptt->req_filters); + mutex_init(&hisi_ptt->filter_lock); + + ret = hisi_ptt_config_trace_buf(hisi_ptt); + if (ret) + return ret; + + /* + * The device range register provides the information about the root + * ports which the RCiEP can control and trace. The RCiEP and the root + * ports which it supports are on the same PCIe core, with same domain + * number but maybe different bus number. The device range register + * will tell us which root ports we can support, Bit[31:16] indicates + * the upper BDF numbers of the root port, while Bit[15:0] indicates + * the lower. + */ + reg = readl(hisi_ptt->iobase + HISI_PTT_DEVICE_RANGE); + hisi_ptt->upper_bdf = FIELD_GET(HISI_PTT_DEVICE_RANGE_UPPER, reg); + hisi_ptt->lower_bdf = FIELD_GET(HISI_PTT_DEVICE_RANGE_LOWER, reg); + + bus = pci_find_bus(pci_domain_nr(pdev->bus), PCI_BUS_NUM(hisi_ptt->upper_bdf)); + if (bus) + pci_walk_bus(bus, hisi_ptt_init_filters, hisi_ptt); + + ret = devm_add_action_or_reset(&pdev->dev, hisi_ptt_release_filters, hisi_ptt); + if (ret) + return ret; + + hisi_ptt->trace_ctrl.on_cpu = -1; + return 0; +} + +static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct hisi_ptt *hisi_ptt = to_hisi_ptt(dev_get_drvdata(dev)); + const cpumask_t *cpumask = cpumask_of_node(dev_to_node(&hisi_ptt->pdev->dev)); + + return cpumap_print_to_pagebuf(true, buf, cpumask); +} +static DEVICE_ATTR_RO(cpumask); + +static struct attribute *hisi_ptt_cpumask_attrs[] = { + &dev_attr_cpumask.attr, + NULL +}; + +static const struct attribute_group hisi_ptt_cpumask_attr_group = { + .attrs = hisi_ptt_cpumask_attrs, +}; + +/* + * Bit 19 indicates the filter type, 1 for Root Port filter and 0 for Requester + * filter. Bit[15:0] indicates the filter value, for Root Port filter it's + * a bit mask of desired ports and for Requester filter it's the Requester ID + * of the desired PCIe function. Bit[18:16] is reserved for extension. + * + * See hisi_ptt.rst documentation for detailed information. + */ +PMU_FORMAT_ATTR(filter, "config:0-19"); +PMU_FORMAT_ATTR(direction, "config:20-23"); +PMU_FORMAT_ATTR(type, "config:24-31"); +PMU_FORMAT_ATTR(format, "config:32-35"); + +static struct attribute *hisi_ptt_pmu_format_attrs[] = { + &format_attr_filter.attr, + &format_attr_direction.attr, + &format_attr_type.attr, + &format_attr_format.attr, + NULL +}; + +static struct attribute_group hisi_ptt_pmu_format_group = { + .name = "format", + .attrs = hisi_ptt_pmu_format_attrs, +}; + +static ssize_t hisi_ptt_filter_multiselect_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct dev_ext_attribute *ext_attr; + + ext_attr = container_of(attr, struct dev_ext_attribute, attr); + return sysfs_emit(buf, "%s\n", (char *)ext_attr->var); +} + +static struct dev_ext_attribute root_port_filters_multiselect = { + .attr = { + .attr = { .name = "multiselect", .mode = 0400 }, + .show = hisi_ptt_filter_multiselect_show, + }, + .var = "1", +}; + +static struct attribute *hisi_ptt_pmu_root_ports_attrs[] = { + &root_port_filters_multiselect.attr.attr, + NULL +}; + +static struct attribute_group hisi_ptt_pmu_root_ports_group = { + .name = HISI_PTT_RP_FILTERS_GRP_NAME, + .attrs = hisi_ptt_pmu_root_ports_attrs, +}; + +static struct dev_ext_attribute requester_filters_multiselect = { + .attr = { + .attr = { .name = "multiselect", .mode = 0400 }, + .show = hisi_ptt_filter_multiselect_show, + }, + .var = "0", +}; + +static struct attribute *hisi_ptt_pmu_requesters_attrs[] = { + &requester_filters_multiselect.attr.attr, + NULL +}; + +static struct attribute_group hisi_ptt_pmu_requesters_group = { + .name = HISI_PTT_REQ_FILTERS_GRP_NAME, + .attrs = hisi_ptt_pmu_requesters_attrs, +}; + +static const struct attribute_group *hisi_ptt_pmu_groups[] = { + &hisi_ptt_cpumask_attr_group, + &hisi_ptt_pmu_format_group, + &hisi_ptt_tune_group, + &hisi_ptt_pmu_root_ports_group, + &hisi_ptt_pmu_requesters_group, + NULL +}; + +static int hisi_ptt_trace_valid_direction(u32 val) +{ + /* + * The direction values have different effects according to the data + * format (specified in the parentheses). TLP set A/B means different + * set of TLP types. See hisi_ptt.rst documentation for more details. + */ + static const u32 hisi_ptt_trace_available_direction[] = { + 0, /* inbound(4DW) or reserved(8DW) */ + 1, /* outbound(4DW) */ + 2, /* {in, out}bound(4DW) or inbound(8DW), TLP set A */ + 3, /* {in, out}bound(4DW) or inbound(8DW), TLP set B */ + }; + int i; + + for (i = 0; i < ARRAY_SIZE(hisi_ptt_trace_available_direction); i++) { + if (val == hisi_ptt_trace_available_direction[i]) + return 0; + } + + return -EINVAL; +} + +static int hisi_ptt_trace_valid_type(u32 val) +{ + /* Different types can be set simultaneously */ + static const u32 hisi_ptt_trace_available_type[] = { + 1, /* posted_request */ + 2, /* non-posted_request */ + 4, /* completion */ + }; + int i; + + if (!val) + return -EINVAL; + + /* + * Walk the available list and clear the valid bits of + * the config. If there is any resident bit after the + * walk then the config is invalid. + */ + for (i = 0; i < ARRAY_SIZE(hisi_ptt_trace_available_type); i++) + val &= ~hisi_ptt_trace_available_type[i]; + + if (val) + return -EINVAL; + + return 0; +} + +static int hisi_ptt_trace_valid_format(u32 val) +{ + static const u32 hisi_ptt_trace_availble_format[] = { + 0, /* 4DW */ + 1, /* 8DW */ + }; + int i; + + for (i = 0; i < ARRAY_SIZE(hisi_ptt_trace_availble_format); i++) { + if (val == hisi_ptt_trace_availble_format[i]) + return 0; + } + + return -EINVAL; +} + +static int hisi_ptt_trace_valid_filter(struct hisi_ptt *hisi_ptt, u64 config) +{ + unsigned long val, port_mask = hisi_ptt->port_mask; + struct hisi_ptt_filter_desc *filter; + int ret = 0; + + hisi_ptt->trace_ctrl.is_port = FIELD_GET(HISI_PTT_PMU_FILTER_IS_PORT, config); + val = FIELD_GET(HISI_PTT_PMU_FILTER_VAL_MASK, config); + + /* + * Port filters are defined as bit mask. For port filters, check + * the bits in the @val are within the range of hisi_ptt->port_mask + * and whether it's empty or not, otherwise user has specified + * some unsupported root ports. + * + * For Requester ID filters, walk the available filter list to see + * whether we have one matched. + */ + mutex_lock(&hisi_ptt->filter_lock); + if (!hisi_ptt->trace_ctrl.is_port) { + list_for_each_entry(filter, &hisi_ptt->req_filters, list) { + if (val == hisi_ptt_get_filter_val(filter->devid, filter->is_port)) + goto out; + } + } else if (bitmap_subset(&val, &port_mask, BITS_PER_LONG)) { + goto out; + } + + ret = -EINVAL; +out: + mutex_unlock(&hisi_ptt->filter_lock); + return ret; +} + +static void hisi_ptt_pmu_init_configs(struct hisi_ptt *hisi_ptt, struct perf_event *event) +{ + struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl; + u32 val; + + val = FIELD_GET(HISI_PTT_PMU_FILTER_VAL_MASK, event->attr.config); + hisi_ptt->trace_ctrl.filter = val; + + val = FIELD_GET(HISI_PTT_PMU_DIRECTION_MASK, event->attr.config); + ctrl->direction = val; + + val = FIELD_GET(HISI_PTT_PMU_TYPE_MASK, event->attr.config); + ctrl->type = val; + + val = FIELD_GET(HISI_PTT_PMU_FORMAT_MASK, event->attr.config); + ctrl->format = val; +} + +static int hisi_ptt_pmu_event_init(struct perf_event *event) +{ + struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu); + int ret; + u32 val; + + if (event->cpu < 0) { + dev_dbg(event->pmu->dev, "Per-task mode not supported\n"); + return -EOPNOTSUPP; + } + + if (event->attr.type != hisi_ptt->hisi_ptt_pmu.type) + return -ENOENT; + + ret = hisi_ptt_trace_valid_filter(hisi_ptt, event->attr.config); + if (ret < 0) + return ret; + + val = FIELD_GET(HISI_PTT_PMU_DIRECTION_MASK, event->attr.config); + ret = hisi_ptt_trace_valid_direction(val); + if (ret < 0) + return ret; + + val = FIELD_GET(HISI_PTT_PMU_TYPE_MASK, event->attr.config); + ret = hisi_ptt_trace_valid_type(val); + if (ret < 0) + return ret; + + val = FIELD_GET(HISI_PTT_PMU_FORMAT_MASK, event->attr.config); + return hisi_ptt_trace_valid_format(val); +} + +static void *hisi_ptt_pmu_setup_aux(struct perf_event *event, void **pages, + int nr_pages, bool overwrite) +{ + struct hisi_ptt_pmu_buf *buf; + struct page **pagelist; + int i; + + if (overwrite) { + dev_warn(event->pmu->dev, "Overwrite mode is not supported\n"); + return NULL; + } + + /* If the pages size less than buffers, we cannot start trace */ + if (nr_pages < HISI_PTT_TRACE_TOTAL_BUF_SIZE / PAGE_SIZE) + return NULL; + + buf = kzalloc(sizeof(*buf), GFP_KERNEL); + if (!buf) + return NULL; + + pagelist = kcalloc(nr_pages, sizeof(*pagelist), GFP_KERNEL); + if (!pagelist) + goto err; + + for (i = 0; i < nr_pages; i++) + pagelist[i] = virt_to_page(pages[i]); + + buf->base = vmap(pagelist, nr_pages, VM_MAP, PAGE_KERNEL); + if (!buf->base) { + kfree(pagelist); + goto err; + } + + buf->nr_pages = nr_pages; + buf->length = nr_pages * PAGE_SIZE; + buf->pos = 0; + + kfree(pagelist); + return buf; +err: + kfree(buf); + return NULL; +} + +static void hisi_ptt_pmu_free_aux(void *aux) +{ + struct hisi_ptt_pmu_buf *buf = aux; + + vunmap(buf->base); + kfree(buf); +} + +static void hisi_ptt_pmu_start(struct perf_event *event, int flags) +{ + struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu); + struct perf_output_handle *handle = &hisi_ptt->trace_ctrl.handle; + struct hw_perf_event *hwc = &event->hw; + struct device *dev = event->pmu->dev; + struct hisi_ptt_pmu_buf *buf; + int cpu = event->cpu; + int ret; + + hwc->state = 0; + + /* Serialize the perf process if user specified several CPUs */ + spin_lock(&hisi_ptt->pmu_lock); + if (hisi_ptt->trace_ctrl.started) { + dev_dbg(dev, "trace has already started\n"); + goto stop; + } + + /* + * Handle the interrupt on the same cpu which starts the trace to avoid + * context mismatch. Otherwise we'll trigger the WARN from the perf + * core in event_function_local(). If CPU passed is offline we'll fail + * here, just log it since we can do nothing here. + */ + ret = irq_set_affinity(hisi_ptt->trace_irq, cpumask_of(cpu)); + if (ret) + dev_warn(dev, "failed to set the affinity of trace interrupt\n"); + + hisi_ptt->trace_ctrl.on_cpu = cpu; + + buf = perf_aux_output_begin(handle, event); + if (!buf) { + dev_dbg(dev, "aux output begin failed\n"); + goto stop; + } + + buf->pos = handle->head % buf->length; + + hisi_ptt_pmu_init_configs(hisi_ptt, event); + + ret = hisi_ptt_trace_start(hisi_ptt); + if (ret) { + dev_dbg(dev, "trace start failed, ret = %d\n", ret); + perf_aux_output_end(handle, 0); + goto stop; + } + + spin_unlock(&hisi_ptt->pmu_lock); + return; +stop: + event->hw.state |= PERF_HES_STOPPED; + spin_unlock(&hisi_ptt->pmu_lock); +} + +static void hisi_ptt_pmu_stop(struct perf_event *event, int flags) +{ + struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu); + struct hw_perf_event *hwc = &event->hw; + + if (hwc->state & PERF_HES_STOPPED) + return; + + spin_lock(&hisi_ptt->pmu_lock); + if (hisi_ptt->trace_ctrl.started) { + hisi_ptt_trace_end(hisi_ptt); + + if (!hisi_ptt_wait_trace_hw_idle(hisi_ptt)) + dev_warn(event->pmu->dev, "Device is still busy\n"); + + hisi_ptt_update_aux(hisi_ptt, hisi_ptt->trace_ctrl.buf_index, true); + } + spin_unlock(&hisi_ptt->pmu_lock); + + hwc->state |= PERF_HES_STOPPED; + perf_event_update_userpage(event); + hwc->state |= PERF_HES_UPTODATE; +} + +static int hisi_ptt_pmu_add(struct perf_event *event, int flags) +{ + struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu); + struct hw_perf_event *hwc = &event->hw; + int cpu = event->cpu; + + /* Only allow the cpus on the device's node to add the event */ + if (!cpumask_test_cpu(cpu, cpumask_of_node(dev_to_node(&hisi_ptt->pdev->dev)))) + return 0; + + hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; + + if (flags & PERF_EF_START) { + hisi_ptt_pmu_start(event, PERF_EF_RELOAD); + if (hwc->state & PERF_HES_STOPPED) + return -EINVAL; + } + + return 0; +} + +static void hisi_ptt_pmu_del(struct perf_event *event, int flags) +{ + hisi_ptt_pmu_stop(event, PERF_EF_UPDATE); +} + +static void hisi_ptt_remove_cpuhp_instance(void *hotplug_node) +{ + cpuhp_state_remove_instance_nocalls(hisi_ptt_pmu_online, hotplug_node); +} + +static void hisi_ptt_unregister_pmu(void *pmu) +{ + perf_pmu_unregister(pmu); +} + +static int hisi_ptt_register_pmu(struct hisi_ptt *hisi_ptt) +{ + u16 core_id, sicl_id; + char *pmu_name; + u32 reg; + int ret; + + ret = cpuhp_state_add_instance_nocalls(hisi_ptt_pmu_online, + &hisi_ptt->hotplug_node); + if (ret) + return ret; + + ret = devm_add_action_or_reset(&hisi_ptt->pdev->dev, + hisi_ptt_remove_cpuhp_instance, + &hisi_ptt->hotplug_node); + if (ret) + return ret; + + mutex_init(&hisi_ptt->tune_lock); + spin_lock_init(&hisi_ptt->pmu_lock); + + hisi_ptt->hisi_ptt_pmu = (struct pmu) { + .module = THIS_MODULE, + .capabilities = PERF_PMU_CAP_EXCLUSIVE | PERF_PMU_CAP_NO_EXCLUDE, + .task_ctx_nr = perf_sw_context, + .attr_groups = hisi_ptt_pmu_groups, + .event_init = hisi_ptt_pmu_event_init, + .setup_aux = hisi_ptt_pmu_setup_aux, + .free_aux = hisi_ptt_pmu_free_aux, + .start = hisi_ptt_pmu_start, + .stop = hisi_ptt_pmu_stop, + .add = hisi_ptt_pmu_add, + .del = hisi_ptt_pmu_del, + }; + + reg = readl(hisi_ptt->iobase + HISI_PTT_LOCATION); + core_id = FIELD_GET(HISI_PTT_CORE_ID, reg); + sicl_id = FIELD_GET(HISI_PTT_SICL_ID, reg); + + pmu_name = devm_kasprintf(&hisi_ptt->pdev->dev, GFP_KERNEL, "hisi_ptt%u_%u", + sicl_id, core_id); + if (!pmu_name) + return -ENOMEM; + + ret = perf_pmu_register(&hisi_ptt->hisi_ptt_pmu, pmu_name, -1); + if (ret) + return ret; + + return devm_add_action_or_reset(&hisi_ptt->pdev->dev, + hisi_ptt_unregister_pmu, + &hisi_ptt->hisi_ptt_pmu); +} + +static void hisi_ptt_unregister_filter_update_notifier(void *data) +{ + struct hisi_ptt *hisi_ptt = data; + + bus_unregister_notifier(&pci_bus_type, &hisi_ptt->hisi_ptt_nb); + + /* Cancel any work that has been queued */ + cancel_delayed_work_sync(&hisi_ptt->work); +} + +/* Register the bus notifier for dynamically updating the filter list */ +static int hisi_ptt_register_filter_update_notifier(struct hisi_ptt *hisi_ptt) +{ + int ret; + + hisi_ptt->hisi_ptt_nb.notifier_call = hisi_ptt_notifier_call; + ret = bus_register_notifier(&pci_bus_type, &hisi_ptt->hisi_ptt_nb); + if (ret) + return ret; + + return devm_add_action_or_reset(&hisi_ptt->pdev->dev, + hisi_ptt_unregister_filter_update_notifier, + hisi_ptt); +} + +/* + * The DMA of PTT trace can only use direct mappings due to some + * hardware restriction. Check whether there is no IOMMU or the + * policy of the IOMMU domain is passthrough, otherwise the trace + * cannot work. + * + * The PTT device is supposed to behind an ARM SMMUv3, which + * should have passthrough the device by a quirk. + */ +static int hisi_ptt_check_iommu_mapping(struct pci_dev *pdev) +{ + struct iommu_domain *iommu_domain; + + iommu_domain = iommu_get_domain_for_dev(&pdev->dev); + if (!iommu_domain || iommu_domain->type == IOMMU_DOMAIN_IDENTITY) + return 0; + + return -EOPNOTSUPP; +} + +static int hisi_ptt_probe(struct pci_dev *pdev, + const struct pci_device_id *id) +{ + struct hisi_ptt *hisi_ptt; + int ret; + + ret = hisi_ptt_check_iommu_mapping(pdev); + if (ret) { + pci_err(pdev, "requires direct DMA mappings\n"); + return ret; + } + + hisi_ptt = devm_kzalloc(&pdev->dev, sizeof(*hisi_ptt), GFP_KERNEL); + if (!hisi_ptt) + return -ENOMEM; + + hisi_ptt->pdev = pdev; + pci_set_drvdata(pdev, hisi_ptt); + + ret = pcim_enable_device(pdev); + if (ret) { + pci_err(pdev, "failed to enable device, ret = %d\n", ret); + return ret; + } + + ret = pcim_iomap_regions(pdev, BIT(2), DRV_NAME); + if (ret) { + pci_err(pdev, "failed to remap io memory, ret = %d\n", ret); + return ret; + } + + hisi_ptt->iobase = pcim_iomap_table(pdev)[2]; + + ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64)); + if (ret) { + pci_err(pdev, "failed to set 64 bit dma mask, ret = %d\n", ret); + return ret; + } + + pci_set_master(pdev); + + ret = hisi_ptt_register_irq(hisi_ptt); + if (ret) + return ret; + + ret = hisi_ptt_init_ctrls(hisi_ptt); + if (ret) { + pci_err(pdev, "failed to init controls, ret = %d\n", ret); + return ret; + } + + ret = hisi_ptt_register_filter_update_notifier(hisi_ptt); + if (ret) + pci_warn(pdev, "failed to register filter update notifier, ret = %d", ret); + + ret = hisi_ptt_register_pmu(hisi_ptt); + if (ret) { + pci_err(pdev, "failed to register PMU device, ret = %d", ret); + return ret; + } + + ret = hisi_ptt_init_filter_attributes(hisi_ptt); + if (ret) { + pci_err(pdev, "failed to init sysfs filter attributes, ret = %d", ret); + return ret; + } + + return 0; +} + +static const struct pci_device_id hisi_ptt_id_tbl[] = { + { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa12e) }, + { } +}; +MODULE_DEVICE_TABLE(pci, hisi_ptt_id_tbl); + +static struct pci_driver hisi_ptt_driver = { + .name = DRV_NAME, + .id_table = hisi_ptt_id_tbl, + .probe = hisi_ptt_probe, +}; + +static int hisi_ptt_cpu_teardown(unsigned int cpu, struct hlist_node *node) +{ + struct hisi_ptt *hisi_ptt; + struct device *dev; + int target, src; + + hisi_ptt = hlist_entry_safe(node, struct hisi_ptt, hotplug_node); + src = hisi_ptt->trace_ctrl.on_cpu; + dev = hisi_ptt->hisi_ptt_pmu.dev; + + if (!hisi_ptt->trace_ctrl.started || src != cpu) + return 0; + + target = cpumask_any_but(cpumask_of_node(dev_to_node(&hisi_ptt->pdev->dev)), cpu); + if (target >= nr_cpu_ids) { + dev_err(dev, "no available cpu for perf context migration\n"); + return 0; + } + + perf_pmu_migrate_context(&hisi_ptt->hisi_ptt_pmu, src, target); + + /* + * Also make sure the interrupt bind to the migrated CPU as well. Warn + * the user on failure here. + */ + if (irq_set_affinity(hisi_ptt->trace_irq, cpumask_of(target))) + dev_warn(dev, "failed to set the affinity of trace interrupt\n"); + + hisi_ptt->trace_ctrl.on_cpu = target; + return 0; +} + +static int __init hisi_ptt_init(void) +{ + int ret; + + ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DRV_NAME, NULL, + hisi_ptt_cpu_teardown); + if (ret < 0) + return ret; + hisi_ptt_pmu_online = ret; + + ret = pci_register_driver(&hisi_ptt_driver); + if (ret) + cpuhp_remove_multi_state(hisi_ptt_pmu_online); + + return ret; +} +module_init(hisi_ptt_init); + +static void __exit hisi_ptt_exit(void) +{ + pci_unregister_driver(&hisi_ptt_driver); + cpuhp_remove_multi_state(hisi_ptt_pmu_online); +} +module_exit(hisi_ptt_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Yicong Yang "); +MODULE_DESCRIPTION("Driver for HiSilicon PCIe tune and trace device"); diff --git a/drivers/hwtracing/ptt/hisi_ptt.h b/drivers/hwtracing/ptt/hisi_ptt.h new file mode 100644 index 0000000000000000000000000000000000000000..c5d93de2953b428bb273eaddce2faa0a6fc2b160 --- /dev/null +++ b/drivers/hwtracing/ptt/hisi_ptt.h @@ -0,0 +1,252 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Driver for HiSilicon PCIe tune and trace device + * + * Copyright (c) 2022 HiSilicon Technologies Co., Ltd. + * Author: Yicong Yang + */ + +#ifndef _HISI_PTT_H +#define _HISI_PTT_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DRV_NAME "hisi_ptt" + +/* + * The definition of the device registers and register fields. + */ +#define HISI_PTT_TUNING_CTRL 0x0000 +#define HISI_PTT_TUNING_CTRL_CODE GENMASK(15, 0) +#define HISI_PTT_TUNING_CTRL_SUB GENMASK(23, 16) +#define HISI_PTT_TUNING_DATA 0x0004 +#define HISI_PTT_TUNING_DATA_VAL_MASK GENMASK(15, 0) +#define HISI_PTT_TRACE_ADDR_SIZE 0x0800 +#define HISI_PTT_TRACE_ADDR_BASE_LO_0 0x0810 +#define HISI_PTT_TRACE_ADDR_BASE_HI_0 0x0814 +#define HISI_PTT_TRACE_ADDR_STRIDE 0x8 +#define HISI_PTT_TRACE_CTRL 0x0850 +#define HISI_PTT_TRACE_CTRL_EN BIT(0) +#define HISI_PTT_TRACE_CTRL_RST BIT(1) +#define HISI_PTT_TRACE_CTRL_RXTX_SEL GENMASK(3, 2) +#define HISI_PTT_TRACE_CTRL_TYPE_SEL GENMASK(7, 4) +#define HISI_PTT_TRACE_CTRL_DATA_FORMAT BIT(14) +#define HISI_PTT_TRACE_CTRL_FILTER_MODE BIT(15) +#define HISI_PTT_TRACE_CTRL_TARGET_SEL GENMASK(31, 16) +#define HISI_PTT_TRACE_INT_STAT 0x0890 +#define HISI_PTT_TRACE_INT_STAT_MASK GENMASK(3, 0) +#define HISI_PTT_TRACE_INT_MASK 0x0894 +#define HISI_PTT_TUNING_INT_STAT 0x0898 +#define HISI_PTT_TUNING_INT_STAT_MASK BIT(0) +#define HISI_PTT_TRACE_WR_STS 0x08a0 +#define HISI_PTT_TRACE_WR_STS_WRITE GENMASK(27, 0) +#define HISI_PTT_TRACE_WR_STS_BUFFER GENMASK(29, 28) +#define HISI_PTT_TRACE_STS 0x08b0 +#define HISI_PTT_TRACE_IDLE BIT(0) +#define HISI_PTT_DEVICE_RANGE 0x0fe0 +#define HISI_PTT_DEVICE_RANGE_UPPER GENMASK(31, 16) +#define HISI_PTT_DEVICE_RANGE_LOWER GENMASK(15, 0) +#define HISI_PTT_LOCATION 0x0fe8 +#define HISI_PTT_CORE_ID GENMASK(15, 0) +#define HISI_PTT_SICL_ID GENMASK(31, 16) + +/* Parameters of PTT trace DMA part. */ +#define HISI_PTT_TRACE_DMA_IRQ 0 +#define HISI_PTT_TRACE_BUF_CNT 4 +#define HISI_PTT_TRACE_BUF_SIZE SZ_4M +#define HISI_PTT_TRACE_TOTAL_BUF_SIZE (HISI_PTT_TRACE_BUF_SIZE * \ + HISI_PTT_TRACE_BUF_CNT) +/* Wait time for hardware DMA to reset */ +#define HISI_PTT_RESET_TIMEOUT_US 10UL +#define HISI_PTT_RESET_POLL_INTERVAL_US 1UL +/* Poll timeout and interval for waiting hardware work to finish */ +#define HISI_PTT_WAIT_TUNE_TIMEOUT_US 1000000UL +#define HISI_PTT_WAIT_TRACE_TIMEOUT_US 100UL +#define HISI_PTT_WAIT_POLL_INTERVAL_US 10UL + +/* FIFO size for dynamically updating the PTT trace filter list. */ +#define HISI_PTT_FILTER_UPDATE_FIFO_SIZE 16 +/* Delay time for filter updating work */ +#define HISI_PTT_WORK_DELAY_MS 100UL + +#define HISI_PCIE_CORE_PORT_ID(devfn) ((PCI_SLOT(devfn) & 0x7) << 1) + +/* Definition of the PMU configs */ +#define HISI_PTT_PMU_FILTER_IS_PORT BIT(19) +#define HISI_PTT_PMU_FILTER_VAL_MASK GENMASK(15, 0) +#define HISI_PTT_PMU_DIRECTION_MASK GENMASK(23, 20) +#define HISI_PTT_PMU_TYPE_MASK GENMASK(31, 24) +#define HISI_PTT_PMU_FORMAT_MASK GENMASK(35, 32) + +/** + * struct hisi_ptt_tune_desc - Describe tune event for PTT tune + * @hisi_ptt: PTT device this tune event belongs to + * @name: name of this event + * @event_code: code of the event + */ +struct hisi_ptt_tune_desc { + struct hisi_ptt *hisi_ptt; + const char *name; + u32 event_code; +}; + +/** + * struct hisi_ptt_dma_buffer - Describe a single trace buffer of PTT trace. + * The detail of the data format is described + * in the documentation of PTT device. + * @dma: DMA address of this buffer visible to the device + * @addr: virtual address of this buffer visible to the cpu + */ +struct hisi_ptt_dma_buffer { + dma_addr_t dma; + void *addr; +}; + +/** + * struct hisi_ptt_trace_ctrl - Control and status of PTT trace + * @trace_buf: array of the trace buffers for holding the trace data. + * the length will be HISI_PTT_TRACE_BUF_CNT. + * @handle: perf output handle of current trace session + * @buf_index: the index of current using trace buffer + * @on_cpu: current tracing cpu + * @started: current trace status, true for started + * @is_port: whether we're tracing root port or not + * @direction: direction of the TLP headers to trace + * @filter: filter value for tracing the TLP headers + * @format: format of the TLP headers to trace + * @type: type of the TLP headers to trace + */ +struct hisi_ptt_trace_ctrl { + struct hisi_ptt_dma_buffer *trace_buf; + struct perf_output_handle handle; + u32 buf_index; + int on_cpu; + bool started; + bool is_port; + u32 direction:2; + u32 filter:16; + u32 format:1; + u32 type:4; +}; + +/* + * sysfs attribute group name for root port filters and requester filters: + * /sys/devices/hisi_ptt_/root_port_filters + * and + * /sys/devices/hisi_ptt_/requester_filters + */ +#define HISI_PTT_RP_FILTERS_GRP_NAME "root_port_filters" +#define HISI_PTT_REQ_FILTERS_GRP_NAME "requester_filters" + +/** + * struct hisi_ptt_filter_desc - Descriptor of the PTT trace filter + * @attr: sysfs attribute of this filter + * @list: entry of this descriptor in the filter list + * @is_port: the PCI device of the filter is a Root Port or not + * @name: name of this filter, same as the name of the related PCI device + * @devid: the PCI device's devid of the filter + */ +struct hisi_ptt_filter_desc { + struct device_attribute attr; + struct list_head list; + bool is_port; + char *name; + u16 devid; +}; + +/** + * struct hisi_ptt_filter_update_info - Information for PTT filter updating + * @is_port: the PCI device to update is a Root Port or not + * @is_add: adding to the filter or not + * @devid: the PCI device's devid of the filter + */ +struct hisi_ptt_filter_update_info { + bool is_port; + bool is_add; + u16 devid; +}; + +/** + * struct hisi_ptt_pmu_buf - Descriptor of the AUX buffer of PTT trace + * @length: size of the AUX buffer + * @nr_pages: number of pages of the AUX buffer + * @base: start address of AUX buffer + * @pos: position in the AUX buffer to commit traced data + */ +struct hisi_ptt_pmu_buf { + size_t length; + int nr_pages; + void *base; + long pos; +}; + +/** + * struct hisi_ptt - Per PTT device data + * @trace_ctrl: the control information of PTT trace + * @hisi_ptt_nb: dynamic filter update notifier + * @hotplug_node: node for register cpu hotplug event + * @hisi_ptt_pmu: the pum device of trace + * @iobase: base IO address of the device + * @pdev: pci_dev of this PTT device + * @tune_lock: lock to serialize the tune process + * @pmu_lock: lock to serialize the perf process + * @req_filters: the filter list of requester ID + * @filter_lock: lock to protect the filters + * @sysfs_inited: whether the filters' sysfs entries has been initialized + * @port_mask: port mask of the managed root ports + * @work: delayed work for filter updating + * @filter_update_lock: spinlock to protect the filter update fifo + * @filter_update_fifo: fifo of the filters waiting to update the filter list + */ +struct hisi_ptt { + struct hisi_ptt_trace_ctrl trace_ctrl; + struct notifier_block hisi_ptt_nb; + struct hlist_node hotplug_node; + struct pmu hisi_ptt_pmu; + void __iomem *iobase; + struct pci_dev *pdev; + struct mutex tune_lock; + spinlock_t pmu_lock; + int trace_irq; + u32 upper_bdf; + u32 lower_bdf; + + /* + * The trace TLP headers can either be filtered by certain + * root port, or by the requester ID. Organize the filters + * by @port_filters and @req_filters here. The mask of all + * the valid ports is also cached for doing sanity check + * of user input. + */ + struct list_head port_filters; + struct list_head req_filters; + struct mutex filter_lock; + bool sysfs_inited; + u16 port_mask; + + /* + * We use a delayed work here to avoid indefinitely waiting for + * the hisi_ptt->mutex which protecting the filter list. The + * work will be delayed only if the mutex can not be held, + * otherwise no delay will be applied. + */ + struct delayed_work work; + spinlock_t filter_update_lock; + DECLARE_KFIFO(filter_update_kfifo, struct hisi_ptt_filter_update_info, + HISI_PTT_FILTER_UPDATE_FIFO_SIZE); +}; + +#define to_hisi_ptt(pmu) container_of(pmu, struct hisi_ptt, hisi_ptt_pmu) + +#endif /* _HISI_PTT_H */ diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig index 854f1b2658b82cb069094cda6f9c011a59278cdd..97e517b7af068e1e0301c2c04db04c43402c13a1 100644 --- a/drivers/i2c/busses/Kconfig +++ b/drivers/i2c/busses/Kconfig @@ -643,6 +643,16 @@ config I2C_HIGHLANDER This driver can also be built as a module. If so, the module will be called i2c-highlander. +config I2C_HISI + tristate "HiSilicon I2C controller" + depends on ARM64 || COMPILE_TEST + help + Say Y here if you want to have Hisilicon I2C controller support + available on the Kunpeng Server. + + This driver can also be built as a module. If so, the module + will be called i2c-hisi. + config I2C_IBM_IIC tristate "IBM PPC 4xx on-chip I2C interface" depends on 4xx diff --git a/drivers/i2c/busses/Makefile b/drivers/i2c/busses/Makefile index 3ab8aebc39c901d746d31ddd8599e8e21cbc5f28..62ef7047805118f3d53f9ebca5d9021d77b71383 100644 --- a/drivers/i2c/busses/Makefile +++ b/drivers/i2c/busses/Makefile @@ -65,6 +65,7 @@ obj-$(CONFIG_I2C_EMEV2) += i2c-emev2.o obj-$(CONFIG_I2C_EXYNOS5) += i2c-exynos5.o obj-$(CONFIG_I2C_GPIO) += i2c-gpio.o obj-$(CONFIG_I2C_HIGHLANDER) += i2c-highlander.o +obj-$(CONFIG_I2C_HISI) += i2c-hisi.o obj-$(CONFIG_I2C_HIX5HD2) += i2c-hix5hd2.o obj-$(CONFIG_I2C_IBM_IIC) += i2c-ibm_iic.o obj-$(CONFIG_I2C_IMG) += i2c-img-scb.o diff --git a/drivers/i2c/busses/i2c-hisi.c b/drivers/i2c/busses/i2c-hisi.c new file mode 100644 index 0000000000000000000000000000000000000000..e13e4b3f59f91f2aa3bcbd770774e537a8e4274f --- /dev/null +++ b/drivers/i2c/busses/i2c-hisi.c @@ -0,0 +1,615 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * HiSilicon I2C Controller Driver for Kunpeng SoC + * + * Copyright (c) 2021 HiSilicon Technologies Co., Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define HISI_I2C_FRAME_CTRL 0x0000 +#define HISI_I2C_FRAME_CTRL_SPEED_MODE GENMASK(1, 0) +#define HISI_I2C_FRAME_CTRL_ADDR_TEN BIT(2) +#define HISI_I2C_SLV_ADDR 0x0004 +#define HISI_I2C_SLV_ADDR_VAL GENMASK(9, 0) +#define HISI_I2C_SLV_ADDR_GC_S_MODE BIT(10) +#define HISI_I2C_SLV_ADDR_GC_S_EN BIT(11) +#define HISI_I2C_CMD_TXDATA 0x0008 +#define HISI_I2C_CMD_TXDATA_DATA GENMASK(7, 0) +#define HISI_I2C_CMD_TXDATA_RW BIT(8) +#define HISI_I2C_CMD_TXDATA_P_EN BIT(9) +#define HISI_I2C_CMD_TXDATA_SR_EN BIT(10) +#define HISI_I2C_RXDATA 0x000c +#define HISI_I2C_RXDATA_DATA GENMASK(7, 0) +#define HISI_I2C_SS_SCL_HCNT 0x0010 +#define HISI_I2C_SS_SCL_LCNT 0x0014 +#define HISI_I2C_FS_SCL_HCNT 0x0018 +#define HISI_I2C_FS_SCL_LCNT 0x001c +#define HISI_I2C_HS_SCL_HCNT 0x0020 +#define HISI_I2C_HS_SCL_LCNT 0x0024 +#define HISI_I2C_FIFO_CTRL 0x0028 +#define HISI_I2C_FIFO_RX_CLR BIT(0) +#define HISI_I2C_FIFO_TX_CLR BIT(1) +#define HISI_I2C_FIFO_RX_AF_THRESH GENMASK(7, 2) +#define HISI_I2C_FIFO_TX_AE_THRESH GENMASK(13, 8) +#define HISI_I2C_FIFO_STATE 0x002c +#define HISI_I2C_FIFO_STATE_RX_RERR BIT(0) +#define HISI_I2C_FIFO_STATE_RX_WERR BIT(1) +#define HISI_I2C_FIFO_STATE_RX_EMPTY BIT(3) +#define HISI_I2C_FIFO_STATE_TX_RERR BIT(6) +#define HISI_I2C_FIFO_STATE_TX_WERR BIT(7) +#define HISI_I2C_FIFO_STATE_TX_FULL BIT(11) +#define HISI_I2C_SDA_HOLD 0x0030 +#define HISI_I2C_SDA_HOLD_TX GENMASK(15, 0) +#define HISI_I2C_SDA_HOLD_RX GENMASK(23, 16) +#define HISI_I2C_FS_SPK_LEN 0x0038 +#define HISI_I2C_FS_SPK_LEN_CNT GENMASK(7, 0) +#define HISI_I2C_HS_SPK_LEN 0x003c +#define HISI_I2C_HS_SPK_LEN_CNT GENMASK(7, 0) +#define HISI_I2C_TX_INT_CLR 0x0040 +#define HISI_I2C_TX_AEMPTY_INT BIT(0) +#define HISI_I2C_INT_MSTAT 0x0044 +#define HISI_I2C_INT_CLR 0x0048 +#define HISI_I2C_INT_MASK 0x004C +#define HISI_I2C_TRANS_STATE 0x0050 +#define HISI_I2C_TRANS_ERR 0x0054 +#define HISI_I2C_VERSION 0x0058 + +#define HISI_I2C_INT_ALL GENMASK(4, 0) +#define HISI_I2C_INT_TRANS_CPLT BIT(0) +#define HISI_I2C_INT_TRANS_ERR BIT(1) +#define HISI_I2C_INT_FIFO_ERR BIT(2) +#define HISI_I2C_INT_RX_FULL BIT(3) +#define HISI_I2C_INT_TX_EMPTY BIT(4) +#define HISI_I2C_INT_ERR \ + (HISI_I2C_INT_TRANS_ERR | HISI_I2C_INT_FIFO_ERR) + +#define HISI_I2C_STD_SPEED_MODE 0 +#define HISI_I2C_FAST_SPEED_MODE 1 +#define HISI_I2C_HIGH_SPEED_MODE 2 + +#define HISI_I2C_TX_FIFO_DEPTH 64 +#define HISI_I2C_RX_FIFO_DEPTH 64 +#define HISI_I2C_TX_F_AE_THRESH 1 +#define HISI_I2C_RX_F_AF_THRESH 60 + +#define HZ_PER_KHZ 1000 + +#define NSEC_TO_CYCLES(ns, clk_rate_khz) \ + DIV_ROUND_UP_ULL((clk_rate_khz) * (ns), NSEC_PER_MSEC) + +struct hisi_i2c_controller { + struct i2c_adapter adapter; + void __iomem *iobase; + struct device *dev; + int irq; + + /* Intermediates for recording the transfer process */ + struct completion *completion; + struct i2c_msg *msgs; + int msg_num; + int msg_tx_idx; + int buf_tx_idx; + int msg_rx_idx; + int buf_rx_idx; + u16 tar_addr; + u32 xfer_err; + + /* I2C bus configuration */ + struct i2c_timings t; + u32 clk_rate_khz; + u32 spk_len; + + /* Bus recovery method */ + struct i2c_bus_recovery_info rinfo; +}; + +static void hisi_i2c_enable_int(struct hisi_i2c_controller *ctlr, u32 mask) +{ + writel_relaxed(mask, ctlr->iobase + HISI_I2C_INT_MASK); +} + +static void hisi_i2c_disable_int(struct hisi_i2c_controller *ctlr, u32 mask) +{ + writel_relaxed((~mask) & HISI_I2C_INT_ALL, ctlr->iobase + HISI_I2C_INT_MASK); +} + +static void hisi_i2c_clear_int(struct hisi_i2c_controller *ctlr, u32 mask) +{ + writel_relaxed(mask, ctlr->iobase + HISI_I2C_INT_CLR); +} + +static void hisi_i2c_clear_tx_int(struct hisi_i2c_controller *ctlr, u32 mask) +{ + writel_relaxed(mask, ctlr->iobase + HISI_I2C_TX_INT_CLR); +} + +static void hisi_i2c_handle_errors(struct hisi_i2c_controller *ctlr) +{ + u32 int_err = ctlr->xfer_err, reg; + + if (int_err & HISI_I2C_INT_FIFO_ERR) { + reg = readl(ctlr->iobase + HISI_I2C_FIFO_STATE); + + if (reg & HISI_I2C_FIFO_STATE_RX_RERR) + dev_err(ctlr->dev, "rx fifo error read\n"); + + if (reg & HISI_I2C_FIFO_STATE_RX_WERR) + dev_err(ctlr->dev, "rx fifo error write\n"); + + if (reg & HISI_I2C_FIFO_STATE_TX_RERR) + dev_err(ctlr->dev, "tx fifo error read\n"); + + if (reg & HISI_I2C_FIFO_STATE_TX_WERR) + dev_err(ctlr->dev, "tx fifo error write\n"); + } +} + +static int hisi_i2c_start_xfer(struct hisi_i2c_controller *ctlr) +{ + struct i2c_msg *msg = ctlr->msgs; + u32 reg; + + reg = readl(ctlr->iobase + HISI_I2C_FRAME_CTRL); + reg &= ~HISI_I2C_FRAME_CTRL_ADDR_TEN; + if (msg->flags & I2C_M_TEN) + reg |= HISI_I2C_FRAME_CTRL_ADDR_TEN; + writel(reg, ctlr->iobase + HISI_I2C_FRAME_CTRL); + + reg = readl(ctlr->iobase + HISI_I2C_SLV_ADDR); + reg &= ~HISI_I2C_SLV_ADDR_VAL; + reg |= FIELD_PREP(HISI_I2C_SLV_ADDR_VAL, msg->addr); + writel(reg, ctlr->iobase + HISI_I2C_SLV_ADDR); + + reg = readl(ctlr->iobase + HISI_I2C_FIFO_CTRL); + reg |= HISI_I2C_FIFO_RX_CLR | HISI_I2C_FIFO_TX_CLR; + writel(reg, ctlr->iobase + HISI_I2C_FIFO_CTRL); + reg &= ~(HISI_I2C_FIFO_RX_CLR | HISI_I2C_FIFO_TX_CLR); + writel(reg, ctlr->iobase + HISI_I2C_FIFO_CTRL); + + hisi_i2c_clear_int(ctlr, HISI_I2C_INT_ALL); + hisi_i2c_clear_tx_int(ctlr, HISI_I2C_TX_AEMPTY_INT); + hisi_i2c_enable_int(ctlr, HISI_I2C_INT_ALL); + + return 0; +} + +static void hisi_i2c_reset_xfer(struct hisi_i2c_controller *ctlr) +{ + ctlr->msg_num = 0; + ctlr->xfer_err = 0; + ctlr->msg_tx_idx = 0; + ctlr->msg_rx_idx = 0; + ctlr->buf_tx_idx = 0; + ctlr->buf_rx_idx = 0; +} + +/* + * Initialize the transfer information and start the I2C bus transfer. + * We only configure the transfer and do some pre/post works here, and + * wait for the transfer done. The major transfer process is performed + * in the IRQ handler. + */ +static int hisi_i2c_master_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, + int num) +{ + struct hisi_i2c_controller *ctlr = i2c_get_adapdata(adap); + DECLARE_COMPLETION_ONSTACK(done); + int ret = num; + + hisi_i2c_reset_xfer(ctlr); + ctlr->completion = &done; + ctlr->msg_num = num; + ctlr->msgs = msgs; + + hisi_i2c_start_xfer(ctlr); + + if (!wait_for_completion_timeout(ctlr->completion, adap->timeout)) { + hisi_i2c_disable_int(ctlr, HISI_I2C_INT_ALL); + synchronize_irq(ctlr->irq); + i2c_recover_bus(&ctlr->adapter); + dev_err(ctlr->dev, "bus transfer timeout\n"); + ret = -EIO; + } + + if (ctlr->xfer_err) { + hisi_i2c_handle_errors(ctlr); + ret = -EIO; + } + + hisi_i2c_reset_xfer(ctlr); + ctlr->completion = NULL; + + return ret; +} + +static u32 hisi_i2c_functionality(struct i2c_adapter *adap) +{ + return I2C_FUNC_I2C | I2C_FUNC_10BIT_ADDR | I2C_FUNC_SMBUS_EMUL; +} + +static const struct i2c_algorithm hisi_i2c_algo = { + .master_xfer = hisi_i2c_master_xfer, + .functionality = hisi_i2c_functionality, +}; + +static int hisi_i2c_read_rx_fifo(struct hisi_i2c_controller *ctlr) +{ + struct i2c_msg *cur_msg; + u32 fifo_state; + + while (ctlr->msg_rx_idx < ctlr->msg_num) { + cur_msg = ctlr->msgs + ctlr->msg_rx_idx; + + if (!(cur_msg->flags & I2C_M_RD)) { + ctlr->msg_rx_idx++; + continue; + } + + fifo_state = readl(ctlr->iobase + HISI_I2C_FIFO_STATE); + while (!(fifo_state & HISI_I2C_FIFO_STATE_RX_EMPTY) && + ctlr->buf_rx_idx < cur_msg->len) { + cur_msg->buf[ctlr->buf_rx_idx++] = readl(ctlr->iobase + HISI_I2C_RXDATA); + fifo_state = readl(ctlr->iobase + HISI_I2C_FIFO_STATE); + } + + if (ctlr->buf_rx_idx == cur_msg->len) { + ctlr->buf_rx_idx = 0; + ctlr->msg_rx_idx++; + } + + if (fifo_state & HISI_I2C_FIFO_STATE_RX_EMPTY) + break; + } + + return 0; +} + +static void hisi_i2c_xfer_msg(struct hisi_i2c_controller *ctlr) +{ + int max_write = HISI_I2C_TX_FIFO_DEPTH - HISI_I2C_TX_F_AE_THRESH; + bool need_restart = false, last_msg; + struct i2c_msg *cur_msg; + u32 cmd, fifo_state; + + while (ctlr->msg_tx_idx < ctlr->msg_num) { + cur_msg = ctlr->msgs + ctlr->msg_tx_idx; + last_msg = (ctlr->msg_tx_idx == ctlr->msg_num - 1); + + /* Signal the SR bit when we start transferring a new message */ + if (ctlr->msg_tx_idx && !ctlr->buf_tx_idx) + need_restart = true; + + fifo_state = readl(ctlr->iobase + HISI_I2C_FIFO_STATE); + while (!(fifo_state & HISI_I2C_FIFO_STATE_TX_FULL) && + ctlr->buf_tx_idx < cur_msg->len && max_write) { + cmd = 0; + + if (need_restart) { + cmd |= HISI_I2C_CMD_TXDATA_SR_EN; + need_restart = false; + } + + /* Signal the STOP bit at the last frame of the last message */ + if (ctlr->buf_tx_idx == cur_msg->len - 1 && last_msg) + cmd |= HISI_I2C_CMD_TXDATA_P_EN; + + if (cur_msg->flags & I2C_M_RD) + cmd |= HISI_I2C_CMD_TXDATA_RW; + else + cmd |= FIELD_PREP(HISI_I2C_CMD_TXDATA_DATA, + cur_msg->buf[ctlr->buf_tx_idx]); + + writel(cmd, ctlr->iobase + HISI_I2C_CMD_TXDATA); + ctlr->buf_tx_idx++; + max_write--; + + fifo_state = readl(ctlr->iobase + HISI_I2C_FIFO_STATE); + } + + /* Update the transfer index after per message transfer is done. */ + if (ctlr->buf_tx_idx == cur_msg->len) { + ctlr->buf_tx_idx = 0; + ctlr->msg_tx_idx++; + } + + if ((fifo_state & HISI_I2C_FIFO_STATE_TX_FULL) || + max_write == 0) + break; + } + + /* + * Disable the TX_EMPTY interrupt after finishing all the messages to + * avoid overwhelming the CPU. + */ + if (ctlr->msg_tx_idx == ctlr->msg_num) + hisi_i2c_disable_int(ctlr, HISI_I2C_INT_TX_EMPTY); + + hisi_i2c_clear_tx_int(ctlr, HISI_I2C_TX_AEMPTY_INT); +} + +static irqreturn_t hisi_i2c_irq(int irq, void *context) +{ + struct hisi_i2c_controller *ctlr = context; + u32 int_stat; + + /* + * Don't handle the interrupt if cltr->completion is NULL. We may + * reach here because the interrupt is spurious or the transfer is + * started by another port (e.g. firmware) rather than us. + */ + if (!ctlr->completion) + return IRQ_NONE; + + int_stat = readl(ctlr->iobase + HISI_I2C_INT_MSTAT); + hisi_i2c_clear_int(ctlr, int_stat); + if (!(int_stat & HISI_I2C_INT_ALL)) + return IRQ_NONE; + + if (int_stat & HISI_I2C_INT_TX_EMPTY) + hisi_i2c_xfer_msg(ctlr); + + if (int_stat & HISI_I2C_INT_ERR) { + ctlr->xfer_err = int_stat; + goto out; + } + + /* Drain the rx fifo before finish the transfer */ + if (int_stat & (HISI_I2C_INT_TRANS_CPLT | HISI_I2C_INT_RX_FULL)) + hisi_i2c_read_rx_fifo(ctlr); + +out: + /* + * Only use TRANS_CPLT to indicate the completion. On error cases we'll + * get two interrupts, INT_ERR first then TRANS_CPLT. + */ + if (int_stat & HISI_I2C_INT_TRANS_CPLT) { + hisi_i2c_disable_int(ctlr, HISI_I2C_INT_ALL); + hisi_i2c_clear_int(ctlr, HISI_I2C_INT_ALL); + hisi_i2c_clear_tx_int(ctlr, HISI_I2C_TX_AEMPTY_INT); + complete(ctlr->completion); + } + + return IRQ_HANDLED; +} + +/* + * Helper function for calculating and configuring the HIGH and LOW + * periods of SCL clock. The caller will pass the ratio of the + * counts (divide / divisor) according to the target speed mode, + * and the target registers. + */ +static void hisi_i2c_set_scl(struct hisi_i2c_controller *ctlr, + u32 divide, u32 divisor, + u32 reg_hcnt, u32 reg_lcnt) +{ + u32 total_cnt, t_scl_hcnt, t_scl_lcnt, scl_fall_cnt, scl_rise_cnt; + u32 scl_hcnt, scl_lcnt; + + /* Total SCL clock cycles per speed period */ + total_cnt = DIV_ROUND_UP_ULL(ctlr->clk_rate_khz * HZ_PER_KHZ, ctlr->t.bus_freq_hz); + /* Total HIGH level SCL clock cycles including edges */ + t_scl_hcnt = DIV_ROUND_UP_ULL(total_cnt * divide, divisor); + /* Total LOW level SCL clock cycles including edges */ + t_scl_lcnt = total_cnt - t_scl_hcnt; + /* Fall edge SCL clock cycles */ + scl_fall_cnt = NSEC_TO_CYCLES(ctlr->t.scl_fall_ns, ctlr->clk_rate_khz); + /* Rise edge SCL clock cycles */ + scl_rise_cnt = NSEC_TO_CYCLES(ctlr->t.scl_rise_ns, ctlr->clk_rate_khz); + + /* Calculated HIGH and LOW periods of SCL clock */ + scl_hcnt = t_scl_hcnt - ctlr->spk_len - 7 - scl_fall_cnt; + scl_lcnt = t_scl_lcnt - 1 - scl_rise_cnt; + + writel(scl_hcnt, ctlr->iobase + reg_hcnt); + writel(scl_lcnt, ctlr->iobase + reg_lcnt); +} + +static void hisi_i2c_configure_bus(struct hisi_i2c_controller *ctlr) +{ + u32 reg, sda_hold_cnt, speed_mode; + + i2c_parse_fw_timings(ctlr->dev, &ctlr->t, true); + ctlr->spk_len = NSEC_TO_CYCLES(ctlr->t.digital_filter_width_ns, ctlr->clk_rate_khz); + + switch (ctlr->t.bus_freq_hz) { + case I2C_MAX_FAST_MODE_FREQ: + speed_mode = HISI_I2C_FAST_SPEED_MODE; + hisi_i2c_set_scl(ctlr, 26, 76, HISI_I2C_FS_SCL_HCNT, HISI_I2C_FS_SCL_LCNT); + break; + case I2C_MAX_HIGH_SPEED_MODE_FREQ: + speed_mode = HISI_I2C_HIGH_SPEED_MODE; + hisi_i2c_set_scl(ctlr, 6, 22, HISI_I2C_HS_SCL_HCNT, HISI_I2C_HS_SCL_LCNT); + break; + case I2C_MAX_STANDARD_MODE_FREQ: + default: + speed_mode = HISI_I2C_STD_SPEED_MODE; + + /* For default condition force the bus speed to standard mode. */ + ctlr->t.bus_freq_hz = I2C_MAX_STANDARD_MODE_FREQ; + hisi_i2c_set_scl(ctlr, 40, 87, HISI_I2C_SS_SCL_HCNT, HISI_I2C_SS_SCL_LCNT); + break; + } + + reg = readl(ctlr->iobase + HISI_I2C_FRAME_CTRL); + reg &= ~HISI_I2C_FRAME_CTRL_SPEED_MODE; + reg |= FIELD_PREP(HISI_I2C_FRAME_CTRL_SPEED_MODE, speed_mode); + writel(reg, ctlr->iobase + HISI_I2C_FRAME_CTRL); + + sda_hold_cnt = NSEC_TO_CYCLES(ctlr->t.sda_hold_ns, ctlr->clk_rate_khz); + + reg = FIELD_PREP(HISI_I2C_SDA_HOLD_TX, sda_hold_cnt); + writel(reg, ctlr->iobase + HISI_I2C_SDA_HOLD); + + writel(ctlr->spk_len, ctlr->iobase + HISI_I2C_FS_SPK_LEN); + + reg = FIELD_PREP(HISI_I2C_FIFO_RX_AF_THRESH, HISI_I2C_RX_F_AF_THRESH); + reg |= FIELD_PREP(HISI_I2C_FIFO_TX_AE_THRESH, HISI_I2C_TX_F_AE_THRESH); + writel(reg, ctlr->iobase + HISI_I2C_FIFO_CTRL); +} + +#ifdef CONFIG_ACPI +#define HISI_I2C_PIN_MUX_METHOD "PMUX" + +/** + * i2c_dw_acpi_pin_mux_change - Change the I2C controller's pin mux through ACPI + * @dev: device owns the SCL/SDA pin + * @to_gpio: true to switch to GPIO, false to switch to SCL/SDA + * + * The function invokes the specific ACPI method "PMUX" for changing the + * pin mux of I2C controller between SCL/SDA and GPIO in order to help on + * the generic GPIO recovery process. + */ +static void i2c_hisi_pin_mux_change(struct device *dev, bool to_gpio) +{ + acpi_handle handle = ACPI_HANDLE(dev); + struct acpi_object_list arg_list; + unsigned long long data; + union acpi_object arg; + + arg.type = ACPI_TYPE_INTEGER; + arg.integer.value = to_gpio; + arg_list.count = 1; + arg_list.pointer = &arg; + + acpi_evaluate_integer(handle, HISI_I2C_PIN_MUX_METHOD, &arg_list, &data); +} + +static void i2c_hisi_prepare_recovery(struct i2c_adapter *adap) +{ + struct hisi_i2c_controller *ctlr = i2c_get_adapdata(adap); + + i2c_hisi_pin_mux_change(ctlr->dev, true); +} + +static void i2c_hisi_unprepare_recovery(struct i2c_adapter *adap) +{ + struct hisi_i2c_controller *ctlr = i2c_get_adapdata(adap); + + i2c_hisi_pin_mux_change(ctlr->dev, false); +} + +static void hisi_i2c_init_recovery_info(struct hisi_i2c_controller *ctlr) +{ + struct i2c_bus_recovery_info *rinfo = &ctlr->rinfo; + struct acpi_device *adev = ACPI_COMPANION(ctlr->dev); + struct gpio_desc *gpio; + + if (acpi_disabled) + return; + + if (!adev || !acpi_has_method(adev->handle, HISI_I2C_PIN_MUX_METHOD)) + return; + + gpio = devm_gpiod_get_optional(ctlr->dev, "scl", GPIOD_OUT_HIGH); + if (IS_ERR_OR_NULL(gpio)) + return; + + rinfo->scl_gpiod = gpio; + + gpio = devm_gpiod_get_optional(ctlr->dev, "sda", GPIOD_IN); + if (IS_ERR(gpio)) + return; + + rinfo->sda_gpiod = gpio; + rinfo->recover_bus = i2c_generic_scl_recovery; + rinfo->prepare_recovery = i2c_hisi_prepare_recovery; + rinfo->unprepare_recovery = i2c_hisi_unprepare_recovery; + + ctlr->adapter.bus_recovery_info = rinfo; +} +#else +static inline void hisi_i2c_init_recovery_info(struct hisi_i2c_controller *ctlr) { } +#endif /* CONFIG_ACPI */ + +static int hisi_i2c_probe(struct platform_device *pdev) +{ + struct hisi_i2c_controller *ctlr; + struct device *dev = &pdev->dev; + struct i2c_adapter *adapter; + u64 clk_rate_hz; + u32 hw_version; + int ret; + + ctlr = devm_kzalloc(dev, sizeof(*ctlr), GFP_KERNEL); + if (!ctlr) + return -ENOMEM; + + ctlr->iobase = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(ctlr->iobase)) + return PTR_ERR(ctlr->iobase); + + ctlr->irq = platform_get_irq(pdev, 0); + if (ctlr->irq < 0) + return ctlr->irq; + + ctlr->dev = dev; + + hisi_i2c_disable_int(ctlr, HISI_I2C_INT_ALL); + + ret = devm_request_irq(dev, ctlr->irq, hisi_i2c_irq, 0, "hisi-i2c", ctlr); + if (ret) { + dev_err(dev, "failed to request irq handler, ret = %d\n", ret); + return ret; + } + + ret = device_property_read_u64(dev, "clk_rate", &clk_rate_hz); + if (ret) { + dev_err(dev, "failed to get clock frequency, ret = %d\n", ret); + return ret; + } + + ctlr->clk_rate_khz = DIV_ROUND_UP_ULL(clk_rate_hz, HZ_PER_KHZ); + + hisi_i2c_configure_bus(ctlr); + + adapter = &ctlr->adapter; + snprintf(adapter->name, sizeof(adapter->name), + "HiSilicon I2C Controller %s", dev_name(dev)); + adapter->owner = THIS_MODULE; + adapter->algo = &hisi_i2c_algo; + adapter->dev.parent = dev; + i2c_set_adapdata(adapter, ctlr); + + hisi_i2c_init_recovery_info(ctlr); + + ret = devm_i2c_add_adapter(dev, adapter); + if (ret) + return ret; + + hw_version = readl(ctlr->iobase + HISI_I2C_VERSION); + dev_info(ctlr->dev, "speed mode is %s. hw version 0x%x\n", + i2c_freq_mode_string(ctlr->t.bus_freq_hz), hw_version); + + return 0; +} + +static const struct acpi_device_id hisi_i2c_acpi_ids[] = { + { "HISI03D1", 0 }, + { } +}; +MODULE_DEVICE_TABLE(acpi, hisi_i2c_acpi_ids); + +static struct platform_driver hisi_i2c_driver = { + .probe = hisi_i2c_probe, + .driver = { + .name = "hisi-i2c", + .acpi_match_table = hisi_i2c_acpi_ids, + }, +}; +module_platform_driver(hisi_i2c_driver); + +MODULE_AUTHOR("Yicong Yang "); +MODULE_DESCRIPTION("HiSilicon I2C Controller Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/i2c/busses/i2c-xgene-slimpro.c b/drivers/i2c/busses/i2c-xgene-slimpro.c index 76e9dcd6385695c9c802fc54041f3e86f60a3148..145aaa1b1bb6fc1f5457c1cb2715637b02d05c5b 100644 --- a/drivers/i2c/busses/i2c-xgene-slimpro.c +++ b/drivers/i2c/busses/i2c-xgene-slimpro.c @@ -104,6 +104,7 @@ struct slimpro_i2c_dev { struct i2c_adapter adapter; struct device *dev; struct mbox_chan *mbox_chan; + struct pcc_mbox_chan *pcc_chan; struct mbox_client mbox_client; int mbox_idx; struct completion rd_complete; @@ -470,7 +471,7 @@ static int xgene_slimpro_i2c_probe(struct platform_device *pdev) return PTR_ERR(ctx->mbox_chan); } } else { - struct acpi_pcct_hw_reduced *cppc_ss; + struct pcc_mbox_chan *pcc_chan; const struct acpi_device_id *acpi_id; int version = XGENE_SLIMPRO_I2C_V1; @@ -487,24 +488,14 @@ static int xgene_slimpro_i2c_probe(struct platform_device *pdev) cl->tx_block = false; cl->rx_callback = slimpro_i2c_pcc_rx_cb; - ctx->mbox_chan = pcc_mbox_request_channel(cl, ctx->mbox_idx); - if (IS_ERR(ctx->mbox_chan)) { + pcc_chan = pcc_mbox_request_channel(cl, ctx->mbox_idx); + if (IS_ERR(pcc_chan)) { dev_err(&pdev->dev, "PCC mailbox channel request failed\n"); - return PTR_ERR(ctx->mbox_chan); + return PTR_ERR(ctx->pcc_chan); } - /* - * The PCC mailbox controller driver should - * have parsed the PCCT (global table of all - * PCC channels) and stored pointers to the - * subspace communication region in con_priv. - */ - cppc_ss = ctx->mbox_chan->con_priv; - if (!cppc_ss) { - dev_err(&pdev->dev, "PPC subspace not found\n"); - rc = -ENOENT; - goto mbox_err; - } + ctx->pcc_chan = pcc_chan; + ctx->mbox_chan = pcc_chan->mchan; if (!ctx->mbox_chan->mbox->txdone_irq) { dev_err(&pdev->dev, "PCC IRQ not supported\n"); @@ -516,17 +507,17 @@ static int xgene_slimpro_i2c_probe(struct platform_device *pdev) * This is the shared communication region * for the OS and Platform to communicate over. */ - ctx->comm_base_addr = cppc_ss->base_address; + ctx->comm_base_addr = pcc_chan->shmem_base_addr; if (ctx->comm_base_addr) { if (version == XGENE_SLIMPRO_I2C_V2) ctx->pcc_comm_addr = memremap( ctx->comm_base_addr, - cppc_ss->length, + pcc_chan->shmem_size, MEMREMAP_WT); else ctx->pcc_comm_addr = memremap( ctx->comm_base_addr, - cppc_ss->length, + pcc_chan->shmem_size, MEMREMAP_WB); } else { dev_err(&pdev->dev, "Failed to get PCC comm region\n"); @@ -565,7 +556,7 @@ static int xgene_slimpro_i2c_probe(struct platform_device *pdev) if (acpi_disabled) mbox_free_channel(ctx->mbox_chan); else - pcc_mbox_free_channel(ctx->mbox_chan); + pcc_mbox_free_channel(ctx->pcc_chan); return rc; } @@ -579,7 +570,7 @@ static int xgene_slimpro_i2c_remove(struct platform_device *pdev) if (acpi_disabled) mbox_free_channel(ctx->mbox_chan); else - pcc_mbox_free_channel(ctx->mbox_chan); + pcc_mbox_free_channel(ctx->pcc_chan); return 0; } diff --git a/drivers/i2c/i2c-core-acpi.c b/drivers/i2c/i2c-core-acpi.c index fe466ee4c49bf6bf7556e5e1653e54770a36e717..17640e2230b61fb92eb9babcaa533f2cd7ca5503 100644 --- a/drivers/i2c/i2c-core-acpi.c +++ b/drivers/i2c/i2c-core-acpi.c @@ -328,7 +328,7 @@ static acpi_status i2c_acpi_lookup_speed(acpi_handle handle, u32 level, lookup->min_speed = lookup->speed; if (acpi_match_device_ids(adev, i2c_acpi_force_400khz_device_ids) == 0) - lookup->force_speed = 400000; + lookup->force_speed = I2C_MAX_FAST_MODE_FREQ; return AE_OK; } diff --git a/drivers/i2c/i2c-core-base.c b/drivers/i2c/i2c-core-base.c index 964e8a29b27b4cfcf879936aeafc44bc61d3ae0a..4b3ac4475a9ebb0ff99ca22e6d7db0ddf0174d4e 100644 --- a/drivers/i2c/i2c-core-base.c +++ b/drivers/i2c/i2c-core-base.c @@ -76,6 +76,27 @@ void i2c_transfer_trace_unreg(void) static_branch_dec(&i2c_trace_msg_key); } +const char *i2c_freq_mode_string(u32 bus_freq_hz) +{ + switch (bus_freq_hz) { + case I2C_MAX_STANDARD_MODE_FREQ: + return "Standard Mode (100 kHz)"; + case I2C_MAX_FAST_MODE_FREQ: + return "Fast Mode (400 kHz)"; + case I2C_MAX_FAST_MODE_PLUS_FREQ: + return "Fast Mode Plus (1.0 MHz)"; + case I2C_MAX_TURBO_MODE_FREQ: + return "Turbo Mode (1.4 MHz)"; + case I2C_MAX_HIGH_SPEED_MODE_FREQ: + return "High Speed Mode (3.4 MHz)"; + case I2C_MAX_ULTRA_FAST_MODE_FREQ: + return "Ultra Fast Mode (5.0 MHz)"; + default: + return "Unknown Mode"; + } +} +EXPORT_SYMBOL_GPL(i2c_freq_mode_string); + const struct i2c_device_id *i2c_match_id(const struct i2c_device_id *id, const struct i2c_client *client) { @@ -1626,6 +1647,32 @@ void i2c_del_adapter(struct i2c_adapter *adap) } EXPORT_SYMBOL(i2c_del_adapter); +static void devm_i2c_del_adapter(void *adapter) +{ + i2c_del_adapter(adapter); +} + +/** + * devm_i2c_add_adapter - device-managed variant of i2c_add_adapter() + * @dev: managing device for adding this I2C adapter + * @adapter: the adapter to add + * Context: can sleep + * + * Add adapter with dynamic bus number, same with i2c_add_adapter() + * but the adapter will be auto deleted on driver detach. + */ +int devm_i2c_add_adapter(struct device *dev, struct i2c_adapter *adapter) +{ + int ret; + + ret = i2c_add_adapter(adapter); + if (ret) + return ret; + + return devm_add_action_or_reset(dev, devm_i2c_del_adapter, adapter); +} +EXPORT_SYMBOL_GPL(devm_i2c_add_adapter); + /** * i2c_parse_fw_timings - get I2C related timing parameters from firmware * @dev: The device to scan for I2C timing properties @@ -1650,13 +1697,13 @@ void i2c_parse_fw_timings(struct device *dev, struct i2c_timings *t, bool use_de ret = device_property_read_u32(dev, "clock-frequency", &t->bus_freq_hz); if (ret && use_defaults) - t->bus_freq_hz = 100000; + t->bus_freq_hz = I2C_MAX_STANDARD_MODE_FREQ; ret = device_property_read_u32(dev, "i2c-scl-rising-time-ns", &t->scl_rise_ns); if (ret && use_defaults) { - if (t->bus_freq_hz <= 100000) + if (t->bus_freq_hz <= I2C_MAX_STANDARD_MODE_FREQ) t->scl_rise_ns = 1000; - else if (t->bus_freq_hz <= 400000) + else if (t->bus_freq_hz <= I2C_MAX_FAST_MODE_FREQ) t->scl_rise_ns = 300; else t->scl_rise_ns = 120; @@ -1664,7 +1711,7 @@ void i2c_parse_fw_timings(struct device *dev, struct i2c_timings *t, bool use_de ret = device_property_read_u32(dev, "i2c-scl-falling-time-ns", &t->scl_fall_ns); if (ret && use_defaults) { - if (t->bus_freq_hz <= 400000) + if (t->bus_freq_hz <= I2C_MAX_FAST_MODE_FREQ) t->scl_fall_ns = 300; else t->scl_fall_ns = 120; @@ -1677,6 +1724,12 @@ void i2c_parse_fw_timings(struct device *dev, struct i2c_timings *t, bool use_de t->sda_fall_ns = t->scl_fall_ns; device_property_read_u32(dev, "i2c-sda-hold-time-ns", &t->sda_hold_ns); + + device_property_read_u32(dev, "i2c-digital-filter-width-ns", + &t->digital_filter_width_ns); + + device_property_read_u32(dev, "i2c-analog-filter-cutoff-frequency", + &t->analog_filter_cutoff_freq_hz); } EXPORT_SYMBOL_GPL(i2c_parse_fw_timings); diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 352d21d628d024fef663e3940682473e639f4140..044121f601749081cc7df25babf1beb41ad2bdc8 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -2823,6 +2823,26 @@ static int arm_smmu_device_domain_type(struct device *dev, unsigned int *type) } #endif +/* + * HiSilicon PCIe tune and trace device can be used to trace TLP headers on the + * PCIe link and save the data to memory by DMA. The hardware is restricted to + * use identity mapping only. + */ +#define IS_HISI_PTT_DEVICE(pdev) ((pdev)->vendor == PCI_VENDOR_ID_HUAWEI && \ + (pdev)->device == 0xa12e) + +static int arm_smmu_def_domain_type(struct device *dev) +{ + if (dev_is_pci(dev)) { + struct pci_dev *pdev = to_pci_dev(dev); + + if (IS_HISI_PTT_DEVICE(pdev)) + return IOMMU_DOMAIN_IDENTITY; + } + + return 0; +} + static struct iommu_ops arm_smmu_ops = { .capable = arm_smmu_capable, .domain_alloc = arm_smmu_domain_alloc, @@ -2841,6 +2861,7 @@ static struct iommu_ops arm_smmu_ops = { .of_xlate = arm_smmu_of_xlate, .get_resv_regions = arm_smmu_get_resv_regions, .put_resv_regions = arm_smmu_put_resv_regions, + .def_domain_type = arm_smmu_def_domain_type, #ifdef CONFIG_SMMU_BYPASS_DEV .device_domain_type = arm_smmu_device_domain_type, #endif diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 628a586be6959d474c280437c11d7af980a6cfd6..f422de073717fd41af4fe17fc74c956c5403b04d 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -11,6 +11,7 @@ #include #include #include +#include /* The anchor node sits above the top of the usable address space */ #define IOVA_ANCHOR ~0UL @@ -774,12 +775,24 @@ split_and_remove_iova(struct iova_domain *iovad, struct iova *iova, * dynamic size tuning described in the paper. */ -#define IOVA_MAG_SIZE 128 +/* + * As kmalloc's buffer size is fixed to power of 2, 127 is chosen to + * assure size of 'iova_magazine' to be 1024 bytes, so that no memory + * will be wasted. Since only full magazines are inserted into the depot, + * we don't need to waste PFN capacity on a separate list head either. + */ +#define IOVA_MAG_SIZE 127 + +#define IOVA_DEPOT_DELAY msecs_to_jiffies(100) struct iova_magazine { - unsigned long size; + union { + unsigned long size; + struct iova_magazine *next; + }; unsigned long pfns[IOVA_MAG_SIZE]; }; +static_assert(!(sizeof(struct iova_magazine) & (sizeof(struct iova_magazine) - 1))); struct iova_cpu_rcache { spinlock_t lock; @@ -859,6 +872,41 @@ static void iova_magazine_push(struct iova_magazine *mag, unsigned long pfn) mag->pfns[mag->size++] = pfn; } +static struct iova_magazine *iova_depot_pop(struct iova_rcache *rcache) +{ + struct iova_magazine *mag = rcache->depot; + + rcache->depot = mag->next; + mag->size = IOVA_MAG_SIZE; + rcache->depot_size--; + return mag; +} + +static void iova_depot_push(struct iova_rcache *rcache, struct iova_magazine *mag) +{ + mag->next = rcache->depot; + rcache->depot = mag; + rcache->depot_size++; +} + +static void iova_depot_work_func(struct work_struct *work) +{ + struct iova_rcache *rcache = container_of(work, typeof(*rcache), work.work); + struct iova_magazine *mag = NULL; + unsigned long flags; + + spin_lock_irqsave(&rcache->lock, flags); + if (rcache->depot_size > num_online_cpus()) + mag = iova_depot_pop(rcache); + spin_unlock_irqrestore(&rcache->lock, flags); + + if (mag) { + iova_magazine_free_pfns(mag, rcache->iovad); + iova_magazine_free(mag); + schedule_delayed_work(&rcache->work, IOVA_DEPOT_DELAY); + } +} + static void init_iova_rcaches(struct iova_domain *iovad) { struct iova_cpu_rcache *cpu_rcache; @@ -869,7 +917,8 @@ static void init_iova_rcaches(struct iova_domain *iovad) for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) { rcache = &iovad->rcaches[i]; spin_lock_init(&rcache->lock); - rcache->depot_size = 0; + rcache->iovad = iovad; + INIT_DELAYED_WORK(&rcache->work, iova_depot_work_func); rcache->cpu_rcaches = __alloc_percpu(sizeof(*cpu_rcache), cache_line_size()); if (WARN_ON(!rcache->cpu_rcaches)) continue; @@ -892,7 +941,6 @@ static bool __iova_rcache_insert(struct iova_domain *iovad, struct iova_rcache *rcache, unsigned long iova_pfn) { - struct iova_magazine *mag_to_free = NULL; struct iova_cpu_rcache *cpu_rcache; bool can_insert = false; unsigned long flags; @@ -910,13 +958,9 @@ static bool __iova_rcache_insert(struct iova_domain *iovad, if (new_mag) { spin_lock(&rcache->lock); - if (rcache->depot_size < MAX_GLOBAL_MAGS) { - rcache->depot[rcache->depot_size++] = - cpu_rcache->loaded; - } else { - mag_to_free = cpu_rcache->loaded; - } + iova_depot_push(rcache, cpu_rcache->loaded); spin_unlock(&rcache->lock); + schedule_delayed_work(&rcache->work, IOVA_DEPOT_DELAY); cpu_rcache->loaded = new_mag; can_insert = true; @@ -928,11 +972,6 @@ static bool __iova_rcache_insert(struct iova_domain *iovad, spin_unlock_irqrestore(&cpu_rcache->lock, flags); - if (mag_to_free) { - iova_magazine_free_pfns(mag_to_free, iovad); - iova_magazine_free(mag_to_free); - } - return can_insert; } @@ -970,9 +1009,9 @@ static unsigned long __iova_rcache_get(struct iova_rcache *rcache, has_pfn = true; } else { spin_lock(&rcache->lock); - if (rcache->depot_size > 0) { + if (rcache->depot) { iova_magazine_free(cpu_rcache->loaded); - cpu_rcache->loaded = rcache->depot[--rcache->depot_size]; + cpu_rcache->loaded = iova_depot_pop(rcache); has_pfn = true; } spin_unlock(&rcache->lock); @@ -1011,7 +1050,7 @@ static void free_iova_rcaches(struct iova_domain *iovad) struct iova_rcache *rcache; struct iova_cpu_rcache *cpu_rcache; unsigned int cpu; - int i, j; + int i; for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) { rcache = &iovad->rcaches[i]; @@ -1021,8 +1060,9 @@ static void free_iova_rcaches(struct iova_domain *iovad) iova_magazine_free(cpu_rcache->prev); } free_percpu(rcache->cpu_rcaches); - for (j = 0; j < rcache->depot_size; ++j) - iova_magazine_free(rcache->depot[j]); + cancel_delayed_work_sync(&rcache->work); + while (rcache->depot) + iova_magazine_free(iova_depot_pop(rcache)); } } diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 75bede37ae544abc0085d72ad38eed8e8833eedd..aa0b5e3f440c3c110d4de3c234c89f7f70d2a322 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -1986,6 +1986,11 @@ static int its_alloc_tables(struct its_node *its) psz, &order, ITS_MAX_VPEID_BITS); break; + case GITS_BASER_TYPE_COLLECTION: + indirect = its_parse_indirect_baser(its, baser, + psz, &order, + order_base_2(num_possible_cpus())); + break; } err = its_setup_baser(its, baser, cache, shr, psz, order, indirect); diff --git a/drivers/mailbox/pcc.c b/drivers/mailbox/pcc.c index 34844b7a367524eb13a27f5331a3c99b14ef1876..978dddfea6070490dbf1c31ae3719653770790c3 100644 --- a/drivers/mailbox/pcc.c +++ b/drivers/mailbox/pcc.c @@ -64,29 +64,24 @@ static struct mbox_chan *pcc_mbox_channels; -/* Array of cached virtual address for doorbell registers */ -static void __iomem **pcc_doorbell_vaddr; -/* Array of cached virtual address for doorbell ack registers */ -static void __iomem **pcc_doorbell_ack_vaddr; -/* Array of doorbell interrupts */ -static int *pcc_doorbell_irq; - -static struct mbox_controller pcc_mbox_ctrl = {}; /** - * get_pcc_channel - Given a PCC subspace idx, get - * the respective mbox_channel. - * @id: PCC subspace index. + * struct pcc_chan_info - PCC channel specific information * - * Return: ERR_PTR(errno) if error, else pointer - * to mbox channel. + * @chan: PCC channel information with Shared Memory Region info + * @db_vaddr: cached virtual address for doorbell register + * @db_ack_vaddr: cached virtual address for doorbell ack register + * @db_irq: doorbell interrupt */ -static struct mbox_chan *get_pcc_channel(int id) -{ - if (id < 0 || id >= pcc_mbox_ctrl.num_chans) - return ERR_PTR(-ENOENT); +struct pcc_chan_info { + struct pcc_mbox_chan chan; + void __iomem *db_vaddr; + void __iomem *db_ack_vaddr; + int db_irq; +}; - return &pcc_mbox_channels[id]; -} +#define to_pcc_chan_info(c) container_of(c, struct pcc_chan_info, chan) +static struct pcc_chan_info *chan_info; +static struct mbox_controller pcc_mbox_ctrl = {}; /* * PCC can be used with perf critical drivers such as CPPC @@ -179,6 +174,7 @@ static irqreturn_t pcc_mbox_irq(int irq, void *p) { struct acpi_generic_address *doorbell_ack; struct acpi_pcct_hw_reduced *pcct_ss; + struct pcc_chan_info *pchan; struct mbox_chan *chan = p; u64 doorbell_ack_preserve; u64 doorbell_ack_write; @@ -193,17 +189,17 @@ static irqreturn_t pcc_mbox_irq(int irq, void *p) struct acpi_pcct_hw_reduced_type2 *pcct2_ss = chan->con_priv; u32 id = chan - pcc_mbox_channels; + pchan = chan_info + id; doorbell_ack = &pcct2_ss->platform_ack_register; doorbell_ack_preserve = pcct2_ss->ack_preserve_mask; doorbell_ack_write = pcct2_ss->ack_write_mask; - ret = read_register(pcc_doorbell_ack_vaddr[id], - &doorbell_ack_val, - doorbell_ack->bit_width); + ret = read_register(pchan->db_ack_vaddr, + &doorbell_ack_val, doorbell_ack->bit_width); if (ret) return IRQ_NONE; - ret = write_register(pcc_doorbell_ack_vaddr[id], + ret = write_register(pchan->db_ack_vaddr, (doorbell_ack_val & doorbell_ack_preserve) | doorbell_ack_write, doorbell_ack->bit_width); @@ -224,25 +220,21 @@ static irqreturn_t pcc_mbox_irq(int irq, void *p) * ACPI package. This is used to lookup the array of PCC * subspaces as parsed by the PCC Mailbox controller. * - * Return: Pointer to the Mailbox Channel if successful or - * ERR_PTR. + * Return: Pointer to the PCC Mailbox Channel if successful or ERR_PTR. */ -struct mbox_chan *pcc_mbox_request_channel(struct mbox_client *cl, - int subspace_id) +struct pcc_mbox_chan * +pcc_mbox_request_channel(struct mbox_client *cl, int subspace_id) { + struct pcc_chan_info *pchan; struct device *dev = pcc_mbox_ctrl.dev; struct mbox_chan *chan; unsigned long flags; - /* - * Each PCC Subspace is a Mailbox Channel. - * The PCC Clients get their PCC Subspace ID - * from their own tables and pass it here. - * This returns a pointer to the PCC subspace - * for the Client to operate on. - */ - chan = get_pcc_channel(subspace_id); + if (subspace_id < 0 || subspace_id >= pcc_mbox_ctrl.num_chans) + return ERR_PTR(-ENOENT); + pchan = chan_info + subspace_id; + chan = pchan->chan.mchan; if (IS_ERR(chan) || chan->cl) { dev_err(dev, "Channel not found for idx: %d\n", subspace_id); return ERR_PTR(-EBUSY); @@ -260,44 +252,40 @@ struct mbox_chan *pcc_mbox_request_channel(struct mbox_client *cl, spin_unlock_irqrestore(&chan->lock, flags); - if (pcc_doorbell_irq[subspace_id] > 0) { + if (pchan->db_irq > 0) { int rc; - rc = devm_request_irq(dev, pcc_doorbell_irq[subspace_id], - pcc_mbox_irq, 0, MBOX_IRQ_NAME, chan); + rc = devm_request_irq(dev, pchan->db_irq, pcc_mbox_irq, 0, + MBOX_IRQ_NAME, chan); if (unlikely(rc)) { dev_err(dev, "failed to register PCC interrupt %d\n", - pcc_doorbell_irq[subspace_id]); - pcc_mbox_free_channel(chan); - chan = ERR_PTR(rc); + pchan->db_irq); + pcc_mbox_free_channel(&pchan->chan); + return ERR_PTR(rc); } } - return chan; + return &pchan->chan; } EXPORT_SYMBOL_GPL(pcc_mbox_request_channel); /** * pcc_mbox_free_channel - Clients call this to free their Channel. * - * @chan: Pointer to the mailbox channel as returned by - * pcc_mbox_request_channel() + * @pchan: Pointer to the PCC mailbox channel as returned by + * pcc_mbox_request_channel() */ -void pcc_mbox_free_channel(struct mbox_chan *chan) +void pcc_mbox_free_channel(struct pcc_mbox_chan *pchan) { - u32 id = chan - pcc_mbox_channels; + struct pcc_chan_info *pchan_info = to_pcc_chan_info(pchan); + struct mbox_chan *chan = pchan->mchan; unsigned long flags; if (!chan || !chan->cl) return; - if (id >= pcc_mbox_ctrl.num_chans) { - pr_debug("pcc_mbox_free_channel: Invalid mbox_chan passed\n"); - return; - } - - if (pcc_doorbell_irq[id] > 0) - devm_free_irq(chan->mbox->dev, pcc_doorbell_irq[id], chan); + if (pchan_info->db_irq > 0) + devm_free_irq(chan->mbox->dev, pchan_info->db_irq, chan); spin_lock_irqsave(&chan->lock, flags); chan->cl = NULL; @@ -325,6 +313,7 @@ static int pcc_send_data(struct mbox_chan *chan, void *data) { struct acpi_pcct_hw_reduced *pcct_ss = chan->con_priv; struct acpi_generic_address *doorbell; + struct pcc_chan_info *pchan; u64 doorbell_preserve; u64 doorbell_val; u64 doorbell_write; @@ -336,19 +325,20 @@ static int pcc_send_data(struct mbox_chan *chan, void *data) return -ENOENT; } + pchan = chan_info + id; doorbell = &pcct_ss->doorbell_register; doorbell_preserve = pcct_ss->preserve_mask; doorbell_write = pcct_ss->write_mask; /* Sync notification from OS to Platform. */ - if (pcc_doorbell_vaddr[id]) { - ret = read_register(pcc_doorbell_vaddr[id], &doorbell_val, - doorbell->bit_width); + if (pchan->db_vaddr) { + ret = read_register(pchan->db_vaddr, &doorbell_val, + doorbell->bit_width); if (ret) return ret; - ret = write_register(pcc_doorbell_vaddr[id], - (doorbell_val & doorbell_preserve) | doorbell_write, - doorbell->bit_width); + ret = write_register(pchan->db_vaddr, + (doorbell_val & doorbell_preserve) + | doorbell_write, doorbell->bit_width); } else { ret = acpi_read(&doorbell_val, doorbell); if (ret) @@ -386,33 +376,41 @@ static int parse_pcc_subspace(union acpi_subtable_headers *header, /** * pcc_parse_subspace_irq - Parse the PCC IRQ and PCC ACK register - * There should be one entry per PCC client. - * @id: PCC subspace index. - * @pcct_ss: Pointer to the ACPI subtable header under the PCCT. + * + * @pchan: Pointer to the PCC channel info structure. + * @pcct_entry: Pointer to the ACPI subtable header. * * Return: 0 for Success, else errno. * - * This gets called for each entry in the PCC table. + * There should be one entry per PCC channel. This gets called for each + * entry in the PCC table. This uses PCCY Type1 structure for all applicable + * types(Type 1-4) to fetch irq */ -static int pcc_parse_subspace_irq(int id, - struct acpi_pcct_hw_reduced *pcct_ss) +static int pcc_parse_subspace_irq(struct pcc_chan_info *pchan, + struct acpi_subtable_header *pcct_entry) { - pcc_doorbell_irq[id] = pcc_map_interrupt(pcct_ss->platform_interrupt, - (u32)pcct_ss->flags); - if (pcc_doorbell_irq[id] <= 0) { + struct acpi_pcct_hw_reduced *pcct_ss; + + if (pcct_entry->type < ACPI_PCCT_TYPE_HW_REDUCED_SUBSPACE || + pcct_entry->type > ACPI_PCCT_TYPE_EXT_PCC_SLAVE_SUBSPACE) + return 0; + + pcct_ss = (struct acpi_pcct_hw_reduced *)pcct_entry; + pchan->db_irq = pcc_map_interrupt(pcct_ss->platform_interrupt, + (u32)pcct_ss->flags); + if (pchan->db_irq <= 0) { pr_err("PCC GSI %d not registered\n", pcct_ss->platform_interrupt); return -EINVAL; } - if (pcct_ss->header.type - == ACPI_PCCT_TYPE_HW_REDUCED_SUBSPACE_TYPE2) { + if (pcct_ss->header.type == ACPI_PCCT_TYPE_HW_REDUCED_SUBSPACE_TYPE2) { struct acpi_pcct_hw_reduced_type2 *pcct2_ss = (void *)pcct_ss; - pcc_doorbell_ack_vaddr[id] = acpi_os_ioremap( - pcct2_ss->platform_ack_register.address, - pcct2_ss->platform_ack_register.bit_width / 8); - if (!pcc_doorbell_ack_vaddr[id]) { + pchan->db_ack_vaddr = + acpi_os_ioremap(pcct2_ss->platform_ack_register.address, + pcct2_ss->platform_ack_register.bit_width / 8); + if (!pchan->db_ack_vaddr) { pr_err("Failed to ioremap PCC ACK register\n"); return -ENOMEM; } @@ -421,6 +419,49 @@ static int pcc_parse_subspace_irq(int id, return 0; } +/** + * pcc_parse_subspace_db_reg - Parse the PCC doorbell register + * + * @pchan: Pointer to the PCC channel info structure. + * @pcct_entry: Pointer to the ACPI subtable header. + * + */ +static void pcc_parse_subspace_db_reg(struct pcc_chan_info *pchan, + struct acpi_subtable_header *pcct_entry) +{ + struct acpi_pcct_subspace *pcct_ss; + struct acpi_generic_address *db_reg; + + pcct_ss = (struct acpi_pcct_subspace *)pcct_entry; + + /* If doorbell is in system memory cache the virt address */ + db_reg = &pcct_ss->doorbell_register; + if (db_reg->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY) + pchan->db_vaddr = acpi_os_ioremap(db_reg->address, + db_reg->bit_width / 8); +} + +/** + * pcc_parse_subspace_shmem - Parse the PCC Shared Memory Region information + * + * @pchan: Pointer to the PCC channel info structure. + * @pcct_entry: Pointer to the ACPI subtable header. + * + */ +static void pcc_parse_subspace_shmem(struct pcc_chan_info *pchan, + struct acpi_subtable_header *pcct_entry) +{ + struct acpi_pcct_subspace *pcct_ss; + + pcct_ss = (struct acpi_pcct_subspace *)pcct_entry; + + pchan->chan.shmem_base_addr = pcct_ss->base_address; + pchan->chan.shmem_size = pcct_ss->length; + pchan->chan.latency = pcct_ss->latency; + pchan->chan.max_access_rate = pcct_ss->max_access_rate; + pchan->chan.min_turnaround_time = pcct_ss->min_turnaround_time; +} + /** * acpi_pcc_probe - Parse the ACPI tree for the PCCT. * @@ -467,24 +508,12 @@ static int __init acpi_pcc_probe(void) return -ENOMEM; } - pcc_doorbell_vaddr = kcalloc(count, sizeof(void *), GFP_KERNEL); - if (!pcc_doorbell_vaddr) { + chan_info = kcalloc(count, sizeof(*chan_info), GFP_KERNEL); + if (!chan_info) { rc = -ENOMEM; goto err_free_mbox; } - pcc_doorbell_ack_vaddr = kcalloc(count, sizeof(void *), GFP_KERNEL); - if (!pcc_doorbell_ack_vaddr) { - rc = -ENOMEM; - goto err_free_db_vaddr; - } - - pcc_doorbell_irq = kcalloc(count, sizeof(int), GFP_KERNEL); - if (!pcc_doorbell_irq) { - rc = -ENOMEM; - goto err_free_db_ack_vaddr; - } - /* Point to the first PCC subspace entry */ pcct_entry = (struct acpi_subtable_header *) ( (unsigned long) pcct_tbl + sizeof(struct acpi_table_pcct)); @@ -494,29 +523,20 @@ static int __init acpi_pcc_probe(void) pcc_mbox_ctrl.txdone_irq = true; for (i = 0; i < count; i++) { - struct acpi_generic_address *db_reg; - struct acpi_pcct_subspace *pcct_ss; + struct pcc_chan_info *pchan = chan_info + i; pcc_mbox_channels[i].con_priv = pcct_entry; - if (pcct_entry->type == ACPI_PCCT_TYPE_HW_REDUCED_SUBSPACE || - pcct_entry->type == ACPI_PCCT_TYPE_HW_REDUCED_SUBSPACE_TYPE2) { - struct acpi_pcct_hw_reduced *pcct_hrss; + pchan->chan.mchan = &pcc_mbox_channels[i]; - pcct_hrss = (struct acpi_pcct_hw_reduced *) pcct_entry; - - if (pcc_mbox_ctrl.txdone_irq) { - rc = pcc_parse_subspace_irq(i, pcct_hrss); - if (rc < 0) - goto err; - } + if (pcc_mbox_ctrl.txdone_irq) { + rc = pcc_parse_subspace_irq(pchan, pcct_entry); + if (rc < 0) + goto err; } - pcct_ss = (struct acpi_pcct_subspace *) pcct_entry; + pcc_parse_subspace_db_reg(pchan, pcct_entry); + + pcc_parse_subspace_shmem(pchan, pcct_entry); - /* If doorbell is in system memory cache the virt address */ - db_reg = &pcct_ss->doorbell_register; - if (db_reg->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY) - pcc_doorbell_vaddr[i] = acpi_os_ioremap(db_reg->address, - db_reg->bit_width/8); pcct_entry = (struct acpi_subtable_header *) ((unsigned long) pcct_entry + pcct_entry->length); } @@ -528,11 +548,7 @@ static int __init acpi_pcc_probe(void) return 0; err: - kfree(pcc_doorbell_irq); -err_free_db_ack_vaddr: - kfree(pcc_doorbell_ack_vaddr); -err_free_db_vaddr: - kfree(pcc_doorbell_vaddr); + kfree(chan_info); err_free_mbox: kfree(pcc_mbox_channels); return rc; diff --git a/drivers/mtd/spi-nor/spi-nor.c b/drivers/mtd/spi-nor/spi-nor.c index dc7f5db544087a827c7c1425a871e219ef4337f8..f4aa90c4eebe201fac9b738181d2463d3e5b4261 100644 --- a/drivers/mtd/spi-nor/spi-nor.c +++ b/drivers/mtd/spi-nor/spi-nor.c @@ -4727,6 +4727,35 @@ static void spi_nor_resume(struct mtd_info *mtd) dev_err(dev, "resume() failed\n"); } +static int spi_nor_get_device(struct mtd_info *mtd) +{ + struct spi_nor *nor = mtd_to_spi_nor(mtd); + struct device *dev; + + if (nor->spimem) + dev = nor->spimem->spi->controller->dev.parent; + else + dev = nor->dev; + + if (!try_module_get(dev->driver->owner)) + return -ENODEV; + + return 0; +} + +static void spi_nor_put_device(struct mtd_info *mtd) +{ + struct spi_nor *nor = mtd_to_spi_nor(mtd); + struct device *dev; + + if (nor->spimem) + dev = nor->spimem->spi->controller->dev.parent; + else + dev = nor->dev; + + module_put(dev->driver->owner); +} + void spi_nor_restore(struct spi_nor *nor) { /* restore the addressing mode */ @@ -4903,6 +4932,8 @@ int spi_nor_scan(struct spi_nor *nor, const char *name, mtd->_erase = spi_nor_erase; mtd->_read = spi_nor_read; mtd->_resume = spi_nor_resume; + mtd->_get_device = spi_nor_get_device; + mtd->_put_device = spi_nor_put_device; if (nor->params.locking_ops) { mtd->_lock = spi_nor_lock; diff --git a/drivers/pci/setup-bus.c b/drivers/pci/setup-bus.c index 44f4866d95d8c4ba6c0811e1e403de5f765816ea..894e4a5232748d19976f605e6633ff2c908f5c26 100644 --- a/drivers/pci/setup-bus.c +++ b/drivers/pci/setup-bus.c @@ -988,7 +988,7 @@ static int pbus_size_mem(struct pci_bus *bus, unsigned long mask, { struct pci_dev *dev; resource_size_t min_align, align, size, size0, size1; - resource_size_t aligns[18]; /* Alignments from 1MB to 128GB */ + resource_size_t aligns[24]; /* Alignments from 1MB to 8TB */ int order, max_order; struct resource *b_res = find_bus_resource_of_type(bus, mask | IORESOURCE_PREFETCH, type); diff --git a/drivers/perf/arm_spe_pmu.c b/drivers/perf/arm_spe_pmu.c index 079701e8de1860e70dd5b66236090726cd3ab796..a462602db408974cbcbec1eb28dad20a3db56e09 100644 --- a/drivers/perf/arm_spe_pmu.c +++ b/drivers/perf/arm_spe_pmu.c @@ -54,7 +54,7 @@ struct arm_spe_pmu { struct hlist_node hotplug_node; int irq; /* PPI */ - + u16 pmsver; u16 min_period; u16 counter_sz; @@ -655,6 +655,18 @@ static irqreturn_t arm_spe_pmu_irq_handler(int irq, void *dev) return IRQ_HANDLED; } +static u64 arm_spe_pmsevfr_res0(u16 pmsver) +{ + switch (pmsver) { + case ID_AA64DFR0_PMSVER_8_2: + return SYS_PMSEVFR_EL1_RES0_8_2; + case ID_AA64DFR0_PMSVER_8_3: + /* Return the highest version we support in default */ + default: + return SYS_PMSEVFR_EL1_RES0_8_3; + } +} + /* Perf callbacks */ static int arm_spe_pmu_event_init(struct perf_event *event) { @@ -670,7 +682,7 @@ static int arm_spe_pmu_event_init(struct perf_event *event) !cpumask_test_cpu(event->cpu, &spe_pmu->supported_cpus)) return -ENOENT; - if (arm_spe_event_to_pmsevfr(event) & SYS_PMSEVFR_EL1_RES0) + if (arm_spe_event_to_pmsevfr(event) & arm_spe_pmsevfr_res0(spe_pmu->pmsver)) return -EOPNOTSUPP; if (attr->exclude_idle) @@ -937,6 +949,7 @@ static void __arm_spe_pmu_dev_probe(void *info) fld, smp_processor_id()); return; } + spe_pmu->pmsver = (u16)fld; /* Read PMBIDR first to determine whether or not we have access */ reg = read_sysreg_s(SYS_PMBIDR_EL1); diff --git a/drivers/perf/hisilicon/hisi_pcie_pmu.c b/drivers/perf/hisilicon/hisi_pcie_pmu.c new file mode 100644 index 0000000000000000000000000000000000000000..e10fc7cb9493a96f4f340b6fa823198bbce8ceab --- /dev/null +++ b/drivers/perf/hisilicon/hisi_pcie_pmu.c @@ -0,0 +1,960 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This driver adds support for PCIe PMU RCiEP device. Related + * perf events are bandwidth, latency etc. + * + * Copyright (C) 2021 HiSilicon Limited + * Author: Qi Liu + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DRV_NAME "hisi_pcie_pmu" +/* Define registers */ +#define HISI_PCIE_GLOBAL_CTRL 0x00 +#define HISI_PCIE_EVENT_CTRL 0x010 +#define HISI_PCIE_CNT 0x090 +#define HISI_PCIE_EXT_CNT 0x110 +#define HISI_PCIE_INT_STAT 0x150 +#define HISI_PCIE_INT_MASK 0x154 +#define HISI_PCIE_REG_BDF 0xfe0 +#define HISI_PCIE_REG_VERSION 0xfe4 +#define HISI_PCIE_REG_INFO 0xfe8 + +/* Define command in HISI_PCIE_GLOBAL_CTRL */ +#define HISI_PCIE_GLOBAL_EN 0x01 +#define HISI_PCIE_GLOBAL_NONE 0 + +/* Define command in HISI_PCIE_EVENT_CTRL */ +#define HISI_PCIE_EVENT_EN BIT_ULL(20) +#define HISI_PCIE_RESET_CNT BIT_ULL(22) +#define HISI_PCIE_INIT_SET BIT_ULL(34) +#define HISI_PCIE_THR_EN BIT_ULL(26) +#define HISI_PCIE_TARGET_EN BIT_ULL(32) +#define HISI_PCIE_TRIG_EN BIT_ULL(52) + +/* Define offsets in HISI_PCIE_EVENT_CTRL */ +#define HISI_PCIE_EVENT_M GENMASK_ULL(15, 0) +#define HISI_PCIE_THR_MODE_M GENMASK_ULL(27, 27) +#define HISI_PCIE_THR_M GENMASK_ULL(31, 28) +#define HISI_PCIE_LEN_M GENMASK_ULL(35, 34) +#define HISI_PCIE_TARGET_M GENMASK_ULL(52, 36) +#define HISI_PCIE_TRIG_MODE_M GENMASK_ULL(53, 53) +#define HISI_PCIE_TRIG_M GENMASK_ULL(59, 56) + +/* Default config of TLP length mode, will count both TLP headers and payloads */ +#define HISI_PCIE_LEN_M_DEFAULT 3ULL + +#define HISI_PCIE_MAX_COUNTERS 8 +#define HISI_PCIE_REG_STEP 8 +#define HISI_PCIE_THR_MAX_VAL 10 +#define HISI_PCIE_TRIG_MAX_VAL 10 +#define HISI_PCIE_MAX_PERIOD (GENMASK_ULL(63, 0)) +#define HISI_PCIE_INIT_VAL BIT_ULL(63) + +struct hisi_pcie_pmu { + struct perf_event *hw_events[HISI_PCIE_MAX_COUNTERS]; + struct hlist_node node; + struct pci_dev *pdev; + struct pmu pmu; + void __iomem *base; + int irq; + u32 identifier; + /* Minimum and maximum BDF of root ports monitored by PMU */ + u16 bdf_min; + u16 bdf_max; + int on_cpu; +}; + +struct hisi_pcie_reg_pair { + u16 lo; + u16 hi; +}; + +#define to_pcie_pmu(p) (container_of((p), struct hisi_pcie_pmu, pmu)) +#define GET_PCI_DEVFN(bdf) ((bdf) & 0xff) + +#define HISI_PCIE_PMU_FILTER_ATTR(_name, _config, _hi, _lo) \ + static u64 hisi_pcie_get_##_name(struct perf_event *event) \ + { \ + return FIELD_GET(GENMASK(_hi, _lo), event->attr._config); \ + } \ + +HISI_PCIE_PMU_FILTER_ATTR(event, config, 16, 0); +HISI_PCIE_PMU_FILTER_ATTR(thr_len, config1, 3, 0); +HISI_PCIE_PMU_FILTER_ATTR(thr_mode, config1, 4, 4); +HISI_PCIE_PMU_FILTER_ATTR(trig_len, config1, 8, 5); +HISI_PCIE_PMU_FILTER_ATTR(trig_mode, config1, 9, 9); +HISI_PCIE_PMU_FILTER_ATTR(len_mode, config1, 11, 10); +HISI_PCIE_PMU_FILTER_ATTR(port, config2, 15, 0); +HISI_PCIE_PMU_FILTER_ATTR(bdf, config2, 31, 16); + +static ssize_t hisi_pcie_format_sysfs_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct dev_ext_attribute *eattr; + + eattr = container_of(attr, struct dev_ext_attribute, attr); + + return sysfs_emit(buf, "%s\n", (char *)eattr->var); +} + +static ssize_t hisi_pcie_event_sysfs_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct perf_pmu_events_attr *pmu_attr = + container_of(attr, struct perf_pmu_events_attr, attr); + + return sysfs_emit(buf, "config=0x%llx\n", pmu_attr->id); +} + +#define HISI_PCIE_PMU_FORMAT_ATTR(_name, _format) \ + (&((struct dev_ext_attribute[]){ \ + { .attr = __ATTR(_name, 0444, hisi_pcie_format_sysfs_show, \ + NULL), \ + .var = (void *)_format } \ + })[0].attr.attr) + +#define HISI_PCIE_PMU_EVENT_ATTR(_name, _id) \ + PMU_EVENT_ATTR_ID(_name, hisi_pcie_event_sysfs_show, _id) + +static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(dev_get_drvdata(dev)); + + return cpumap_print_to_pagebuf(true, buf, cpumask_of(pcie_pmu->on_cpu)); +} +static DEVICE_ATTR_RO(cpumask); + +static ssize_t identifier_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(dev_get_drvdata(dev)); + + return sysfs_emit(buf, "%#x\n", pcie_pmu->identifier); +} +static DEVICE_ATTR_RO(identifier); + +static ssize_t bus_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(dev_get_drvdata(dev)); + + return sysfs_emit(buf, "%#04x\n", PCI_BUS_NUM(pcie_pmu->bdf_min)); +} +static DEVICE_ATTR_RO(bus); + +static struct hisi_pcie_reg_pair +hisi_pcie_parse_reg_value(struct hisi_pcie_pmu *pcie_pmu, u32 reg_off) +{ + u32 val = readl_relaxed(pcie_pmu->base + reg_off); + struct hisi_pcie_reg_pair regs = { + .lo = val, + .hi = val >> 16, + }; + + return regs; +} + +/* + * Hardware counter and ext_counter work together for bandwidth, latency, bus + * utilization and buffer occupancy events. For example, RX memory write latency + * events(index = 0x0010), counter counts total delay cycles and ext_counter + * counts RX memory write PCIe packets number. + * + * As we don't want PMU driver to process these two data, "delay cycles" can + * be treated as an independent event(index = 0x0010), "RX memory write packets + * number" as another(index = 0x10010). BIT 16 is used to distinguish and 0-15 + * bits are "real" event index, which can be used to set HISI_PCIE_EVENT_CTRL. + */ +#define EXT_COUNTER_IS_USED(idx) ((idx) & BIT(16)) + +static u32 hisi_pcie_get_real_event(struct perf_event *event) +{ + return hisi_pcie_get_event(event) & GENMASK(15, 0); +} + +static u32 hisi_pcie_pmu_get_offset(u32 offset, u32 idx) +{ + return offset + HISI_PCIE_REG_STEP * idx; +} + +static u32 hisi_pcie_pmu_readl(struct hisi_pcie_pmu *pcie_pmu, u32 reg_offset, + u32 idx) +{ + u32 offset = hisi_pcie_pmu_get_offset(reg_offset, idx); + + return readl_relaxed(pcie_pmu->base + offset); +} + +static void hisi_pcie_pmu_writel(struct hisi_pcie_pmu *pcie_pmu, u32 reg_offset, u32 idx, u32 val) +{ + u32 offset = hisi_pcie_pmu_get_offset(reg_offset, idx); + + writel_relaxed(val, pcie_pmu->base + offset); +} + +static u64 hisi_pcie_pmu_readq(struct hisi_pcie_pmu *pcie_pmu, u32 reg_offset, u32 idx) +{ + u32 offset = hisi_pcie_pmu_get_offset(reg_offset, idx); + + return readq_relaxed(pcie_pmu->base + offset); +} + +static void hisi_pcie_pmu_writeq(struct hisi_pcie_pmu *pcie_pmu, u32 reg_offset, u32 idx, u64 val) +{ + u32 offset = hisi_pcie_pmu_get_offset(reg_offset, idx); + + writeq_relaxed(val, pcie_pmu->base + offset); +} + +static void hisi_pcie_pmu_config_filter(struct perf_event *event) +{ + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); + struct hw_perf_event *hwc = &event->hw; + u64 port, trig_len, thr_len, len_mode; + u64 reg = HISI_PCIE_INIT_SET; + + /* Config HISI_PCIE_EVENT_CTRL according to event. */ + reg |= FIELD_PREP(HISI_PCIE_EVENT_M, hisi_pcie_get_real_event(event)); + + /* Config HISI_PCIE_EVENT_CTRL according to root port or EP device. */ + port = hisi_pcie_get_port(event); + if (port) + reg |= FIELD_PREP(HISI_PCIE_TARGET_M, port); + else + reg |= HISI_PCIE_TARGET_EN | + FIELD_PREP(HISI_PCIE_TARGET_M, hisi_pcie_get_bdf(event)); + + /* Config HISI_PCIE_EVENT_CTRL according to trigger condition. */ + trig_len = hisi_pcie_get_trig_len(event); + if (trig_len) { + reg |= FIELD_PREP(HISI_PCIE_TRIG_M, trig_len); + reg |= FIELD_PREP(HISI_PCIE_TRIG_MODE_M, hisi_pcie_get_trig_mode(event)); + reg |= HISI_PCIE_TRIG_EN; + } + + /* Config HISI_PCIE_EVENT_CTRL according to threshold condition. */ + thr_len = hisi_pcie_get_thr_len(event); + if (thr_len) { + reg |= FIELD_PREP(HISI_PCIE_THR_M, thr_len); + reg |= FIELD_PREP(HISI_PCIE_THR_MODE_M, hisi_pcie_get_thr_mode(event)); + reg |= HISI_PCIE_THR_EN; + } + + len_mode = hisi_pcie_get_len_mode(event); + if (len_mode) + reg |= FIELD_PREP(HISI_PCIE_LEN_M, len_mode); + else + reg |= FIELD_PREP(HISI_PCIE_LEN_M, HISI_PCIE_LEN_M_DEFAULT); + + hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, hwc->idx, reg); +} + +static void hisi_pcie_pmu_clear_filter(struct perf_event *event) +{ + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); + struct hw_perf_event *hwc = &event->hw; + + hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, hwc->idx, HISI_PCIE_INIT_SET); +} + +static bool hisi_pcie_pmu_valid_requester_id(struct hisi_pcie_pmu *pcie_pmu, u32 bdf) +{ + struct pci_dev *root_port, *pdev; + u16 rp_bdf; + + pdev = pci_get_domain_bus_and_slot(pci_domain_nr(pcie_pmu->pdev->bus), PCI_BUS_NUM(bdf), + GET_PCI_DEVFN(bdf)); + if (!pdev) + return false; + + root_port = pcie_find_root_port(pdev); + if (!root_port) { + pci_dev_put(pdev); + return false; + } + + pci_dev_put(pdev); + rp_bdf = pci_dev_id(root_port); + return rp_bdf >= pcie_pmu->bdf_min && rp_bdf <= pcie_pmu->bdf_max; +} + +static bool hisi_pcie_pmu_valid_filter(struct perf_event *event, + struct hisi_pcie_pmu *pcie_pmu) +{ + u32 requester_id = hisi_pcie_get_bdf(event); + + if (hisi_pcie_get_thr_len(event) > HISI_PCIE_THR_MAX_VAL) + return false; + + if (hisi_pcie_get_trig_len(event) > HISI_PCIE_TRIG_MAX_VAL) + return false; + + if (requester_id) { + if (!hisi_pcie_pmu_valid_requester_id(pcie_pmu, requester_id)) + return false; + } + + return true; +} + +static bool hisi_pcie_pmu_cmp_event(struct perf_event *target, + struct perf_event *event) +{ + return hisi_pcie_get_real_event(target) == hisi_pcie_get_real_event(event); +} + +static bool hisi_pcie_pmu_validate_event_group(struct perf_event *event) +{ + struct perf_event *sibling, *leader = event->group_leader; + struct perf_event *event_group[HISI_PCIE_MAX_COUNTERS]; + int counters = 1; + int num; + + event_group[0] = leader; + if (!is_software_event(leader)) { + if (leader->pmu != event->pmu) + return false; + + if (leader != event && !hisi_pcie_pmu_cmp_event(leader, event)) + event_group[counters++] = event; + } + + for_each_sibling_event(sibling, event->group_leader) { + if (is_software_event(sibling)) + continue; + + if (sibling->pmu != event->pmu) + return false; + + for (num = 0; num < counters; num++) { + if (hisi_pcie_pmu_cmp_event(event_group[num], sibling)) + break; + } + + if (num == counters) + event_group[counters++] = sibling; + } + + return counters <= HISI_PCIE_MAX_COUNTERS; +} + +static int hisi_pcie_pmu_event_init(struct perf_event *event) +{ + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); + struct hw_perf_event *hwc = &event->hw; + + event->cpu = pcie_pmu->on_cpu; + + if (EXT_COUNTER_IS_USED(hisi_pcie_get_event(event))) + hwc->event_base = HISI_PCIE_EXT_CNT; + else + hwc->event_base = HISI_PCIE_CNT; + + if (event->attr.type != event->pmu->type) + return -ENOENT; + + /* Sampling is not supported. */ + if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK) + return -EOPNOTSUPP; + + if (!hisi_pcie_pmu_valid_filter(event, pcie_pmu)) + return -EINVAL; + + if (!hisi_pcie_pmu_validate_event_group(event)) + return -EINVAL; + + return 0; +} + +static u64 hisi_pcie_pmu_read_counter(struct perf_event *event) +{ + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); + u32 idx = event->hw.idx; + + return hisi_pcie_pmu_readq(pcie_pmu, event->hw.event_base, idx); +} + +static int hisi_pcie_pmu_find_related_event(struct hisi_pcie_pmu *pcie_pmu, + struct perf_event *event) +{ + struct perf_event *sibling; + int idx; + + for (idx = 0; idx < HISI_PCIE_MAX_COUNTERS; idx++) { + sibling = pcie_pmu->hw_events[idx]; + if (!sibling) + continue; + + if (!hisi_pcie_pmu_cmp_event(sibling, event)) + continue; + + /* Related events must be used in group */ + if (sibling->group_leader == event->group_leader) + return idx; + else + return -EINVAL; + } + + return idx; +} + +static int hisi_pcie_pmu_get_event_idx(struct hisi_pcie_pmu *pcie_pmu) +{ + int idx; + + for (idx = 0; idx < HISI_PCIE_MAX_COUNTERS; idx++) { + if (!pcie_pmu->hw_events[idx]) + return idx; + } + + return -EINVAL; +} + +static void hisi_pcie_pmu_event_update(struct perf_event *event) +{ + struct hw_perf_event *hwc = &event->hw; + u64 new_cnt, prev_cnt, delta; + + do { + prev_cnt = local64_read(&hwc->prev_count); + new_cnt = hisi_pcie_pmu_read_counter(event); + } while (local64_cmpxchg(&hwc->prev_count, prev_cnt, + new_cnt) != prev_cnt); + + delta = (new_cnt - prev_cnt) & HISI_PCIE_MAX_PERIOD; + local64_add(delta, &event->count); +} + +static void hisi_pcie_pmu_read(struct perf_event *event) +{ + hisi_pcie_pmu_event_update(event); +} + +static void hisi_pcie_pmu_set_period(struct perf_event *event) +{ + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); + struct hw_perf_event *hwc = &event->hw; + int idx = hwc->idx; + + local64_set(&hwc->prev_count, HISI_PCIE_INIT_VAL); + hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_CNT, idx, HISI_PCIE_INIT_VAL); + hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EXT_CNT, idx, HISI_PCIE_INIT_VAL); +} + +static void hisi_pcie_pmu_enable_counter(struct hisi_pcie_pmu *pcie_pmu, struct hw_perf_event *hwc) +{ + u32 idx = hwc->idx; + u64 val; + + val = hisi_pcie_pmu_readq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx); + val |= HISI_PCIE_EVENT_EN; + hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx, val); +} + +static void hisi_pcie_pmu_disable_counter(struct hisi_pcie_pmu *pcie_pmu, struct hw_perf_event *hwc) +{ + u32 idx = hwc->idx; + u64 val; + + val = hisi_pcie_pmu_readq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx); + val &= ~HISI_PCIE_EVENT_EN; + hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx, val); +} + +static void hisi_pcie_pmu_enable_int(struct hisi_pcie_pmu *pcie_pmu, struct hw_perf_event *hwc) +{ + u32 idx = hwc->idx; + + hisi_pcie_pmu_writel(pcie_pmu, HISI_PCIE_INT_MASK, idx, 0); +} + +static void hisi_pcie_pmu_disable_int(struct hisi_pcie_pmu *pcie_pmu, struct hw_perf_event *hwc) +{ + u32 idx = hwc->idx; + + hisi_pcie_pmu_writel(pcie_pmu, HISI_PCIE_INT_MASK, idx, 1); +} + +static void hisi_pcie_pmu_reset_counter(struct hisi_pcie_pmu *pcie_pmu, int idx) +{ + hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx, HISI_PCIE_RESET_CNT); + hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx, HISI_PCIE_INIT_SET); +} + +static void hisi_pcie_pmu_start(struct perf_event *event, int flags) +{ + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); + struct hw_perf_event *hwc = &event->hw; + int idx = hwc->idx; + u64 prev_cnt; + + if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED))) + return; + + WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); + hwc->state = 0; + + hisi_pcie_pmu_config_filter(event); + hisi_pcie_pmu_enable_counter(pcie_pmu, hwc); + hisi_pcie_pmu_enable_int(pcie_pmu, hwc); + hisi_pcie_pmu_set_period(event); + + if (flags & PERF_EF_RELOAD) { + prev_cnt = local64_read(&hwc->prev_count); + hisi_pcie_pmu_writeq(pcie_pmu, hwc->event_base, idx, prev_cnt); + } + + perf_event_update_userpage(event); +} + +static void hisi_pcie_pmu_stop(struct perf_event *event, int flags) +{ + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); + struct hw_perf_event *hwc = &event->hw; + + hisi_pcie_pmu_event_update(event); + hisi_pcie_pmu_disable_int(pcie_pmu, hwc); + hisi_pcie_pmu_disable_counter(pcie_pmu, hwc); + hisi_pcie_pmu_clear_filter(event); + WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); + hwc->state |= PERF_HES_STOPPED; + + if (hwc->state & PERF_HES_UPTODATE) + return; + + hwc->state |= PERF_HES_UPTODATE; +} + +static int hisi_pcie_pmu_add(struct perf_event *event, int flags) +{ + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); + struct hw_perf_event *hwc = &event->hw; + int idx; + + hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; + + /* Check all working events to find a related event. */ + idx = hisi_pcie_pmu_find_related_event(pcie_pmu, event); + if (idx < 0) + return idx; + + /* Current event shares an enabled counter with the related event */ + if (idx < HISI_PCIE_MAX_COUNTERS) { + hwc->idx = idx; + goto start_count; + } + + idx = hisi_pcie_pmu_get_event_idx(pcie_pmu); + if (idx < 0) + return idx; + + hwc->idx = idx; + pcie_pmu->hw_events[idx] = event; + /* Reset Counter to avoid previous statistic interference. */ + hisi_pcie_pmu_reset_counter(pcie_pmu, idx); + +start_count: + if (flags & PERF_EF_START) + hisi_pcie_pmu_start(event, PERF_EF_RELOAD); + + return 0; +} + +static void hisi_pcie_pmu_del(struct perf_event *event, int flags) +{ + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); + struct hw_perf_event *hwc = &event->hw; + + hisi_pcie_pmu_stop(event, PERF_EF_UPDATE); + pcie_pmu->hw_events[hwc->idx] = NULL; + perf_event_update_userpage(event); +} + +static void hisi_pcie_pmu_enable(struct pmu *pmu) +{ + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(pmu); + int num; + + for (num = 0; num < HISI_PCIE_MAX_COUNTERS; num++) { + if (pcie_pmu->hw_events[num]) + break; + } + + if (num == HISI_PCIE_MAX_COUNTERS) + return; + + writel(HISI_PCIE_GLOBAL_EN, pcie_pmu->base + HISI_PCIE_GLOBAL_CTRL); +} + +static void hisi_pcie_pmu_disable(struct pmu *pmu) +{ + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(pmu); + + writel(HISI_PCIE_GLOBAL_NONE, pcie_pmu->base + HISI_PCIE_GLOBAL_CTRL); +} + +static irqreturn_t hisi_pcie_pmu_irq(int irq, void *data) +{ + struct hisi_pcie_pmu *pcie_pmu = data; + irqreturn_t ret = IRQ_NONE; + struct perf_event *event; + u32 overflown; + int idx; + + for (idx = 0; idx < HISI_PCIE_MAX_COUNTERS; idx++) { + overflown = hisi_pcie_pmu_readl(pcie_pmu, HISI_PCIE_INT_STAT, idx); + if (!overflown) + continue; + + /* Clear status of interrupt. */ + hisi_pcie_pmu_writel(pcie_pmu, HISI_PCIE_INT_STAT, idx, 1); + event = pcie_pmu->hw_events[idx]; + if (!event) + continue; + + hisi_pcie_pmu_event_update(event); + hisi_pcie_pmu_set_period(event); + ret = IRQ_HANDLED; + } + + return ret; +} + +static int hisi_pcie_pmu_irq_register(struct pci_dev *pdev, struct hisi_pcie_pmu *pcie_pmu) +{ + int irq, ret; + + ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI); + if (ret < 0) { + pci_err(pdev, "Failed to enable MSI vectors: %d\n", ret); + return ret; + } + + irq = pci_irq_vector(pdev, 0); + ret = request_irq(irq, hisi_pcie_pmu_irq, IRQF_NOBALANCING | IRQF_NO_THREAD, DRV_NAME, + pcie_pmu); + if (ret) { + pci_err(pdev, "Failed to register IRQ: %d\n", ret); + pci_free_irq_vectors(pdev); + return ret; + } + + pcie_pmu->irq = irq; + + return 0; +} + +static void hisi_pcie_pmu_irq_unregister(struct pci_dev *pdev, struct hisi_pcie_pmu *pcie_pmu) +{ + free_irq(pcie_pmu->irq, pcie_pmu); + pci_free_irq_vectors(pdev); +} + +static int hisi_pcie_pmu_online_cpu(unsigned int cpu, struct hlist_node *node) +{ + struct hisi_pcie_pmu *pcie_pmu = hlist_entry_safe(node, struct hisi_pcie_pmu, node); + + if (pcie_pmu->on_cpu == -1) { + pcie_pmu->on_cpu = cpu; + WARN_ON(irq_set_affinity(pcie_pmu->irq, cpumask_of(cpu))); + } + + return 0; +} + +static int hisi_pcie_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) +{ + struct hisi_pcie_pmu *pcie_pmu = hlist_entry_safe(node, struct hisi_pcie_pmu, node); + unsigned int target; + + /* Nothing to do if this CPU doesn't own the PMU */ + if (pcie_pmu->on_cpu != cpu) + return 0; + + pcie_pmu->on_cpu = -1; + /* Choose a new CPU from all online cpus. */ + target = cpumask_any_but(cpu_online_mask, cpu); + if (target >= nr_cpu_ids) { + pci_err(pcie_pmu->pdev, "There is no CPU to set\n"); + return 0; + } + + perf_pmu_migrate_context(&pcie_pmu->pmu, cpu, target); + /* Use this CPU for event counting */ + pcie_pmu->on_cpu = target; + WARN_ON(irq_set_affinity(pcie_pmu->irq, cpumask_of(target))); + + return 0; +} + +static struct attribute *hisi_pcie_pmu_events_attr[] = { + HISI_PCIE_PMU_EVENT_ATTR(rx_mwr_latency, 0x0010), + HISI_PCIE_PMU_EVENT_ATTR(rx_mwr_cnt, 0x10010), + HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_latency, 0x0210), + HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_cnt, 0x10210), + HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_latency, 0x0011), + HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_cnt, 0x10011), + HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_flux, 0x0804), + HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_time, 0x10804), + HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_flux, 0x0405), + HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_time, 0x10405), + NULL +}; + +static struct attribute_group hisi_pcie_pmu_events_group = { + .name = "events", + .attrs = hisi_pcie_pmu_events_attr, +}; + +static struct attribute *hisi_pcie_pmu_format_attr[] = { + HISI_PCIE_PMU_FORMAT_ATTR(event, "config:0-16"), + HISI_PCIE_PMU_FORMAT_ATTR(thr_len, "config1:0-3"), + HISI_PCIE_PMU_FORMAT_ATTR(thr_mode, "config1:4"), + HISI_PCIE_PMU_FORMAT_ATTR(trig_len, "config1:5-8"), + HISI_PCIE_PMU_FORMAT_ATTR(trig_mode, "config1:9"), + HISI_PCIE_PMU_FORMAT_ATTR(len_mode, "config1:10-11"), + HISI_PCIE_PMU_FORMAT_ATTR(port, "config2:0-15"), + HISI_PCIE_PMU_FORMAT_ATTR(bdf, "config2:16-31"), + NULL +}; + +static const struct attribute_group hisi_pcie_pmu_format_group = { + .name = "format", + .attrs = hisi_pcie_pmu_format_attr, +}; + +static struct attribute *hisi_pcie_pmu_bus_attrs[] = { + &dev_attr_bus.attr, + NULL +}; + +static const struct attribute_group hisi_pcie_pmu_bus_attr_group = { + .attrs = hisi_pcie_pmu_bus_attrs, +}; + +static struct attribute *hisi_pcie_pmu_cpumask_attrs[] = { + &dev_attr_cpumask.attr, + NULL +}; + +static const struct attribute_group hisi_pcie_pmu_cpumask_attr_group = { + .attrs = hisi_pcie_pmu_cpumask_attrs, +}; + +static struct attribute *hisi_pcie_pmu_identifier_attrs[] = { + &dev_attr_identifier.attr, + NULL +}; + +static const struct attribute_group hisi_pcie_pmu_identifier_attr_group = { + .attrs = hisi_pcie_pmu_identifier_attrs, +}; + +static const struct attribute_group *hisi_pcie_pmu_attr_groups[] = { + &hisi_pcie_pmu_events_group, + &hisi_pcie_pmu_format_group, + &hisi_pcie_pmu_bus_attr_group, + &hisi_pcie_pmu_cpumask_attr_group, + &hisi_pcie_pmu_identifier_attr_group, + NULL +}; + +static int hisi_pcie_alloc_pmu(struct pci_dev *pdev, struct hisi_pcie_pmu *pcie_pmu) +{ + struct hisi_pcie_reg_pair regs; + u16 sicl_id, core_id; + char *name; + + regs = hisi_pcie_parse_reg_value(pcie_pmu, HISI_PCIE_REG_BDF); + pcie_pmu->bdf_min = regs.lo; + pcie_pmu->bdf_max = regs.hi; + + regs = hisi_pcie_parse_reg_value(pcie_pmu, HISI_PCIE_REG_INFO); + sicl_id = regs.hi; + core_id = regs.lo; + + name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "hisi_pcie%u_core%u", sicl_id, core_id); + if (!name) + return -ENOMEM; + + pcie_pmu->pdev = pdev; + pcie_pmu->on_cpu = -1; + pcie_pmu->identifier = readl(pcie_pmu->base + HISI_PCIE_REG_VERSION); + pcie_pmu->pmu = (struct pmu) { + .name = name, + .module = THIS_MODULE, + .event_init = hisi_pcie_pmu_event_init, + .pmu_enable = hisi_pcie_pmu_enable, + .pmu_disable = hisi_pcie_pmu_disable, + .add = hisi_pcie_pmu_add, + .del = hisi_pcie_pmu_del, + .start = hisi_pcie_pmu_start, + .stop = hisi_pcie_pmu_stop, + .read = hisi_pcie_pmu_read, + .task_ctx_nr = perf_invalid_context, + .attr_groups = hisi_pcie_pmu_attr_groups, + .capabilities = PERF_PMU_CAP_NO_EXCLUDE, + }; + + return 0; +} + +static int hisi_pcie_init_pmu(struct pci_dev *pdev, struct hisi_pcie_pmu *pcie_pmu) +{ + int ret; + + pcie_pmu->base = pci_ioremap_bar(pdev, 2); + if (!pcie_pmu->base) { + pci_err(pdev, "Ioremap failed for pcie_pmu resource\n"); + return -ENOMEM; + } + + ret = hisi_pcie_alloc_pmu(pdev, pcie_pmu); + if (ret) + goto err_iounmap; + + ret = hisi_pcie_pmu_irq_register(pdev, pcie_pmu); + if (ret) + goto err_iounmap; + + ret = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE, &pcie_pmu->node); + if (ret) { + pci_err(pdev, "Failed to register hotplug: %d\n", ret); + goto err_irq_unregister; + } + + ret = perf_pmu_register(&pcie_pmu->pmu, pcie_pmu->pmu.name, -1); + if (ret) { + pci_err(pdev, "Failed to register PCIe PMU: %d\n", ret); + goto err_hotplug_unregister; + } + + return ret; + +err_hotplug_unregister: + cpuhp_state_remove_instance_nocalls( + CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE, &pcie_pmu->node); + +err_irq_unregister: + hisi_pcie_pmu_irq_unregister(pdev, pcie_pmu); + +err_iounmap: + iounmap(pcie_pmu->base); + + return ret; +} + +static void hisi_pcie_uninit_pmu(struct pci_dev *pdev) +{ + struct hisi_pcie_pmu *pcie_pmu = pci_get_drvdata(pdev); + + perf_pmu_unregister(&pcie_pmu->pmu); + cpuhp_state_remove_instance_nocalls( + CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE, &pcie_pmu->node); + hisi_pcie_pmu_irq_unregister(pdev, pcie_pmu); + iounmap(pcie_pmu->base); +} + +static int hisi_pcie_init_dev(struct pci_dev *pdev) +{ + int ret; + + ret = pcim_enable_device(pdev); + if (ret) { + pci_err(pdev, "Failed to enable PCI device: %d\n", ret); + return ret; + } + + ret = pcim_iomap_regions(pdev, BIT(2), DRV_NAME); + if (ret < 0) { + pci_err(pdev, "Failed to request PCI mem regions: %d\n", ret); + return ret; + } + + pci_set_master(pdev); + + return 0; +} + +static int hisi_pcie_pmu_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct hisi_pcie_pmu *pcie_pmu; + int ret; + + pcie_pmu = devm_kzalloc(&pdev->dev, sizeof(*pcie_pmu), GFP_KERNEL); + if (!pcie_pmu) + return -ENOMEM; + + ret = hisi_pcie_init_dev(pdev); + if (ret) + return ret; + + ret = hisi_pcie_init_pmu(pdev, pcie_pmu); + if (ret) + return ret; + + pci_set_drvdata(pdev, pcie_pmu); + + return ret; +} + +static void hisi_pcie_pmu_remove(struct pci_dev *pdev) +{ + hisi_pcie_uninit_pmu(pdev); + pci_set_drvdata(pdev, NULL); +} + +static const struct pci_device_id hisi_pcie_pmu_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa12d) }, + { 0, } +}; +MODULE_DEVICE_TABLE(pci, hisi_pcie_pmu_ids); + +static struct pci_driver hisi_pcie_pmu_driver = { + .name = DRV_NAME, + .id_table = hisi_pcie_pmu_ids, + .probe = hisi_pcie_pmu_probe, + .remove = hisi_pcie_pmu_remove, +}; + +static int __init hisi_pcie_module_init(void) +{ + int ret; + + ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE, + "AP_PERF_ARM_HISI_PCIE_PMU_ONLINE", + hisi_pcie_pmu_online_cpu, + hisi_pcie_pmu_offline_cpu); + if (ret) { + pr_err("Failed to setup PCIe PMU hotplug: %d\n", ret); + return ret; + } + + ret = pci_register_driver(&hisi_pcie_pmu_driver); + if (ret) + cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE); + + return ret; +} +module_init(hisi_pcie_module_init); + +static void __exit hisi_pcie_module_exit(void) +{ + pci_unregister_driver(&hisi_pcie_pmu_driver); + cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE); +} +module_exit(hisi_pcie_module_exit); + +MODULE_DESCRIPTION("HiSilicon PCIe PMU driver"); +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Qi Liu "); diff --git a/drivers/scsi/aic94xx/aic94xx_scb.c b/drivers/scsi/aic94xx/aic94xx_scb.c index 4a80ec08f0c968c26d0599659b30c38b1da1fd0f..cdeebd2ffe22593a10a8a7cadfc4d975d2bc91eb 100644 --- a/drivers/scsi/aic94xx/aic94xx_scb.c +++ b/drivers/scsi/aic94xx/aic94xx_scb.c @@ -68,7 +68,6 @@ static void asd_phy_event_tasklet(struct asd_ascb *ascb, struct done_list_struct *dl) { struct asd_ha_struct *asd_ha = ascb->ha; - struct sas_ha_struct *sas_ha = &asd_ha->sas_ha; int phy_id = dl->status_block[0] & DL_PHY_MASK; struct asd_phy *phy = &asd_ha->phys[phy_id]; @@ -81,7 +80,8 @@ static void asd_phy_event_tasklet(struct asd_ascb *ascb, ASD_DPRINTK("phy%d: device unplugged\n", phy_id); asd_turn_led(asd_ha, phy_id, 0); sas_phy_disconnected(&phy->sas_phy); - sas_ha->notify_phy_event(&phy->sas_phy, PHYE_LOSS_OF_SIGNAL); + sas_notify_phy_event(&phy->sas_phy, PHYE_LOSS_OF_SIGNAL, + GFP_ATOMIC); break; case CURRENT_OOB_DONE: /* hot plugged device */ @@ -89,12 +89,13 @@ static void asd_phy_event_tasklet(struct asd_ascb *ascb, get_lrate_mode(phy, oob_mode); ASD_DPRINTK("phy%d device plugged: lrate:0x%x, proto:0x%x\n", phy_id, phy->sas_phy.linkrate, phy->sas_phy.iproto); - sas_ha->notify_phy_event(&phy->sas_phy, PHYE_OOB_DONE); + sas_notify_phy_event(&phy->sas_phy, PHYE_OOB_DONE, GFP_ATOMIC); break; case CURRENT_SPINUP_HOLD: /* hot plug SATA, no COMWAKE sent */ asd_turn_led(asd_ha, phy_id, 1); - sas_ha->notify_phy_event(&phy->sas_phy, PHYE_SPINUP_HOLD); + sas_notify_phy_event(&phy->sas_phy, PHYE_SPINUP_HOLD, + GFP_ATOMIC); break; case CURRENT_GTO_TIMEOUT: case CURRENT_OOB_ERROR: @@ -102,7 +103,7 @@ static void asd_phy_event_tasklet(struct asd_ascb *ascb, dl->status_block[1]); asd_turn_led(asd_ha, phy_id, 0); sas_phy_disconnected(&phy->sas_phy); - sas_ha->notify_phy_event(&phy->sas_phy, PHYE_OOB_ERROR); + sas_notify_phy_event(&phy->sas_phy, PHYE_OOB_ERROR, GFP_ATOMIC); break; } } @@ -222,7 +223,6 @@ static void asd_bytes_dmaed_tasklet(struct asd_ascb *ascb, int edb_el = edb_id + ascb->edb_index; struct asd_dma_tok *edb = ascb->ha->seq.edb_arr[edb_el]; struct asd_phy *phy = &ascb->ha->phys[phy_id]; - struct sas_ha_struct *sas_ha = phy->sas_phy.ha; u16 size = ((dl->status_block[3] & 7) << 8) | dl->status_block[2]; size = min(size, (u16) sizeof(phy->frame_rcvd)); @@ -234,7 +234,7 @@ static void asd_bytes_dmaed_tasklet(struct asd_ascb *ascb, spin_unlock_irqrestore(&phy->sas_phy.frame_rcvd_lock, flags); asd_dump_frame_rcvd(phy, dl); asd_form_port(ascb->ha, phy); - sas_ha->notify_port_event(&phy->sas_phy, PORTE_BYTES_DMAED); + sas_notify_port_event(&phy->sas_phy, PORTE_BYTES_DMAED, GFP_ATOMIC); } static void asd_link_reset_err_tasklet(struct asd_ascb *ascb, @@ -270,7 +270,7 @@ static void asd_link_reset_err_tasklet(struct asd_ascb *ascb, asd_turn_led(asd_ha, phy_id, 0); sas_phy_disconnected(sas_phy); asd_deform_port(asd_ha, phy); - sas_ha->notify_port_event(sas_phy, PORTE_LINK_RESET_ERR); + sas_notify_port_event(sas_phy, PORTE_LINK_RESET_ERR, GFP_ATOMIC); if (retries_left == 0) { int num = 1; @@ -315,7 +315,8 @@ static void asd_primitive_rcvd_tasklet(struct asd_ascb *ascb, spin_lock_irqsave(&sas_phy->sas_prim_lock, flags); sas_phy->sas_prim = ffs(cont); spin_unlock_irqrestore(&sas_phy->sas_prim_lock, flags); - sas_ha->notify_port_event(sas_phy,PORTE_BROADCAST_RCVD); + sas_notify_port_event(sas_phy, PORTE_BROADCAST_RCVD, + GFP_ATOMIC); break; case LmUNKNOWNP: @@ -336,7 +337,8 @@ static void asd_primitive_rcvd_tasklet(struct asd_ascb *ascb, /* The sequencer disables all phys on that port. * We have to re-enable the phys ourselves. */ asd_deform_port(asd_ha, phy); - sas_ha->notify_port_event(sas_phy, PORTE_HARD_RESET); + sas_notify_port_event(sas_phy, PORTE_HARD_RESET, + GFP_ATOMIC); break; default: @@ -567,7 +569,7 @@ static void escb_tasklet_complete(struct asd_ascb *ascb, /* the device is gone */ sas_phy_disconnected(sas_phy); asd_deform_port(asd_ha, phy); - sas_ha->notify_port_event(sas_phy, PORTE_TIMER_EVENT); + sas_notify_port_event(sas_phy, PORTE_TIMER_EVENT, GFP_ATOMIC); break; default: ASD_DPRINTK("%s: phy%d: unknown event:0x%x\n", __func__, diff --git a/drivers/scsi/hisi_sas/Kconfig b/drivers/scsi/hisi_sas/Kconfig index 90a17452a50dc3beabf87ce2334e9c1f2c39a06f..4ba3a8eadb7735bacc258a024257f826c0d3810c 100644 --- a/drivers/scsi/hisi_sas/Kconfig +++ b/drivers/scsi/hisi_sas/Kconfig @@ -6,6 +6,7 @@ config SCSI_HISI_SAS select SCSI_SAS_LIBSAS select BLK_DEV_INTEGRITY depends on ATA + select SATA_HOST help This driver supports HiSilicon's SAS HBA, including support based on platform device @@ -14,5 +15,12 @@ config SCSI_HISI_SAS_PCI tristate "HiSilicon SAS on PCI bus" depends on SCSI_HISI_SAS depends on PCI + depends on ACPI help This driver supports HiSilicon's SAS HBA based on PCI device + +config SCSI_HISI_SAS_DEBUGFS_DEFAULT_ENABLE + bool "HiSilicon SAS debugging default enable" + depends on SCSI_HISI_SAS + help + Set Y to default enable DEBUGFS for SCSI_HISI_SAS diff --git a/drivers/scsi/hisi_sas/hisi_sas.h b/drivers/scsi/hisi_sas/hisi_sas.h index 720c4d6be939df480265e6db0b8bd54727b34090..62f86b9f1ad41fb5d4bb17aa05ce6b669a59d4ff 100644 --- a/drivers/scsi/hisi_sas/hisi_sas.h +++ b/drivers/scsi/hisi_sas/hisi_sas.h @@ -19,8 +19,10 @@ #include #include #include +#include #include #include +#include #include #include @@ -31,6 +33,8 @@ #define HISI_SAS_MAX_DEVICES HISI_SAS_MAX_ITCT_ENTRIES #define HISI_SAS_RESET_BIT 0 #define HISI_SAS_REJECT_CMD_BIT 1 +#define HISI_SAS_PM_BIT 2 +#define HISI_SAS_HW_FAULT_BIT 3 #define HISI_SAS_MAX_COMMANDS (HISI_SAS_QUEUE_SLOTS) #define HISI_SAS_RESERVED_IPTT 96 #define HISI_SAS_UNRESERVED_IPTT \ @@ -38,6 +42,7 @@ #define HISI_SAS_IOST_ITCT_CACHE_NUM 64 #define HISI_SAS_IOST_ITCT_CACHE_DW_SZ 10 +#define HISI_SAS_FIFO_DATA_DW_SIZE 32 #define HISI_SAS_STATUS_BUF_SZ (sizeof(struct hisi_sas_status_buffer)) #define HISI_SAS_COMMAND_TABLE_SZ (sizeof(union hisi_sas_command_table)) @@ -83,7 +88,8 @@ #define HISI_SAS_PROT_MASK (HISI_SAS_DIF_PROT_MASK | HISI_SAS_DIX_PROT_MASK) -#define HISI_SAS_WAIT_PHYUP_TIMEOUT 20 +#define HISI_SAS_WAIT_PHYUP_TIMEOUT (30 * HZ) +#define HISI_SAS_CLEAR_ITCT_TIMEOUT (20 * HZ) struct hisi_hba; @@ -95,6 +101,7 @@ enum { enum dev_status { HISI_SAS_DEV_INIT, HISI_SAS_DEV_NORMAL, + HISI_SAS_DEV_NCQ_ERR, }; enum { @@ -144,9 +151,20 @@ enum hisi_sas_bit_err_type { enum hisi_sas_phy_event { HISI_PHYE_PHY_UP = 0U, HISI_PHYE_LINK_RESET, + HISI_PHYE_PHY_UP_PM, HISI_PHYES_NUM, }; +struct hisi_sas_debugfs_fifo { + u32 signal_sel; + u32 dump_msk; + u32 dump_mode; + u32 trigger; + u32 trigger_msk; + u32 trigger_mode; + u32 rd_data[HISI_SAS_FIFO_DATA_DW_SIZE]; +}; + struct hisi_sas_phy { struct work_struct works[HISI_PHYES_NUM]; struct hisi_hba *hisi_hba; @@ -167,6 +185,11 @@ struct hisi_sas_phy { enum sas_linkrate minimum_linkrate; enum sas_linkrate maximum_linkrate; int enable; + int wait_phyup_cnt; + atomic_t down_cnt; + + /* Trace FIFO */ + struct hisi_sas_debugfs_fifo fifo; }; struct hisi_sas_port { @@ -177,10 +200,10 @@ struct hisi_sas_port { struct hisi_sas_cq { struct hisi_hba *hisi_hba; - const struct cpumask *pci_irq_mask; - struct tasklet_struct tasklet; + const struct cpumask *irq_mask; int rd_point; int id; + int irq_no; }; struct hisi_sas_dq { @@ -236,24 +259,6 @@ struct hisi_sas_slot { u16 idx; }; -#define HISI_SAS_DEBUGFS_REG(x) {#x, x} - -struct hisi_sas_debugfs_reg_lu { - char *name; - int off; -}; - -struct hisi_sas_debugfs_reg { - const struct hisi_sas_debugfs_reg_lu *lu; - int count; - int base_off; - union { - u32 (*read_global_reg)(struct hisi_hba *hisi_hba, u32 off); - u32 (*read_port_reg)(struct hisi_hba *hisi_hba, int port, - u32 off); - }; -}; - struct hisi_sas_iost_itct_cache { u32 data[HISI_SAS_IOST_ITCT_CACHE_DW_SZ]; }; @@ -270,6 +275,39 @@ enum hisi_sas_debugfs_cache_type { HISI_SAS_IOST_CACHE, }; +enum hisi_sas_debugfs_bist_ffe_cfg { + FFE_SAS_1_5_GBPS, + FFE_SAS_3_0_GBPS, + FFE_SAS_6_0_GBPS, + FFE_SAS_12_0_GBPS, + FFE_RESV, + FFE_SATA_1_5_GBPS, + FFE_SATA_3_0_GBPS, + FFE_SATA_6_0_GBPS, + FFE_CFG_MAX +}; + +enum hisi_sas_debugfs_bist_fixed_code { + FIXED_CODE, + FIXED_CODE_1, + FIXED_CODE_MAX +}; + +enum { + HISI_SAS_BIST_CODE_MODE_PRBS7, + HISI_SAS_BIST_CODE_MODE_PRBS23, + HISI_SAS_BIST_CODE_MODE_PRBS31, + HISI_SAS_BIST_CODE_MODE_JTPAT, + HISI_SAS_BIST_CODE_MODE_CJTPAT, + HISI_SAS_BIST_CODE_MODE_SCRAMBED_0, + HISI_SAS_BIST_CODE_MODE_TRAIN, + HISI_SAS_BIST_CODE_MODE_TRAIN_DONE, + HISI_SAS_BIST_CODE_MODE_HFTP, + HISI_SAS_BIST_CODE_MODE_MFTP, + HISI_SAS_BIST_CODE_MODE_LFTP, + HISI_SAS_BIST_CODE_MODE_FIXED_DATA, +}; + struct hisi_sas_hw { int (*hw_init)(struct hisi_hba *hisi_hba); void (*setup_itct)(struct hisi_hba *hisi_hba, @@ -296,8 +334,8 @@ struct hisi_sas_hw { void (*phy_set_linkrate)(struct hisi_hba *hisi_hba, int phy_no, struct sas_phy_linkrates *linkrates); enum sas_linkrate (*phy_get_max_linkrate)(void); - void (*clear_itct)(struct hisi_hba *hisi_hba, - struct hisi_sas_device *dev); + int (*clear_itct)(struct hisi_hba *hisi_hba, + struct hisi_sas_device *dev); void (*free_device)(struct hisi_sas_device *sas_dev); int (*get_wideport_bitmap)(struct hisi_hba *hisi_hba, int port_id); void (*dereg_device)(struct hisi_hba *hisi_hba, @@ -308,17 +346,47 @@ struct hisi_sas_hw { u8 reg_index, u8 reg_count, u8 *write_data); void (*wait_cmds_complete_timeout)(struct hisi_hba *hisi_hba, int delay_ms, int timeout_ms); - void (*snapshot_prepare)(struct hisi_hba *hisi_hba); - void (*snapshot_restore)(struct hisi_hba *hisi_hba); - int (*set_bist)(struct hisi_hba *hisi_hba, bool enable); - void (*read_iost_itct_cache)(struct hisi_hba *hisi_hba, - enum hisi_sas_debugfs_cache_type type, - u32 *cache); + int (*debugfs_snapshot_regs)(struct hisi_hba *hisi_hba); int complete_hdr_size; struct scsi_host_template *sht; +}; + +#define HISI_SAS_MAX_DEBUGFS_DUMP (50) - const struct hisi_sas_debugfs_reg *debugfs_reg_array[DEBUGFS_REGS_NUM]; - const struct hisi_sas_debugfs_reg *debugfs_reg_port; +struct hisi_sas_debugfs_cq { + struct hisi_sas_cq *cq; + void *complete_hdr; +}; + +struct hisi_sas_debugfs_dq { + struct hisi_sas_dq *dq; + struct hisi_sas_cmd_hdr *hdr; +}; + +struct hisi_sas_debugfs_regs { + struct hisi_hba *hisi_hba; + u32 *data; +}; + +struct hisi_sas_debugfs_port { + struct hisi_sas_phy *phy; + u32 *data; +}; + +struct hisi_sas_debugfs_iost { + struct hisi_sas_iost *iost; +}; + +struct hisi_sas_debugfs_itct { + struct hisi_sas_itct *itct; +}; + +struct hisi_sas_debugfs_iost_cache { + struct hisi_sas_iost_itct_cache *cache; +}; + +struct hisi_sas_debugfs_itct_cache { + struct hisi_sas_iost_itct_cache *cache; }; struct hisi_hba { @@ -384,7 +452,6 @@ struct hisi_hba { const struct hisi_sas_hw *hw; /* Low level hw interface */ unsigned long sata_dev_bitmap[BITS_TO_LONGS(HISI_SAS_MAX_DEVICES)]; struct work_struct rst_work; - struct work_struct debugfs_work; u32 phy_state; u32 intr_coal_ticks; /* Time of interrupt coalesce in us */ u32 intr_coal_count; /* Interrupt count to coalesce */ @@ -399,22 +466,26 @@ struct hisi_hba { int debugfs_bist_mode; u32 debugfs_bist_cnt; int debugfs_bist_enable; + u32 debugfs_bist_ffe[HISI_SAS_MAX_PHYS][FFE_CFG_MAX]; + u32 debugfs_bist_fixed_code[FIXED_CODE_MAX]; /* debugfs memories */ /* Put Global AXI and RAS Register into register array */ - u32 *debugfs_regs[DEBUGFS_REGS_NUM]; - u32 *debugfs_port_reg[HISI_SAS_MAX_PHYS]; - void *debugfs_complete_hdr[HISI_SAS_MAX_QUEUES]; - struct hisi_sas_cmd_hdr *debugfs_cmd_hdr[HISI_SAS_MAX_QUEUES]; - struct hisi_sas_iost *debugfs_iost; - struct hisi_sas_itct *debugfs_itct; - u64 *debugfs_iost_cache; - u64 *debugfs_itct_cache; - + struct hisi_sas_debugfs_regs debugfs_regs[HISI_SAS_MAX_DEBUGFS_DUMP][DEBUGFS_REGS_NUM]; + struct hisi_sas_debugfs_port debugfs_port_reg[HISI_SAS_MAX_DEBUGFS_DUMP][HISI_SAS_MAX_PHYS]; + struct hisi_sas_debugfs_cq debugfs_cq[HISI_SAS_MAX_DEBUGFS_DUMP][HISI_SAS_MAX_QUEUES]; + struct hisi_sas_debugfs_dq debugfs_dq[HISI_SAS_MAX_DEBUGFS_DUMP][HISI_SAS_MAX_QUEUES]; + struct hisi_sas_debugfs_iost debugfs_iost[HISI_SAS_MAX_DEBUGFS_DUMP]; + struct hisi_sas_debugfs_itct debugfs_itct[HISI_SAS_MAX_DEBUGFS_DUMP]; + struct hisi_sas_debugfs_iost_cache debugfs_iost_cache[HISI_SAS_MAX_DEBUGFS_DUMP]; + struct hisi_sas_debugfs_itct_cache debugfs_itct_cache[HISI_SAS_MAX_DEBUGFS_DUMP]; + + u64 debugfs_timestamp[HISI_SAS_MAX_DEBUGFS_DUMP]; + int debugfs_dump_index; struct dentry *debugfs_dir; struct dentry *debugfs_dump_dentry; struct dentry *debugfs_bist_dentry; - bool debugfs_snapshot; + struct dentry *debugfs_fifo_dentry; }; /* Generic HW DMA host memory structures */ @@ -556,6 +627,7 @@ struct hisi_sas_slot_dif_buf_table { extern struct scsi_transport_template *hisi_sas_stt; extern bool hisi_sas_debugfs_enable; +extern u32 hisi_sas_debugfs_dump_count; extern struct dentry *hisi_sas_debugfs_dir; extern void hisi_sas_stop_phys(struct hisi_hba *hisi_hba); @@ -572,19 +644,22 @@ extern int hisi_sas_probe(struct platform_device *pdev, extern int hisi_sas_remove(struct platform_device *pdev); extern int hisi_sas_slave_configure(struct scsi_device *sdev); +extern int hisi_sas_slave_alloc(struct scsi_device *sdev); extern int hisi_sas_scan_finished(struct Scsi_Host *shost, unsigned long time); extern void hisi_sas_scan_start(struct Scsi_Host *shost); extern int hisi_sas_host_reset(struct Scsi_Host *shost, int reset_type); extern void hisi_sas_phy_enable(struct hisi_hba *hisi_hba, int phy_no, int enable); -extern void hisi_sas_phy_down(struct hisi_hba *hisi_hba, int phy_no, int rdy); +extern void hisi_sas_phy_down(struct hisi_hba *hisi_hba, int phy_no, int rdy, + gfp_t gfp_flags); extern void hisi_sas_slot_task_free(struct hisi_hba *hisi_hba, struct sas_task *task, - struct hisi_sas_slot *slot); + struct hisi_sas_slot *slot, + bool need_lock); extern void hisi_sas_init_mem(struct hisi_hba *hisi_hba); extern void hisi_sas_rst_work_handler(struct work_struct *work); extern void hisi_sas_sync_rst_work_handler(struct work_struct *work); -extern void hisi_sas_kill_tasklets(struct hisi_hba *hisi_hba); +extern void hisi_sas_sync_irqs(struct hisi_hba *hisi_hba); extern void hisi_sas_phy_oob_ready(struct hisi_hba *hisi_hba, int phy_no); extern bool hisi_sas_notify_phy_event(struct hisi_sas_phy *phy, enum hisi_sas_phy_event event); @@ -592,7 +667,4 @@ extern void hisi_sas_release_tasks(struct hisi_hba *hisi_hba); extern u8 hisi_sas_get_prog_phy_linkrate_mask(enum sas_linkrate max); extern void hisi_sas_controller_reset_prepare(struct hisi_hba *hisi_hba); extern void hisi_sas_controller_reset_done(struct hisi_hba *hisi_hba); -extern void hisi_sas_debugfs_init(struct hisi_hba *hisi_hba); -extern void hisi_sas_debugfs_exit(struct hisi_hba *hisi_hba); -extern void hisi_sas_debugfs_work_handler(struct work_struct *work); #endif diff --git a/drivers/scsi/hisi_sas/hisi_sas_main.c b/drivers/scsi/hisi_sas/hisi_sas_main.c index 7135bbe5abb8a2a247da5f5bc2c24d32d90bca52..fbf866af77e008f4443915c46caa42355aee63c3 100644 --- a/drivers/scsi/hisi_sas/hisi_sas_main.c +++ b/drivers/scsi/hisi_sas/hisi_sas_main.c @@ -15,7 +15,7 @@ static int hisi_sas_debug_issue_ssp_tmf(struct domain_device *device, static int hisi_sas_internal_task_abort(struct hisi_hba *hisi_hba, struct domain_device *device, - int abort_flag, int tag); + int abort_flag, int tag, bool rst_to_recover); static int hisi_sas_softreset_ata_disk(struct domain_device *device); static int hisi_sas_control_phy(struct asd_sas_phy *sas_phy, enum phy_func func, void *funcdata); @@ -163,13 +163,11 @@ static void hisi_sas_slot_index_clear(struct hisi_hba *hisi_hba, int slot_idx) static void hisi_sas_slot_index_free(struct hisi_hba *hisi_hba, int slot_idx) { - unsigned long flags; - if (hisi_hba->hw->slot_index_alloc || slot_idx >= HISI_SAS_UNRESERVED_IPTT) { - spin_lock_irqsave(&hisi_hba->lock, flags); + spin_lock(&hisi_hba->lock); hisi_sas_slot_index_clear(hisi_hba, slot_idx); - spin_unlock_irqrestore(&hisi_hba->lock, flags); + spin_unlock(&hisi_hba->lock); } } @@ -185,12 +183,11 @@ static int hisi_sas_slot_index_alloc(struct hisi_hba *hisi_hba, { int index; void *bitmap = hisi_hba->slot_index_tags; - unsigned long flags; if (scsi_cmnd) return scsi_cmnd->request->tag; - spin_lock_irqsave(&hisi_hba->lock, flags); + spin_lock(&hisi_hba->lock); index = find_next_zero_bit(bitmap, hisi_hba->slot_index_count, hisi_hba->last_slot_index + 1); if (index >= hisi_hba->slot_index_count) { @@ -198,13 +195,13 @@ static int hisi_sas_slot_index_alloc(struct hisi_hba *hisi_hba, hisi_hba->slot_index_count, HISI_SAS_UNRESERVED_IPTT); if (index >= hisi_hba->slot_index_count) { - spin_unlock_irqrestore(&hisi_hba->lock, flags); + spin_unlock(&hisi_hba->lock); return -SAS_QUEUE_FULL; } } hisi_sas_slot_index_set(hisi_hba, index); hisi_hba->last_slot_index = index; - spin_unlock_irqrestore(&hisi_hba->lock, flags); + spin_unlock(&hisi_hba->lock); return index; } @@ -218,9 +215,8 @@ static void hisi_sas_slot_index_init(struct hisi_hba *hisi_hba) } void hisi_sas_slot_task_free(struct hisi_hba *hisi_hba, struct sas_task *task, - struct hisi_sas_slot *slot) + struct hisi_sas_slot *slot, bool need_lock) { - unsigned long flags; int device_id = slot->device_id; struct hisi_sas_device *sas_dev = &hisi_hba->devices[device_id]; @@ -233,23 +229,28 @@ void hisi_sas_slot_task_free(struct hisi_hba *hisi_hba, struct sas_task *task, task->lldd_task = NULL; if (!sas_protocol_ata(task->task_proto)) { - struct sas_ssp_task *ssp_task = &task->ssp_task; - struct scsi_cmnd *scsi_cmnd = ssp_task->cmd; - if (slot->n_elem) dma_unmap_sg(dev, task->scatter, task->num_scatter, task->data_dir); - if (slot->n_elem_dif) + if (slot->n_elem_dif) { + struct sas_ssp_task *ssp_task = &task->ssp_task; + struct scsi_cmnd *scsi_cmnd = ssp_task->cmd; + dma_unmap_sg(dev, scsi_prot_sglist(scsi_cmnd), scsi_prot_sg_count(scsi_cmnd), task->data_dir); + } } } - spin_lock_irqsave(&sas_dev->lock, flags); - list_del_init(&slot->entry); - spin_unlock_irqrestore(&sas_dev->lock, flags); + if (need_lock) { + spin_lock(&sas_dev->lock); + list_del_init(&slot->entry); + spin_unlock(&sas_dev->lock); + } else { + list_del_init(&slot->entry); + } memset(slot, 0, offsetof(struct hisi_sas_slot, buf)); @@ -338,7 +339,7 @@ static int hisi_sas_dma_map(struct hisi_hba *hisi_hba, } if (*n_elem > HISI_SAS_SGE_PAGE_CNT) { - dev_err(dev, "task prep: n_elem(%d) > HISI_SAS_SGE_PAGE_CNT", + dev_err(dev, "task prep: n_elem(%d) > HISI_SAS_SGE_PAGE_CNT\n", *n_elem); rc = -EINVAL; goto err_out_dma_unmap; @@ -489,14 +490,14 @@ static int hisi_sas_task_prep(struct sas_task *task, slot_idx = rc; slot = &hisi_hba->slot_info[slot_idx]; - spin_lock_irqsave(&dq->lock, flags); + spin_lock(&dq->lock); wr_q_index = dq->wr_point; dq->wr_point = (dq->wr_point + 1) % HISI_SAS_QUEUE_SLOTS; list_add_tail(&slot->delivery, &dq->list); - spin_unlock_irqrestore(&dq->lock, flags); - spin_lock_irqsave(&sas_dev->lock, flags); + spin_unlock(&dq->lock); + spin_lock(&sas_dev->lock); list_add_tail(&slot->entry, &sas_dev->list); - spin_unlock_irqrestore(&sas_dev->lock, flags); + spin_unlock(&sas_dev->lock); dlvry_queue = dq->id; dlvry_queue_slot = wr_q_index; @@ -562,7 +563,6 @@ static int hisi_sas_task_exec(struct sas_task *task, gfp_t gfp_flags, { u32 rc; u32 pass = 0; - unsigned long flags; struct hisi_hba *hisi_hba; struct device *dev; struct domain_device *device = task->dev; @@ -587,13 +587,7 @@ static int hisi_sas_task_exec(struct sas_task *task, gfp_t gfp_flags, dev = hisi_hba->dev; if (unlikely(test_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags))) { - /* - * For IOs from upper layer, it may already disable preempt - * in the IO path, if disable preempt again in down(), - * function schedule() will report schedule_bug(), so check - * preemptible() before goto down(). - */ - if (!preemptible()) + if (!gfpflags_allow_blocking(gfp_flags)) return -EINVAL; down(&hisi_hba->sem); @@ -606,25 +600,24 @@ static int hisi_sas_task_exec(struct sas_task *task, gfp_t gfp_flags, dev_err(dev, "task exec: failed[%d]!\n", rc); if (likely(pass)) { - spin_lock_irqsave(&dq->lock, flags); + spin_lock(&dq->lock); hisi_hba->hw->start_delivery(dq); - spin_unlock_irqrestore(&dq->lock, flags); + spin_unlock(&dq->lock); } return rc; } -static void hisi_sas_bytes_dmaed(struct hisi_hba *hisi_hba, int phy_no) +static void hisi_sas_bytes_dmaed(struct hisi_hba *hisi_hba, int phy_no, + gfp_t gfp_flags) { struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no]; struct asd_sas_phy *sas_phy = &phy->sas_phy; - struct sas_ha_struct *sas_ha; if (!phy->phy_attached) return; - sas_ha = &hisi_hba->sha; - sas_ha->notify_phy_event(sas_phy, PHYE_OOB_DONE); + sas_notify_phy_event(sas_phy, PHYE_OOB_DONE, gfp_flags); if (sas_phy->phy) { struct sas_phy *sphy = sas_phy->phy; @@ -652,19 +645,18 @@ static void hisi_sas_bytes_dmaed(struct hisi_hba *hisi_hba, int phy_no) } sas_phy->frame_rcvd_size = phy->frame_rcvd_size; - sas_ha->notify_port_event(sas_phy, PORTE_BYTES_DMAED); + sas_notify_port_event(sas_phy, PORTE_BYTES_DMAED, gfp_flags); } static struct hisi_sas_device *hisi_sas_alloc_dev(struct domain_device *device) { struct hisi_hba *hisi_hba = dev_to_hisi_hba(device); struct hisi_sas_device *sas_dev = NULL; - unsigned long flags; int last = hisi_hba->last_dev_id; int first = (hisi_hba->last_dev_id + 1) % HISI_SAS_MAX_DEVICES; int i; - spin_lock_irqsave(&hisi_hba->lock, flags); + spin_lock(&hisi_hba->lock); for (i = first; i != last; i %= HISI_SAS_MAX_DEVICES) { if (hisi_hba->devices[i].dev_type == SAS_PHY_UNUSED) { int queue = i % hisi_hba->queue_count; @@ -684,7 +676,7 @@ static struct hisi_sas_device *hisi_sas_alloc_dev(struct domain_device *device) i++; } hisi_hba->last_dev_id = i; - spin_unlock_irqrestore(&hisi_hba->lock, flags); + spin_unlock(&hisi_hba->lock); return sas_dev; } @@ -697,14 +689,12 @@ static int hisi_sas_init_device(struct domain_device *device) struct hisi_sas_tmf_task tmf_task; int retry = HISI_SAS_DISK_RECOVER_CNT; struct hisi_hba *hisi_hba = dev_to_hisi_hba(device); - struct device *dev = hisi_hba->dev; - struct sas_phy *local_phy; switch (device->dev_type) { case SAS_END_DEVICE: int_to_scsilun(0, &lun); - tmf_task.tmf = TMF_CLEAR_TASK_SET; + tmf_task.tmf = TMF_ABORT_TASK_SET; while (retry-- > 0) { rc = hisi_sas_debug_issue_ssp_tmf(device, lun.scsi_lun, &tmf_task); @@ -719,30 +709,18 @@ static int hisi_sas_init_device(struct domain_device *device) case SAS_SATA_PM_PORT: case SAS_SATA_PENDING: /* - * send HARD RESET to clear previous affiliation of - * STP target port + * If an expander is swapped when a SATA disk is attached then + * we should issue a hard reset to clear previous affiliation + * of STP target port, see SPL (chapter 6.19.4). + * + * However we don't need to issue a hard reset here for these + * reasons: + * a. When probing the device, libsas/libata already issues a + * hard reset in sas_probe_sata() -> ata_sas_async_probe(). + * Note that in hisi_sas_debug_I_T_nexus_reset() we take care + * to issue a hard reset by checking the dev status (== INIT). + * b. When resetting the controller, this is simply unnecessary. */ - local_phy = sas_get_local_phy(device); - if (!scsi_is_sas_phy_local(local_phy) && - !test_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags)) { - unsigned long deadline = ata_deadline(jiffies, 20000); - struct sata_device *sata_dev = &device->sata_dev; - struct ata_host *ata_host = sata_dev->ata_host; - struct ata_port_operations *ops = ata_host->ops; - struct ata_port *ap = sata_dev->ap; - struct ata_link *link; - unsigned int classes; - - ata_for_each_link(link, ap, EDGE) - rc = ops->hardreset(link, &classes, - deadline); - } - sas_put_local_phy(local_phy); - if (rc) { - dev_warn(dev, "SATA disk hardreset fail: %d\n", rc); - return rc; - } - while (retry-- > 0) { rc = hisi_sas_softreset_ata_disk(device); if (!rc) @@ -756,6 +734,24 @@ static int hisi_sas_init_device(struct domain_device *device) return rc; } +int hisi_sas_slave_alloc(struct scsi_device *sdev) +{ + struct domain_device *ddev = sdev_to_domain_dev(sdev); + struct hisi_sas_device *sas_dev = ddev->lldd_dev; + int rc; + + rc = sas_slave_alloc(sdev); + if (rc) + return rc; + + rc = hisi_sas_init_device(ddev); + if (rc) + return rc; + sas_dev->dev_status = HISI_SAS_DEV_NORMAL; + return 0; +} +EXPORT_SYMBOL_GPL(hisi_sas_slave_alloc); + static int hisi_sas_dev_found(struct domain_device *device) { struct hisi_hba *hisi_hba = dev_to_hisi_hba(device); @@ -802,10 +798,6 @@ static int hisi_sas_dev_found(struct domain_device *device) dev_info(dev, "dev[%d:%x] found\n", sas_dev->device_id, sas_dev->dev_type); - rc = hisi_sas_init_device(device); - if (rc) - goto err_out; - sas_dev->dev_status = HISI_SAS_DEV_NORMAL; return 0; err_out: @@ -849,17 +841,24 @@ int hisi_sas_scan_finished(struct Scsi_Host *shost, unsigned long time) } EXPORT_SYMBOL_GPL(hisi_sas_scan_finished); -static void hisi_sas_phyup_work(struct work_struct *work) +static void hisi_sas_phyup_work_common(struct work_struct *work, + enum hisi_sas_phy_event event) { struct hisi_sas_phy *phy = - container_of(work, typeof(*phy), works[HISI_PHYE_PHY_UP]); + container_of(work, typeof(*phy), works[event]); struct hisi_hba *hisi_hba = phy->hisi_hba; struct asd_sas_phy *sas_phy = &phy->sas_phy; int phy_no = sas_phy->id; + phy->wait_phyup_cnt = 0; if (phy->identify.target_port_protocols == SAS_PROTOCOL_SSP) hisi_hba->hw->sl_notify_ssp(hisi_hba, phy_no); - hisi_sas_bytes_dmaed(hisi_hba, phy_no); + hisi_sas_bytes_dmaed(hisi_hba, phy_no, GFP_KERNEL); +} + +static void hisi_sas_phyup_work(struct work_struct *work) +{ + hisi_sas_phyup_work_common(work, HISI_PHYE_PHY_UP); } static void hisi_sas_linkreset_work(struct work_struct *work) @@ -871,9 +870,21 @@ static void hisi_sas_linkreset_work(struct work_struct *work) hisi_sas_control_phy(sas_phy, PHY_FUNC_LINK_RESET, NULL); } +static void hisi_sas_phyup_pm_work(struct work_struct *work) +{ + struct hisi_sas_phy *phy = + container_of(work, typeof(*phy), works[HISI_PHYE_PHY_UP_PM]); + struct hisi_hba *hisi_hba = phy->hisi_hba; + struct device *dev = hisi_hba->dev; + + hisi_sas_phyup_work_common(work, HISI_PHYE_PHY_UP_PM); + pm_runtime_put_sync(dev); +} + static const work_func_t hisi_sas_phye_fns[HISI_PHYES_NUM] = { [HISI_PHYE_PHY_UP] = hisi_sas_phyup_work, [HISI_PHYE_LINK_RESET] = hisi_sas_linkreset_work, + [HISI_PHYE_PHY_UP_PM] = hisi_sas_phyup_pm_work, }; bool hisi_sas_notify_phy_event(struct hisi_sas_phy *phy, @@ -899,20 +910,38 @@ static void hisi_sas_wait_phyup_timedout(struct timer_list *t) hisi_sas_notify_phy_event(phy, HISI_PHYE_LINK_RESET); } +#define HISI_SAS_WAIT_PHYUP_RETRIES 10 + void hisi_sas_phy_oob_ready(struct hisi_hba *hisi_hba, int phy_no) { struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no]; struct device *dev = hisi_hba->dev; + unsigned long flags; dev_dbg(dev, "phy%d OOB ready\n", phy_no); - if (phy->phy_attached) + spin_lock_irqsave(&phy->lock, flags); + if (phy->phy_attached) { + spin_unlock_irqrestore(&phy->lock, flags); return; + } if (!timer_pending(&phy->timer)) { - phy->timer.expires = jiffies + HISI_SAS_WAIT_PHYUP_TIMEOUT * HZ; - add_timer(&phy->timer); + if (phy->wait_phyup_cnt < HISI_SAS_WAIT_PHYUP_RETRIES) { + phy->wait_phyup_cnt++; + phy->timer.expires = jiffies + + HISI_SAS_WAIT_PHYUP_TIMEOUT; + add_timer(&phy->timer); + spin_unlock_irqrestore(&phy->lock, flags); + return; + } + + dev_warn(dev, "phy%d failed to come up %d times, giving up\n", + phy_no, phy->wait_phyup_cnt); + phy->wait_phyup_cnt = 0; } + spin_unlock_irqrestore(&phy->lock, flags); } + EXPORT_SYMBOL_GPL(hisi_sas_phy_oob_ready); static void hisi_sas_phy_init(struct hisi_hba *hisi_hba, int phy_no) @@ -993,7 +1022,7 @@ static void hisi_sas_port_notify_formed(struct asd_sas_phy *sas_phy) } static void hisi_sas_do_release_task(struct hisi_hba *hisi_hba, struct sas_task *task, - struct hisi_sas_slot *slot) + struct hisi_sas_slot *slot, bool need_lock) { if (task) { unsigned long flags; @@ -1011,7 +1040,7 @@ static void hisi_sas_do_release_task(struct hisi_hba *hisi_hba, struct sas_task spin_unlock_irqrestore(&task->task_state_lock, flags); } - hisi_sas_slot_task_free(hisi_hba, task, slot); + hisi_sas_slot_task_free(hisi_hba, task, slot, need_lock); } static void hisi_sas_release_task(struct hisi_hba *hisi_hba, @@ -1020,8 +1049,11 @@ static void hisi_sas_release_task(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot, *slot2; struct hisi_sas_device *sas_dev = device->lldd_dev; + spin_lock(&sas_dev->lock); list_for_each_entry_safe(slot, slot2, &sas_dev->list, entry) - hisi_sas_do_release_task(hisi_hba, slot->task, slot); + hisi_sas_do_release_task(hisi_hba, slot->task, slot, false); + + spin_unlock(&sas_dev->lock); } void hisi_sas_release_tasks(struct hisi_hba *hisi_hba) @@ -1055,6 +1087,7 @@ static void hisi_sas_dev_gone(struct domain_device *device) struct hisi_sas_device *sas_dev = device->lldd_dev; struct hisi_hba *hisi_hba = dev_to_hisi_hba(device); struct device *dev = hisi_hba->dev; + int ret = 0; dev_info(dev, "dev[%d:%x] is gone\n", sas_dev->device_id, sas_dev->dev_type); @@ -1062,17 +1095,20 @@ static void hisi_sas_dev_gone(struct domain_device *device) down(&hisi_hba->sem); if (!test_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags)) { hisi_sas_internal_task_abort(hisi_hba, device, - HISI_SAS_INT_ABT_DEV, 0); + HISI_SAS_INT_ABT_DEV, 0, true); hisi_sas_dereg_device(hisi_hba, device); - hisi_hba->hw->clear_itct(hisi_hba, sas_dev); + ret = hisi_hba->hw->clear_itct(hisi_hba, sas_dev); device->lldd_dev = NULL; } if (hisi_hba->hw->free_device) hisi_hba->hw->free_device(sas_dev); - sas_dev->dev_type = SAS_PHY_UNUSED; + + /* Don't mark it as SAS_PHY_UNUSED if failed to clear ITCT */ + if (!ret) + sas_dev->dev_type = SAS_PHY_UNUSED; sas_dev->sas_device = NULL; up(&hisi_hba->sem); } @@ -1120,9 +1156,18 @@ static int hisi_sas_phy_set_linkrate(struct hisi_hba *hisi_hba, int phy_no, static int hisi_sas_control_phy(struct asd_sas_phy *sas_phy, enum phy_func func, void *funcdata) { + struct hisi_sas_phy *phy = container_of(sas_phy, + struct hisi_sas_phy, sas_phy); struct sas_ha_struct *sas_ha = sas_phy->ha; struct hisi_hba *hisi_hba = sas_ha->lldd_ha; + struct device *dev = hisi_hba->dev; + DECLARE_COMPLETION_ONSTACK(completion); int phy_no = sas_phy->id; + u8 sts = phy->phy_attached; + int ret = 0; + + down(&hisi_hba->sem); + phy->reset_completion = &completion; switch (func) { case PHY_FUNC_HARD_RESET: @@ -1137,26 +1182,42 @@ static int hisi_sas_control_phy(struct asd_sas_phy *sas_phy, enum phy_func func, case PHY_FUNC_DISABLE: hisi_sas_phy_enable(hisi_hba, phy_no, 0); - break; + goto out; case PHY_FUNC_SET_LINK_RATE: - return hisi_sas_phy_set_linkrate(hisi_hba, phy_no, funcdata); + ret = hisi_sas_phy_set_linkrate(hisi_hba, phy_no, funcdata); + break; + case PHY_FUNC_GET_EVENTS: if (hisi_hba->hw->get_events) { hisi_hba->hw->get_events(hisi_hba, phy_no); - break; + goto out; } /* fallthru */ case PHY_FUNC_RELEASE_SPINUP_HOLD: default: - return -EOPNOTSUPP; + ret = -EOPNOTSUPP; + goto out; } - return 0; + + if (sts && !wait_for_completion_timeout(&completion, + HISI_SAS_WAIT_PHYUP_TIMEOUT)) { + dev_warn(dev, "phy%d wait phyup timed out for func %d\n", + phy_no, func); + if (phy->in_reset) + ret = -ETIMEDOUT; + } + +out: + phy->reset_completion = NULL; + + up(&hisi_hba->sem); + return ret; } static void hisi_sas_task_done(struct sas_task *task) { - del_timer(&task->slow_task->timer); + del_timer_sync(&task->slow_task->timer); complete(&task->slow_task->completion); } @@ -1178,9 +1239,9 @@ static void hisi_sas_tmf_timedout(struct timer_list *t) complete(&task->slow_task->completion); } -#define TASK_TIMEOUT 20 -#define TASK_RETRY 3 -#define INTERNAL_ABORT_TIMEOUT 6 +#define TASK_TIMEOUT (20 * HZ) +#define TASK_RETRY 3 +#define INTERNAL_ABORT_TIMEOUT (6 * HZ) static int hisi_sas_exec_internal_tmf_task(struct domain_device *device, void *parameter, u32 para_len, struct hisi_sas_tmf_task *tmf) @@ -1208,13 +1269,13 @@ static int hisi_sas_exec_internal_tmf_task(struct domain_device *device, task->task_done = hisi_sas_task_done; task->slow_task->timer.function = hisi_sas_tmf_timedout; - task->slow_task->timer.expires = jiffies + TASK_TIMEOUT * HZ; + task->slow_task->timer.expires = jiffies + TASK_TIMEOUT; add_timer(&task->slow_task->timer); res = hisi_sas_task_exec(task, GFP_KERNEL, 1, tmf); if (res) { - del_timer(&task->slow_task->timer); + del_timer_sync(&task->slow_task->timer); dev_err(dev, "abort tmf: executing internal task failed: %d\n", res); goto ex_err; @@ -1232,10 +1293,10 @@ static int hisi_sas_exec_internal_tmf_task(struct domain_device *device, struct hisi_sas_cq *cq = &hisi_hba->cq[slot->dlvry_queue]; /* - * flush tasklet to avoid free'ing task + * sync irq to avoid free'ing task * before using task in IO completion */ - tasklet_kill(&cq->tasklet); + synchronize_irq(cq->irq_no); slot->task = NULL; } @@ -1337,10 +1398,12 @@ static int hisi_sas_softreset_ata_disk(struct domain_device *device) rc = hisi_sas_exec_internal_tmf_task(device, fis, s, NULL); if (rc != TMF_RESP_FUNC_COMPLETE) - dev_err(dev, "ata disk de-reset failed\n"); + dev_err(dev, "ata disk %016llx de-reset failed\n", + SAS_ADDR(device->sas_addr)); } } else { - dev_err(dev, "ata disk reset failed\n"); + dev_err(dev, "ata disk %016llx reset failed\n", + SAS_ADDR(device->sas_addr)); } if (rc == TMF_RESP_FUNC_COMPLETE) @@ -1383,11 +1446,13 @@ static void hisi_sas_refresh_port_id(struct hisi_hba *hisi_hba) sas_port = device->port; port = to_hisi_sas_port(sas_port); + spin_lock(&sas_port->phy_list_lock); list_for_each_entry(sas_phy, &sas_port->phy_list, port_phy_el) if (state & BIT(sas_phy->id)) { phy = sas_phy->lldd_phy; break; } + spin_unlock(&sas_port->phy_list_lock); if (phy) { port->id = phy->port_id; @@ -1404,7 +1469,6 @@ static void hisi_sas_refresh_port_id(struct hisi_hba *hisi_hba) static void hisi_sas_rescan_topology(struct hisi_hba *hisi_hba, u32 state) { - struct sas_ha_struct *sas_ha = &hisi_hba->sha; struct asd_sas_port *_sas_port = NULL; int phy_no; @@ -1412,7 +1476,7 @@ static void hisi_sas_rescan_topology(struct hisi_hba *hisi_hba, u32 state) struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no]; struct asd_sas_phy *sas_phy = &phy->sas_phy; struct asd_sas_port *sas_port = sas_phy->port; - bool do_port_check = !!(_sas_port != sas_port); + bool do_port_check = _sas_port != sas_port; if (!sas_phy->phy->enabled) continue; @@ -1425,13 +1489,13 @@ static void hisi_sas_rescan_topology(struct hisi_hba *hisi_hba, u32 state) _sas_port = sas_port; if (dev_is_expander(dev->dev_type)) - sas_ha->notify_port_event(sas_phy, - PORTE_BROADCAST_RCVD); + sas_notify_port_event(sas_phy, + PORTE_BROADCAST_RCVD, + GFP_KERNEL); } } else { - hisi_sas_phy_down(hisi_hba, phy_no, 0); + hisi_sas_phy_down(hisi_hba, phy_no, 0, GFP_KERNEL); } - } } @@ -1461,26 +1525,24 @@ static void hisi_sas_send_ata_reset_each_phy(struct hisi_hba *hisi_hba, struct device *dev = hisi_hba->dev; int s = sizeof(struct host_to_dev_fis); int rc = TMF_RESP_FUNC_FAILED; - struct asd_sas_phy *sas_phy; struct ata_link *link; u8 fis[20] = {0}; - u32 state; + int i; - state = hisi_hba->hw->get_phys_state(hisi_hba); - list_for_each_entry(sas_phy, &sas_port->phy_list, port_phy_el) { - if (!(state & BIT(sas_phy->id))) + for (i = 0; i < hisi_hba->n_phy; i++) { + if (!(sas_port->phy_mask & BIT(i))) continue; ata_for_each_link(link, ap, EDGE) { int pmp = sata_srst_pmp(link); - tmf_task.phy_id = sas_phy->id; + tmf_task.phy_id = i; hisi_sas_fill_ata_reset_cmd(link->device, 1, pmp, fis); rc = hisi_sas_exec_internal_tmf_task(device, fis, s, &tmf_task); if (rc != TMF_RESP_FUNC_COMPLETE) { dev_err(dev, "phy%d ata reset failed rc=%d\n", - sas_phy->id, rc); + i, rc); break; } } @@ -1500,7 +1562,8 @@ static void hisi_sas_terminate_stp_reject(struct hisi_hba *hisi_hba) continue; rc = hisi_sas_internal_task_abort(hisi_hba, device, - HISI_SAS_INT_ABT_DEV, 0); + HISI_SAS_INT_ABT_DEV, 0, + false); if (rc < 0) dev_err(dev, "STP reject: abort dev failed %d\n", rc); } @@ -1531,57 +1594,89 @@ void hisi_sas_controller_reset_prepare(struct hisi_hba *hisi_hba) { struct Scsi_Host *shost = hisi_hba->shost; - down(&hisi_hba->sem); hisi_hba->phy_state = hisi_hba->hw->get_phys_state(hisi_hba); scsi_block_requests(shost); hisi_hba->hw->wait_cmds_complete_timeout(hisi_hba, 100, 5000); - if (timer_pending(&hisi_hba->timer)) - del_timer_sync(&hisi_hba->timer); + del_timer_sync(&hisi_hba->timer); set_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags); } EXPORT_SYMBOL_GPL(hisi_sas_controller_reset_prepare); +static void hisi_sas_async_init_wait_phyup(void *data, async_cookie_t cookie) +{ + struct hisi_sas_phy *phy = data; + struct hisi_hba *hisi_hba = phy->hisi_hba; + struct device *dev = hisi_hba->dev; + DECLARE_COMPLETION_ONSTACK(completion); + int phy_no = phy->sas_phy.id; + + phy->reset_completion = &completion; + hisi_sas_phy_enable(hisi_hba, phy_no, 1); + if (!wait_for_completion_timeout(&completion, + HISI_SAS_WAIT_PHYUP_TIMEOUT)) + dev_warn(dev, "phy%d wait phyup timed out\n", phy_no); + + phy->reset_completion = NULL; +} + void hisi_sas_controller_reset_done(struct hisi_hba *hisi_hba) { struct Scsi_Host *shost = hisi_hba->shost; - u32 state; + ASYNC_DOMAIN_EXCLUSIVE(async); + int phy_no; /* Init and wait for PHYs to come up and all libsas event finished. */ - hisi_hba->hw->phys_init(hisi_hba); - msleep(1000); + for (phy_no = 0; phy_no < hisi_hba->n_phy; phy_no++) { + struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no]; + + if (!(hisi_hba->phy_state & BIT(phy_no))) + continue; + + async_schedule_domain(hisi_sas_async_init_wait_phyup, + phy, &async); + } + + async_synchronize_full_domain(&async); hisi_sas_refresh_port_id(hisi_hba); clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags); if (hisi_hba->reject_stp_links_msk) hisi_sas_terminate_stp_reject(hisi_hba); hisi_sas_reset_init_all_devices(hisi_hba); - up(&hisi_hba->sem); scsi_unblock_requests(shost); clear_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags); + up(&hisi_hba->sem); - state = hisi_hba->hw->get_phys_state(hisi_hba); - hisi_sas_rescan_topology(hisi_hba, state); + hisi_sas_rescan_topology(hisi_hba, hisi_hba->phy_state); } EXPORT_SYMBOL_GPL(hisi_sas_controller_reset_done); +static int hisi_sas_controller_prereset(struct hisi_hba *hisi_hba) +{ + if (!hisi_hba->hw->soft_reset) + return -ENOENT; + + down(&hisi_hba->sem); + if (test_and_set_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags)) { + up(&hisi_hba->sem); + return -EPERM; + } + + if (hisi_sas_debugfs_enable) + hisi_hba->hw->debugfs_snapshot_regs(hisi_hba); + + return 0; +} + static int hisi_sas_controller_reset(struct hisi_hba *hisi_hba) { struct device *dev = hisi_hba->dev; struct Scsi_Host *shost = hisi_hba->shost; int rc; - if (hisi_sas_debugfs_enable && hisi_hba->debugfs_itct) - queue_work(hisi_hba->wq, &hisi_hba->debugfs_work); - - if (!hisi_hba->hw->soft_reset) - return -1; - - if (test_and_set_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags)) - return -1; - dev_info(dev, "controller resetting...\n"); hisi_sas_controller_reset_prepare(hisi_hba); @@ -1596,6 +1691,7 @@ static int hisi_sas_controller_reset(struct hisi_hba *hisi_hba) } hisi_sas_controller_reset_done(hisi_hba); + clear_bit(HISI_SAS_HW_FAULT_BIT, &hisi_hba->flags); dev_info(dev, "controller reset complete\n"); return 0; @@ -1607,6 +1703,7 @@ static int hisi_sas_abort_task(struct sas_task *task) struct hisi_sas_tmf_task tmf_task; struct domain_device *device = task->dev; struct hisi_sas_device *sas_dev = device->lldd_dev; + struct hisi_sas_slot *slot = task->lldd_task; struct hisi_hba *hisi_hba; struct device *dev; int rc = TMF_RESP_FUNC_FAILED; @@ -1620,16 +1717,15 @@ static int hisi_sas_abort_task(struct sas_task *task) spin_lock_irqsave(&task->task_state_lock, flags); if (task->task_state_flags & SAS_TASK_STATE_DONE) { - struct hisi_sas_slot *slot = task->lldd_task; struct hisi_sas_cq *cq; if (slot) { /* - * flush tasklet to avoid free'ing task + * sync irq to avoid free'ing task * before using task in IO completion */ cq = &hisi_hba->cq[slot->dlvry_queue]; - tasklet_kill(&cq->tasklet); + synchronize_irq(cq->irq_no); } spin_unlock_irqrestore(&task->task_state_lock, flags); rc = TMF_RESP_FUNC_COMPLETE; @@ -1638,9 +1734,11 @@ static int hisi_sas_abort_task(struct sas_task *task) task->task_state_flags |= SAS_TASK_STATE_ABORTED; spin_unlock_irqrestore(&task->task_state_lock, flags); - if (task->lldd_task && task->task_proto & SAS_PROTOCOL_SSP) { + if (!slot) + goto out; + + if (task->task_proto & SAS_PROTOCOL_SSP) { struct scsi_cmnd *cmnd = task->uldd_task; - struct hisi_sas_slot *slot = task->lldd_task; u16 tag = slot->idx; int rc2; @@ -1652,7 +1750,8 @@ static int hisi_sas_abort_task(struct sas_task *task) &tmf_task); rc2 = hisi_sas_internal_task_abort(hisi_hba, device, - HISI_SAS_INT_ABT_CMD, tag); + HISI_SAS_INT_ABT_CMD, tag, + false); if (rc2 < 0) { dev_err(dev, "abort task: internal abort (%d)\n", rc2); return TMF_RESP_FUNC_FAILED; @@ -1667,36 +1766,49 @@ static int hisi_sas_abort_task(struct sas_task *task) */ if (rc == TMF_RESP_FUNC_COMPLETE && rc2 != TMF_RESP_FUNC_SUCC) { if (task->lldd_task) - hisi_sas_do_release_task(hisi_hba, task, slot); + hisi_sas_do_release_task(hisi_hba, task, slot, true); } } else if (task->task_proto & SAS_PROTOCOL_SATA || task->task_proto & SAS_PROTOCOL_STP) { if (task->dev->dev_type == SAS_SATA_DEV) { + struct ata_queued_cmd *qc = task->uldd_task; + rc = hisi_sas_internal_task_abort(hisi_hba, device, HISI_SAS_INT_ABT_DEV, - 0); + 0, false); if (rc < 0) { dev_err(dev, "abort task: internal abort failed\n"); goto out; } hisi_sas_dereg_device(hisi_hba, device); - rc = hisi_sas_softreset_ata_disk(device); + + /* + * If an ATA internal command times out in ATA EH, it + * need to execute soft reset, so check the scsicmd + */ + if ((sas_dev->dev_status == HISI_SAS_DEV_NCQ_ERR) && + qc && qc->scsicmd) { + hisi_sas_do_release_task(hisi_hba, task, slot, true); + rc = TMF_RESP_FUNC_COMPLETE; + } else { + rc = hisi_sas_softreset_ata_disk(device); + } } - } else if (task->lldd_task && task->task_proto & SAS_PROTOCOL_SMP) { + } else if (task->task_proto & SAS_PROTOCOL_SMP) { /* SMP */ - struct hisi_sas_slot *slot = task->lldd_task; u32 tag = slot->idx; struct hisi_sas_cq *cq = &hisi_hba->cq[slot->dlvry_queue]; rc = hisi_sas_internal_task_abort(hisi_hba, device, - HISI_SAS_INT_ABT_CMD, tag); + HISI_SAS_INT_ABT_CMD, tag, + false); if (((rc < 0) || (rc == TMF_RESP_FUNC_FAILED)) && task->lldd_task) { /* - * flush tasklet to avoid free'ing task + * sync irq to avoid free'ing task * before using task in IO completion */ - tasklet_kill(&cq->tasklet); + synchronize_irq(cq->irq_no); slot->task = NULL; } } @@ -1715,7 +1827,7 @@ static int hisi_sas_abort_task_set(struct domain_device *device, u8 *lun) int rc; rc = hisi_sas_internal_task_abort(hisi_hba, device, - HISI_SAS_INT_ABT_DEV, 0); + HISI_SAS_INT_ABT_DEV, 0, false); if (rc < 0) { dev_err(dev, "abort task set: internal abort rc=%d\n", rc); return TMF_RESP_FUNC_FAILED; @@ -1742,13 +1854,14 @@ static int hisi_sas_clear_aca(struct domain_device *device, u8 *lun) return rc; } +#define I_T_NEXUS_RESET_PHYUP_TIMEOUT (2 * HZ) + static int hisi_sas_debug_I_T_nexus_reset(struct domain_device *device) { struct sas_phy *local_phy = sas_get_local_phy(device); struct hisi_sas_device *sas_dev = device->lldd_dev; struct hisi_hba *hisi_hba = dev_to_hisi_hba(device); struct sas_ha_struct *sas_ha = &hisi_hba->sha; - DECLARE_COMPLETION_ONSTACK(phyreset); int rc, reset_type; if (!local_phy->enabled) { @@ -1761,8 +1874,11 @@ static int hisi_sas_debug_I_T_nexus_reset(struct domain_device *device) sas_ha->sas_phy[local_phy->number]; struct hisi_sas_phy *phy = container_of(sas_phy, struct hisi_sas_phy, sas_phy); + unsigned long flags; + + spin_lock_irqsave(&phy->lock, flags); phy->in_reset = 1; - phy->reset_completion = &phyreset; + spin_unlock_irqrestore(&phy->lock, flags); } reset_type = (sas_dev->dev_status == HISI_SAS_DEV_INIT || @@ -1776,24 +1892,29 @@ static int hisi_sas_debug_I_T_nexus_reset(struct domain_device *device) sas_ha->sas_phy[local_phy->number]; struct hisi_sas_phy *phy = container_of(sas_phy, struct hisi_sas_phy, sas_phy); - int ret = wait_for_completion_timeout(&phyreset, 2 * HZ); unsigned long flags; spin_lock_irqsave(&phy->lock, flags); - phy->reset_completion = NULL; phy->in_reset = 0; spin_unlock_irqrestore(&phy->lock, flags); /* report PHY down if timed out */ - if (!ret) - hisi_sas_phy_down(hisi_hba, sas_phy->id, 0); - } else if (sas_dev->dev_status != HISI_SAS_DEV_INIT) { - /* - * If in init state, we rely on caller to wait for link to be - * ready; otherwise, except phy reset is fail, delay. - */ - if (!rc) - msleep(2000); + if (rc == -ETIMEDOUT) + hisi_sas_phy_down(hisi_hba, sas_phy->id, 0, GFP_KERNEL); + return rc; + } + + /* Remote phy */ + if (rc) + return rc; + + if (dev_is_sata(device)) { + struct ata_link *link = &device->sata_dev.ap->link; + + rc = ata_wait_after_reset(link, HISI_SAS_WAIT_PHYUP_TIMEOUT, + smp_ata_check_ready_type); + } else { + msleep(2000); } return rc; @@ -1803,24 +1924,47 @@ static int hisi_sas_I_T_nexus_reset(struct domain_device *device) { struct hisi_hba *hisi_hba = dev_to_hisi_hba(device); struct device *dev = hisi_hba->dev; + struct hisi_sas_device *sas_dev = device->lldd_dev; int rc; + if (sas_dev->dev_status == HISI_SAS_DEV_NCQ_ERR) + sas_dev->dev_status = HISI_SAS_DEV_NORMAL; + rc = hisi_sas_internal_task_abort(hisi_hba, device, - HISI_SAS_INT_ABT_DEV, 0); + HISI_SAS_INT_ABT_DEV, 0, false); if (rc < 0) { dev_err(dev, "I_T nexus reset: internal abort (%d)\n", rc); return TMF_RESP_FUNC_FAILED; } hisi_sas_dereg_device(hisi_hba, device); - if (dev_is_sata(device)) { + rc = hisi_sas_debug_I_T_nexus_reset(device); + if (rc == TMF_RESP_FUNC_COMPLETE && dev_is_sata(device)) { + struct sas_phy *local_phy; + rc = hisi_sas_softreset_ata_disk(device); - if (rc == TMF_RESP_FUNC_FAILED) - return TMF_RESP_FUNC_FAILED; + switch (rc) { + case -ECOMM: + rc = -ENODEV; + break; + case TMF_RESP_FUNC_FAILED: + case -EMSGSIZE: + case -EIO: + local_phy = sas_get_local_phy(device); + rc = sas_phy_enable(local_phy, 0); + if (!rc) { + local_phy->enabled = 0; + dev_err(dev, "Disabled local phy of ATA disk %016llx due to softreset fail (%d)\n", + SAS_ADDR(device->sas_addr), rc); + rc = -ENODEV; + } + sas_put_local_phy(local_phy); + break; + default: + break; + } } - rc = hisi_sas_debug_I_T_nexus_reset(device); - if ((rc == TMF_RESP_FUNC_COMPLETE) || (rc == -ENODEV)) hisi_sas_release_task(hisi_hba, device); @@ -1836,7 +1980,7 @@ static int hisi_sas_lu_reset(struct domain_device *device, u8 *lun) /* Clear internal IO and then lu reset */ rc = hisi_sas_internal_task_abort(hisi_hba, device, - HISI_SAS_INT_ABT_DEV, 0); + HISI_SAS_INT_ABT_DEV, 0, false); if (rc < 0) { dev_err(dev, "lu_reset: internal abort failed\n"); goto out; @@ -1964,14 +2108,14 @@ hisi_sas_internal_abort_task_exec(struct hisi_hba *hisi_hba, int device_id, slot_idx = rc; slot = &hisi_hba->slot_info[slot_idx]; - spin_lock_irqsave(&dq->lock, flags); + spin_lock(&dq->lock); wr_q_index = dq->wr_point; dq->wr_point = (dq->wr_point + 1) % HISI_SAS_QUEUE_SLOTS; list_add_tail(&slot->delivery, &dq->list); - spin_unlock_irqrestore(&dq->lock, flags); - spin_lock_irqsave(&sas_dev->lock, flags); + spin_unlock(&dq->lock); + spin_lock(&sas_dev->lock); list_add_tail(&slot->entry, &sas_dev->list); - spin_unlock_irqrestore(&sas_dev->lock, flags); + spin_unlock(&sas_dev->lock); dlvry_queue = dq->id; dlvry_queue_slot = wr_q_index; @@ -2000,9 +2144,9 @@ hisi_sas_internal_abort_task_exec(struct hisi_hba *hisi_hba, int device_id, spin_unlock_irqrestore(&task->task_state_lock, flags); WRITE_ONCE(slot->ready, 1); /* send abort command to the chip */ - spin_lock_irqsave(&dq->lock, flags); + spin_lock(&dq->lock); hisi_hba->hw->start_delivery(dq); - spin_unlock_irqrestore(&dq->lock, flags); + spin_unlock(&dq->lock); return 0; @@ -2021,11 +2165,13 @@ hisi_sas_internal_abort_task_exec(struct hisi_hba *hisi_hba, int device_id, * @tag: tag of IO to be aborted (only relevant to single * IO mode) * @dq: delivery queue for this internal abort command + * @rst_to_recover: If rst_to_recover set, queue a controller + * reset if an internal abort times out. */ static int _hisi_sas_internal_task_abort(struct hisi_hba *hisi_hba, struct domain_device *device, int abort_flag, - int tag, struct hisi_sas_dq *dq) + int tag, struct hisi_sas_dq *dq, bool rst_to_recover) { struct sas_task *task; struct hisi_sas_device *sas_dev = device->lldd_dev; @@ -2041,6 +2187,9 @@ _hisi_sas_internal_task_abort(struct hisi_hba *hisi_hba, if (!hisi_hba->hw->prep_abort) return TMF_RESP_FUNC_FAILED; + if (test_bit(HISI_SAS_HW_FAULT_BIT, &hisi_hba->flags)) + return -EIO; + task = sas_alloc_slow_task(GFP_KERNEL); if (!task) return -ENOMEM; @@ -2049,13 +2198,13 @@ _hisi_sas_internal_task_abort(struct hisi_hba *hisi_hba, task->task_proto = device->tproto; task->task_done = hisi_sas_task_done; task->slow_task->timer.function = hisi_sas_tmf_timedout; - task->slow_task->timer.expires = jiffies + INTERNAL_ABORT_TIMEOUT * HZ; + task->slow_task->timer.expires = jiffies + INTERNAL_ABORT_TIMEOUT; add_timer(&task->slow_task->timer); res = hisi_sas_internal_abort_task_exec(hisi_hba, sas_dev->device_id, task, abort_flag, tag, dq); if (res) { - del_timer(&task->slow_task->timer); + del_timer_sync(&task->slow_task->timer); dev_err(dev, "internal task abort: executing internal task failed: %d\n", res); goto exit; @@ -2065,23 +2214,43 @@ _hisi_sas_internal_task_abort(struct hisi_hba *hisi_hba, /* Internal abort timed out */ if ((task->task_state_flags & SAS_TASK_STATE_ABORTED)) { - if (hisi_sas_debugfs_enable && hisi_hba->debugfs_itct) - queue_work(hisi_hba->wq, &hisi_hba->debugfs_work); + if (hisi_sas_debugfs_enable) { + /* + * If timeout occurs in device gone scenario, to avoid + * circular dependency like: + * hisi_sas_dev_gone() -> down() -> ... -> + * hisi_sas_internal_abort_timeout() -> down(). + */ + if (!rst_to_recover) + down(&hisi_hba->sem); + + hisi_hba->hw->debugfs_snapshot_regs(hisi_hba); + if (!rst_to_recover) + up(&hisi_hba->sem); + } if (!(task->task_state_flags & SAS_TASK_STATE_DONE)) { struct hisi_sas_slot *slot = task->lldd_task; + set_bit(HISI_SAS_HW_FAULT_BIT, &hisi_hba->flags); + if (slot) { struct hisi_sas_cq *cq = &hisi_hba->cq[slot->dlvry_queue]; /* - * flush tasklet to avoid free'ing task + * sync irq to avoid free'ing task * before using task in IO completion */ - tasklet_kill(&cq->tasklet); + synchronize_irq(cq->irq_no); slot->task = NULL; } - dev_err(dev, "internal task abort: timeout and not done.\n"); + + if (rst_to_recover) { + dev_err(dev, "internal task abort: timeout and not done. Queuing reset.\n"); + queue_work(hisi_hba->wq, &hisi_hba->rst_work); + } else { + dev_err(dev, "internal task abort: timeout and not done.\n"); + } res = -EIO; goto exit; @@ -2114,7 +2283,7 @@ _hisi_sas_internal_task_abort(struct hisi_hba *hisi_hba, static int hisi_sas_internal_task_abort(struct hisi_hba *hisi_hba, struct domain_device *device, - int abort_flag, int tag) + int abort_flag, int tag, bool rst_to_recover) { struct hisi_sas_slot *slot; struct device *dev = hisi_hba->dev; @@ -2126,18 +2295,19 @@ hisi_sas_internal_task_abort(struct hisi_hba *hisi_hba, slot = &hisi_hba->slot_info[tag]; dq = &hisi_hba->dq[slot->dlvry_queue]; return _hisi_sas_internal_task_abort(hisi_hba, device, - abort_flag, tag, dq); + abort_flag, tag, dq, + rst_to_recover); case HISI_SAS_INT_ABT_DEV: for (i = 0; i < hisi_hba->cq_nvecs; i++) { struct hisi_sas_cq *cq = &hisi_hba->cq[i]; - const struct cpumask *mask = cq->pci_irq_mask; + const struct cpumask *mask = cq->irq_mask; if (mask && !cpumask_intersects(cpu_online_mask, mask)) continue; dq = &hisi_hba->dq[i]; rc = _hisi_sas_internal_task_abort(hisi_hba, device, abort_flag, tag, - dq); + dq, rst_to_recover); if (rc) return rc; } @@ -2186,16 +2356,16 @@ static void hisi_sas_phy_disconnected(struct hisi_sas_phy *phy) spin_unlock_irqrestore(&phy->lock, flags); } -void hisi_sas_phy_down(struct hisi_hba *hisi_hba, int phy_no, int rdy) +void hisi_sas_phy_down(struct hisi_hba *hisi_hba, int phy_no, int rdy, + gfp_t gfp_flags) { struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no]; struct asd_sas_phy *sas_phy = &phy->sas_phy; - struct sas_ha_struct *sas_ha = &hisi_hba->sha; struct device *dev = hisi_hba->dev; if (rdy) { /* Phy down but ready */ - hisi_sas_bytes_dmaed(hisi_hba, phy_no); + hisi_sas_bytes_dmaed(hisi_hba, phy_no, gfp_flags); hisi_sas_port_notify_formed(sas_phy); } else { struct hisi_sas_port *port = phy->port; @@ -2206,7 +2376,7 @@ void hisi_sas_phy_down(struct hisi_hba *hisi_hba, int phy_no, int rdy) return; } /* Phy down and not ready */ - sas_ha->notify_phy_event(sas_phy, PHYE_LOSS_OF_SIGNAL); + sas_notify_phy_event(sas_phy, PHYE_LOSS_OF_SIGNAL, gfp_flags); sas_phy_disconnected(sas_phy); if (port) { @@ -2224,17 +2394,17 @@ void hisi_sas_phy_down(struct hisi_hba *hisi_hba, int phy_no, int rdy) } EXPORT_SYMBOL_GPL(hisi_sas_phy_down); -void hisi_sas_kill_tasklets(struct hisi_hba *hisi_hba) +void hisi_sas_sync_irqs(struct hisi_hba *hisi_hba) { int i; for (i = 0; i < hisi_hba->cq_nvecs; i++) { struct hisi_sas_cq *cq = &hisi_hba->cq[i]; - tasklet_kill(&cq->tasklet); + synchronize_irq(cq->irq_no); } } -EXPORT_SYMBOL_GPL(hisi_sas_kill_tasklets); +EXPORT_SYMBOL_GPL(hisi_sas_sync_irqs); int hisi_sas_host_reset(struct Scsi_Host *shost, int reset_type) { @@ -2470,6 +2640,9 @@ void hisi_sas_rst_work_handler(struct work_struct *work) struct hisi_hba *hisi_hba = container_of(work, struct hisi_hba, rst_work); + if (hisi_sas_controller_prereset(hisi_hba)) + return; + hisi_sas_controller_reset(hisi_hba); } EXPORT_SYMBOL_GPL(hisi_sas_rst_work_handler); @@ -2479,8 +2652,12 @@ void hisi_sas_sync_rst_work_handler(struct work_struct *work) struct hisi_sas_rst *rst = container_of(work, struct hisi_sas_rst, work); + if (hisi_sas_controller_prereset(rst->hisi_hba)) + goto rst_complete; + if (!hisi_sas_controller_reset(rst->hisi_hba)) rst->done = true; +rst_complete: complete(rst->completion); } EXPORT_SYMBOL_GPL(hisi_sas_sync_rst_work_handler); @@ -2677,1220 +2854,76 @@ int hisi_sas_probe(struct platform_device *pdev, rc = hisi_hba->hw->hw_init(hisi_hba); if (rc) - goto err_out_register_ha; + goto err_out_hw_init; scsi_scan_host(shost); return 0; +err_out_hw_init: + sas_unregister_ha(sha); err_out_register_ha: scsi_remove_host(shost); err_out_ha: - hisi_sas_debugfs_exit(hisi_hba); hisi_sas_free(hisi_hba); scsi_host_put(shost); return rc; } EXPORT_SYMBOL_GPL(hisi_sas_probe); -struct dentry *hisi_sas_debugfs_dir; - -static void hisi_sas_debugfs_snapshot_cq_reg(struct hisi_hba *hisi_hba) -{ - int queue_entry_size = hisi_hba->hw->complete_hdr_size; - int i; - - for (i = 0; i < hisi_hba->queue_count; i++) - memcpy(hisi_hba->debugfs_complete_hdr[i], - hisi_hba->complete_hdr[i], - HISI_SAS_QUEUE_SLOTS * queue_entry_size); -} - -static void hisi_sas_debugfs_snapshot_dq_reg(struct hisi_hba *hisi_hba) -{ - int queue_entry_size = sizeof(struct hisi_sas_cmd_hdr); - int i; - - for (i = 0; i < hisi_hba->queue_count; i++) { - struct hisi_sas_cmd_hdr *debugfs_cmd_hdr, *cmd_hdr; - int j; - - debugfs_cmd_hdr = hisi_hba->debugfs_cmd_hdr[i]; - cmd_hdr = hisi_hba->cmd_hdr[i]; - - for (j = 0; j < HISI_SAS_QUEUE_SLOTS; j++) - memcpy(&debugfs_cmd_hdr[j], &cmd_hdr[j], - queue_entry_size); - } -} - -static void hisi_sas_debugfs_snapshot_port_reg(struct hisi_hba *hisi_hba) -{ - const struct hisi_sas_debugfs_reg *port = - hisi_hba->hw->debugfs_reg_port; - int i, phy_cnt; - u32 offset; - u32 *databuf; - - for (phy_cnt = 0; phy_cnt < hisi_hba->n_phy; phy_cnt++) { - databuf = (u32 *)hisi_hba->debugfs_port_reg[phy_cnt]; - for (i = 0; i < port->count; i++, databuf++) { - offset = port->base_off + 4 * i; - *databuf = port->read_port_reg(hisi_hba, phy_cnt, - offset); - } - } -} - -static void hisi_sas_debugfs_snapshot_global_reg(struct hisi_hba *hisi_hba) -{ - u32 *databuf = hisi_hba->debugfs_regs[DEBUGFS_GLOBAL]; - const struct hisi_sas_hw *hw = hisi_hba->hw; - const struct hisi_sas_debugfs_reg *global = - hw->debugfs_reg_array[DEBUGFS_GLOBAL]; - int i; - - for (i = 0; i < global->count; i++, databuf++) - *databuf = global->read_global_reg(hisi_hba, 4 * i); -} - -static void hisi_sas_debugfs_snapshot_axi_reg(struct hisi_hba *hisi_hba) +int hisi_sas_remove(struct platform_device *pdev) { - u32 *databuf = hisi_hba->debugfs_regs[DEBUGFS_AXI]; - const struct hisi_sas_hw *hw = hisi_hba->hw; - const struct hisi_sas_debugfs_reg *axi = - hw->debugfs_reg_array[DEBUGFS_AXI]; - int i; + struct sas_ha_struct *sha = platform_get_drvdata(pdev); + struct hisi_hba *hisi_hba = sha->lldd_ha; + struct Scsi_Host *shost = sha->core.shost; - for (i = 0; i < axi->count; i++, databuf++) - *databuf = axi->read_global_reg(hisi_hba, - 4 * i + axi->base_off); -} + del_timer_sync(&hisi_hba->timer); -static void hisi_sas_debugfs_snapshot_ras_reg(struct hisi_hba *hisi_hba) -{ - u32 *databuf = hisi_hba->debugfs_regs[DEBUGFS_RAS]; - const struct hisi_sas_hw *hw = hisi_hba->hw; - const struct hisi_sas_debugfs_reg *ras = - hw->debugfs_reg_array[DEBUGFS_RAS]; - int i; + sas_unregister_ha(sha); + sas_remove_host(sha->core.shost); - for (i = 0; i < ras->count; i++, databuf++) - *databuf = ras->read_global_reg(hisi_hba, - 4 * i + ras->base_off); + hisi_sas_free(hisi_hba); + scsi_host_put(shost); + return 0; } +EXPORT_SYMBOL_GPL(hisi_sas_remove); -static void hisi_sas_debugfs_snapshot_itct_reg(struct hisi_hba *hisi_hba) -{ - void *cachebuf = hisi_hba->debugfs_itct_cache; - void *databuf = hisi_hba->debugfs_itct; - struct hisi_sas_itct *itct; - int i; +#if IS_ENABLED(CONFIG_SCSI_HISI_SAS_DEBUGFS_DEFAULT_ENABLE) +#define DEBUGFS_ENABLE_DEFAULT "enabled" +bool hisi_sas_debugfs_enable = true; +u32 hisi_sas_debugfs_dump_count = 50; +#else +#define DEBUGFS_ENABLE_DEFAULT "disabled" +bool hisi_sas_debugfs_enable; +u32 hisi_sas_debugfs_dump_count = 1; +#endif - hisi_hba->hw->read_iost_itct_cache(hisi_hba, HISI_SAS_ITCT_CACHE, - cachebuf); +EXPORT_SYMBOL_GPL(hisi_sas_debugfs_enable); +module_param_named(debugfs_enable, hisi_sas_debugfs_enable, bool, 0444); +MODULE_PARM_DESC(hisi_sas_debugfs_enable, + "Enable driver debugfs (default "DEBUGFS_ENABLE_DEFAULT")"); - itct = hisi_hba->itct; +EXPORT_SYMBOL_GPL(hisi_sas_debugfs_dump_count); +module_param_named(debugfs_dump_count, hisi_sas_debugfs_dump_count, uint, 0444); +MODULE_PARM_DESC(hisi_sas_debugfs_dump_count, "Number of debugfs dumps to allow"); - for (i = 0; i < HISI_SAS_MAX_ITCT_ENTRIES; i++, itct++) { - memcpy(databuf, itct, sizeof(struct hisi_sas_itct)); - databuf += sizeof(struct hisi_sas_itct); - } -} +struct dentry *hisi_sas_debugfs_dir; +EXPORT_SYMBOL_GPL(hisi_sas_debugfs_dir); -static void hisi_sas_debugfs_snapshot_iost_reg(struct hisi_hba *hisi_hba) +static __init int hisi_sas_init(void) { - int max_command_entries = HISI_SAS_MAX_COMMANDS; - void *cachebuf = hisi_hba->debugfs_iost_cache; - void *databuf = hisi_hba->debugfs_iost; - struct hisi_sas_iost *iost; - int i; - - hisi_hba->hw->read_iost_itct_cache(hisi_hba, HISI_SAS_IOST_CACHE, - cachebuf); - - iost = hisi_hba->iost; - - for (i = 0; i < max_command_entries; i++, iost++) { - memcpy(databuf, iost, sizeof(struct hisi_sas_iost)); - databuf += sizeof(struct hisi_sas_iost); - } -} + hisi_sas_stt = sas_domain_attach_transport(&hisi_sas_transport_ops); + if (!hisi_sas_stt) + return -ENOMEM; -static const char * -hisi_sas_debugfs_to_reg_name(int off, int base_off, - const struct hisi_sas_debugfs_reg_lu *lu) -{ - for (; lu->name; lu++) { - if (off == lu->off - base_off) - return lu->name; + if (hisi_sas_debugfs_enable) { + hisi_sas_debugfs_dir = debugfs_create_dir("hisi_sas", NULL); + if (hisi_sas_debugfs_dump_count > HISI_SAS_MAX_DEBUGFS_DUMP) { + pr_info("hisi_sas: Limiting debugfs dump count\n"); + hisi_sas_debugfs_dump_count = HISI_SAS_MAX_DEBUGFS_DUMP; + } } - return NULL; -} - -static void hisi_sas_debugfs_print_reg(u32 *regs_val, const void *ptr, - struct seq_file *s) -{ - const struct hisi_sas_debugfs_reg *reg = ptr; - int i; - - for (i = 0; i < reg->count; i++) { - int off = i * 4; - const char *name; - - name = hisi_sas_debugfs_to_reg_name(off, reg->base_off, - reg->lu); - - if (name) - seq_printf(s, "0x%08x 0x%08x %s\n", off, - regs_val[i], name); - else - seq_printf(s, "0x%08x 0x%08x\n", off, - regs_val[i]); - } -} - -static int hisi_sas_debugfs_global_show(struct seq_file *s, void *p) -{ - struct hisi_hba *hisi_hba = s->private; - const struct hisi_sas_hw *hw = hisi_hba->hw; - const void *reg_global = hw->debugfs_reg_array[DEBUGFS_GLOBAL]; - - hisi_sas_debugfs_print_reg(hisi_hba->debugfs_regs[DEBUGFS_GLOBAL], - reg_global, s); - - return 0; -} - -static int hisi_sas_debugfs_global_open(struct inode *inode, struct file *filp) -{ - return single_open(filp, hisi_sas_debugfs_global_show, - inode->i_private); -} - -static const struct file_operations hisi_sas_debugfs_global_fops = { - .open = hisi_sas_debugfs_global_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .owner = THIS_MODULE, -}; - -static int hisi_sas_debugfs_axi_show(struct seq_file *s, void *p) -{ - struct hisi_hba *hisi_hba = s->private; - const struct hisi_sas_hw *hw = hisi_hba->hw; - const void *reg_axi = hw->debugfs_reg_array[DEBUGFS_AXI]; - - hisi_sas_debugfs_print_reg(hisi_hba->debugfs_regs[DEBUGFS_AXI], - reg_axi, s); - - return 0; -} - -static int hisi_sas_debugfs_axi_open(struct inode *inode, struct file *filp) -{ - return single_open(filp, hisi_sas_debugfs_axi_show, - inode->i_private); -} - -static const struct file_operations hisi_sas_debugfs_axi_fops = { - .open = hisi_sas_debugfs_axi_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .owner = THIS_MODULE, -}; - -static int hisi_sas_debugfs_ras_show(struct seq_file *s, void *p) -{ - struct hisi_hba *hisi_hba = s->private; - const struct hisi_sas_hw *hw = hisi_hba->hw; - const void *reg_ras = hw->debugfs_reg_array[DEBUGFS_RAS]; - - hisi_sas_debugfs_print_reg(hisi_hba->debugfs_regs[DEBUGFS_RAS], - reg_ras, s); - - return 0; -} - -static int hisi_sas_debugfs_ras_open(struct inode *inode, struct file *filp) -{ - return single_open(filp, hisi_sas_debugfs_ras_show, - inode->i_private); -} - -static const struct file_operations hisi_sas_debugfs_ras_fops = { - .open = hisi_sas_debugfs_ras_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .owner = THIS_MODULE, -}; - -static int hisi_sas_debugfs_port_show(struct seq_file *s, void *p) -{ - struct hisi_sas_phy *phy = s->private; - struct hisi_hba *hisi_hba = phy->hisi_hba; - const struct hisi_sas_hw *hw = hisi_hba->hw; - const struct hisi_sas_debugfs_reg *reg_port = hw->debugfs_reg_port; - u32 *databuf = hisi_hba->debugfs_port_reg[phy->sas_phy.id]; - - hisi_sas_debugfs_print_reg(databuf, reg_port, s); - - return 0; -} - -static int hisi_sas_debugfs_port_open(struct inode *inode, struct file *filp) -{ - return single_open(filp, hisi_sas_debugfs_port_show, inode->i_private); -} - -static const struct file_operations hisi_sas_debugfs_port_fops = { - .open = hisi_sas_debugfs_port_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .owner = THIS_MODULE, -}; - -static void hisi_sas_show_row_64(struct seq_file *s, int index, - int sz, __le64 *ptr) -{ - int i; - - /* completion header size not fixed per HW version */ - seq_printf(s, "index %04d:\n\t", index); - for (i = 1; i <= sz / 8; i++, ptr++) { - seq_printf(s, " 0x%016llx", le64_to_cpu(*ptr)); - if (!(i % 2)) - seq_puts(s, "\n\t"); - } - - seq_puts(s, "\n"); -} - -static void hisi_sas_show_row_32(struct seq_file *s, int index, - int sz, __le32 *ptr) -{ - int i; - - /* completion header size not fixed per HW version */ - seq_printf(s, "index %04d:\n\t", index); - for (i = 1; i <= sz / 4; i++, ptr++) { - seq_printf(s, " 0x%08x", le32_to_cpu(*ptr)); - if (!(i % 4)) - seq_puts(s, "\n\t"); - } - seq_puts(s, "\n"); -} - -static void hisi_sas_cq_show_slot(struct seq_file *s, int slot, void *cq_ptr) -{ - struct hisi_sas_cq *cq = cq_ptr; - struct hisi_hba *hisi_hba = cq->hisi_hba; - void *complete_queue = hisi_hba->debugfs_complete_hdr[cq->id]; - __le32 *complete_hdr = complete_queue + - (hisi_hba->hw->complete_hdr_size * slot); - - hisi_sas_show_row_32(s, slot, - hisi_hba->hw->complete_hdr_size, - complete_hdr); -} - -static int hisi_sas_debugfs_cq_show(struct seq_file *s, void *p) -{ - struct hisi_sas_cq *cq = s->private; - int slot; - - for (slot = 0; slot < HISI_SAS_QUEUE_SLOTS; slot++) { - hisi_sas_cq_show_slot(s, slot, cq); - } - return 0; -} - -static int hisi_sas_debugfs_cq_open(struct inode *inode, struct file *filp) -{ - return single_open(filp, hisi_sas_debugfs_cq_show, inode->i_private); -} - -static const struct file_operations hisi_sas_debugfs_cq_fops = { - .open = hisi_sas_debugfs_cq_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .owner = THIS_MODULE, -}; - -static void hisi_sas_dq_show_slot(struct seq_file *s, int slot, void *dq_ptr) -{ - struct hisi_sas_dq *dq = dq_ptr; - struct hisi_hba *hisi_hba = dq->hisi_hba; - void *cmd_queue = hisi_hba->debugfs_cmd_hdr[dq->id]; - __le32 *cmd_hdr = cmd_queue + - sizeof(struct hisi_sas_cmd_hdr) * slot; - - hisi_sas_show_row_32(s, slot, sizeof(struct hisi_sas_cmd_hdr), cmd_hdr); -} - -static int hisi_sas_debugfs_dq_show(struct seq_file *s, void *p) -{ - int slot; - - for (slot = 0; slot < HISI_SAS_QUEUE_SLOTS; slot++) { - hisi_sas_dq_show_slot(s, slot, s->private); - } - return 0; -} - -static int hisi_sas_debugfs_dq_open(struct inode *inode, struct file *filp) -{ - return single_open(filp, hisi_sas_debugfs_dq_show, inode->i_private); -} - -static const struct file_operations hisi_sas_debugfs_dq_fops = { - .open = hisi_sas_debugfs_dq_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .owner = THIS_MODULE, -}; - -static int hisi_sas_debugfs_iost_show(struct seq_file *s, void *p) -{ - struct hisi_hba *hisi_hba = s->private; - struct hisi_sas_iost *debugfs_iost = hisi_hba->debugfs_iost; - int i, max_command_entries = HISI_SAS_MAX_COMMANDS; - - for (i = 0; i < max_command_entries; i++, debugfs_iost++) { - __le64 *iost = &debugfs_iost->qw0; - - hisi_sas_show_row_64(s, i, sizeof(*debugfs_iost), iost); - } - - return 0; -} - -static int hisi_sas_debugfs_iost_open(struct inode *inode, struct file *filp) -{ - return single_open(filp, hisi_sas_debugfs_iost_show, inode->i_private); -} - -static const struct file_operations hisi_sas_debugfs_iost_fops = { - .open = hisi_sas_debugfs_iost_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .owner = THIS_MODULE, -}; - -static int hisi_sas_debugfs_iost_cache_show(struct seq_file *s, void *p) -{ - struct hisi_hba *hisi_hba = s->private; - struct hisi_sas_iost_itct_cache *iost_cache = - (struct hisi_sas_iost_itct_cache *)hisi_hba->debugfs_iost_cache; - u32 cache_size = HISI_SAS_IOST_ITCT_CACHE_DW_SZ * 4; - int i, tab_idx; - __le64 *iost; - - for (i = 0; i < HISI_SAS_IOST_ITCT_CACHE_NUM; i++, iost_cache++) { - /* - * Data struct of IOST cache: - * Data[1]: BIT0~15: Table index - * Bit16: Valid mask - * Data[2]~[9]: IOST table - */ - tab_idx = (iost_cache->data[1] & 0xffff); - iost = (__le64 *)iost_cache; - - hisi_sas_show_row_64(s, tab_idx, cache_size, iost); - } - - return 0; -} - -static int hisi_sas_debugfs_iost_cache_open(struct inode *inode, - struct file *filp) -{ - return single_open(filp, hisi_sas_debugfs_iost_cache_show, - inode->i_private); -} - -static const struct file_operations hisi_sas_debugfs_iost_cache_fops = { - .open = hisi_sas_debugfs_iost_cache_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .owner = THIS_MODULE, -}; - -static int hisi_sas_debugfs_itct_show(struct seq_file *s, void *p) -{ - int i; - struct hisi_hba *hisi_hba = s->private; - struct hisi_sas_itct *debugfs_itct = hisi_hba->debugfs_itct; - - for (i = 0; i < HISI_SAS_MAX_ITCT_ENTRIES; i++, debugfs_itct++) { - __le64 *itct = &debugfs_itct->qw0; - - hisi_sas_show_row_64(s, i, sizeof(*debugfs_itct), itct); - } - - return 0; -} - -static int hisi_sas_debugfs_itct_open(struct inode *inode, struct file *filp) -{ - return single_open(filp, hisi_sas_debugfs_itct_show, inode->i_private); -} - -static const struct file_operations hisi_sas_debugfs_itct_fops = { - .open = hisi_sas_debugfs_itct_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .owner = THIS_MODULE, -}; - -static int hisi_sas_debugfs_itct_cache_show(struct seq_file *s, void *p) -{ - struct hisi_hba *hisi_hba = s->private; - struct hisi_sas_iost_itct_cache *itct_cache = - (struct hisi_sas_iost_itct_cache *)hisi_hba->debugfs_itct_cache; - u32 cache_size = HISI_SAS_IOST_ITCT_CACHE_DW_SZ * 4; - int i, tab_idx; - __le64 *itct; - - for (i = 0; i < HISI_SAS_IOST_ITCT_CACHE_NUM; i++, itct_cache++) { - /* - * Data struct of ITCT cache: - * Data[1]: BIT0~15: Table index - * Bit16: Valid mask - * Data[2]~[9]: ITCT table - */ - tab_idx = itct_cache->data[1] & 0xffff; - itct = (__le64 *)itct_cache; - - hisi_sas_show_row_64(s, tab_idx, cache_size, itct); - } - - return 0; -} - -static int hisi_sas_debugfs_itct_cache_open(struct inode *inode, - struct file *filp) -{ - return single_open(filp, hisi_sas_debugfs_itct_cache_show, - inode->i_private); -} - -static const struct file_operations hisi_sas_debugfs_itct_cache_fops = { - .open = hisi_sas_debugfs_itct_cache_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .owner = THIS_MODULE, -}; - -static void hisi_sas_debugfs_create_files(struct hisi_hba *hisi_hba) -{ - struct dentry *dump_dentry; - struct dentry *dentry; - char name[256]; - int p; - int c; - int d; - - /* Create dump dir inside device dir */ - dump_dentry = debugfs_create_dir("dump", hisi_hba->debugfs_dir); - hisi_hba->debugfs_dump_dentry = dump_dentry; - - debugfs_create_file("global", 0400, dump_dentry, hisi_hba, - &hisi_sas_debugfs_global_fops); - - /* Create port dir and files */ - dentry = debugfs_create_dir("port", dump_dentry); - for (p = 0; p < hisi_hba->n_phy; p++) { - snprintf(name, 256, "%d", p); - - debugfs_create_file(name, 0400, dentry, &hisi_hba->phy[p], - &hisi_sas_debugfs_port_fops); - } - - /* Create CQ dir and files */ - dentry = debugfs_create_dir("cq", dump_dentry); - for (c = 0; c < hisi_hba->queue_count; c++) { - snprintf(name, 256, "%d", c); - - debugfs_create_file(name, 0400, dentry, &hisi_hba->cq[c], - &hisi_sas_debugfs_cq_fops); - } - - /* Create DQ dir and files */ - dentry = debugfs_create_dir("dq", dump_dentry); - for (d = 0; d < hisi_hba->queue_count; d++) { - snprintf(name, 256, "%d", d); - - debugfs_create_file(name, 0400, dentry, &hisi_hba->dq[d], - &hisi_sas_debugfs_dq_fops); - } - - debugfs_create_file("iost", 0400, dump_dentry, hisi_hba, - &hisi_sas_debugfs_iost_fops); - - debugfs_create_file("iost_cache", 0400, dump_dentry, hisi_hba, - &hisi_sas_debugfs_iost_cache_fops); - - debugfs_create_file("itct", 0400, dump_dentry, hisi_hba, - &hisi_sas_debugfs_itct_fops); - - debugfs_create_file("itct_cache", 0400, dump_dentry, hisi_hba, - &hisi_sas_debugfs_itct_cache_fops); - - debugfs_create_file("axi", 0400, dump_dentry, hisi_hba, - &hisi_sas_debugfs_axi_fops); - - debugfs_create_file("ras", 0400, dump_dentry, hisi_hba, - &hisi_sas_debugfs_ras_fops); - - return; -} - -static void hisi_sas_debugfs_snapshot_regs(struct hisi_hba *hisi_hba) -{ - hisi_hba->hw->snapshot_prepare(hisi_hba); - - hisi_sas_debugfs_snapshot_global_reg(hisi_hba); - hisi_sas_debugfs_snapshot_port_reg(hisi_hba); - hisi_sas_debugfs_snapshot_axi_reg(hisi_hba); - hisi_sas_debugfs_snapshot_ras_reg(hisi_hba); - hisi_sas_debugfs_snapshot_cq_reg(hisi_hba); - hisi_sas_debugfs_snapshot_dq_reg(hisi_hba); - hisi_sas_debugfs_snapshot_itct_reg(hisi_hba); - hisi_sas_debugfs_snapshot_iost_reg(hisi_hba); - - hisi_sas_debugfs_create_files(hisi_hba); - - hisi_hba->hw->snapshot_restore(hisi_hba); -} - -static ssize_t hisi_sas_debugfs_trigger_dump_write(struct file *file, - const char __user *user_buf, - size_t count, loff_t *ppos) -{ - struct hisi_hba *hisi_hba = file->f_inode->i_private; - char buf[8]; - - /* A bit racy, but don't care too much since it's only debugfs */ - if (hisi_hba->debugfs_snapshot) - return -EFAULT; - - if (count > 8) - return -EFAULT; - - if (copy_from_user(buf, user_buf, count)) - return -EFAULT; - - if (buf[0] != '1') - return -EFAULT; - - queue_work(hisi_hba->wq, &hisi_hba->debugfs_work); - - return count; -} - -static const struct file_operations hisi_sas_debugfs_trigger_dump_fops = { - .write = &hisi_sas_debugfs_trigger_dump_write, - .owner = THIS_MODULE, -}; - -enum { - HISI_SAS_BIST_LOOPBACK_MODE_DIGITAL = 0, - HISI_SAS_BIST_LOOPBACK_MODE_SERDES, - HISI_SAS_BIST_LOOPBACK_MODE_REMOTE, -}; - -enum { - HISI_SAS_BIST_CODE_MODE_PRBS7 = 0, - HISI_SAS_BIST_CODE_MODE_PRBS23, - HISI_SAS_BIST_CODE_MODE_PRBS31, - HISI_SAS_BIST_CODE_MODE_JTPAT, - HISI_SAS_BIST_CODE_MODE_CJTPAT, - HISI_SAS_BIST_CODE_MODE_SCRAMBED_0, - HISI_SAS_BIST_CODE_MODE_TRAIN, - HISI_SAS_BIST_CODE_MODE_TRAIN_DONE, - HISI_SAS_BIST_CODE_MODE_HFTP, - HISI_SAS_BIST_CODE_MODE_MFTP, - HISI_SAS_BIST_CODE_MODE_LFTP, - HISI_SAS_BIST_CODE_MODE_FIXED_DATA, -}; - -static const struct { - int value; - char *name; -} hisi_sas_debugfs_loop_linkrate[] = { - { SAS_LINK_RATE_1_5_GBPS, "1.5 Gbit" }, - { SAS_LINK_RATE_3_0_GBPS, "3.0 Gbit" }, - { SAS_LINK_RATE_6_0_GBPS, "6.0 Gbit" }, - { SAS_LINK_RATE_12_0_GBPS, "12.0 Gbit" }, -}; - -static int hisi_sas_debugfs_bist_linkrate_show(struct seq_file *s, void *p) -{ - struct hisi_hba *hisi_hba = s->private; - int i; - - for (i = 0; i < ARRAY_SIZE(hisi_sas_debugfs_loop_linkrate); i++) { - int match = (hisi_hba->debugfs_bist_linkrate == - hisi_sas_debugfs_loop_linkrate[i].value); - - seq_printf(s, "%s%s%s ", match ? "[" : "", - hisi_sas_debugfs_loop_linkrate[i].name, - match ? "]" : ""); - } - seq_puts(s, "\n"); - - return 0; -} - -static ssize_t hisi_sas_debugfs_bist_linkrate_write(struct file *filp, - const char __user *buf, - size_t count, loff_t *ppos) -{ - struct seq_file *m = filp->private_data; - struct hisi_hba *hisi_hba = m->private; - char kbuf[16] = {}, *pkbuf; - bool found = false; - int i; - - if (hisi_hba->debugfs_bist_enable) - return -EPERM; - - if (count >= sizeof(kbuf)) - return -EOVERFLOW; - - if (copy_from_user(kbuf, buf, count)) - return -EINVAL; - - pkbuf = strstrip(kbuf); - - for (i = 0; i < ARRAY_SIZE(hisi_sas_debugfs_loop_linkrate); i++) { - if (!strncmp(hisi_sas_debugfs_loop_linkrate[i].name, - pkbuf, 16)) { - hisi_hba->debugfs_bist_linkrate = - hisi_sas_debugfs_loop_linkrate[i].value; - found = true; - break; - } - } - - if (!found) - return -EINVAL; - - return count; -} - -static int hisi_sas_debugfs_bist_linkrate_open(struct inode *inode, - struct file *filp) -{ - return single_open(filp, hisi_sas_debugfs_bist_linkrate_show, - inode->i_private); -} - -static const struct file_operations hisi_sas_debugfs_bist_linkrate_ops = { - .open = hisi_sas_debugfs_bist_linkrate_open, - .read = seq_read, - .write = hisi_sas_debugfs_bist_linkrate_write, - .llseek = seq_lseek, - .release = single_release, - .owner = THIS_MODULE, -}; - -static const struct { - int value; - char *name; -} hisi_sas_debugfs_loop_code_mode[] = { - { HISI_SAS_BIST_CODE_MODE_PRBS7, "PRBS7" }, - { HISI_SAS_BIST_CODE_MODE_PRBS23, "PRBS23" }, - { HISI_SAS_BIST_CODE_MODE_PRBS31, "PRBS31" }, - { HISI_SAS_BIST_CODE_MODE_JTPAT, "JTPAT" }, - { HISI_SAS_BIST_CODE_MODE_CJTPAT, "CJTPAT" }, - { HISI_SAS_BIST_CODE_MODE_SCRAMBED_0, "SCRAMBED_0" }, - { HISI_SAS_BIST_CODE_MODE_TRAIN, "TRAIN" }, - { HISI_SAS_BIST_CODE_MODE_TRAIN_DONE, "TRAIN_DONE" }, - { HISI_SAS_BIST_CODE_MODE_HFTP, "HFTP" }, - { HISI_SAS_BIST_CODE_MODE_MFTP, "MFTP" }, - { HISI_SAS_BIST_CODE_MODE_LFTP, "LFTP" }, - { HISI_SAS_BIST_CODE_MODE_FIXED_DATA, "FIXED_DATA" }, -}; - -static int hisi_sas_debugfs_bist_code_mode_show(struct seq_file *s, void *p) -{ - struct hisi_hba *hisi_hba = s->private; - int i; - - for (i = 0; i < ARRAY_SIZE(hisi_sas_debugfs_loop_code_mode); i++) { - int match = (hisi_hba->debugfs_bist_code_mode == - hisi_sas_debugfs_loop_code_mode[i].value); - - seq_printf(s, "%s%s%s ", match ? "[" : "", - hisi_sas_debugfs_loop_code_mode[i].name, - match ? "]" : ""); - } - seq_puts(s, "\n"); - - return 0; -} - -static ssize_t hisi_sas_debugfs_bist_code_mode_write(struct file *filp, - const char __user *buf, - size_t count, - loff_t *ppos) -{ - struct seq_file *m = filp->private_data; - struct hisi_hba *hisi_hba = m->private; - char kbuf[16] = {}, *pkbuf; - bool found = false; - int i; - - if (hisi_hba->debugfs_bist_enable) - return -EPERM; - - if (count >= sizeof(kbuf)) - return -EINVAL; - - if (copy_from_user(kbuf, buf, count)) - return -EOVERFLOW; - - pkbuf = strstrip(kbuf); - - for (i = 0; i < ARRAY_SIZE(hisi_sas_debugfs_loop_code_mode); i++) { - if (!strncmp(hisi_sas_debugfs_loop_code_mode[i].name, - pkbuf, 16)) { - hisi_hba->debugfs_bist_code_mode = - hisi_sas_debugfs_loop_code_mode[i].value; - found = true; - break; - } - } - - if (!found) - return -EINVAL; - - return count; -} - -static int hisi_sas_debugfs_bist_code_mode_open(struct inode *inode, - struct file *filp) -{ - return single_open(filp, hisi_sas_debugfs_bist_code_mode_show, - inode->i_private); -} - -static const struct file_operations hisi_sas_debugfs_bist_code_mode_ops = { - .open = hisi_sas_debugfs_bist_code_mode_open, - .read = seq_read, - .write = hisi_sas_debugfs_bist_code_mode_write, - .llseek = seq_lseek, - .release = single_release, - .owner = THIS_MODULE, -}; - -static ssize_t hisi_sas_debugfs_bist_phy_write(struct file *filp, - const char __user *buf, - size_t count, loff_t *ppos) -{ - struct seq_file *m = filp->private_data; - struct hisi_hba *hisi_hba = m->private; - unsigned int phy_no; - int val; - - if (hisi_hba->debugfs_bist_enable) - return -EPERM; - - val = kstrtouint_from_user(buf, count, 0, &phy_no); - if (val) - return val; - - if (phy_no >= hisi_hba->n_phy) - return -EINVAL; - - hisi_hba->debugfs_bist_phy_no = phy_no; - - return count; -} - -static int hisi_sas_debugfs_bist_phy_show(struct seq_file *s, void *p) -{ - struct hisi_hba *hisi_hba = s->private; - - seq_printf(s, "%d\n", hisi_hba->debugfs_bist_phy_no); - - return 0; -} - -static int hisi_sas_debugfs_bist_phy_open(struct inode *inode, - struct file *filp) -{ - return single_open(filp, hisi_sas_debugfs_bist_phy_show, - inode->i_private); -} - -static const struct file_operations hisi_sas_debugfs_bist_phy_ops = { - .open = hisi_sas_debugfs_bist_phy_open, - .read = seq_read, - .write = hisi_sas_debugfs_bist_phy_write, - .llseek = seq_lseek, - .release = single_release, - .owner = THIS_MODULE, -}; - -static const struct { - int value; - char *name; -} hisi_sas_debugfs_loop_modes[] = { - { HISI_SAS_BIST_LOOPBACK_MODE_DIGITAL, "digial" }, - { HISI_SAS_BIST_LOOPBACK_MODE_SERDES, "serdes" }, - { HISI_SAS_BIST_LOOPBACK_MODE_REMOTE, "remote" }, -}; - -static int hisi_sas_debugfs_bist_mode_show(struct seq_file *s, void *p) -{ - struct hisi_hba *hisi_hba = s->private; - int i; - - for (i = 0; i < ARRAY_SIZE(hisi_sas_debugfs_loop_modes); i++) { - int match = (hisi_hba->debugfs_bist_mode == - hisi_sas_debugfs_loop_modes[i].value); - - seq_printf(s, "%s%s%s ", match ? "[" : "", - hisi_sas_debugfs_loop_modes[i].name, - match ? "]" : ""); - } - seq_puts(s, "\n"); - - return 0; -} - -static ssize_t hisi_sas_debugfs_bist_mode_write(struct file *filp, - const char __user *buf, - size_t count, loff_t *ppos) -{ - struct seq_file *m = filp->private_data; - struct hisi_hba *hisi_hba = m->private; - char kbuf[16] = {}, *pkbuf; - bool found = false; - int i; - - if (hisi_hba->debugfs_bist_enable) - return -EPERM; - - if (count >= sizeof(kbuf)) - return -EINVAL; - - if (copy_from_user(kbuf, buf, count)) - return -EOVERFLOW; - - pkbuf = strstrip(kbuf); - - for (i = 0; i < ARRAY_SIZE(hisi_sas_debugfs_loop_modes); i++) { - if (!strncmp(hisi_sas_debugfs_loop_modes[i].name, pkbuf, 16)) { - hisi_hba->debugfs_bist_mode = - hisi_sas_debugfs_loop_modes[i].value; - found = true; - break; - } - } - - if (!found) - return -EINVAL; - - return count; -} - -static int hisi_sas_debugfs_bist_mode_open(struct inode *inode, - struct file *filp) -{ - return single_open(filp, hisi_sas_debugfs_bist_mode_show, - inode->i_private); -} - -static const struct file_operations hisi_sas_debugfs_bist_mode_ops = { - .open = hisi_sas_debugfs_bist_mode_open, - .read = seq_read, - .write = hisi_sas_debugfs_bist_mode_write, - .llseek = seq_lseek, - .release = single_release, - .owner = THIS_MODULE, -}; - -static ssize_t hisi_sas_debugfs_bist_enable_write(struct file *filp, - const char __user *buf, - size_t count, loff_t *ppos) -{ - struct seq_file *m = filp->private_data; - struct hisi_hba *hisi_hba = m->private; - unsigned int enable; - int val; - - val = kstrtouint_from_user(buf, count, 0, &enable); - if (val) - return val; - - if (enable > 1) - return -EINVAL; - - if (enable == hisi_hba->debugfs_bist_enable) - return count; - - if (!hisi_hba->hw->set_bist) - return -EPERM; - - val = hisi_hba->hw->set_bist(hisi_hba, enable); - if (val < 0) - return val; - - hisi_hba->debugfs_bist_enable = enable; - - return count; -} - -static int hisi_sas_debugfs_bist_enable_show(struct seq_file *s, void *p) -{ - struct hisi_hba *hisi_hba = s->private; - - seq_printf(s, "%d\n", hisi_hba->debugfs_bist_enable); - - return 0; -} - -static int hisi_sas_debugfs_bist_enable_open(struct inode *inode, - struct file *filp) -{ - return single_open(filp, hisi_sas_debugfs_bist_enable_show, - inode->i_private); -} - -static const struct file_operations hisi_sas_debugfs_bist_enable_ops = { - .open = hisi_sas_debugfs_bist_enable_open, - .read = seq_read, - .write = hisi_sas_debugfs_bist_enable_write, - .llseek = seq_lseek, - .release = single_release, - .owner = THIS_MODULE, -}; - -void hisi_sas_debugfs_work_handler(struct work_struct *work) -{ - struct hisi_hba *hisi_hba = - container_of(work, struct hisi_hba, debugfs_work); - - if (hisi_hba->debugfs_snapshot) - return; - hisi_hba->debugfs_snapshot = true; - - hisi_sas_debugfs_snapshot_regs(hisi_hba); -} -EXPORT_SYMBOL_GPL(hisi_sas_debugfs_work_handler); - -static void hisi_sas_debugfs_release(struct hisi_hba *hisi_hba) -{ - struct device *dev = hisi_hba->dev; - int i; - - devm_kfree(dev, hisi_hba->debugfs_iost_cache); - devm_kfree(dev, hisi_hba->debugfs_itct_cache); - devm_kfree(dev, hisi_hba->debugfs_iost); - - for (i = 0; i < hisi_hba->queue_count; i++) - devm_kfree(dev, hisi_hba->debugfs_cmd_hdr[i]); - - for (i = 0; i < hisi_hba->queue_count; i++) - devm_kfree(dev, hisi_hba->debugfs_complete_hdr[i]); - - for (i = 0; i < DEBUGFS_REGS_NUM; i++) - devm_kfree(dev, hisi_hba->debugfs_regs[i]); - - for (i = 0; i < hisi_hba->n_phy; i++) - devm_kfree(dev, hisi_hba->debugfs_port_reg[i]); -} - -static int hisi_sas_debugfs_alloc(struct hisi_hba *hisi_hba) -{ - const struct hisi_sas_hw *hw = hisi_hba->hw; - struct device *dev = hisi_hba->dev; - int p, c, d; - size_t sz; - - sz = hw->debugfs_reg_array[DEBUGFS_GLOBAL]->count * 4; - hisi_hba->debugfs_regs[DEBUGFS_GLOBAL] = - devm_kmalloc(dev, sz, GFP_KERNEL); - - if (!hisi_hba->debugfs_regs[DEBUGFS_GLOBAL]) - goto fail; - - sz = hw->debugfs_reg_port->count * 4; - for (p = 0; p < hisi_hba->n_phy; p++) { - hisi_hba->debugfs_port_reg[p] = - devm_kmalloc(dev, sz, GFP_KERNEL); - - if (!hisi_hba->debugfs_port_reg[p]) - goto fail; - } - - sz = hw->debugfs_reg_array[DEBUGFS_AXI]->count * 4; - hisi_hba->debugfs_regs[DEBUGFS_AXI] = - devm_kmalloc(dev, sz, GFP_KERNEL); - - if (!hisi_hba->debugfs_regs[DEBUGFS_AXI]) - goto fail; - - sz = hw->debugfs_reg_array[DEBUGFS_RAS]->count * 4; - hisi_hba->debugfs_regs[DEBUGFS_RAS] = - devm_kmalloc(dev, sz, GFP_KERNEL); - - if (!hisi_hba->debugfs_regs[DEBUGFS_RAS]) - goto fail; - - sz = hw->complete_hdr_size * HISI_SAS_QUEUE_SLOTS; - for (c = 0; c < hisi_hba->queue_count; c++) { - hisi_hba->debugfs_complete_hdr[c] = - devm_kmalloc(dev, sz, GFP_KERNEL); - - if (!hisi_hba->debugfs_complete_hdr[c]) - goto fail; - } - - sz = sizeof(struct hisi_sas_cmd_hdr) * HISI_SAS_QUEUE_SLOTS; - for (d = 0; d < hisi_hba->queue_count; d++) { - hisi_hba->debugfs_cmd_hdr[d] = - devm_kmalloc(dev, sz, GFP_KERNEL); - - if (!hisi_hba->debugfs_cmd_hdr[d]) - goto fail; - } - - sz = HISI_SAS_MAX_COMMANDS * sizeof(struct hisi_sas_iost); - - hisi_hba->debugfs_iost = devm_kmalloc(dev, sz, GFP_KERNEL); - if (!hisi_hba->debugfs_iost) - goto fail; - - sz = HISI_SAS_IOST_ITCT_CACHE_NUM * - sizeof(struct hisi_sas_iost_itct_cache); - - hisi_hba->debugfs_iost_cache = devm_kmalloc(dev, sz, GFP_KERNEL); - if (!hisi_hba->debugfs_iost_cache) - goto fail; - - sz = HISI_SAS_IOST_ITCT_CACHE_NUM * - sizeof(struct hisi_sas_iost_itct_cache); - - hisi_hba->debugfs_itct_cache = devm_kmalloc(dev, sz, GFP_KERNEL); - if (!hisi_hba->debugfs_itct_cache) - goto fail; - - /* New memory allocation must be locate before itct */ - sz = HISI_SAS_MAX_ITCT_ENTRIES * sizeof(struct hisi_sas_itct); - - hisi_hba->debugfs_itct = devm_kmalloc(dev, sz, GFP_KERNEL); - if (!hisi_hba->debugfs_itct) - goto fail; - - return 0; -fail: - hisi_sas_debugfs_release(hisi_hba); - return -ENOMEM; -} - -static void hisi_sas_debugfs_bist_init(struct hisi_hba *hisi_hba) -{ - hisi_hba->debugfs_bist_dentry = - debugfs_create_dir("bist", hisi_hba->debugfs_dir); - debugfs_create_file("link_rate", 0600, - hisi_hba->debugfs_bist_dentry, hisi_hba, - &hisi_sas_debugfs_bist_linkrate_ops); - - debugfs_create_file("code_mode", 0600, - hisi_hba->debugfs_bist_dentry, hisi_hba, - &hisi_sas_debugfs_bist_code_mode_ops); - - debugfs_create_file("phy_id", 0600, hisi_hba->debugfs_bist_dentry, - hisi_hba, &hisi_sas_debugfs_bist_phy_ops); - - debugfs_create_u32("cnt", 0600, hisi_hba->debugfs_bist_dentry, - &hisi_hba->debugfs_bist_cnt); - - debugfs_create_file("loopback_mode", 0600, - hisi_hba->debugfs_bist_dentry, - hisi_hba, &hisi_sas_debugfs_bist_mode_ops); - - debugfs_create_file("enable", 0600, hisi_hba->debugfs_bist_dentry, - hisi_hba, &hisi_sas_debugfs_bist_enable_ops); - - hisi_hba->debugfs_bist_linkrate = SAS_LINK_RATE_1_5_GBPS; -} - -void hisi_sas_debugfs_init(struct hisi_hba *hisi_hba) -{ - struct device *dev = hisi_hba->dev; - - hisi_hba->debugfs_dir = debugfs_create_dir(dev_name(dev), - hisi_sas_debugfs_dir); - debugfs_create_file("trigger_dump", 0600, - hisi_hba->debugfs_dir, - hisi_hba, - &hisi_sas_debugfs_trigger_dump_fops); - - /* create bist structures */ - hisi_sas_debugfs_bist_init(hisi_hba); - - if (hisi_sas_debugfs_alloc(hisi_hba)) { - debugfs_remove_recursive(hisi_hba->debugfs_dir); - dev_dbg(dev, "failed to init debugfs!\n"); - } -} -EXPORT_SYMBOL_GPL(hisi_sas_debugfs_init); - -void hisi_sas_debugfs_exit(struct hisi_hba *hisi_hba) -{ - debugfs_remove_recursive(hisi_hba->debugfs_dir); -} -EXPORT_SYMBOL_GPL(hisi_sas_debugfs_exit); - -int hisi_sas_remove(struct platform_device *pdev) -{ - struct sas_ha_struct *sha = platform_get_drvdata(pdev); - struct hisi_hba *hisi_hba = sha->lldd_ha; - struct Scsi_Host *shost = sha->core.shost; - - if (timer_pending(&hisi_hba->timer)) - del_timer(&hisi_hba->timer); - - sas_unregister_ha(sha); - sas_remove_host(sha->core.shost); - - hisi_sas_free(hisi_hba); - scsi_host_put(shost); - return 0; -} -EXPORT_SYMBOL_GPL(hisi_sas_remove); - -bool hisi_sas_debugfs_enable; -EXPORT_SYMBOL_GPL(hisi_sas_debugfs_enable); -module_param_named(debugfs_enable, hisi_sas_debugfs_enable, bool, 0444); -MODULE_PARM_DESC(hisi_sas_debugfs_enable, "Enable driver debugfs (default disabled)"); - -static __init int hisi_sas_init(void) -{ - hisi_sas_stt = sas_domain_attach_transport(&hisi_sas_transport_ops); - if (!hisi_sas_stt) - return -ENOMEM; - - if (hisi_sas_debugfs_enable) - hisi_sas_debugfs_dir = debugfs_create_dir("hisi_sas", NULL); - return 0; } diff --git a/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c index 1443c803d8f753b730533db64153ecfa2b7cd3de..a7d0cc647a395ce542c8a1358a013dbe4577aeac 100644 --- a/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c +++ b/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c @@ -531,8 +531,8 @@ static void setup_itct_v1_hw(struct hisi_hba *hisi_hba, (0xff00ULL << ITCT_HDR_REJ_OPEN_TL_OFF)); } -static void clear_itct_v1_hw(struct hisi_hba *hisi_hba, - struct hisi_sas_device *sas_dev) +static int clear_itct_v1_hw(struct hisi_hba *hisi_hba, + struct hisi_sas_device *sas_dev) { u64 dev_id = sas_dev->device_id; struct hisi_sas_itct *itct = &hisi_hba->itct[dev_id]; @@ -551,6 +551,8 @@ static void clear_itct_v1_hw(struct hisi_hba *hisi_hba, qw0 = le64_to_cpu(itct->qw0); qw0 &= ~ITCT_HDR_VALID_MSK; itct->qw0 = cpu_to_le64(qw0); + + return 0; } static int reset_hw_v1_hw(struct hisi_hba *hisi_hba) @@ -750,7 +752,7 @@ static int hw_init_v1_hw(struct hisi_hba *hisi_hba) rc = reset_hw_v1_hw(hisi_hba); if (rc) { - dev_err(dev, "hisi_sas_reset_hw failed, rc=%d", rc); + dev_err(dev, "hisi_sas_reset_hw failed, rc=%d\n", rc); return rc; } @@ -1164,7 +1166,7 @@ static void slot_err_v1_hw(struct hisi_hba *hisi_hba, case SAS_PROTOCOL_STP: case SAS_PROTOCOL_SATA | SAS_PROTOCOL_STP: { - dev_err(dev, "slot err: SATA/STP not supported"); + dev_err(dev, "slot err: SATA/STP not supported\n"); } break; default: @@ -1173,15 +1175,14 @@ static void slot_err_v1_hw(struct hisi_hba *hisi_hba, } -static int slot_complete_v1_hw(struct hisi_hba *hisi_hba, - struct hisi_sas_slot *slot) +static void slot_complete_v1_hw(struct hisi_hba *hisi_hba, + struct hisi_sas_slot *slot) { struct sas_task *task = slot->task; struct hisi_sas_device *sas_dev; struct device *dev = hisi_hba->dev; struct task_status_struct *ts; struct domain_device *device; - enum exec_status sts; struct hisi_sas_complete_v1_hdr *complete_queue = hisi_hba->complete_hdr[slot->cmplt_queue]; struct hisi_sas_complete_v1_hdr *complete_hdr; @@ -1192,7 +1193,7 @@ static int slot_complete_v1_hw(struct hisi_hba *hisi_hba, cmplt_hdr_data = le32_to_cpu(complete_hdr->data); if (unlikely(!task || !task->lldd_task || !task->dev)) - return -EINVAL; + return; ts = &task->task_status; device = task->dev; @@ -1217,35 +1218,35 @@ static int slot_complete_v1_hw(struct hisi_hba *hisi_hba, u32 info_reg = hisi_sas_read32(hisi_hba, HGC_INVLD_DQE_INFO); if (info_reg & HGC_INVLD_DQE_INFO_DQ_MSK) - dev_err(dev, "slot complete: [%d:%d] has dq IPTT err", + dev_err(dev, "slot complete: [%d:%d] has dq IPTT err\n", slot->cmplt_queue, slot->cmplt_queue_slot); if (info_reg & HGC_INVLD_DQE_INFO_TYPE_MSK) - dev_err(dev, "slot complete: [%d:%d] has dq type err", + dev_err(dev, "slot complete: [%d:%d] has dq type err\n", slot->cmplt_queue, slot->cmplt_queue_slot); if (info_reg & HGC_INVLD_DQE_INFO_FORCE_MSK) - dev_err(dev, "slot complete: [%d:%d] has dq force phy err", + dev_err(dev, "slot complete: [%d:%d] has dq force phy err\n", slot->cmplt_queue, slot->cmplt_queue_slot); if (info_reg & HGC_INVLD_DQE_INFO_PHY_MSK) - dev_err(dev, "slot complete: [%d:%d] has dq phy id err", + dev_err(dev, "slot complete: [%d:%d] has dq phy id err\n", slot->cmplt_queue, slot->cmplt_queue_slot); if (info_reg & HGC_INVLD_DQE_INFO_ABORT_MSK) - dev_err(dev, "slot complete: [%d:%d] has dq abort flag err", + dev_err(dev, "slot complete: [%d:%d] has dq abort flag err\n", slot->cmplt_queue, slot->cmplt_queue_slot); if (info_reg & HGC_INVLD_DQE_INFO_IPTT_OF_MSK) - dev_err(dev, "slot complete: [%d:%d] has dq IPTT or ICT err", + dev_err(dev, "slot complete: [%d:%d] has dq IPTT or ICT err\n", slot->cmplt_queue, slot->cmplt_queue_slot); if (info_reg & HGC_INVLD_DQE_INFO_SSP_ERR_MSK) - dev_err(dev, "slot complete: [%d:%d] has dq SSP frame type err", + dev_err(dev, "slot complete: [%d:%d] has dq SSP frame type err\n", slot->cmplt_queue, slot->cmplt_queue_slot); if (info_reg & HGC_INVLD_DQE_INFO_OFL_MSK) - dev_err(dev, "slot complete: [%d:%d] has dq order frame len err", + dev_err(dev, "slot complete: [%d:%d] has dq order frame len err\n", slot->cmplt_queue, slot->cmplt_queue_slot); ts->stat = SAS_OPEN_REJECT; @@ -1257,8 +1258,14 @@ static int slot_complete_v1_hw(struct hisi_hba *hisi_hba, !(cmplt_hdr_data & CMPLT_HDR_RSPNS_XFRD_MSK)) { slot_err_v1_hw(hisi_hba, task, slot); - if (unlikely(slot->abort)) - return ts->stat; + if (unlikely(slot->abort)) { + if (dev_is_sata(device) && task->ata_task.use_ncq) + sas_ata_device_link_abort(device, true); + else + sas_task_abort(task); + + return; + } goto out; } @@ -1291,7 +1298,7 @@ static int slot_complete_v1_hw(struct hisi_hba *hisi_hba, case SAS_PROTOCOL_SATA: case SAS_PROTOCOL_STP: case SAS_PROTOCOL_SATA | SAS_PROTOCOL_STP: - dev_err(dev, "slot complete: SATA/STP not supported"); + dev_err(dev, "slot complete: SATA/STP not supported\n"); break; default: @@ -1306,13 +1313,10 @@ static int slot_complete_v1_hw(struct hisi_hba *hisi_hba, } out: - hisi_sas_slot_task_free(hisi_hba, task, slot); - sts = ts->stat; + hisi_sas_slot_task_free(hisi_hba, task, slot, true); if (task->task_done) task->task_done(task); - - return sts; } /* Interrupts */ @@ -1327,7 +1331,6 @@ static irqreturn_t int_phyup_v1_hw(int irq_no, void *p) u32 *frame_rcvd = (u32 *)sas_phy->frame_rcvd; struct sas_identify_frame *id = (struct sas_identify_frame *)frame_rcvd; irqreturn_t res = IRQ_HANDLED; - unsigned long flags; irq_value = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT2); if (!(irq_value & CHL_INT2_SL_PHY_ENA_MSK)) { @@ -1380,15 +1383,9 @@ static irqreturn_t int_phyup_v1_hw(int irq_no, void *p) phy->identify.target_port_protocols = SAS_PROTOCOL_SMP; hisi_sas_notify_phy_event(phy, HISI_PHYE_PHY_UP); - - spin_lock_irqsave(&phy->lock, flags); - if (phy->reset_completion) { - phy->in_reset = 0; - complete(phy->reset_completion); - } - spin_unlock_irqrestore(&phy->lock, flags); - end: + if (phy->reset_completion) + complete(phy->reset_completion); hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT2, CHL_INT2_SL_PHY_ENA_MSK); @@ -1408,7 +1405,6 @@ static irqreturn_t int_bcast_v1_hw(int irq, void *p) struct hisi_sas_phy *phy = p; struct hisi_hba *hisi_hba = phy->hisi_hba; struct asd_sas_phy *sas_phy = &phy->sas_phy; - struct sas_ha_struct *sha = &hisi_hba->sha; struct device *dev = hisi_hba->dev; int phy_no = sas_phy->id; u32 irq_value; @@ -1417,14 +1413,15 @@ static irqreturn_t int_bcast_v1_hw(int irq, void *p) irq_value = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT2); if (!(irq_value & CHL_INT2_SL_RX_BC_ACK_MSK)) { - dev_err(dev, "bcast: irq_value = %x not set enable bit", + dev_err(dev, "bcast: irq_value = %x not set enable bit\n", irq_value); res = IRQ_NONE; goto end; } if (!test_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags)) - sha->notify_port_event(sas_phy, PORTE_BROADCAST_RCVD); + sas_notify_port_event(sas_phy, PORTE_BROADCAST_RCVD, + GFP_ATOMIC); end: hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT2, @@ -1453,7 +1450,8 @@ static irqreturn_t int_abnormal_v1_hw(int irq, void *p) u32 phy_state = hisi_sas_read32(hisi_hba, PHY_STATE); hisi_sas_phy_down(hisi_hba, phy_no, - (phy_state & 1 << phy_no) ? 1 : 0); + (phy_state & 1 << phy_no) ? 1 : 0, + GFP_ATOMIC); } if (irq_value & CHL_INT0_ID_TIMEOUT_MSK) @@ -1755,6 +1753,7 @@ static struct device_attribute *host_attrs_v1_hw[] = { static struct scsi_host_template sht_v1_hw = { .name = DRV_NAME, + .proc_name = DRV_NAME, .module = THIS_MODULE, .queuecommand = sas_queuecommand, .target_alloc = sas_target_alloc, @@ -1768,7 +1767,7 @@ static struct scsi_host_template sht_v1_hw = { .max_sectors = SCSI_DEFAULT_MAX_SECTORS, .eh_device_reset_handler = sas_eh_device_reset_handler, .eh_target_reset_handler = sas_eh_target_reset_handler, - .slave_alloc = sas_slave_alloc, + .slave_alloc = hisi_sas_slave_alloc, .target_destroy = sas_target_destroy, .ioctl = sas_ioctl, .shost_attrs = host_attrs_v1_hw, diff --git a/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c index 11c75881bd892a7d7771691acc64e86809bcdd25..72d441eebd6f54ebdc3c03bc9775fa3de1dbd7bd 100644 --- a/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c +++ b/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c @@ -773,7 +773,6 @@ slot_index_alloc_quirk_v2_hw(struct hisi_hba *hisi_hba, struct hisi_sas_device *sas_dev = device->lldd_dev; int sata_idx = sas_dev->sata_idx; int start, end; - unsigned long flags; if (!sata_dev) { /* @@ -797,12 +796,12 @@ slot_index_alloc_quirk_v2_hw(struct hisi_hba *hisi_hba, end = 64 * (sata_idx + 2); } - spin_lock_irqsave(&hisi_hba->lock, flags); + spin_lock(&hisi_hba->lock); while (1) { start = find_next_zero_bit(bitmap, hisi_hba->slot_index_count, start); if (start >= end) { - spin_unlock_irqrestore(&hisi_hba->lock, flags); + spin_unlock(&hisi_hba->lock); return -SAS_QUEUE_FULL; } /* @@ -814,7 +813,7 @@ slot_index_alloc_quirk_v2_hw(struct hisi_hba *hisi_hba, } set_bit(start, bitmap); - spin_unlock_irqrestore(&hisi_hba->lock, flags); + spin_unlock(&hisi_hba->lock); return start; } @@ -843,9 +842,8 @@ hisi_sas_device *alloc_dev_quirk_v2_hw(struct domain_device *device) struct hisi_sas_device *sas_dev = NULL; int i, sata_dev = dev_is_sata(device); int sata_idx = -1; - unsigned long flags; - spin_lock_irqsave(&hisi_hba->lock, flags); + spin_lock(&hisi_hba->lock); if (sata_dev) if (!sata_index_alloc_v2_hw(hisi_hba, &sata_idx)) @@ -876,7 +874,7 @@ hisi_sas_device *alloc_dev_quirk_v2_hw(struct domain_device *device) } out: - spin_unlock_irqrestore(&hisi_hba->lock, flags); + spin_unlock(&hisi_hba->lock); return sas_dev; } @@ -974,13 +972,14 @@ static void setup_itct_v2_hw(struct hisi_hba *hisi_hba, (0x1ULL << ITCT_HDR_RTOLT_OFF)); } -static void clear_itct_v2_hw(struct hisi_hba *hisi_hba, - struct hisi_sas_device *sas_dev) +static int clear_itct_v2_hw(struct hisi_hba *hisi_hba, + struct hisi_sas_device *sas_dev) { DECLARE_COMPLETION_ONSTACK(completion); u64 dev_id = sas_dev->device_id; struct hisi_sas_itct *itct = &hisi_hba->itct[dev_id]; u32 reg_val = hisi_sas_read32(hisi_hba, ENT_INT_SRC3); + struct device *dev = hisi_hba->dev; int i; sas_dev->completion = &completion; @@ -990,13 +989,19 @@ static void clear_itct_v2_hw(struct hisi_hba *hisi_hba, hisi_sas_write32(hisi_hba, ENT_INT_SRC3, ENT_INT_SRC3_ITC_INT_MSK); + /* need to set register twice to clear ITCT for v2 hw */ for (i = 0; i < 2; i++) { reg_val = ITCT_CLR_EN_MSK | (dev_id & ITCT_DEV_MSK); hisi_sas_write32(hisi_hba, ITCT_CLR, reg_val); - wait_for_completion(sas_dev->completion); + if (!wait_for_completion_timeout(sas_dev->completion, + HISI_SAS_CLEAR_ITCT_TIMEOUT)) { + dev_warn(dev, "failed to clear ITCT\n"); + return -ETIMEDOUT; + } memset(itct, 0, sizeof(struct hisi_sas_itct)); } + return 0; } static void free_device_v2_hw(struct hisi_sas_device *sas_dev) @@ -1197,7 +1202,7 @@ static void init_reg_v2_hw(struct hisi_hba *hisi_hba) hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0x7ffe20fe); hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0xfff00c30); for (i = 0; i < hisi_hba->queue_count; i++) - hisi_sas_write32(hisi_hba, OQ0_INT_SRC_MSK+0x4*i, 0); + hisi_sas_write32(hisi_hba, OQ0_INT_SRC_MSK + 0x4 * i, 0); hisi_sas_write32(hisi_hba, AXI_AHB_CLK_CFG, 1); hisi_sas_write32(hisi_hba, HYPER_STREAM_ID_EN_CFG, 1); @@ -1377,7 +1382,7 @@ static int hw_init_v2_hw(struct hisi_hba *hisi_hba) rc = reset_hw_v2_hw(hisi_hba); if (rc) { - dev_err(dev, "hisi_sas_reset_hw failed, rc=%d", rc); + dev_err(dev, "hisi_sas_reset_hw failed, rc=%d\n", rc); return rc; } @@ -2021,6 +2026,11 @@ static void slot_err_v2_hw(struct hisi_hba *hisi_hba, u16 dma_tx_err_type = le16_to_cpu(err_record->dma_tx_err_type); u16 sipc_rx_err_type = le16_to_cpu(err_record->sipc_rx_err_type); u32 dma_rx_err_type = le32_to_cpu(err_record->dma_rx_err_type); + struct hisi_sas_complete_v2_hdr *complete_queue = + hisi_hba->complete_hdr[slot->cmplt_queue]; + struct hisi_sas_complete_v2_hdr *complete_hdr = + &complete_queue[slot->cmplt_queue_slot]; + u32 dw0 = le32_to_cpu(complete_hdr->dw0); int error = -1; if (err_phase == 1) { @@ -2305,7 +2315,8 @@ static void slot_err_v2_hw(struct hisi_hba *hisi_hba, break; } } - hisi_sas_sata_done(task, slot); + if (dw0 & CMPLT_HDR_RSPNS_XFRD_MSK) + hisi_sas_sata_done(task, slot); } break; default: @@ -2313,8 +2324,8 @@ static void slot_err_v2_hw(struct hisi_hba *hisi_hba, } } -static int -slot_complete_v2_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot) +static void slot_complete_v2_hw(struct hisi_hba *hisi_hba, + struct hisi_sas_slot *slot) { struct sas_task *task = slot->task; struct hisi_sas_device *sas_dev; @@ -2322,7 +2333,6 @@ slot_complete_v2_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot) struct task_status_struct *ts; struct domain_device *device; struct sas_ha_struct *ha; - enum exec_status sts; struct hisi_sas_complete_v2_hdr *complete_queue = hisi_hba->complete_hdr[slot->cmplt_queue]; struct hisi_sas_complete_v2_hdr *complete_hdr = @@ -2332,7 +2342,7 @@ slot_complete_v2_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot) u32 dw0; if (unlikely(!task || !task->lldd_task || !task->dev)) - return -EINVAL; + return; ts = &task->task_status; device = task->dev; @@ -2364,18 +2374,18 @@ slot_complete_v2_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot) case STAT_IO_COMPLETE: /* internal abort command complete */ ts->stat = TMF_RESP_FUNC_SUCC; - del_timer(&slot->internal_abort_timer); + del_timer_sync(&slot->internal_abort_timer); goto out; case STAT_IO_NO_DEVICE: ts->stat = TMF_RESP_FUNC_COMPLETE; - del_timer(&slot->internal_abort_timer); + del_timer_sync(&slot->internal_abort_timer); goto out; case STAT_IO_NOT_VALID: /* abort single io, controller don't find * the io need to abort */ ts->stat = TMF_RESP_FUNC_FAILED; - del_timer(&slot->internal_abort_timer); + del_timer_sync(&slot->internal_abort_timer); goto out; default: break; @@ -2400,8 +2410,14 @@ slot_complete_v2_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot) error_info[0], error_info[1], error_info[2], error_info[3]); - if (unlikely(slot->abort)) - return ts->stat; + if (unlikely(slot->abort)) { + if (dev_is_sata(device) && task->ata_task.use_ncq) + sas_ata_device_link_abort(device, true); + else + sas_task_abort(task); + + return; + } goto out; } @@ -2436,7 +2452,8 @@ slot_complete_v2_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot) case SAS_PROTOCOL_SATA | SAS_PROTOCOL_STP: { ts->stat = SAM_STAT_GOOD; - hisi_sas_sata_done(task, slot); + if (dw0 & CMPLT_HDR_RSPNS_XFRD_MSK) + hisi_sas_sata_done(task, slot); break; } default: @@ -2451,16 +2468,15 @@ slot_complete_v2_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot) } out: - sts = ts->stat; spin_lock_irqsave(&task->task_state_lock, flags); if (task->task_state_flags & SAS_TASK_STATE_ABORTED) { spin_unlock_irqrestore(&task->task_state_lock, flags); dev_info(dev, "slot complete: task(%pK) aborted\n", task); - return SAS_ABORTED_TASK; + return; } task->task_state_flags |= SAS_TASK_STATE_DONE; spin_unlock_irqrestore(&task->task_state_lock, flags); - hisi_sas_slot_task_free(hisi_hba, task, slot); + hisi_sas_slot_task_free(hisi_hba, task, slot, true); if (!is_internal && (task->task_proto != SAS_PROTOCOL_SMP)) { spin_lock_irqsave(&device->done_lock, flags); @@ -2468,15 +2484,13 @@ slot_complete_v2_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot) spin_unlock_irqrestore(&device->done_lock, flags); dev_info(dev, "slot complete: task(%pK) ignored\n", task); - return sts; + return; } spin_unlock_irqrestore(&device->done_lock, flags); } if (task->task_done) task->task_done(task); - - return sts; } static void prep_ata_v2_hw(struct hisi_hba *hisi_hba, @@ -2638,7 +2652,6 @@ static int phy_up_v2_hw(int phy_no, struct hisi_hba *hisi_hba) struct device *dev = hisi_hba->dev; u32 *frame_rcvd = (u32 *)sas_phy->frame_rcvd; struct sas_identify_frame *id = (struct sas_identify_frame *)frame_rcvd; - unsigned long flags; hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_PHY_ENA_MSK, 1); @@ -2693,14 +2706,9 @@ static int phy_up_v2_hw(int phy_no, struct hisi_hba *hisi_hba) set_link_timer_quirk(hisi_hba); } hisi_sas_notify_phy_event(phy, HISI_PHYE_PHY_UP); - spin_lock_irqsave(&phy->lock, flags); - if (phy->reset_completion) { - phy->in_reset = 0; - complete(phy->reset_completion); - } - spin_unlock_irqrestore(&phy->lock, flags); - end: + if (phy->reset_completion) + complete(phy->reset_completion); hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0, CHL_INT0_SL_PHY_ENABLE_MSK); hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_PHY_ENA_MSK, 0); @@ -2731,7 +2739,8 @@ static int phy_down_v2_hw(int phy_no, struct hisi_hba *hisi_hba) phy_state = hisi_sas_read32(hisi_hba, PHY_STATE); dev_info(dev, "phydown: phy%d phy_state=0x%x\n", phy_no, phy_state); - hisi_sas_phy_down(hisi_hba, phy_no, (phy_state & 1 << phy_no) ? 1 : 0); + hisi_sas_phy_down(hisi_hba, phy_no, (phy_state & 1 << phy_no) ? 1 : 0, + GFP_ATOMIC); sl_ctrl = hisi_sas_phy_read32(hisi_hba, phy_no, SL_CONTROL); hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL, @@ -2815,14 +2824,14 @@ static void phy_bcast_v2_hw(int phy_no, struct hisi_hba *hisi_hba) { struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no]; struct asd_sas_phy *sas_phy = &phy->sas_phy; - struct sas_ha_struct *sas_ha = &hisi_hba->sha; u32 bcast_status; hisi_sas_phy_write32(hisi_hba, phy_no, SL_RX_BCAST_CHK_MSK, 1); bcast_status = hisi_sas_phy_read32(hisi_hba, phy_no, RX_PRIMS_STATUS); if ((bcast_status & RX_BCAST_CHG_MSK) && !test_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags)) - sas_ha->notify_port_event(sas_phy, PORTE_BROADCAST_RCVD); + sas_notify_port_event(sas_phy, PORTE_BROADCAST_RCVD, + GFP_ATOMIC); hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0, CHL_INT0_SL_RX_BCST_ACK_MSK); hisi_sas_phy_write32(hisi_hba, phy_no, SL_RX_BCAST_CHK_MSK, 0); @@ -3104,9 +3113,9 @@ static irqreturn_t fatal_axi_int_v2_hw(int irq_no, void *p) return IRQ_HANDLED; } -static void cq_tasklet_v2_hw(unsigned long val) +static irqreturn_t cq_thread_v2_hw(int irq_no, void *p) { - struct hisi_sas_cq *cq = (struct hisi_sas_cq *)val; + struct hisi_sas_cq *cq = p; struct hisi_hba *hisi_hba = cq->hisi_hba; struct hisi_sas_slot *slot; struct hisi_sas_itct *itct; @@ -3174,6 +3183,8 @@ static void cq_tasklet_v2_hw(unsigned long val) /* update rd_point */ cq->rd_point = rd_point; hisi_sas_write32(hisi_hba, COMPL_Q_0_RD_PTR + (0x14 * queue), rd_point); + + return IRQ_HANDLED; } static irqreturn_t cq_interrupt_v2_hw(int irq_no, void *p) @@ -3184,9 +3195,7 @@ static irqreturn_t cq_interrupt_v2_hw(int irq_no, void *p) hisi_sas_write32(hisi_hba, OQ_INT_SRC, 1 << queue); - tasklet_schedule(&cq->tasklet); - - return IRQ_HANDLED; + return IRQ_WAKE_THREAD; } static irqreturn_t sata_int_v2_hw(int irq_no, void *p) @@ -3200,7 +3209,6 @@ static irqreturn_t sata_int_v2_hw(int irq_no, void *p) u32 ent_tmp, ent_msk, ent_int, port_id, link_rate, hard_phy_linkrate; irqreturn_t res = IRQ_HANDLED; u8 attached_sas_addr[SAS_ADDR_SIZE] = {0}; - unsigned long flags; int phy_no, offset; del_timer(&phy->timer); @@ -3276,12 +3284,8 @@ static irqreturn_t sata_int_v2_hw(int irq_no, void *p) phy->identify.target_port_protocols = SAS_PROTOCOL_SATA; hisi_sas_notify_phy_event(phy, HISI_PHYE_PHY_UP); - spin_lock_irqsave(&phy->lock, flags); - if (phy->reset_completion) { - phy->in_reset = 0; + if (phy->reset_completion) complete(phy->reset_completion); - } - spin_unlock_irqrestore(&phy->lock, flags); end: hisi_sas_write32(hisi_hba, ENT_INT_SRC1 + offset, ent_tmp); hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1 + offset, ent_msk); @@ -3299,7 +3303,7 @@ static irq_handler_t fatal_interrupts[HISI_SAS_FATAL_INT_NR] = { fatal_axi_int_v2_hw }; -/** +/* * There is a limitation in the hip06 chipset that we need * to map in all mbigen interrupts, even if they are not used. */ @@ -3353,18 +3357,18 @@ static int interrupt_init_v2_hw(struct hisi_hba *hisi_hba) for (queue_no = 0; queue_no < hisi_hba->queue_count; queue_no++) { struct hisi_sas_cq *cq = &hisi_hba->cq[queue_no]; - struct tasklet_struct *t = &cq->tasklet; - irq = irq_map[queue_no + 96]; - rc = devm_request_irq(dev, irq, cq_interrupt_v2_hw, 0, - DRV_NAME " cq", cq); + cq->irq_no = irq_map[queue_no + 96]; + rc = devm_request_threaded_irq(dev, cq->irq_no, + cq_interrupt_v2_hw, + cq_thread_v2_hw, IRQF_ONESHOT, + DRV_NAME " cq", cq); if (rc) { dev_err(dev, "irq init: could not request cq interrupt %d, rc=%d\n", irq, rc); rc = -ENOENT; goto err_out; } - tasklet_init(t, cq_tasklet_v2_hw, (unsigned long)cq); } hisi_hba->cq_nvecs = hisi_hba->queue_count; @@ -3425,7 +3429,6 @@ static int soft_reset_v2_hw(struct hisi_hba *hisi_hba) interrupt_disable_v2_hw(hisi_hba); hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, 0x0); - hisi_sas_kill_tasklets(hisi_hba); hisi_sas_stop_phys(hisi_hba); @@ -3529,6 +3532,7 @@ static struct device_attribute *host_attrs_v2_hw[] = { static struct scsi_host_template sht_v2_hw = { .name = DRV_NAME, + .proc_name = DRV_NAME, .module = THIS_MODULE, .queuecommand = sas_queuecommand, .target_alloc = sas_target_alloc, @@ -3542,7 +3546,7 @@ static struct scsi_host_template sht_v2_hw = { .max_sectors = SCSI_DEFAULT_MAX_SECTORS, .eh_device_reset_handler = sas_eh_device_reset_handler, .eh_target_reset_handler = sas_eh_target_reset_handler, - .slave_alloc = sas_slave_alloc, + .slave_alloc = hisi_sas_slave_alloc, .target_destroy = sas_target_destroy, .ioctl = sas_ioctl, .shost_attrs = host_attrs_v2_hw, @@ -3580,28 +3584,11 @@ static const struct hisi_sas_hw hisi_sas_v2_hw = { static int hisi_sas_v2_probe(struct platform_device *pdev) { - /* - * Check if we should defer the probe before we probe the - * upper layer, as it's hard to defer later on. - */ - int ret = platform_get_irq(pdev, 0); - - if (ret < 0) { - if (ret != -EPROBE_DEFER) - dev_err(&pdev->dev, "cannot obtain irq\n"); - return ret; - } - return hisi_sas_probe(pdev, &hisi_sas_v2_hw); } static int hisi_sas_v2_remove(struct platform_device *pdev) { - struct sas_ha_struct *sha = platform_get_drvdata(pdev); - struct hisi_hba *hisi_hba = sha->lldd_ha; - - hisi_sas_kill_tasklets(hisi_hba); - return hisi_sas_remove(pdev); } diff --git a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c index a86aae52d94f4b036be51d900a19a1a233ea66d0..d04b20e4bf112fc01150973cdc6812096a2f9952 100644 --- a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c +++ b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c @@ -29,6 +29,7 @@ #define SATA_INITI_D2H_STORE_ADDR_LO 0x60 #define SATA_INITI_D2H_STORE_ADDR_HI 0x64 #define CFG_MAX_TAG 0x68 +#define TRANS_LOCK_ICT_TIME 0X70 #define HGC_SAS_TX_OPEN_FAIL_RETRY_CTRL 0x84 #define HGC_SAS_TXFAIL_RETRY_CTRL 0x88 #define HGC_GET_ITV_TIME 0x90 @@ -191,8 +192,10 @@ #define PHY_CFG_PHY_RST_OFF 3 #define PHY_CFG_PHY_RST_MSK (0x1 << PHY_CFG_PHY_RST_OFF) #define PROG_PHY_LINK_RATE (PORT_BASE + 0x8) -#define CFG_PROG_PHY_LINK_RATE_OFF 8 -#define CFG_PROG_PHY_LINK_RATE_MSK (0xf << CFG_PROG_PHY_LINK_RATE_OFF) +#define CFG_PROG_PHY_LINK_RATE_OFF 0 +#define CFG_PROG_PHY_LINK_RATE_MSK (0xff << CFG_PROG_PHY_LINK_RATE_OFF) +#define CFG_PROG_OOB_PHY_LINK_RATE_OFF 8 +#define CFG_PROG_OOB_PHY_LINK_RATE_MSK (0xf << CFG_PROG_OOB_PHY_LINK_RATE_OFF) #define PHY_CTRL (PORT_BASE + 0x14) #define PHY_CTRL_RESET_OFF 0 #define PHY_CTRL_RESET_MSK (0x1 << PHY_CTRL_RESET_OFF) @@ -295,11 +298,25 @@ #define DMA_RX_STATUS_BUSY_MSK (0x1 << DMA_RX_STATUS_BUSY_OFF) #define COARSETUNE_TIME (PORT_BASE + 0x304) +#define TXDEEMPH_G1 (PORT_BASE + 0x350) #define ERR_CNT_DWS_LOST (PORT_BASE + 0x380) #define ERR_CNT_RESET_PROB (PORT_BASE + 0x384) #define ERR_CNT_INVLD_DW (PORT_BASE + 0x390) #define ERR_CNT_CODE_ERR (PORT_BASE + 0x394) #define ERR_CNT_DISP_ERR (PORT_BASE + 0x398) +#define DFX_FIFO_CTRL (PORT_BASE + 0x3a0) +#define DFX_FIFO_CTRL_TRIGGER_MODE_OFF 0 +#define DFX_FIFO_CTRL_TRIGGER_MODE_MSK (0x7 << DFX_FIFO_CTRL_TRIGGER_MODE_OFF) +#define DFX_FIFO_CTRL_DUMP_MODE_OFF 3 +#define DFX_FIFO_CTRL_DUMP_MODE_MSK (0x7 << DFX_FIFO_CTRL_DUMP_MODE_OFF) +#define DFX_FIFO_CTRL_SIGNAL_SEL_OFF 6 +#define DFX_FIFO_CTRL_SIGNAL_SEL_MSK (0xF << DFX_FIFO_CTRL_SIGNAL_SEL_OFF) +#define DFX_FIFO_CTRL_DUMP_DISABLE_OFF 10 +#define DFX_FIFO_CTRL_DUMP_DISABLE_MSK (0x1 << DFX_FIFO_CTRL_DUMP_DISABLE_OFF) +#define DFX_FIFO_TRIGGER (PORT_BASE + 0x3a4) +#define DFX_FIFO_TRIGGER_MSK (PORT_BASE + 0x3a8) +#define DFX_FIFO_DUMP_MSK (PORT_BASE + 0x3aC) +#define DFX_FIFO_RD_DATA (PORT_BASE + 0x3b0) #define DEFAULT_ITCT_HW 2048 /* reset value, not reprogrammed */ #if (HISI_SAS_MAX_DEVICES > DEFAULT_ITCT_HW) @@ -387,8 +404,15 @@ #define CMPLT_HDR_CMPLT_MSK (0x3 << CMPLT_HDR_CMPLT_OFF) #define CMPLT_HDR_ERROR_PHASE_OFF 2 #define CMPLT_HDR_ERROR_PHASE_MSK (0xff << CMPLT_HDR_ERROR_PHASE_OFF) +/* bit[9:2] Error Phase */ +#define ERR_PHASE_RESPONSE_FRAME_REV_STAGE_OFF \ + 8 +#define ERR_PHASE_RESPONSE_FRAME_REV_STAGE_MSK \ + (0x1 << ERR_PHASE_RESPONSE_FRAME_REV_STAGE_OFF) #define CMPLT_HDR_RSPNS_XFRD_OFF 10 #define CMPLT_HDR_RSPNS_XFRD_MSK (0x1 << CMPLT_HDR_RSPNS_XFRD_OFF) +#define CMPLT_HDR_RSPNS_GOOD_OFF 11 +#define CMPLT_HDR_RSPNS_GOOD_MSK (0x1 << CMPLT_HDR_RSPNS_GOOD_OFF) #define CMPLT_HDR_ERX_OFF 12 #define CMPLT_HDR_ERX_MSK (0x1 << CMPLT_HDR_ERX_OFF) #define CMPLT_HDR_ABORT_STAT_OFF 13 @@ -404,8 +428,17 @@ #define CMPLT_HDR_DEV_ID_OFF 16 #define CMPLT_HDR_DEV_ID_MSK (0xffff << CMPLT_HDR_DEV_ID_OFF) /* dw3 */ +#define SATA_DISK_IN_ERROR_STATUS_OFF 8 +#define SATA_DISK_IN_ERROR_STATUS_MSK (0x1 << SATA_DISK_IN_ERROR_STATUS_OFF) +#define CMPLT_HDR_SATA_DISK_ERR_OFF 16 +#define CMPLT_HDR_SATA_DISK_ERR_MSK (0x1 << CMPLT_HDR_SATA_DISK_ERR_OFF) #define CMPLT_HDR_IO_IN_TARGET_OFF 17 #define CMPLT_HDR_IO_IN_TARGET_MSK (0x1 << CMPLT_HDR_IO_IN_TARGET_OFF) +/* bit[23:18] ERR_FIS_ATA_STATUS */ +#define FIS_ATA_STATUS_ERR_OFF 18 +#define FIS_ATA_STATUS_ERR_MSK (0x1 << FIS_ATA_STATUS_ERR_OFF) +#define FIS_TYPE_SDB_OFF 31 +#define FIS_TYPE_SDB_MSK (0x1 << FIS_TYPE_SDB_OFF) /* ITCT header */ /* qw0 */ @@ -462,6 +495,9 @@ struct hisi_sas_err_record_v3 { #define RX_DATA_LEN_UNDERFLOW_OFF 6 #define RX_DATA_LEN_UNDERFLOW_MSK (1 << RX_DATA_LEN_UNDERFLOW_OFF) +#define RX_FIS_STATUS_ERR_OFF 0 +#define RX_FIS_STATUS_ERR_MSK (1 << RX_FIS_STATUS_ERR_OFF) + #define HISI_SAS_COMMAND_ENTRIES_V3_HW 4096 #define HISI_SAS_MSI_COUNT_V3_HW 32 @@ -495,6 +531,15 @@ struct hisi_sas_err_record_v3 { #define BASE_VECTORS_V3_HW 16 #define MIN_AFFINE_VECTORS_V3_HW (BASE_VECTORS_V3_HW + 1) +#define CHNL_INT_STS_MSK 0xeeeeeeee +#define CHNL_INT_STS_PHY_MSK 0xe +#define CHNL_INT_STS_INT0_MSK BIT(1) +#define CHNL_INT_STS_INT1_MSK BIT(2) +#define CHNL_INT_STS_INT2_MSK BIT(3) +#define CHNL_WIDTH 4 + +#define BAR_NO_V3_HW 5 + enum { DSM_FUNC_ERR_HANDLE_MSI = 0, }; @@ -512,6 +557,8 @@ module_param(auto_affine_msi_experimental, bool, 0444); MODULE_PARM_DESC(auto_affine_msi_experimental, "Enable auto-affinity of MSI IRQs as experimental:\n" "default is off"); +static int debugfs_snapshot_regs_v3_hw(struct hisi_hba *hisi_hba); + static u32 hisi_sas_read32(struct hisi_hba *hisi_hba, u32 off) { void __iomem *regs = hisi_hba->regs + off; @@ -556,15 +603,38 @@ static u32 hisi_sas_phy_read32(struct hisi_hba *hisi_hba, readl_poll_timeout_atomic(regs, val, cond, delay_us, timeout_us);\ }) -static void init_reg_v3_hw(struct hisi_hba *hisi_hba) +static void interrupt_enable_v3_hw(struct hisi_hba *hisi_hba) { int i; + for (i = 0; i < hisi_hba->queue_count; i++) + hisi_sas_write32(hisi_hba, OQ0_INT_SRC_MSK + 0x4 * i, 0); + + hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1, 0xfefefefe); + hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK2, 0xfefefefe); + hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xffc220ff); + hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0x155555); + + for (i = 0; i < hisi_hba->n_phy; i++) { + hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK, 0xf2057fff); + hisi_sas_phy_write32(hisi_hba, i, CHL_INT2_MSK, 0xffffbfe); + hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_NOT_RDY_MSK, 0x0); + hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_PHY_ENA_MSK, 0x0); + hisi_sas_phy_write32(hisi_hba, i, SL_RX_BCAST_CHK_MSK, 0x0); + } +} + +static void init_reg_v3_hw(struct hisi_hba *hisi_hba) +{ + struct pci_dev *pdev = hisi_hba->pci_dev; + int i, j; + /* Global registers init */ hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, (u32)((1ULL << hisi_hba->queue_count) - 1)); - hisi_sas_write32(hisi_hba, SAS_AXI_USER3, 0); hisi_sas_write32(hisi_hba, CFG_MAX_TAG, 0xfff0400); + /* time / CLK_AHB = 2.5s / 2ns = 0x4A817C80 */ + hisi_sas_write32(hisi_hba, TRANS_LOCK_ICT_TIME, 0x4A817C80); hisi_sas_write32(hisi_hba, HGC_SAS_TXFAIL_RETRY_CTRL, 0x108); hisi_sas_write32(hisi_hba, CFG_AGING_TIME, 0x1); hisi_sas_write32(hisi_hba, INT_COAL_EN, 0x1); @@ -576,59 +646,61 @@ static void init_reg_v3_hw(struct hisi_hba *hisi_hba) hisi_sas_write32(hisi_hba, ENT_INT_SRC1, 0xffffffff); hisi_sas_write32(hisi_hba, ENT_INT_SRC2, 0xffffffff); hisi_sas_write32(hisi_hba, ENT_INT_SRC3, 0xffffffff); - hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1, 0xfefefefe); - hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK2, 0xfefefefe); - hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xffc220ff); hisi_sas_write32(hisi_hba, CHNL_PHYUPDOWN_INT_MSK, 0x0); hisi_sas_write32(hisi_hba, CHNL_ENT_INT_MSK, 0x0); hisi_sas_write32(hisi_hba, HGC_COM_INT_MSK, 0x0); - hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0x155555); hisi_sas_write32(hisi_hba, AWQOS_AWCACHE_CFG, 0xf0f0); hisi_sas_write32(hisi_hba, ARQOS_ARCACHE_CFG, 0xf0f0); - for (i = 0; i < hisi_hba->queue_count; i++) - hisi_sas_write32(hisi_hba, OQ0_INT_SRC_MSK+0x4*i, 0); - hisi_sas_write32(hisi_hba, HYPER_STREAM_ID_EN_CFG, 1); + if (pdev->revision < 0x30) + hisi_sas_write32(hisi_hba, SAS_AXI_USER3, 0); + + interrupt_enable_v3_hw(hisi_hba); for (i = 0; i < hisi_hba->n_phy; i++) { + enum sas_linkrate max; struct hisi_sas_phy *phy = &hisi_hba->phy[i]; struct asd_sas_phy *sas_phy = &phy->sas_phy; - u32 prog_phy_link_rate = 0x800; + u32 prog_phy_link_rate = hisi_sas_phy_read32(hisi_hba, i, + PROG_PHY_LINK_RATE); + prog_phy_link_rate &= ~CFG_PROG_PHY_LINK_RATE_MSK; if (!sas_phy->phy || (sas_phy->phy->maximum_linkrate < - SAS_LINK_RATE_1_5_GBPS)) { - prog_phy_link_rate = 0x855; - } else { - enum sas_linkrate max = sas_phy->phy->maximum_linkrate; - - prog_phy_link_rate = - hisi_sas_get_prog_phy_linkrate_mask(max) | - 0x800; - } + SAS_LINK_RATE_1_5_GBPS)) + max = SAS_LINK_RATE_12_0_GBPS; + else + max = sas_phy->phy->maximum_linkrate; + prog_phy_link_rate |= hisi_sas_get_prog_phy_linkrate_mask(max); hisi_sas_phy_write32(hisi_hba, i, PROG_PHY_LINK_RATE, prog_phy_link_rate); - hisi_sas_phy_write32(hisi_hba, i, SERDES_CFG, 0xffc00); hisi_sas_phy_write32(hisi_hba, i, SAS_RX_TRAIN_TIMER, 0x13e80); hisi_sas_phy_write32(hisi_hba, i, CHL_INT0, 0xffffffff); hisi_sas_phy_write32(hisi_hba, i, CHL_INT1, 0xffffffff); hisi_sas_phy_write32(hisi_hba, i, CHL_INT2, 0xffffffff); hisi_sas_phy_write32(hisi_hba, i, RXOP_CHECK_CFG_H, 0x1000); - hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK, 0xf2057fff); - hisi_sas_phy_write32(hisi_hba, i, CHL_INT2_MSK, 0xffffbfe); hisi_sas_phy_write32(hisi_hba, i, PHY_CTRL_RDY_MSK, 0x0); - hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_NOT_RDY_MSK, 0x0); hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_DWS_RESET_MSK, 0x0); - hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_PHY_ENA_MSK, 0x0); - hisi_sas_phy_write32(hisi_hba, i, SL_RX_BCAST_CHK_MSK, 0x0); hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_OOB_RESTART_MSK, 0x1); hisi_sas_phy_write32(hisi_hba, i, STP_LINK_TIMER, 0x7f7a120); hisi_sas_phy_write32(hisi_hba, i, CON_CFG_DRIVER, 0x2a0a01); - hisi_sas_phy_write32(hisi_hba, i, SAS_SSP_CON_TIMER_CFG, 0x32); hisi_sas_phy_write32(hisi_hba, i, SAS_EC_INT_COAL_TIME, 0x30f4240); - /* used for 12G negotiate */ - hisi_sas_phy_write32(hisi_hba, i, COARSETUNE_TIME, 0x1e); hisi_sas_phy_write32(hisi_hba, i, AIP_LIMIT, 0x2ffff); + + /* set value through firmware for 920B and later version */ + if (pdev->revision < 0x30) { + hisi_sas_phy_write32(hisi_hba, i, SAS_SSP_CON_TIMER_CFG, 0x32); + hisi_sas_phy_write32(hisi_hba, i, SERDES_CFG, 0xffc00); + /* used for 12G negotiate */ + hisi_sas_phy_write32(hisi_hba, i, COARSETUNE_TIME, 0x1e); + } + + /* get default FFE configuration for BIST */ + for (j = 0; j < FFE_CFG_MAX; j++) { + u32 val = hisi_sas_phy_read32(hisi_hba, i, + TXDEEMPH_G1 + (j * 0x4)); + hisi_hba->debugfs_bist_ffe[i][j] = val; + } } for (i = 0; i < hisi_hba->queue_count; i++) { @@ -795,13 +867,14 @@ static void setup_itct_v3_hw(struct hisi_hba *hisi_hba, (0x1ULL << ITCT_HDR_RTOLT_OFF)); } -static void clear_itct_v3_hw(struct hisi_hba *hisi_hba, - struct hisi_sas_device *sas_dev) +static int clear_itct_v3_hw(struct hisi_hba *hisi_hba, + struct hisi_sas_device *sas_dev) { DECLARE_COMPLETION_ONSTACK(completion); u64 dev_id = sas_dev->device_id; struct hisi_sas_itct *itct = &hisi_hba->itct[dev_id]; u32 reg_val = hisi_sas_read32(hisi_hba, ENT_INT_SRC3); + struct device *dev = hisi_hba->dev; sas_dev->completion = &completion; @@ -814,8 +887,14 @@ static void clear_itct_v3_hw(struct hisi_hba *hisi_hba, reg_val = ITCT_CLR_EN_MSK | (dev_id & ITCT_DEV_MSK); hisi_sas_write32(hisi_hba, ITCT_CLR, reg_val); - wait_for_completion(sas_dev->completion); + if (!wait_for_completion_timeout(sas_dev->completion, + HISI_SAS_CLEAR_ITCT_TIMEOUT)) { + dev_warn(dev, "failed to clear ITCT\n"); + return -ETIMEDOUT; + } + memset(itct, 0, sizeof(struct hisi_sas_itct)); + return 0; } static void dereg_device_v3_hw(struct hisi_hba *hisi_hba, @@ -827,6 +906,7 @@ static void dereg_device_v3_hw(struct hisi_hba *hisi_hba, cfg_abt_set_query_iptt = hisi_sas_read32(hisi_hba, CFG_ABT_SET_QUERY_IPTT); + spin_lock(&sas_dev->lock); list_for_each_entry_safe(slot, slot2, &sas_dev->list, entry) { cfg_abt_set_query_iptt &= ~CFG_SET_ABORTED_IPTT_MSK; cfg_abt_set_query_iptt |= (1 << CFG_SET_ABORTED_EN_OFF) | @@ -834,6 +914,7 @@ static void dereg_device_v3_hw(struct hisi_hba *hisi_hba, hisi_sas_write32(hisi_hba, CFG_ABT_SET_QUERY_IPTT, cfg_abt_set_query_iptt); } + spin_unlock(&sas_dev->lock); cfg_abt_set_query_iptt &= ~(1 << CFG_SET_ABORTED_EN_OFF); hisi_sas_write32(hisi_hba, CFG_ABT_SET_QUERY_IPTT, cfg_abt_set_query_iptt); @@ -880,13 +961,14 @@ static int reset_hw_v3_hw(struct hisi_hba *hisi_hba) static int hw_init_v3_hw(struct hisi_hba *hisi_hba) { struct device *dev = hisi_hba->dev; + struct acpi_device *acpi_dev; union acpi_object *obj; guid_t guid; int rc; rc = reset_hw_v3_hw(hisi_hba); if (rc) { - dev_err(dev, "hisi_sas_reset_hw failed, rc=%d", rc); + dev_err(dev, "hisi_sas_reset_hw failed, rc=%d\n", rc); return rc; } @@ -898,14 +980,21 @@ static int hw_init_v3_hw(struct hisi_hba *hisi_hba) return -EINVAL; } - /* Switch over to MSI handling , from PCI AER default */ + /* + * This DSM handles some hardware-related configurations: + * 1. Switch over to MSI error handling in kernel + * 2. BIOS *may* reset some register values through this method + */ obj = acpi_evaluate_dsm(ACPI_HANDLE(dev), &guid, 0, DSM_FUNC_ERR_HANDLE_MSI, NULL); if (!obj) - dev_warn(dev, "Switch over to MSI handling failed\n"); + dev_warn(dev, "can not find DSM method, ignore\n"); else ACPI_FREE(obj); + acpi_dev = ACPI_COMPANION(dev); + if (!acpi_device_power_manageable(acpi_dev)) + dev_notice(dev, "neither _PS0 nor _PR0 is defined\n"); return 0; } @@ -1323,7 +1412,6 @@ static void prep_smp_v3_hw(struct hisi_hba *hisi_hba, hdr->cmd_table_addr = cpu_to_le64(req_dma_addr); hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot)); - } static void prep_ata_v3_hw(struct hisi_hba *hisi_hba, @@ -1429,7 +1517,6 @@ static void prep_abort_v3_hw(struct hisi_hba *hisi_hba, /* dw7 */ hdr->dw7 = cpu_to_le32(tag_to_abort << CMD_HDR_ABORT_IPTT_OFF); hdr->transfer_tags = cpu_to_le32(slot->idx); - } static irqreturn_t phy_up_v3_hw(int phy_no, struct hisi_hba *hisi_hba) @@ -1440,9 +1527,7 @@ static irqreturn_t phy_up_v3_hw(int phy_no, struct hisi_hba *hisi_hba) struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no]; struct asd_sas_phy *sas_phy = &phy->sas_phy; struct device *dev = hisi_hba->dev; - unsigned long flags; - del_timer(&phy->timer); hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_PHY_ENA_MSK, 1); port_id = hisi_sas_read32(hisi_hba, PHY_PORT_NUM_MA); @@ -1519,16 +1604,27 @@ static irqreturn_t phy_up_v3_hw(int phy_no, struct hisi_hba *hisi_hba) } phy->port_id = port_id; + spin_lock(&phy->lock); + /* Delete timer and set phy_attached atomically */ + del_timer(&phy->timer); phy->phy_attached = 1; - hisi_sas_notify_phy_event(phy, HISI_PHYE_PHY_UP); + spin_unlock(&phy->lock); + + /* + * Call pm_runtime_get_noresume() which pairs with + * hisi_sas_phyup_pm_work() -> pm_runtime_put_sync(). + * For failure call pm_runtime_put() as we are in a hardirq context. + */ + pm_runtime_get_noresume(dev); + res = hisi_sas_notify_phy_event(phy, HISI_PHYE_PHY_UP_PM); + if (!res) + pm_runtime_put(dev); + res = IRQ_HANDLED; - spin_lock_irqsave(&phy->lock, flags); - if (phy->reset_completion) { - phy->in_reset = 0; - complete(phy->reset_completion); - } - spin_unlock_irqrestore(&phy->lock, flags); + end: + if (phy->reset_completion) + complete(phy->reset_completion); hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0, CHL_INT0_SL_PHY_ENABLE_MSK); hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_PHY_ENA_MSK, 0); @@ -1542,12 +1638,15 @@ static irqreturn_t phy_down_v3_hw(int phy_no, struct hisi_hba *hisi_hba) u32 phy_state, sl_ctrl, txid_auto; struct device *dev = hisi_hba->dev; + atomic_inc(&phy->down_cnt); + del_timer(&phy->timer); hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_NOT_RDY_MSK, 1); phy_state = hisi_sas_read32(hisi_hba, PHY_STATE); dev_info(dev, "phydown: phy%d phy_state=0x%x\n", phy_no, phy_state); - hisi_sas_phy_down(hisi_hba, phy_no, (phy_state & 1 << phy_no) ? 1 : 0); + hisi_sas_phy_down(hisi_hba, phy_no, (phy_state & 1 << phy_no) ? 1 : 0, + GFP_ATOMIC); sl_ctrl = hisi_sas_phy_read32(hisi_hba, phy_no, SL_CONTROL); hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL, @@ -1567,14 +1666,14 @@ static irqreturn_t phy_bcast_v3_hw(int phy_no, struct hisi_hba *hisi_hba) { struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no]; struct asd_sas_phy *sas_phy = &phy->sas_phy; - struct sas_ha_struct *sas_ha = &hisi_hba->sha; u32 bcast_status; hisi_sas_phy_write32(hisi_hba, phy_no, SL_RX_BCAST_CHK_MSK, 1); bcast_status = hisi_sas_phy_read32(hisi_hba, phy_no, RX_PRIMS_STATUS); if ((bcast_status & RX_BCAST_CHG_MSK) && !test_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags)) - sas_ha->notify_port_event(sas_phy, PORTE_BROADCAST_RCVD); + sas_notify_port_event(sas_phy, PORTE_BROADCAST_RCVD, + GFP_ATOMIC); hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0, CHL_INT0_SL_RX_BCST_ACK_MSK); hisi_sas_phy_write32(hisi_hba, phy_no, SL_RX_BCAST_CHK_MSK, 0); @@ -1675,8 +1774,11 @@ static void handle_chl_int1_v3_hw(struct hisi_hba *hisi_hba, int phy_no) int i; irq_value &= ~irq_msk; - if (!irq_value) + if (!irq_value) { + dev_warn(dev, "phy%d channel int 1 received with status bits cleared\n", + phy_no); return; + } for (i = 0; i < ARRAY_SIZE(port_axi_error); i++) { const struct hisi_sas_hw_error *error = &port_axi_error[i]; @@ -1737,8 +1839,11 @@ static void handle_chl_int2_v3_hw(struct hisi_hba *hisi_hba, int phy_no) BIT(CHL_INT2_RX_INVLD_DW_OFF); irq_value &= ~irq_msk; - if (!irq_value) + if (!irq_value) { + dev_warn(dev, "phy%d channel int 2 received with status bits cleared\n", + phy_no); return; + } if (irq_value & BIT(CHL_INT2_SL_IDAF_TOUT_CONF_OFF)) { dev_warn(dev, "phy%d identify timeout\n", phy_no); @@ -1810,19 +1915,19 @@ static irqreturn_t int_chnl_int_v3_hw(int irq_no, void *p) int phy_no = 0; irq_msk = hisi_sas_read32(hisi_hba, CHNL_INT_STATUS) - & 0xeeeeeeee; + & CHNL_INT_STS_MSK; while (irq_msk) { - if (irq_msk & (2 << (phy_no * 4))) + if (irq_msk & (CHNL_INT_STS_INT0_MSK << (phy_no * CHNL_WIDTH))) handle_chl_int0_v3_hw(hisi_hba, phy_no); - if (irq_msk & (4 << (phy_no * 4))) + if (irq_msk & (CHNL_INT_STS_INT1_MSK << (phy_no * CHNL_WIDTH))) handle_chl_int1_v3_hw(hisi_hba, phy_no); - if (irq_msk & (8 << (phy_no * 4))) + if (irq_msk & (CHNL_INT_STS_INT2_MSK << (phy_no * CHNL_WIDTH))) handle_chl_int2_v3_hw(hisi_hba, phy_no); - irq_msk &= ~(0xe << (phy_no * 4)); + irq_msk &= ~(CHNL_INT_STS_PHY_MSK << (phy_no * CHNL_WIDTH)); phy_no++; } @@ -2085,7 +2190,19 @@ static irqreturn_t fatal_axi_int_v3_hw(int irq_no, void *p) return IRQ_HANDLED; } -static void +static bool is_ncq_err_v3_hw(struct hisi_sas_complete_v3_hdr *complete_hdr) +{ + u32 dw0, dw3; + + dw0 = le32_to_cpu(complete_hdr->dw0); + dw3 = le32_to_cpu(complete_hdr->dw3); + + return (dw0 & ERR_PHASE_RESPONSE_FRAME_REV_STAGE_MSK) && + (dw3 & FIS_TYPE_SDB_MSK) && + (dw3 & FIS_ATA_STATUS_ERR_MSK); +} + +static bool slot_err_v3_hw(struct hisi_hba *hisi_hba, struct sas_task *task, struct hisi_sas_slot *slot) { @@ -2098,11 +2215,22 @@ slot_err_v3_hw(struct hisi_hba *hisi_hba, struct sas_task *task, hisi_sas_status_buf_addr_mem(slot); u32 dma_rx_err_type = le32_to_cpu(record->dma_rx_err_type); u32 trans_tx_fail_type = le32_to_cpu(record->trans_tx_fail_type); + u16 sipc_rx_err_type = le16_to_cpu(record->sipc_rx_err_type); u32 dw3 = le32_to_cpu(complete_hdr->dw3); + u32 dw0 = le32_to_cpu(complete_hdr->dw0); switch (task->task_proto) { case SAS_PROTOCOL_SSP: if (dma_rx_err_type & RX_DATA_LEN_UNDERFLOW_MSK) { + /* + * If returned response frame is incorrect because of data underflow, + * but I/O information has been written to the host memory, we examine + * response IU. + */ + if (!(dw0 & CMPLT_HDR_RSPNS_GOOD_MSK) && + (dw0 & CMPLT_HDR_RSPNS_XFRD_MSK)) + return false; + ts->residual = trans_tx_fail_type; ts->stat = SAS_DATA_UNDERRUN; } else if (dw3 & CMPLT_HDR_IO_IN_TARGET_MSK) { @@ -2116,17 +2244,31 @@ slot_err_v3_hw(struct hisi_hba *hisi_hba, struct sas_task *task, case SAS_PROTOCOL_SATA: case SAS_PROTOCOL_STP: case SAS_PROTOCOL_SATA | SAS_PROTOCOL_STP: - if (dma_rx_err_type & RX_DATA_LEN_UNDERFLOW_MSK) { + if ((dw0 & CMPLT_HDR_RSPNS_XFRD_MSK) && + (sipc_rx_err_type & RX_FIS_STATUS_ERR_MSK)) { + if (task->ata_task.use_ncq) { + struct domain_device *device = task->dev; + struct hisi_sas_device *sas_dev = + device->lldd_dev; + sas_dev->dev_status = HISI_SAS_DEV_NCQ_ERR; + slot->abort = 1; + } else { + ts->stat = SAS_PROTO_RESPONSE; + } + } else if (dma_rx_err_type & RX_DATA_LEN_UNDERFLOW_MSK) { ts->residual = trans_tx_fail_type; ts->stat = SAS_DATA_UNDERRUN; - } else if (dw3 & CMPLT_HDR_IO_IN_TARGET_MSK) { + } else if ((dw3 & CMPLT_HDR_IO_IN_TARGET_MSK) || + (dw3 & SATA_DISK_IN_ERROR_STATUS_MSK)) { ts->stat = SAS_PHY_DOWN; slot->abort = 1; } else { ts->stat = SAS_OPEN_REJECT; ts->open_rej_reason = SAS_OREJ_RSVD_RETRY; } - hisi_sas_sata_done(task, slot); + + if (dw0 & CMPLT_HDR_RSPNS_XFRD_MSK) + hisi_sas_sata_done(task, slot); break; case SAS_PROTOCOL_SMP: ts->stat = SAM_STAT_CHECK_CONDITION; @@ -2134,10 +2276,11 @@ slot_err_v3_hw(struct hisi_hba *hisi_hba, struct sas_task *task, default: break; } + return true; } -static int -slot_complete_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot) +static void slot_complete_v3_hw(struct hisi_hba *hisi_hba, + struct hisi_sas_slot *slot) { struct sas_task *task = slot->task; struct hisi_sas_device *sas_dev; @@ -2145,7 +2288,6 @@ slot_complete_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot) struct task_status_struct *ts; struct domain_device *device; struct sas_ha_struct *ha; - enum exec_status sts; struct hisi_sas_complete_v3_hdr *complete_queue = hisi_hba->complete_hdr[slot->cmplt_queue]; struct hisi_sas_complete_v3_hdr *complete_hdr = @@ -2155,7 +2297,7 @@ slot_complete_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot) u32 dw0, dw1, dw3; if (unlikely(!task || !task->lldd_task || !task->dev)) - return -EINVAL; + return; ts = &task->task_status; device = task->dev; @@ -2209,16 +2351,24 @@ slot_complete_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot) if ((dw0 & CMPLT_HDR_CMPLT_MSK) == 0x3) { u32 *error_info = hisi_sas_status_buf_addr_mem(slot); - slot_err_v3_hw(hisi_hba, task, slot); - if (ts->stat != SAS_DATA_UNDERRUN) - dev_info(dev, "erroneous completion iptt=%d task=%pK dev id=%d CQ hdr: 0x%x 0x%x 0x%x 0x%x Error info: 0x%x 0x%x 0x%x 0x%x\n", - slot->idx, task, sas_dev->device_id, - dw0, dw1, complete_hdr->act, dw3, - error_info[0], error_info[1], - error_info[2], error_info[3]); - if (unlikely(slot->abort)) - return ts->stat; - goto out; + if (slot_err_v3_hw(hisi_hba, task, slot)) { + if (ts->stat != SAS_DATA_UNDERRUN) + dev_info(dev, "erroneous completion iptt=%d task=%pK dev id=%d addr=%016llx CQ hdr: 0x%x 0x%x 0x%x 0x%x Error info: 0x%x 0x%x 0x%x 0x%x\n", + slot->idx, task, sas_dev->device_id, + SAS_ADDR(device->sas_addr), + dw0, dw1, complete_hdr->act, dw3, + error_info[0], error_info[1], + error_info[2], error_info[3]); + if (unlikely(slot->abort)) { + if (dev_is_sata(device) && task->ata_task.use_ncq) + sas_ata_device_link_abort(device, true); + else + sas_task_abort(task); + + return; + } + goto out; + } } switch (task->task_proto) { @@ -2248,7 +2398,8 @@ slot_complete_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot) case SAS_PROTOCOL_STP: case SAS_PROTOCOL_SATA | SAS_PROTOCOL_STP: ts->stat = SAM_STAT_GOOD; - hisi_sas_sata_done(task, slot); + if (dw0 & CMPLT_HDR_RSPNS_XFRD_MSK) + hisi_sas_sata_done(task, slot); break; default: ts->stat = SAM_STAT_CHECK_CONDITION; @@ -2262,16 +2413,15 @@ slot_complete_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot) } out: - sts = ts->stat; spin_lock_irqsave(&task->task_state_lock, flags); if (task->task_state_flags & SAS_TASK_STATE_ABORTED) { spin_unlock_irqrestore(&task->task_state_lock, flags); dev_info(dev, "slot complete: task(%pK) aborted\n", task); - return SAS_ABORTED_TASK; + return; } task->task_state_flags |= SAS_TASK_STATE_DONE; spin_unlock_irqrestore(&task->task_state_lock, flags); - hisi_sas_slot_task_free(hisi_hba, task, slot); + hisi_sas_slot_task_free(hisi_hba, task, slot, true); if (!is_internal && (task->task_proto != SAS_PROTOCOL_SMP)) { spin_lock_irqsave(&device->done_lock, flags); @@ -2279,20 +2429,18 @@ slot_complete_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot) spin_unlock_irqrestore(&device->done_lock, flags); dev_info(dev, "slot complete: task(%pK) ignored\n ", task); - return sts; + return; } spin_unlock_irqrestore(&device->done_lock, flags); } if (task->task_done) task->task_done(task); - - return sts; } -static void cq_tasklet_v3_hw(unsigned long val) +static irqreturn_t cq_thread_v3_hw(int irq_no, void *p) { - struct hisi_sas_cq *cq = (struct hisi_sas_cq *)val; + struct hisi_sas_cq *cq = p; struct hisi_hba *hisi_hba = cq->hisi_hba; struct hisi_sas_slot *slot; struct hisi_sas_complete_v3_hdr *complete_queue; @@ -2307,14 +2455,34 @@ static void cq_tasklet_v3_hw(unsigned long val) while (rd_point != wr_point) { struct hisi_sas_complete_v3_hdr *complete_hdr; struct device *dev = hisi_hba->dev; - u32 dw1; + u32 dw0, dw1, dw3; int iptt; complete_hdr = &complete_queue[rd_point]; + dw0 = le32_to_cpu(complete_hdr->dw0); dw1 = le32_to_cpu(complete_hdr->dw1); + dw3 = le32_to_cpu(complete_hdr->dw3); iptt = dw1 & CMPLT_HDR_IPTT_MSK; - if (likely(iptt < HISI_SAS_COMMAND_ENTRIES_V3_HW)) { + if (unlikely((dw0 & CMPLT_HDR_CMPLT_MSK) == 0x3) && + (dw3 & CMPLT_HDR_SATA_DISK_ERR_MSK)) { + int device_id = (dw1 & CMPLT_HDR_DEV_ID_MSK) >> + CMPLT_HDR_DEV_ID_OFF; + struct hisi_sas_itct *itct = + &hisi_hba->itct[device_id]; + struct hisi_sas_device *sas_dev = + &hisi_hba->devices[device_id]; + struct domain_device *device = sas_dev->sas_device; + + dev_err(dev, "erroneous completion disk err dev id=%d sas_addr=0x%llx CQ hdr: 0x%x 0x%x 0x%x 0x%x\n", + device_id, itct->sas_addr, dw0, dw1, + complete_hdr->act, dw3); + + if (is_ncq_err_v3_hw(complete_hdr)) + sas_dev->dev_status = HISI_SAS_DEV_NCQ_ERR; + + sas_ata_device_link_abort(device, true); + } else if (likely(iptt < HISI_SAS_COMMAND_ENTRIES_V3_HW)) { slot = &hisi_hba->slot_info[iptt]; slot->cmplt_queue_slot = rd_point; slot->cmplt_queue = queue; @@ -2329,6 +2497,8 @@ static void cq_tasklet_v3_hw(unsigned long val) /* update rd_point */ cq->rd_point = rd_point; hisi_sas_write32(hisi_hba, COMPL_Q_0_RD_PTR + (0x14 * queue), rd_point); + + return IRQ_HANDLED; } static irqreturn_t cq_interrupt_v3_hw(int irq_no, void *p) @@ -2339,9 +2509,7 @@ static irqreturn_t cq_interrupt_v3_hw(int irq_no, void *p) hisi_sas_write32(hisi_hba, OQ_INT_SRC, 1 << queue); - tasklet_schedule(&cq->tasklet); - - return IRQ_HANDLED; + return IRQ_WAKE_THREAD; } static void setup_reply_map_v3_hw(struct hisi_hba *hisi_hba, int nvecs) @@ -2356,7 +2524,7 @@ static void setup_reply_map_v3_hw(struct hisi_hba *hisi_hba, int nvecs) BASE_VECTORS_V3_HW); if (!mask) goto fallback; - cq->pci_irq_mask = mask; + cq->irq_mask = mask; for_each_cpu(cpu, mask) hisi_hba->reply_map[cpu] = queue; } @@ -2380,6 +2548,8 @@ static int interrupt_init_v3_hw(struct hisi_hba *hisi_hba) .pre_vectors = BASE_VECTORS_V3_HW, }; + dev_info(dev, "Enable MSI auto-affinity\n"); + min_msi = MIN_AFFINE_VECTORS_V3_HW; hisi_hba->reply_map = devm_kcalloc(dev, nr_cpu_ids, @@ -2410,8 +2580,7 @@ static int interrupt_init_v3_hw(struct hisi_hba *hisi_hba) DRV_NAME " phy", hisi_hba); if (rc) { dev_err(dev, "could not request phy interrupt, rc=%d\n", rc); - rc = -ENOENT; - goto free_irq_vectors; + return -ENOENT; } rc = devm_request_irq(dev, pci_irq_vector(pdev, 2), @@ -2419,8 +2588,7 @@ static int interrupt_init_v3_hw(struct hisi_hba *hisi_hba) DRV_NAME " channel", hisi_hba); if (rc) { dev_err(dev, "could not request chnl interrupt, rc=%d\n", rc); - rc = -ENOENT; - goto free_irq_vectors; + return -ENOENT; } rc = devm_request_irq(dev, pci_irq_vector(pdev, 11), @@ -2428,35 +2596,32 @@ static int interrupt_init_v3_hw(struct hisi_hba *hisi_hba) DRV_NAME " fatal", hisi_hba); if (rc) { dev_err(dev, "could not request fatal interrupt, rc=%d\n", rc); - rc = -ENOENT; - goto free_irq_vectors; + return -ENOENT; } - /* Init tasklets for cq only */ + if (hisi_sas_intr_conv) + dev_info(dev, "Enable interrupt converge\n"); + for (i = 0; i < hisi_hba->cq_nvecs; i++) { struct hisi_sas_cq *cq = &hisi_hba->cq[i]; - struct tasklet_struct *t = &cq->tasklet; int nr = hisi_sas_intr_conv ? 16 : 16 + i; - unsigned long irqflags = hisi_sas_intr_conv ? IRQF_SHARED : 0; - - rc = devm_request_irq(dev, pci_irq_vector(pdev, nr), - cq_interrupt_v3_hw, irqflags, + unsigned long irqflags = hisi_sas_intr_conv ? IRQF_SHARED : + IRQF_ONESHOT; + + cq->irq_no = pci_irq_vector(pdev, nr); + rc = devm_request_threaded_irq(dev, cq->irq_no, + cq_interrupt_v3_hw, + cq_thread_v3_hw, + irqflags, DRV_NAME " cq", cq); if (rc) { dev_err(dev, "could not request cq%d interrupt, rc=%d\n", i, rc); - rc = -ENOENT; - goto free_irq_vectors; + return -ENOENT; } - - tasklet_init(t, cq_tasklet_v3_hw, (unsigned long)cq); } return 0; - -free_irq_vectors: - pci_free_irq_vectors(pdev); - return rc; } static int hisi_sas_v3_init(struct hisi_hba *hisi_hba) @@ -2478,8 +2643,10 @@ static void phy_set_linkrate_v3_hw(struct hisi_hba *hisi_hba, int phy_no, struct sas_phy_linkrates *r) { enum sas_linkrate max = r->maximum_linkrate; - u32 prog_phy_link_rate = 0x800; + u32 prog_phy_link_rate = hisi_sas_phy_read32(hisi_hba, phy_no, + PROG_PHY_LINK_RATE); + prog_phy_link_rate &= ~CFG_PROG_PHY_LINK_RATE_MSK; prog_phy_link_rate |= hisi_sas_get_prog_phy_linkrate_mask(max); hisi_sas_phy_write32(hisi_hba, phy_no, PROG_PHY_LINK_RATE, prog_phy_link_rate); @@ -2493,10 +2660,11 @@ static void interrupt_disable_v3_hw(struct hisi_hba *hisi_hba) synchronize_irq(pci_irq_vector(pdev, 1)); synchronize_irq(pci_irq_vector(pdev, 2)); synchronize_irq(pci_irq_vector(pdev, 11)); - for (i = 0; i < hisi_hba->queue_count; i++) { + for (i = 0; i < hisi_hba->queue_count; i++) hisi_sas_write32(hisi_hba, OQ0_INT_SRC_MSK + 0x4 * i, 0x1); + + for (i = 0; i < hisi_hba->cq_nvecs; i++) synchronize_irq(pci_irq_vector(pdev, i + 16)); - } hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1, 0xffffffff); hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK2, 0xffffffff); @@ -2523,9 +2691,7 @@ static int disable_host_v3_hw(struct hisi_hba *hisi_hba) u32 status, reg_val; int rc; - interrupt_disable_v3_hw(hisi_hba); hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, 0x0); - hisi_sas_kill_tasklets(hisi_hba); hisi_sas_stop_phys(hisi_hba); @@ -2554,6 +2720,7 @@ static int soft_reset_v3_hw(struct hisi_hba *hisi_hba) struct device *dev = hisi_hba->dev; int rc; + interrupt_disable_v3_hw(hisi_hba); rc = disable_host_v3_hw(hisi_hba); if (rc) { dev_err(dev, "soft reset: disable host failed rc=%d\n", rc); @@ -2722,6 +2889,43 @@ static ssize_t intr_coal_count_v3_hw_store(struct device *dev, } static DEVICE_ATTR_RW(intr_coal_count_v3_hw); +static int slave_configure_v3_hw(struct scsi_device *sdev) +{ + struct Scsi_Host *shost = dev_to_shost(&sdev->sdev_gendev); + struct domain_device *ddev = sdev_to_domain_dev(sdev); + struct hisi_hba *hisi_hba = shost_priv(shost); + struct device *dev = hisi_hba->dev; + int ret = sas_slave_configure(sdev); + + if (ret) + return ret; + if (!dev_is_sata(ddev)) + sas_change_queue_depth(sdev, 64); + + if (sdev->type == TYPE_ENCLOSURE) + return 0; + + if (!device_link_add(&sdev->sdev_gendev, dev, + DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME | + DL_FLAG_RPM_ACTIVE)) { + if (pm_runtime_enabled(dev)) { + dev_info(dev, "add device link failed, disable runtime PM for the host\n"); + pm_runtime_disable(dev); + } + } + + return 0; +} + +static void slave_destroy_v3_hw(struct scsi_device *sdev) +{ + struct Scsi_Host *shost = dev_to_shost(&sdev->sdev_gendev); + struct hisi_hba *hisi_hba = shost_priv(shost); + struct device *dev = hisi_hba->dev; + + device_link_remove(&sdev->sdev_gendev, dev); +} + static struct device_attribute *host_attrs_v3_hw[] = { &dev_attr_phy_event_threshold, &dev_attr_intr_conv_v3_hw, @@ -2730,6 +2934,19 @@ static struct device_attribute *host_attrs_v3_hw[] = { NULL }; +#define HISI_SAS_DEBUGFS_REG(x) {#x, x} + +struct hisi_sas_debugfs_reg_lu { + char *name; + int off; +}; + +struct hisi_sas_debugfs_reg { + const struct hisi_sas_debugfs_reg_lu *lu; + int count; + int base_off; +}; + static const struct hisi_sas_debugfs_reg_lu debugfs_port_reg_lu[] = { HISI_SAS_DEBUGFS_REG(PHY_CFG), HISI_SAS_DEBUGFS_REG(HARD_PHY_LINKRATE), @@ -2785,7 +3002,6 @@ static const struct hisi_sas_debugfs_reg debugfs_port_reg = { .lu = debugfs_port_reg_lu, .count = 0x100, .base_off = PORT_BASE, - .read_port_reg = hisi_sas_phy_read32, }; static const struct hisi_sas_debugfs_reg_lu debugfs_global_reg_lu[] = { @@ -2800,6 +3016,7 @@ static const struct hisi_sas_debugfs_reg_lu debugfs_global_reg_lu[] = { HISI_SAS_DEBUGFS_REG(SATA_INITI_D2H_STORE_ADDR_LO), HISI_SAS_DEBUGFS_REG(SATA_INITI_D2H_STORE_ADDR_HI), HISI_SAS_DEBUGFS_REG(CFG_MAX_TAG), + HISI_SAS_DEBUGFS_REG(TRANS_LOCK_ICT_TIME), HISI_SAS_DEBUGFS_REG(HGC_SAS_TX_OPEN_FAIL_RETRY_CTRL), HISI_SAS_DEBUGFS_REG(HGC_SAS_TXFAIL_RETRY_CTRL), HISI_SAS_DEBUGFS_REG(HGC_GET_ITV_TIME), @@ -2858,7 +3075,6 @@ static const struct hisi_sas_debugfs_reg_lu debugfs_global_reg_lu[] = { static const struct hisi_sas_debugfs_reg debugfs_global_reg = { .lu = debugfs_global_reg_lu, .count = 0x800, - .read_global_reg = hisi_sas_read32, }; static const struct hisi_sas_debugfs_reg_lu debugfs_axi_reg_lu[] = { @@ -2873,10 +3089,10 @@ static const struct hisi_sas_debugfs_reg debugfs_axi_reg = { .lu = debugfs_axi_reg_lu, .count = 0x61, .base_off = AXI_MASTER_CFG_BASE, - .read_global_reg = hisi_sas_read32, }; static const struct hisi_sas_debugfs_reg_lu debugfs_ras_reg_lu[] = { + HISI_SAS_DEBUGFS_REG(SAS_RAS_INTR0), HISI_SAS_DEBUGFS_REG(SAS_RAS_INTR1), HISI_SAS_DEBUGFS_REG(SAS_RAS_INTR0_MASK), HISI_SAS_DEBUGFS_REG(SAS_RAS_INTR1_MASK), @@ -2890,26 +3106,28 @@ static const struct hisi_sas_debugfs_reg debugfs_ras_reg = { .lu = debugfs_ras_reg_lu, .count = 0x10, .base_off = RAS_BASE, - .read_global_reg = hisi_sas_read32, }; static void debugfs_snapshot_prepare_v3_hw(struct hisi_hba *hisi_hba) { - set_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags); - - hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, 0); + struct Scsi_Host *shost = hisi_hba->shost; + scsi_block_requests(shost); wait_cmds_complete_timeout_v3_hw(hisi_hba, 100, 5000); - - hisi_sas_kill_tasklets(hisi_hba); + set_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags); + hisi_sas_sync_irqs(hisi_hba); + hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, 0); } static void debugfs_snapshot_restore_v3_hw(struct hisi_hba *hisi_hba) { + struct Scsi_Host *shost = hisi_hba->shost; + hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, (u32)((1ULL << hisi_hba->queue_count) - 1)); clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags); + scsi_unblock_requests(shost); } static void read_iost_itct_cache_v3_hw(struct hisi_hba *hisi_hba, @@ -2918,6 +3136,7 @@ static void read_iost_itct_cache_v3_hw(struct hisi_hba *hisi_hba, { u32 cache_dw_size = HISI_SAS_IOST_ITCT_CACHE_DW_SZ * HISI_SAS_IOST_ITCT_CACHE_NUM; + struct device *dev = hisi_hba->dev; u32 *buf = cache; u32 i, val; @@ -2930,7 +3149,7 @@ static void read_iost_itct_cache_v3_hw(struct hisi_hba *hisi_hba, } if (val != 0xffffffff) { - pr_err("Issue occur when reading IOST/ITCT cache!\n"); + dev_err(dev, "Issue occurred in reading IOST/ITCT cache!\n"); return; } @@ -2944,42 +3163,48 @@ static void read_iost_itct_cache_v3_hw(struct hisi_hba *hisi_hba, static void hisi_sas_bist_test_prep_v3_hw(struct hisi_hba *hisi_hba) { u32 reg_val; - int phy_id = hisi_hba->debugfs_bist_phy_no; + int phy_no = hisi_hba->debugfs_bist_phy_no; + int i; /* disable PHY */ - hisi_sas_phy_enable(hisi_hba, phy_id, 0); + hisi_sas_phy_enable(hisi_hba, phy_no, 0); + + /* update FFE */ + for (i = 0; i < FFE_CFG_MAX; i++) + hisi_sas_phy_write32(hisi_hba, phy_no, TXDEEMPH_G1 + (i * 0x4), + hisi_hba->debugfs_bist_ffe[phy_no][i]); /* disable ALOS */ - reg_val = hisi_sas_phy_read32(hisi_hba, phy_id, SERDES_CFG); + reg_val = hisi_sas_phy_read32(hisi_hba, phy_no, SERDES_CFG); reg_val |= CFG_ALOS_CHK_DISABLE_MSK; - hisi_sas_phy_write32(hisi_hba, phy_id, SERDES_CFG, reg_val); + hisi_sas_phy_write32(hisi_hba, phy_no, SERDES_CFG, reg_val); } static void hisi_sas_bist_test_restore_v3_hw(struct hisi_hba *hisi_hba) { u32 reg_val; - int phy_id = hisi_hba->debugfs_bist_phy_no; + int phy_no = hisi_hba->debugfs_bist_phy_no; /* disable loopback */ - reg_val = hisi_sas_phy_read32(hisi_hba, phy_id, SAS_PHY_BIST_CTRL); + reg_val = hisi_sas_phy_read32(hisi_hba, phy_no, SAS_PHY_BIST_CTRL); reg_val &= ~(CFG_RX_BIST_EN_MSK | CFG_TX_BIST_EN_MSK | CFG_BIST_TEST_MSK); - hisi_sas_phy_write32(hisi_hba, phy_id, SAS_PHY_BIST_CTRL, reg_val); + hisi_sas_phy_write32(hisi_hba, phy_no, SAS_PHY_BIST_CTRL, reg_val); /* enable ALOS */ - reg_val = hisi_sas_phy_read32(hisi_hba, phy_id, SERDES_CFG); + reg_val = hisi_sas_phy_read32(hisi_hba, phy_no, SERDES_CFG); reg_val &= ~CFG_ALOS_CHK_DISABLE_MSK; - hisi_sas_phy_write32(hisi_hba, phy_id, SERDES_CFG, reg_val); + hisi_sas_phy_write32(hisi_hba, phy_no, SERDES_CFG, reg_val); /* restore the linkrate */ - reg_val = hisi_sas_phy_read32(hisi_hba, phy_id, PROG_PHY_LINK_RATE); + reg_val = hisi_sas_phy_read32(hisi_hba, phy_no, PROG_PHY_LINK_RATE); /* init OOB link rate as 1.5 Gbits */ - reg_val &= ~CFG_PROG_PHY_LINK_RATE_MSK; - reg_val |= (0x8 << CFG_PROG_PHY_LINK_RATE_OFF); - hisi_sas_phy_write32(hisi_hba, phy_id, PROG_PHY_LINK_RATE, reg_val); + reg_val &= ~CFG_PROG_OOB_PHY_LINK_RATE_MSK; + reg_val |= (0x8 << CFG_PROG_OOB_PHY_LINK_RATE_OFF); + hisi_sas_phy_write32(hisi_hba, phy_no, PROG_PHY_LINK_RATE, reg_val); /* enable PHY */ - hisi_sas_phy_enable(hisi_hba, phy_id, 1); + hisi_sas_phy_enable(hisi_hba, phy_no, 1); } #define SAS_PHY_BIST_CODE_INIT 0x1 @@ -2988,60 +3213,75 @@ static int debugfs_set_bist_v3_hw(struct hisi_hba *hisi_hba, bool enable) { u32 reg_val, mode_tmp; u32 linkrate = hisi_hba->debugfs_bist_linkrate; - u32 phy_id = hisi_hba->debugfs_bist_phy_no; + u32 phy_no = hisi_hba->debugfs_bist_phy_no; + u32 *ffe = hisi_hba->debugfs_bist_ffe[phy_no]; u32 code_mode = hisi_hba->debugfs_bist_code_mode; u32 path_mode = hisi_hba->debugfs_bist_mode; + u32 *fix_code = &hisi_hba->debugfs_bist_fixed_code[0]; struct device *dev = hisi_hba->dev; - dev_info(dev, "BIST info:linkrate=%d phy_id=%d code_mode=%d path_mode=%d\n", - linkrate, phy_id, code_mode, path_mode); + dev_info(dev, "BIST info:phy%d link_rate=%d code_mode=%d path_mode=%d ffe={0x%x, 0x%x, 0x%x, 0x%x, 0x%x, 0x%x, 0x%x} fixed_code={0x%x, 0x%x}\n", + phy_no, linkrate, code_mode, path_mode, + ffe[FFE_SAS_1_5_GBPS], ffe[FFE_SAS_3_0_GBPS], + ffe[FFE_SAS_6_0_GBPS], ffe[FFE_SAS_12_0_GBPS], + ffe[FFE_SATA_1_5_GBPS], ffe[FFE_SATA_3_0_GBPS], + ffe[FFE_SATA_6_0_GBPS], fix_code[FIXED_CODE], + fix_code[FIXED_CODE_1]); mode_tmp = path_mode ? 2 : 1; if (enable) { /* some preparations before bist test */ hisi_sas_bist_test_prep_v3_hw(hisi_hba); /* set linkrate of bit test*/ - reg_val = hisi_sas_phy_read32(hisi_hba, phy_id, + reg_val = hisi_sas_phy_read32(hisi_hba, phy_no, PROG_PHY_LINK_RATE); - reg_val &= ~CFG_PROG_PHY_LINK_RATE_MSK; - reg_val |= (linkrate << CFG_PROG_PHY_LINK_RATE_OFF); - hisi_sas_phy_write32(hisi_hba, phy_id, - PROG_PHY_LINK_RATE, reg_val); + reg_val &= ~CFG_PROG_OOB_PHY_LINK_RATE_MSK; + reg_val |= (linkrate << CFG_PROG_OOB_PHY_LINK_RATE_OFF); + hisi_sas_phy_write32(hisi_hba, phy_no, PROG_PHY_LINK_RATE, + reg_val); /* set code mode of bit test */ - reg_val = hisi_sas_phy_read32(hisi_hba, phy_id, + reg_val = hisi_sas_phy_read32(hisi_hba, phy_no, SAS_PHY_BIST_CTRL); - reg_val &= ~(CFG_BIST_MODE_SEL_MSK | - CFG_LOOP_TEST_MODE_MSK | - CFG_RX_BIST_EN_MSK | - CFG_TX_BIST_EN_MSK | - CFG_BIST_TEST_MSK); + reg_val &= ~(CFG_BIST_MODE_SEL_MSK | CFG_LOOP_TEST_MODE_MSK | + CFG_RX_BIST_EN_MSK | CFG_TX_BIST_EN_MSK | + CFG_BIST_TEST_MSK); reg_val |= ((code_mode << CFG_BIST_MODE_SEL_OFF) | (mode_tmp << CFG_LOOP_TEST_MODE_OFF) | CFG_BIST_TEST_MSK); - hisi_sas_phy_write32(hisi_hba, phy_id, - SAS_PHY_BIST_CTRL, reg_val); + hisi_sas_phy_write32(hisi_hba, phy_no, SAS_PHY_BIST_CTRL, + reg_val); /* set the bist init value */ - hisi_sas_phy_write32(hisi_hba, phy_id, - SAS_PHY_BIST_CODE, - SAS_PHY_BIST_CODE_INIT); - hisi_sas_phy_write32(hisi_hba, phy_id, - SAS_PHY_BIST_CODE1, - SAS_PHY_BIST_CODE1_INIT); + if (code_mode == HISI_SAS_BIST_CODE_MODE_FIXED_DATA) { + reg_val = hisi_hba->debugfs_bist_fixed_code[0]; + hisi_sas_phy_write32(hisi_hba, phy_no, + SAS_PHY_BIST_CODE, reg_val); + + reg_val = hisi_hba->debugfs_bist_fixed_code[1]; + hisi_sas_phy_write32(hisi_hba, phy_no, + SAS_PHY_BIST_CODE1, reg_val); + } else { + hisi_sas_phy_write32(hisi_hba, phy_no, + SAS_PHY_BIST_CODE, + SAS_PHY_BIST_CODE_INIT); + hisi_sas_phy_write32(hisi_hba, phy_no, + SAS_PHY_BIST_CODE1, + SAS_PHY_BIST_CODE1_INIT); + } mdelay(100); reg_val |= (CFG_RX_BIST_EN_MSK | CFG_TX_BIST_EN_MSK); - hisi_sas_phy_write32(hisi_hba, phy_id, - SAS_PHY_BIST_CTRL, reg_val); + hisi_sas_phy_write32(hisi_hba, phy_no, SAS_PHY_BIST_CTRL, + reg_val); /* clear error bit */ mdelay(100); - hisi_sas_phy_read32(hisi_hba, phy_id, SAS_BIST_ERR_CNT); + hisi_sas_phy_read32(hisi_hba, phy_no, SAS_BIST_ERR_CNT); } else { /* disable bist test and recover it */ hisi_hba->debugfs_bist_cnt += hisi_sas_phy_read32(hisi_hba, - phy_id, SAS_BIST_ERR_CNT); + phy_no, SAS_BIST_ERR_CNT); hisi_sas_bist_test_restore_v3_hw(hisi_hba); } @@ -3050,10 +3290,11 @@ static int debugfs_set_bist_v3_hw(struct hisi_hba *hisi_hba, bool enable) static struct scsi_host_template sht_v3_hw = { .name = DRV_NAME, + .proc_name = DRV_NAME, .module = THIS_MODULE, .queuecommand = sas_queuecommand, .target_alloc = sas_target_alloc, - .slave_configure = hisi_sas_slave_configure, + .slave_configure = slave_configure_v3_hw, .scan_finished = hisi_sas_scan_finished, .scan_start = hisi_sas_scan_start, .change_queue_depth = sas_change_queue_depth, @@ -3064,7 +3305,8 @@ static struct scsi_host_template sht_v3_hw = { .max_sectors = SCSI_DEFAULT_MAX_SECTORS, .eh_device_reset_handler = sas_eh_device_reset_handler, .eh_target_reset_handler = sas_eh_target_reset_handler, - .slave_alloc = sas_slave_alloc, + .slave_alloc = hisi_sas_slave_alloc, + .slave_destroy = slave_destroy_v3_hw, .target_destroy = sas_target_destroy, .ioctl = sas_ioctl, .shost_attrs = host_attrs_v3_hw, @@ -3073,7 +3315,6 @@ static struct scsi_host_template sht_v3_hw = { }; static const struct hisi_sas_hw hisi_sas_v3_hw = { - .hw_init = hisi_sas_v3_init, .setup_itct = setup_itct_v3_hw, .get_wideport_bitmap = get_wideport_bitmap_v3_hw, .complete_hdr_size = sizeof(struct hisi_sas_complete_v3_hdr), @@ -3096,14 +3337,7 @@ static const struct hisi_sas_hw hisi_sas_v3_hw = { .get_events = phy_get_events_v3_hw, .write_gpio = write_gpio_v3_hw, .wait_cmds_complete_timeout = wait_cmds_complete_timeout_v3_hw, - .debugfs_reg_array[DEBUGFS_GLOBAL] = &debugfs_global_reg, - .debugfs_reg_array[DEBUGFS_AXI] = &debugfs_axi_reg, - .debugfs_reg_array[DEBUGFS_RAS] = &debugfs_ras_reg, - .debugfs_reg_port = &debugfs_port_reg, - .snapshot_prepare = debugfs_snapshot_prepare_v3_hw, - .snapshot_restore = debugfs_snapshot_restore_v3_hw, - .read_iost_itct_cache = read_iost_itct_cache_v3_hw, - .set_bist = debugfs_set_bist_v3_hw, + .debugfs_snapshot_regs = debugfs_snapshot_regs_v3_hw, }; static struct Scsi_Host * @@ -3121,7 +3355,6 @@ hisi_sas_shost_alloc_pci(struct pci_dev *pdev) hisi_hba = shost_priv(shost); INIT_WORK(&hisi_hba->rst_work, hisi_sas_rst_work_handler); - INIT_WORK(&hisi_hba->debugfs_work, hisi_sas_debugfs_work_handler); hisi_hba->hw = &hisi_sas_v3_hw; hisi_hba->pci_dev = pdev; hisi_hba->dev = dev; @@ -3149,123 +3382,1605 @@ hisi_sas_shost_alloc_pci(struct pci_dev *pdev) return NULL; } -static int -hisi_sas_v3_probe(struct pci_dev *pdev, const struct pci_device_id *id) +static void debugfs_snapshot_cq_reg_v3_hw(struct hisi_hba *hisi_hba) { - struct Scsi_Host *shost; - struct hisi_hba *hisi_hba; - struct device *dev = &pdev->dev; - struct asd_sas_phy **arr_phy; - struct asd_sas_port **arr_port; - struct sas_ha_struct *sha; - int rc, phy_nr, port_nr, i; + int queue_entry_size = hisi_hba->hw->complete_hdr_size; + int dump_index = hisi_hba->debugfs_dump_index; + int i; - rc = pci_enable_device(pdev); - if (rc) - goto err_out; + for (i = 0; i < hisi_hba->queue_count; i++) + memcpy(hisi_hba->debugfs_cq[dump_index][i].complete_hdr, + hisi_hba->complete_hdr[i], + HISI_SAS_QUEUE_SLOTS * queue_entry_size); +} - pci_set_master(pdev); +static void debugfs_snapshot_dq_reg_v3_hw(struct hisi_hba *hisi_hba) +{ + int queue_entry_size = sizeof(struct hisi_sas_cmd_hdr); + int dump_index = hisi_hba->debugfs_dump_index; + int i; - rc = pci_request_regions(pdev, DRV_NAME); - if (rc) - goto err_out_disable_device; + for (i = 0; i < hisi_hba->queue_count; i++) { + struct hisi_sas_cmd_hdr *debugfs_cmd_hdr, *cmd_hdr; + int j; - rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); - if (rc) - rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); - if (rc) { - dev_err(dev, "No usable DMA addressing method\n"); - rc = -ENODEV; - goto err_out_regions; + debugfs_cmd_hdr = hisi_hba->debugfs_dq[dump_index][i].hdr; + cmd_hdr = hisi_hba->cmd_hdr[i]; + + for (j = 0; j < HISI_SAS_QUEUE_SLOTS; j++) + memcpy(&debugfs_cmd_hdr[j], &cmd_hdr[j], + queue_entry_size); } +} - shost = hisi_sas_shost_alloc_pci(pdev); - if (!shost) { - rc = -ENOMEM; - goto err_out_regions; +static void debugfs_snapshot_port_reg_v3_hw(struct hisi_hba *hisi_hba) +{ + int dump_index = hisi_hba->debugfs_dump_index; + const struct hisi_sas_debugfs_reg *port = &debugfs_port_reg; + int i, phy_cnt; + u32 offset; + u32 *databuf; + + for (phy_cnt = 0; phy_cnt < hisi_hba->n_phy; phy_cnt++) { + databuf = hisi_hba->debugfs_port_reg[dump_index][phy_cnt].data; + for (i = 0; i < port->count; i++, databuf++) { + offset = port->base_off + 4 * i; + *databuf = hisi_sas_phy_read32(hisi_hba, phy_cnt, + offset); + } } +} - sha = SHOST_TO_SAS_HA(shost); - hisi_hba = shost_priv(shost); - dev_set_drvdata(dev, sha); +static void debugfs_snapshot_global_reg_v3_hw(struct hisi_hba *hisi_hba) +{ + int dump_index = hisi_hba->debugfs_dump_index; + u32 *databuf = hisi_hba->debugfs_regs[dump_index][DEBUGFS_GLOBAL].data; + int i; - hisi_hba->regs = pcim_iomap(pdev, 5, 0); - if (!hisi_hba->regs) { - dev_err(dev, "cannot map register\n"); - rc = -ENOMEM; - goto err_out_ha; - } + for (i = 0; i < debugfs_axi_reg.count; i++, databuf++) + *databuf = hisi_sas_read32(hisi_hba, 4 * i); +} - phy_nr = port_nr = hisi_hba->n_phy; +static void debugfs_snapshot_axi_reg_v3_hw(struct hisi_hba *hisi_hba) +{ + int dump_index = hisi_hba->debugfs_dump_index; + u32 *databuf = hisi_hba->debugfs_regs[dump_index][DEBUGFS_AXI].data; + const struct hisi_sas_debugfs_reg *axi = &debugfs_axi_reg; + int i; - arr_phy = devm_kcalloc(dev, phy_nr, sizeof(void *), GFP_KERNEL); - arr_port = devm_kcalloc(dev, port_nr, sizeof(void *), GFP_KERNEL); - if (!arr_phy || !arr_port) { - rc = -ENOMEM; - goto err_out_ha; - } + for (i = 0; i < axi->count; i++, databuf++) + *databuf = hisi_sas_read32(hisi_hba, 4 * i + axi->base_off); +} - sha->sas_phy = arr_phy; - sha->sas_port = arr_port; - sha->core.shost = shost; - sha->lldd_ha = hisi_hba; +static void debugfs_snapshot_ras_reg_v3_hw(struct hisi_hba *hisi_hba) +{ + int dump_index = hisi_hba->debugfs_dump_index; + u32 *databuf = hisi_hba->debugfs_regs[dump_index][DEBUGFS_RAS].data; + const struct hisi_sas_debugfs_reg *ras = &debugfs_ras_reg; + int i; - shost->transportt = hisi_sas_stt; - shost->max_id = HISI_SAS_MAX_DEVICES; - shost->max_lun = ~0; - shost->max_channel = 1; - shost->max_cmd_len = 16; - shost->can_queue = HISI_SAS_UNRESERVED_IPTT; - shost->cmd_per_lun = HISI_SAS_UNRESERVED_IPTT; + for (i = 0; i < ras->count; i++, databuf++) + *databuf = hisi_sas_read32(hisi_hba, 4 * i + ras->base_off); +} - sha->sas_ha_name = DRV_NAME; - sha->dev = dev; - sha->lldd_module = THIS_MODULE; - sha->sas_addr = &hisi_hba->sas_addr[0]; - sha->num_phys = hisi_hba->n_phy; +static void debugfs_snapshot_itct_reg_v3_hw(struct hisi_hba *hisi_hba) +{ + int dump_index = hisi_hba->debugfs_dump_index; + void *cachebuf = hisi_hba->debugfs_itct_cache[dump_index].cache; + void *databuf = hisi_hba->debugfs_itct[dump_index].itct; + struct hisi_sas_itct *itct; + int i; - for (i = 0; i < hisi_hba->n_phy; i++) { - sha->sas_phy[i] = &hisi_hba->phy[i].sas_phy; - sha->sas_port[i] = &hisi_hba->port[i].sas_port; - } + read_iost_itct_cache_v3_hw(hisi_hba, HISI_SAS_ITCT_CACHE, cachebuf); - if (hisi_hba->prot_mask) { - dev_info(dev, "Registering for DIF/DIX prot_mask=0x%x\n", - prot_mask); - scsi_host_set_prot(hisi_hba->shost, prot_mask); - if (hisi_hba->prot_mask & HISI_SAS_DIX_PROT_MASK) - scsi_host_set_guard(hisi_hba->shost, - SHOST_DIX_GUARD_CRC); + itct = hisi_hba->itct; + + for (i = 0; i < HISI_SAS_MAX_ITCT_ENTRIES; i++, itct++) { + memcpy(databuf, itct, sizeof(struct hisi_sas_itct)); + databuf += sizeof(struct hisi_sas_itct); } +} - if (hisi_sas_debugfs_enable) - hisi_sas_debugfs_init(hisi_hba); +static void debugfs_snapshot_iost_reg_v3_hw(struct hisi_hba *hisi_hba) +{ + int dump_index = hisi_hba->debugfs_dump_index; + int max_command_entries = HISI_SAS_MAX_COMMANDS; + void *cachebuf = hisi_hba->debugfs_iost_cache[dump_index].cache; + void *databuf = hisi_hba->debugfs_iost[dump_index].iost; + struct hisi_sas_iost *iost; + int i; - rc = scsi_add_host(shost, dev); - if (rc) - goto err_out_ha; + read_iost_itct_cache_v3_hw(hisi_hba, HISI_SAS_IOST_CACHE, cachebuf); - rc = sas_register_ha(sha); - if (rc) - goto err_out_register_ha; + iost = hisi_hba->iost; - rc = hisi_hba->hw->hw_init(hisi_hba); - if (rc) - goto err_out_register_ha; + for (i = 0; i < max_command_entries; i++, iost++) { + memcpy(databuf, iost, sizeof(struct hisi_sas_iost)); + databuf += sizeof(struct hisi_sas_iost); + } +} - scsi_scan_host(shost); +static const char * +debugfs_to_reg_name_v3_hw(int off, int base_off, + const struct hisi_sas_debugfs_reg_lu *lu) +{ + for (; lu->name; lu++) { + if (off == lu->off - base_off) + return lu->name; + } - return 0; + return NULL; +} -err_out_register_ha: +static void debugfs_print_reg_v3_hw(u32 *regs_val, struct seq_file *s, + const struct hisi_sas_debugfs_reg *reg) +{ + int i; + + for (i = 0; i < reg->count; i++) { + int off = i * 4; + const char *name; + + name = debugfs_to_reg_name_v3_hw(off, reg->base_off, + reg->lu); + + if (name) + seq_printf(s, "0x%08x 0x%08x %s\n", off, + regs_val[i], name); + else + seq_printf(s, "0x%08x 0x%08x\n", off, + regs_val[i]); + } +} + +static int debugfs_global_v3_hw_show(struct seq_file *s, void *p) +{ + struct hisi_sas_debugfs_regs *global = s->private; + + debugfs_print_reg_v3_hw(global->data, s, + &debugfs_global_reg); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(debugfs_global_v3_hw); + +static int debugfs_axi_v3_hw_show(struct seq_file *s, void *p) +{ + struct hisi_sas_debugfs_regs *axi = s->private; + + debugfs_print_reg_v3_hw(axi->data, s, + &debugfs_axi_reg); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(debugfs_axi_v3_hw); + +static int debugfs_ras_v3_hw_show(struct seq_file *s, void *p) +{ + struct hisi_sas_debugfs_regs *ras = s->private; + + debugfs_print_reg_v3_hw(ras->data, s, + &debugfs_ras_reg); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(debugfs_ras_v3_hw); + +static int debugfs_port_v3_hw_show(struct seq_file *s, void *p) +{ + struct hisi_sas_debugfs_port *port = s->private; + const struct hisi_sas_debugfs_reg *reg_port = &debugfs_port_reg; + + debugfs_print_reg_v3_hw(port->data, s, reg_port); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(debugfs_port_v3_hw); + +static void debugfs_show_row_64_v3_hw(struct seq_file *s, int index, + int sz, __le64 *ptr) +{ + int i; + + /* completion header size not fixed per HW version */ + seq_printf(s, "index %04d:\n\t", index); + for (i = 1; i <= sz / 8; i++, ptr++) { + seq_printf(s, " 0x%016llx", le64_to_cpu(*ptr)); + if (!(i % 2)) + seq_puts(s, "\n\t"); + } + + seq_puts(s, "\n"); +} + +static void debugfs_show_row_32_v3_hw(struct seq_file *s, int index, + int sz, __le32 *ptr) +{ + int i; + + /* completion header size not fixed per HW version */ + seq_printf(s, "index %04d:\n\t", index); + for (i = 1; i <= sz / 4; i++, ptr++) { + seq_printf(s, " 0x%08x", le32_to_cpu(*ptr)); + if (!(i % 4)) + seq_puts(s, "\n\t"); + } + seq_puts(s, "\n"); +} + +static void debugfs_cq_show_slot_v3_hw(struct seq_file *s, int slot, + struct hisi_sas_debugfs_cq *debugfs_cq) +{ + struct hisi_sas_cq *cq = debugfs_cq->cq; + struct hisi_hba *hisi_hba = cq->hisi_hba; + __le32 *complete_hdr = debugfs_cq->complete_hdr + + (hisi_hba->hw->complete_hdr_size * slot); + + debugfs_show_row_32_v3_hw(s, slot, + hisi_hba->hw->complete_hdr_size, + complete_hdr); +} + +static int debugfs_cq_v3_hw_show(struct seq_file *s, void *p) +{ + struct hisi_sas_debugfs_cq *debugfs_cq = s->private; + int slot; + + for (slot = 0; slot < HISI_SAS_QUEUE_SLOTS; slot++) + debugfs_cq_show_slot_v3_hw(s, slot, debugfs_cq); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(debugfs_cq_v3_hw); + +static void debugfs_dq_show_slot_v3_hw(struct seq_file *s, int slot, + void *dq_ptr) +{ + struct hisi_sas_debugfs_dq *debugfs_dq = dq_ptr; + void *cmd_queue = debugfs_dq->hdr; + __le32 *cmd_hdr = cmd_queue + + sizeof(struct hisi_sas_cmd_hdr) * slot; + + debugfs_show_row_32_v3_hw(s, slot, sizeof(struct hisi_sas_cmd_hdr), + cmd_hdr); +} + +static int debugfs_dq_v3_hw_show(struct seq_file *s, void *p) +{ + int slot; + + for (slot = 0; slot < HISI_SAS_QUEUE_SLOTS; slot++) + debugfs_dq_show_slot_v3_hw(s, slot, s->private); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(debugfs_dq_v3_hw); + +static int debugfs_iost_v3_hw_show(struct seq_file *s, void *p) +{ + struct hisi_sas_debugfs_iost *debugfs_iost = s->private; + struct hisi_sas_iost *iost = debugfs_iost->iost; + int i, max_command_entries = HISI_SAS_MAX_COMMANDS; + + for (i = 0; i < max_command_entries; i++, iost++) { + __le64 *data = &iost->qw0; + + debugfs_show_row_64_v3_hw(s, i, sizeof(*iost), data); + } + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(debugfs_iost_v3_hw); + +static int debugfs_iost_cache_v3_hw_show(struct seq_file *s, void *p) +{ + struct hisi_sas_debugfs_iost_cache *debugfs_iost_cache = s->private; + struct hisi_sas_iost_itct_cache *iost_cache = + debugfs_iost_cache->cache; + u32 cache_size = HISI_SAS_IOST_ITCT_CACHE_DW_SZ * 4; + int i, tab_idx; + __le64 *iost; + + for (i = 0; i < HISI_SAS_IOST_ITCT_CACHE_NUM; i++, iost_cache++) { + /* + * Data struct of IOST cache: + * Data[1]: BIT0~15: Table index + * Bit16: Valid mask + * Data[2]~[9]: IOST table + */ + tab_idx = (iost_cache->data[1] & 0xffff); + iost = (__le64 *)iost_cache; + + debugfs_show_row_64_v3_hw(s, tab_idx, cache_size, iost); + } + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(debugfs_iost_cache_v3_hw); + +static int debugfs_itct_v3_hw_show(struct seq_file *s, void *p) +{ + int i; + struct hisi_sas_debugfs_itct *debugfs_itct = s->private; + struct hisi_sas_itct *itct = debugfs_itct->itct; + + for (i = 0; i < HISI_SAS_MAX_ITCT_ENTRIES; i++, itct++) { + __le64 *data = &itct->qw0; + + debugfs_show_row_64_v3_hw(s, i, sizeof(*itct), data); + } + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(debugfs_itct_v3_hw); + +static int debugfs_itct_cache_v3_hw_show(struct seq_file *s, void *p) +{ + struct hisi_sas_debugfs_itct_cache *debugfs_itct_cache = s->private; + struct hisi_sas_iost_itct_cache *itct_cache = + debugfs_itct_cache->cache; + u32 cache_size = HISI_SAS_IOST_ITCT_CACHE_DW_SZ * 4; + int i, tab_idx; + __le64 *itct; + + for (i = 0; i < HISI_SAS_IOST_ITCT_CACHE_NUM; i++, itct_cache++) { + /* + * Data struct of ITCT cache: + * Data[1]: BIT0~15: Table index + * Bit16: Valid mask + * Data[2]~[9]: ITCT table + */ + tab_idx = itct_cache->data[1] & 0xffff; + itct = (__le64 *)itct_cache; + + debugfs_show_row_64_v3_hw(s, tab_idx, cache_size, itct); + } + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(debugfs_itct_cache_v3_hw); + +static void debugfs_create_files_v3_hw(struct hisi_hba *hisi_hba) +{ + u64 *debugfs_timestamp; + int dump_index = hisi_hba->debugfs_dump_index; + struct dentry *dump_dentry; + struct dentry *dentry; + char name[256]; + int p; + int c; + int d; + + snprintf(name, 256, "%d", dump_index); + + dump_dentry = debugfs_create_dir(name, hisi_hba->debugfs_dump_dentry); + + debugfs_timestamp = &hisi_hba->debugfs_timestamp[dump_index]; + + debugfs_create_u64("timestamp", 0400, dump_dentry, + debugfs_timestamp); + + debugfs_create_file("global", 0400, dump_dentry, + &hisi_hba->debugfs_regs[dump_index][DEBUGFS_GLOBAL], + &debugfs_global_v3_hw_fops); + + /* Create port dir and files */ + dentry = debugfs_create_dir("port", dump_dentry); + for (p = 0; p < hisi_hba->n_phy; p++) { + snprintf(name, 256, "%d", p); + + debugfs_create_file(name, 0400, dentry, + &hisi_hba->debugfs_port_reg[dump_index][p], + &debugfs_port_v3_hw_fops); + } + + /* Create CQ dir and files */ + dentry = debugfs_create_dir("cq", dump_dentry); + for (c = 0; c < hisi_hba->queue_count; c++) { + snprintf(name, 256, "%d", c); + + debugfs_create_file(name, 0400, dentry, + &hisi_hba->debugfs_cq[dump_index][c], + &debugfs_cq_v3_hw_fops); + } + + /* Create DQ dir and files */ + dentry = debugfs_create_dir("dq", dump_dentry); + for (d = 0; d < hisi_hba->queue_count; d++) { + snprintf(name, 256, "%d", d); + + debugfs_create_file(name, 0400, dentry, + &hisi_hba->debugfs_dq[dump_index][d], + &debugfs_dq_v3_hw_fops); + } + + debugfs_create_file("iost", 0400, dump_dentry, + &hisi_hba->debugfs_iost[dump_index], + &debugfs_iost_v3_hw_fops); + + debugfs_create_file("iost_cache", 0400, dump_dentry, + &hisi_hba->debugfs_iost_cache[dump_index], + &debugfs_iost_cache_v3_hw_fops); + + debugfs_create_file("itct", 0400, dump_dentry, + &hisi_hba->debugfs_itct[dump_index], + &debugfs_itct_v3_hw_fops); + + debugfs_create_file("itct_cache", 0400, dump_dentry, + &hisi_hba->debugfs_itct_cache[dump_index], + &debugfs_itct_cache_v3_hw_fops); + + debugfs_create_file("axi", 0400, dump_dentry, + &hisi_hba->debugfs_regs[dump_index][DEBUGFS_AXI], + &debugfs_axi_v3_hw_fops); + + debugfs_create_file("ras", 0400, dump_dentry, + &hisi_hba->debugfs_regs[dump_index][DEBUGFS_RAS], + &debugfs_ras_v3_hw_fops); +} + +static ssize_t debugfs_trigger_dump_v3_hw_write(struct file *file, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct hisi_hba *hisi_hba = file->f_inode->i_private; + char buf[8]; + + if (count > 8) + return -EFAULT; + + if (copy_from_user(buf, user_buf, count)) + return -EFAULT; + + if (buf[0] != '1') + return -EFAULT; + + down(&hisi_hba->sem); + if (debugfs_snapshot_regs_v3_hw(hisi_hba)) { + up(&hisi_hba->sem); + return -EFAULT; + } + up(&hisi_hba->sem); + + return count; +} + +static const struct file_operations debugfs_trigger_dump_v3_hw_fops = { + .write = &debugfs_trigger_dump_v3_hw_write, + .owner = THIS_MODULE, +}; + +enum { + HISI_SAS_BIST_LOOPBACK_MODE_DIGITAL = 0, + HISI_SAS_BIST_LOOPBACK_MODE_SERDES, + HISI_SAS_BIST_LOOPBACK_MODE_REMOTE, +}; + +static const struct { + int value; + char *name; +} debugfs_loop_linkrate_v3_hw[] = { + { SAS_LINK_RATE_1_5_GBPS, "1.5 Gbit" }, + { SAS_LINK_RATE_3_0_GBPS, "3.0 Gbit" }, + { SAS_LINK_RATE_6_0_GBPS, "6.0 Gbit" }, + { SAS_LINK_RATE_12_0_GBPS, "12.0 Gbit" }, +}; + +static int debugfs_bist_linkrate_v3_hw_show(struct seq_file *s, void *p) +{ + struct hisi_hba *hisi_hba = s->private; + int i; + + for (i = 0; i < ARRAY_SIZE(debugfs_loop_linkrate_v3_hw); i++) { + int match = (hisi_hba->debugfs_bist_linkrate == + debugfs_loop_linkrate_v3_hw[i].value); + + seq_printf(s, "%s%s%s ", match ? "[" : "", + debugfs_loop_linkrate_v3_hw[i].name, + match ? "]" : ""); + } + seq_puts(s, "\n"); + + return 0; +} + +static ssize_t debugfs_bist_linkrate_v3_hw_write(struct file *filp, + const char __user *buf, + size_t count, loff_t *ppos) +{ + struct seq_file *m = filp->private_data; + struct hisi_hba *hisi_hba = m->private; + char kbuf[16] = {}, *pkbuf; + bool found = false; + int i; + + if (hisi_hba->debugfs_bist_enable) + return -EPERM; + + if (count >= sizeof(kbuf)) + return -EOVERFLOW; + + if (copy_from_user(kbuf, buf, count)) + return -EINVAL; + + pkbuf = strstrip(kbuf); + + for (i = 0; i < ARRAY_SIZE(debugfs_loop_linkrate_v3_hw); i++) { + if (!strncmp(debugfs_loop_linkrate_v3_hw[i].name, + pkbuf, 16)) { + hisi_hba->debugfs_bist_linkrate = + debugfs_loop_linkrate_v3_hw[i].value; + found = true; + break; + } + } + + if (!found) + return -EINVAL; + + return count; +} + +static int debugfs_bist_linkrate_v3_hw_open(struct inode *inode, + struct file *filp) +{ + return single_open(filp, debugfs_bist_linkrate_v3_hw_show, + inode->i_private); +} + +static const struct file_operations debugfs_bist_linkrate_v3_hw_fops = { + .open = debugfs_bist_linkrate_v3_hw_open, + .read = seq_read, + .write = debugfs_bist_linkrate_v3_hw_write, + .llseek = seq_lseek, + .release = single_release, + .owner = THIS_MODULE, +}; + +static const struct { + int value; + char *name; +} debugfs_loop_code_mode_v3_hw[] = { + { HISI_SAS_BIST_CODE_MODE_PRBS7, "PRBS7" }, + { HISI_SAS_BIST_CODE_MODE_PRBS23, "PRBS23" }, + { HISI_SAS_BIST_CODE_MODE_PRBS31, "PRBS31" }, + { HISI_SAS_BIST_CODE_MODE_JTPAT, "JTPAT" }, + { HISI_SAS_BIST_CODE_MODE_CJTPAT, "CJTPAT" }, + { HISI_SAS_BIST_CODE_MODE_SCRAMBED_0, "SCRAMBED_0" }, + { HISI_SAS_BIST_CODE_MODE_TRAIN, "TRAIN" }, + { HISI_SAS_BIST_CODE_MODE_TRAIN_DONE, "TRAIN_DONE" }, + { HISI_SAS_BIST_CODE_MODE_HFTP, "HFTP" }, + { HISI_SAS_BIST_CODE_MODE_MFTP, "MFTP" }, + { HISI_SAS_BIST_CODE_MODE_LFTP, "LFTP" }, + { HISI_SAS_BIST_CODE_MODE_FIXED_DATA, "FIXED_DATA" }, +}; + +static int debugfs_bist_code_mode_v3_hw_show(struct seq_file *s, void *p) +{ + struct hisi_hba *hisi_hba = s->private; + int i; + + for (i = 0; i < ARRAY_SIZE(debugfs_loop_code_mode_v3_hw); i++) { + int match = (hisi_hba->debugfs_bist_code_mode == + debugfs_loop_code_mode_v3_hw[i].value); + + seq_printf(s, "%s%s%s ", match ? "[" : "", + debugfs_loop_code_mode_v3_hw[i].name, + match ? "]" : ""); + } + seq_puts(s, "\n"); + + return 0; +} + +static ssize_t debugfs_bist_code_mode_v3_hw_write(struct file *filp, + const char __user *buf, + size_t count, + loff_t *ppos) +{ + struct seq_file *m = filp->private_data; + struct hisi_hba *hisi_hba = m->private; + char kbuf[16] = {}, *pkbuf; + bool found = false; + int i; + + if (hisi_hba->debugfs_bist_enable) + return -EPERM; + + if (count >= sizeof(kbuf)) + return -EINVAL; + + if (copy_from_user(kbuf, buf, count)) + return -EOVERFLOW; + + pkbuf = strstrip(kbuf); + + for (i = 0; i < ARRAY_SIZE(debugfs_loop_code_mode_v3_hw); i++) { + if (!strncmp(debugfs_loop_code_mode_v3_hw[i].name, + pkbuf, 16)) { + hisi_hba->debugfs_bist_code_mode = + debugfs_loop_code_mode_v3_hw[i].value; + found = true; + break; + } + } + + if (!found) + return -EINVAL; + + return count; +} + +static int debugfs_bist_code_mode_v3_hw_open(struct inode *inode, + struct file *filp) +{ + return single_open(filp, debugfs_bist_code_mode_v3_hw_show, + inode->i_private); +} + +static const struct file_operations debugfs_bist_code_mode_v3_hw_fops = { + .open = debugfs_bist_code_mode_v3_hw_open, + .read = seq_read, + .write = debugfs_bist_code_mode_v3_hw_write, + .llseek = seq_lseek, + .release = single_release, + .owner = THIS_MODULE, +}; + +static ssize_t debugfs_bist_phy_v3_hw_write(struct file *filp, + const char __user *buf, + size_t count, loff_t *ppos) +{ + struct seq_file *m = filp->private_data; + struct hisi_hba *hisi_hba = m->private; + unsigned int phy_no; + int val; + + if (hisi_hba->debugfs_bist_enable) + return -EPERM; + + val = kstrtouint_from_user(buf, count, 0, &phy_no); + if (val) + return val; + + if (phy_no >= hisi_hba->n_phy) + return -EINVAL; + + hisi_hba->debugfs_bist_phy_no = phy_no; + + return count; +} + +static int debugfs_bist_phy_v3_hw_show(struct seq_file *s, void *p) +{ + struct hisi_hba *hisi_hba = s->private; + + seq_printf(s, "%d\n", hisi_hba->debugfs_bist_phy_no); + + return 0; +} + +static int debugfs_bist_phy_v3_hw_open(struct inode *inode, + struct file *filp) +{ + return single_open(filp, debugfs_bist_phy_v3_hw_show, + inode->i_private); +} + +static const struct file_operations debugfs_bist_phy_v3_hw_fops = { + .open = debugfs_bist_phy_v3_hw_open, + .read = seq_read, + .write = debugfs_bist_phy_v3_hw_write, + .llseek = seq_lseek, + .release = single_release, + .owner = THIS_MODULE, +}; + +static ssize_t debugfs_bist_cnt_v3_hw_write(struct file *filp, + const char __user *buf, + size_t count, loff_t *ppos) +{ + struct seq_file *m = filp->private_data; + struct hisi_hba *hisi_hba = m->private; + unsigned int cnt; + int val; + + if (hisi_hba->debugfs_bist_enable) + return -EPERM; + + val = kstrtouint_from_user(buf, count, 0, &cnt); + if (val) + return val; + + if (cnt) + return -EINVAL; + + hisi_hba->debugfs_bist_cnt = 0; + return count; +} + +static int debugfs_bist_cnt_v3_hw_show(struct seq_file *s, void *p) +{ + struct hisi_hba *hisi_hba = s->private; + + seq_printf(s, "%u\n", hisi_hba->debugfs_bist_cnt); + + return 0; +} + +static int debugfs_bist_cnt_v3_hw_open(struct inode *inode, + struct file *filp) +{ + return single_open(filp, debugfs_bist_cnt_v3_hw_show, + inode->i_private); +} + +static const struct file_operations debugfs_bist_cnt_v3_hw_ops = { + .open = debugfs_bist_cnt_v3_hw_open, + .read = seq_read, + .write = debugfs_bist_cnt_v3_hw_write, + .llseek = seq_lseek, + .release = single_release, + .owner = THIS_MODULE, +}; + +static const struct { + int value; + char *name; +} debugfs_loop_modes_v3_hw[] = { + { HISI_SAS_BIST_LOOPBACK_MODE_DIGITAL, "digital" }, + { HISI_SAS_BIST_LOOPBACK_MODE_SERDES, "serdes" }, + { HISI_SAS_BIST_LOOPBACK_MODE_REMOTE, "remote" }, +}; + +static int debugfs_bist_mode_v3_hw_show(struct seq_file *s, void *p) +{ + struct hisi_hba *hisi_hba = s->private; + int i; + + for (i = 0; i < ARRAY_SIZE(debugfs_loop_modes_v3_hw); i++) { + int match = (hisi_hba->debugfs_bist_mode == + debugfs_loop_modes_v3_hw[i].value); + + seq_printf(s, "%s%s%s ", match ? "[" : "", + debugfs_loop_modes_v3_hw[i].name, + match ? "]" : ""); + } + seq_puts(s, "\n"); + + return 0; +} + +static ssize_t debugfs_bist_mode_v3_hw_write(struct file *filp, + const char __user *buf, + size_t count, loff_t *ppos) +{ + struct seq_file *m = filp->private_data; + struct hisi_hba *hisi_hba = m->private; + char kbuf[16] = {}, *pkbuf; + bool found = false; + int i; + + if (hisi_hba->debugfs_bist_enable) + return -EPERM; + + if (count >= sizeof(kbuf)) + return -EINVAL; + + if (copy_from_user(kbuf, buf, count)) + return -EOVERFLOW; + + pkbuf = strstrip(kbuf); + + for (i = 0; i < ARRAY_SIZE(debugfs_loop_modes_v3_hw); i++) { + if (!strncmp(debugfs_loop_modes_v3_hw[i].name, pkbuf, 16)) { + hisi_hba->debugfs_bist_mode = + debugfs_loop_modes_v3_hw[i].value; + found = true; + break; + } + } + + if (!found) + return -EINVAL; + + return count; +} + +static int debugfs_bist_mode_v3_hw_open(struct inode *inode, + struct file *filp) +{ + return single_open(filp, debugfs_bist_mode_v3_hw_show, + inode->i_private); +} + +static const struct file_operations debugfs_bist_mode_v3_hw_fops = { + .open = debugfs_bist_mode_v3_hw_open, + .read = seq_read, + .write = debugfs_bist_mode_v3_hw_write, + .llseek = seq_lseek, + .release = single_release, + .owner = THIS_MODULE, +}; + +static ssize_t debugfs_bist_enable_v3_hw_write(struct file *filp, + const char __user *buf, + size_t count, loff_t *ppos) +{ + struct seq_file *m = filp->private_data; + struct hisi_hba *hisi_hba = m->private; + unsigned int enable; + int val; + + val = kstrtouint_from_user(buf, count, 0, &enable); + if (val) + return val; + + if (enable > 1) + return -EINVAL; + + if (enable == hisi_hba->debugfs_bist_enable) + return count; + + val = debugfs_set_bist_v3_hw(hisi_hba, enable); + if (val < 0) + return val; + + hisi_hba->debugfs_bist_enable = enable; + + return count; +} + +static int debugfs_bist_enable_v3_hw_show(struct seq_file *s, void *p) +{ + struct hisi_hba *hisi_hba = s->private; + + seq_printf(s, "%d\n", hisi_hba->debugfs_bist_enable); + + return 0; +} + +static int debugfs_bist_enable_v3_hw_open(struct inode *inode, + struct file *filp) +{ + return single_open(filp, debugfs_bist_enable_v3_hw_show, + inode->i_private); +} + +static const struct file_operations debugfs_bist_enable_v3_hw_fops = { + .open = debugfs_bist_enable_v3_hw_open, + .read = seq_read, + .write = debugfs_bist_enable_v3_hw_write, + .llseek = seq_lseek, + .release = single_release, + .owner = THIS_MODULE, +}; + +static const struct { + char *name; +} debugfs_ffe_name_v3_hw[FFE_CFG_MAX] = { + { "SAS_1_5_GBPS" }, + { "SAS_3_0_GBPS" }, + { "SAS_6_0_GBPS" }, + { "SAS_12_0_GBPS" }, + { "FFE_RESV" }, + { "SATA_1_5_GBPS" }, + { "SATA_3_0_GBPS" }, + { "SATA_6_0_GBPS" }, +}; + +static ssize_t debugfs_v3_hw_write(struct file *filp, + const char __user *buf, + size_t count, loff_t *ppos) +{ + struct seq_file *m = filp->private_data; + u32 *val = m->private; + int res; + + res = kstrtouint_from_user(buf, count, 0, val); + if (res) + return res; + + return count; +} + +static int debugfs_v3_hw_show(struct seq_file *s, void *p) +{ + u32 *val = s->private; + + seq_printf(s, "0x%x\n", *val); + + return 0; +} + +static int debugfs_v3_hw_open(struct inode *inode, struct file *filp) +{ + return single_open(filp, debugfs_v3_hw_show, + inode->i_private); +} + +static const struct file_operations debugfs_v3_hw_fops = { + .open = debugfs_v3_hw_open, + .read = seq_read, + .write = debugfs_v3_hw_write, + .llseek = seq_lseek, + .release = single_release, + .owner = THIS_MODULE, +}; + +static ssize_t debugfs_phy_down_cnt_v3_hw_write(struct file *filp, + const char __user *buf, + size_t count, loff_t *ppos) +{ + struct seq_file *s = filp->private_data; + struct hisi_sas_phy *phy = s->private; + unsigned int set_val; + int res; + + res = kstrtouint_from_user(buf, count, 0, &set_val); + if (res) + return res; + + if (set_val > 0) + return -EINVAL; + + atomic_set(&phy->down_cnt, 0); + + return count; +} + +static int debugfs_phy_down_cnt_v3_hw_show(struct seq_file *s, void *p) +{ + struct hisi_sas_phy *phy = s->private; + + seq_printf(s, "%d\n", atomic_read(&phy->down_cnt)); + + return 0; +} + +static int debugfs_phy_down_cnt_v3_hw_open(struct inode *inode, + struct file *filp) +{ + return single_open(filp, debugfs_phy_down_cnt_v3_hw_show, + inode->i_private); +} + +static const struct file_operations debugfs_phy_down_cnt_v3_hw_fops = { + .open = debugfs_phy_down_cnt_v3_hw_open, + .read = seq_read, + .write = debugfs_phy_down_cnt_v3_hw_write, + .llseek = seq_lseek, + .release = single_release, + .owner = THIS_MODULE, +}; + +enum fifo_dump_mode_v3_hw { + FIFO_DUMP_FORVER = (1U << 0), + FIFO_DUMP_AFTER_TRIGGER = (1U << 1), + FIFO_DUMP_UNTILL_TRIGGER = (1U << 2), +}; + +enum fifo_trigger_mode_v3_hw { + FIFO_TRIGGER_EDGE = (1U << 0), + FIFO_TRIGGER_SAME_LEVEL = (1U << 1), + FIFO_TRIGGER_DIFF_LEVEL = (1U << 2), +}; + +static int debugfs_is_fifo_config_valid_v3_hw(struct hisi_sas_phy *phy) +{ + struct hisi_hba *hisi_hba = phy->hisi_hba; + + if (phy->fifo.signal_sel > 0xf) { + dev_info(hisi_hba->dev, "Invalid signal select: %u\n", + phy->fifo.signal_sel); + return -EINVAL; + } + + switch (phy->fifo.dump_mode) { + case FIFO_DUMP_FORVER: + case FIFO_DUMP_AFTER_TRIGGER: + case FIFO_DUMP_UNTILL_TRIGGER: + break; + default: + dev_info(hisi_hba->dev, "Invalid dump mode: %u\n", + phy->fifo.dump_mode); + return -EINVAL; + } + + /* when FIFO_DUMP_FORVER, no need to check trigger_mode */ + if (phy->fifo.dump_mode == FIFO_DUMP_FORVER) + return 0; + + switch (phy->fifo.trigger_mode) { + case FIFO_TRIGGER_EDGE: + case FIFO_TRIGGER_SAME_LEVEL: + case FIFO_TRIGGER_DIFF_LEVEL: + break; + default: + dev_info(hisi_hba->dev, "Invalid trigger mode: %u\n", + phy->fifo.trigger_mode); + return -EINVAL; + } + return 0; +} + +static int debugfs_update_fifo_config_v3_hw(struct hisi_sas_phy *phy) +{ + u32 trigger_mode = phy->fifo.trigger_mode; + u32 signal_sel = phy->fifo.signal_sel; + u32 dump_mode = phy->fifo.dump_mode; + struct hisi_hba *hisi_hba = phy->hisi_hba; + int phy_no = phy->sas_phy.id; + u32 reg_val; + int res; + + /* Check the validity of trace FIFO configuration */ + res = debugfs_is_fifo_config_valid_v3_hw(phy); + if (res) + return res; + + reg_val = hisi_sas_phy_read32(hisi_hba, phy_no, DFX_FIFO_CTRL); + /* Disable trace FIFO before update configuration */ + reg_val |= DFX_FIFO_CTRL_DUMP_DISABLE_MSK; + + /* Update trace FIFO configuration */ + reg_val &= ~(DFX_FIFO_CTRL_DUMP_MODE_MSK | + DFX_FIFO_CTRL_SIGNAL_SEL_MSK | + DFX_FIFO_CTRL_TRIGGER_MODE_MSK); + + reg_val |= ((trigger_mode << DFX_FIFO_CTRL_TRIGGER_MODE_OFF) | + (dump_mode << DFX_FIFO_CTRL_DUMP_MODE_OFF) | + (signal_sel << DFX_FIFO_CTRL_SIGNAL_SEL_OFF)); + hisi_sas_phy_write32(hisi_hba, phy_no, DFX_FIFO_CTRL, reg_val); + + hisi_sas_phy_write32(hisi_hba, phy_no, DFX_FIFO_DUMP_MSK, + phy->fifo.dump_msk); + + hisi_sas_phy_write32(hisi_hba, phy_no, DFX_FIFO_TRIGGER, + phy->fifo.trigger); + + hisi_sas_phy_write32(hisi_hba, phy_no, DFX_FIFO_TRIGGER_MSK, + phy->fifo.trigger_msk); + + /* Enable trace FIFO after updated configuration */ + reg_val = hisi_sas_phy_read32(hisi_hba, phy_no, DFX_FIFO_CTRL); + reg_val &= ~DFX_FIFO_CTRL_DUMP_DISABLE_MSK; + hisi_sas_phy_write32(hisi_hba, phy_no, DFX_FIFO_CTRL, reg_val); + + return 0; +} + +static ssize_t debugfs_fifo_update_cfg_v3_hw_write(struct file *filp, + const char __user *buf, + size_t count, loff_t *ppos) +{ + struct hisi_sas_phy *phy = filp->private_data; + bool update; + int val; + + val = kstrtobool_from_user(buf, count, &update); + if (val) + return val; + + if (update != 1) + return -EINVAL; + + val = debugfs_update_fifo_config_v3_hw(phy); + if (val) + return val; + + return count; +} + +static const struct file_operations debugfs_fifo_update_cfg_v3_hw_fops = { + .open = simple_open, + .write = debugfs_fifo_update_cfg_v3_hw_write, + .owner = THIS_MODULE, +}; + +static void debugfs_read_fifo_data_v3_hw(struct hisi_sas_phy *phy) +{ + struct hisi_hba *hisi_hba = phy->hisi_hba; + u32 *buf = phy->fifo.rd_data; + int phy_no = phy->sas_phy.id; + u32 val; + int i; + + memset(buf, 0, sizeof(phy->fifo.rd_data)); + + /* Disable trace FIFO before read data */ + val = hisi_sas_phy_read32(hisi_hba, phy_no, DFX_FIFO_CTRL); + val |= DFX_FIFO_CTRL_DUMP_DISABLE_MSK; + hisi_sas_phy_write32(hisi_hba, phy_no, DFX_FIFO_CTRL, val); + + for (i = 0; i < HISI_SAS_FIFO_DATA_DW_SIZE; i++) { + val = hisi_sas_phy_read32(hisi_hba, phy_no, + DFX_FIFO_RD_DATA); + buf[i] = val; + } + + /* Enable trace FIFO after read data */ + val = hisi_sas_phy_read32(hisi_hba, phy_no, DFX_FIFO_CTRL); + val &= ~DFX_FIFO_CTRL_DUMP_DISABLE_MSK; + hisi_sas_phy_write32(hisi_hba, phy_no, DFX_FIFO_CTRL, val); +} + +static int debugfs_fifo_data_v3_hw_show(struct seq_file *s, void *p) +{ + struct hisi_sas_phy *phy = s->private; + + debugfs_read_fifo_data_v3_hw(phy); + + debugfs_show_row_32_v3_hw(s, 0, HISI_SAS_FIFO_DATA_DW_SIZE * 4, + phy->fifo.rd_data); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(debugfs_fifo_data_v3_hw); + +static void debugfs_fifo_init_v3_hw(struct hisi_hba *hisi_hba) +{ + int phy_no; + + hisi_hba->debugfs_fifo_dentry = + debugfs_create_dir("fifo", hisi_hba->debugfs_dir); + + for (phy_no = 0; phy_no < hisi_hba->n_phy; phy_no++) { + struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no]; + struct dentry *port_dentry; + char name[256]; + u32 val; + + /* get default configuration for trace FIFO */ + val = hisi_sas_phy_read32(hisi_hba, phy_no, DFX_FIFO_CTRL); + val &= DFX_FIFO_CTRL_DUMP_MODE_MSK; + val >>= DFX_FIFO_CTRL_DUMP_MODE_OFF; + phy->fifo.dump_mode = val; + + val = hisi_sas_phy_read32(hisi_hba, phy_no, DFX_FIFO_CTRL); + val &= DFX_FIFO_CTRL_TRIGGER_MODE_MSK; + val >>= DFX_FIFO_CTRL_TRIGGER_MODE_OFF; + phy->fifo.trigger_mode = val; + + val = hisi_sas_phy_read32(hisi_hba, phy_no, DFX_FIFO_CTRL); + val &= DFX_FIFO_CTRL_SIGNAL_SEL_MSK; + val >>= DFX_FIFO_CTRL_SIGNAL_SEL_OFF; + phy->fifo.signal_sel = val; + + val = hisi_sas_phy_read32(hisi_hba, phy_no, DFX_FIFO_DUMP_MSK); + phy->fifo.dump_msk = val; + + val = hisi_sas_phy_read32(hisi_hba, phy_no, DFX_FIFO_TRIGGER); + phy->fifo.trigger = val; + val = hisi_sas_phy_read32(hisi_hba, phy_no, DFX_FIFO_TRIGGER_MSK); + phy->fifo.trigger_msk = val; + + snprintf(name, 256, "%d", phy_no); + port_dentry = debugfs_create_dir(name, + hisi_hba->debugfs_fifo_dentry); + + debugfs_create_file("update_config", 0200, port_dentry, phy, + &debugfs_fifo_update_cfg_v3_hw_fops); + + debugfs_create_file("signal_sel", 0600, port_dentry, + &phy->fifo.signal_sel, + &debugfs_v3_hw_fops); + + debugfs_create_file("dump_msk", 0600, port_dentry, + &phy->fifo.dump_msk, + &debugfs_v3_hw_fops); + + debugfs_create_file("dump_mode", 0600, port_dentry, + &phy->fifo.dump_mode, + &debugfs_v3_hw_fops); + + debugfs_create_file("trigger_mode", 0600, port_dentry, + &phy->fifo.trigger_mode, + &debugfs_v3_hw_fops); + + debugfs_create_file("trigger", 0600, port_dentry, + &phy->fifo.trigger, + &debugfs_v3_hw_fops); + + debugfs_create_file("trigger_msk", 0600, port_dentry, + &phy->fifo.trigger_msk, + &debugfs_v3_hw_fops); + + debugfs_create_file("fifo_data", 0400, port_dentry, phy, + &debugfs_fifo_data_v3_hw_fops); + } +} + +static void debugfs_release_v3_hw(struct hisi_hba *hisi_hba, int dump_index) +{ + struct device *dev = hisi_hba->dev; + int i; + + devm_kfree(dev, hisi_hba->debugfs_iost_cache[dump_index].cache); + devm_kfree(dev, hisi_hba->debugfs_itct_cache[dump_index].cache); + devm_kfree(dev, hisi_hba->debugfs_iost[dump_index].iost); + devm_kfree(dev, hisi_hba->debugfs_itct[dump_index].itct); + + for (i = 0; i < hisi_hba->queue_count; i++) + devm_kfree(dev, hisi_hba->debugfs_dq[dump_index][i].hdr); + + for (i = 0; i < hisi_hba->queue_count; i++) + devm_kfree(dev, + hisi_hba->debugfs_cq[dump_index][i].complete_hdr); + + for (i = 0; i < DEBUGFS_REGS_NUM; i++) + devm_kfree(dev, hisi_hba->debugfs_regs[dump_index][i].data); + + for (i = 0; i < hisi_hba->n_phy; i++) + devm_kfree(dev, hisi_hba->debugfs_port_reg[dump_index][i].data); +} + +static const struct hisi_sas_debugfs_reg *debugfs_reg_array_v3_hw[DEBUGFS_REGS_NUM] = { + [DEBUGFS_GLOBAL] = &debugfs_global_reg, + [DEBUGFS_AXI] = &debugfs_axi_reg, + [DEBUGFS_RAS] = &debugfs_ras_reg, +}; + +static int debugfs_alloc_v3_hw(struct hisi_hba *hisi_hba, int dump_index) +{ + const struct hisi_sas_hw *hw = hisi_hba->hw; + struct device *dev = hisi_hba->dev; + int p, c, d, r; + size_t sz; + + for (r = 0; r < DEBUGFS_REGS_NUM; r++) { + struct hisi_sas_debugfs_regs *regs = + &hisi_hba->debugfs_regs[dump_index][r]; + + sz = debugfs_reg_array_v3_hw[r]->count * 4; + regs->data = devm_kmalloc(dev, sz, GFP_KERNEL); + if (!regs->data) + goto fail; + regs->hisi_hba = hisi_hba; + } + + sz = debugfs_port_reg.count * 4; + for (p = 0; p < hisi_hba->n_phy; p++) { + struct hisi_sas_debugfs_port *port = + &hisi_hba->debugfs_port_reg[dump_index][p]; + + port->data = devm_kmalloc(dev, sz, GFP_KERNEL); + if (!port->data) + goto fail; + port->phy = &hisi_hba->phy[p]; + } + + sz = hw->complete_hdr_size * HISI_SAS_QUEUE_SLOTS; + for (c = 0; c < hisi_hba->queue_count; c++) { + struct hisi_sas_debugfs_cq *cq = + &hisi_hba->debugfs_cq[dump_index][c]; + + cq->complete_hdr = devm_kmalloc(dev, sz, GFP_KERNEL); + if (!cq->complete_hdr) + goto fail; + cq->cq = &hisi_hba->cq[c]; + } + + sz = sizeof(struct hisi_sas_cmd_hdr) * HISI_SAS_QUEUE_SLOTS; + for (d = 0; d < hisi_hba->queue_count; d++) { + struct hisi_sas_debugfs_dq *dq = + &hisi_hba->debugfs_dq[dump_index][d]; + + dq->hdr = devm_kmalloc(dev, sz, GFP_KERNEL); + if (!dq->hdr) + goto fail; + dq->dq = &hisi_hba->dq[d]; + } + + sz = HISI_SAS_MAX_COMMANDS * sizeof(struct hisi_sas_iost); + + hisi_hba->debugfs_iost[dump_index].iost = + devm_kmalloc(dev, sz, GFP_KERNEL); + if (!hisi_hba->debugfs_iost[dump_index].iost) + goto fail; + + sz = HISI_SAS_IOST_ITCT_CACHE_NUM * + sizeof(struct hisi_sas_iost_itct_cache); + + hisi_hba->debugfs_iost_cache[dump_index].cache = + devm_kmalloc(dev, sz, GFP_KERNEL); + if (!hisi_hba->debugfs_iost_cache[dump_index].cache) + goto fail; + + sz = HISI_SAS_IOST_ITCT_CACHE_NUM * + sizeof(struct hisi_sas_iost_itct_cache); + + hisi_hba->debugfs_itct_cache[dump_index].cache = + devm_kmalloc(dev, sz, GFP_KERNEL); + if (!hisi_hba->debugfs_itct_cache[dump_index].cache) + goto fail; + + /* New memory allocation must be locate before itct */ + sz = HISI_SAS_MAX_ITCT_ENTRIES * sizeof(struct hisi_sas_itct); + + hisi_hba->debugfs_itct[dump_index].itct = + devm_kmalloc(dev, sz, GFP_KERNEL); + if (!hisi_hba->debugfs_itct[dump_index].itct) + goto fail; + + return 0; +fail: + debugfs_release_v3_hw(hisi_hba, dump_index); + return -ENOMEM; +} + +static int debugfs_snapshot_regs_v3_hw(struct hisi_hba *hisi_hba) +{ + int debugfs_dump_index = hisi_hba->debugfs_dump_index; + struct device *dev = hisi_hba->dev; + u64 timestamp = local_clock(); + + if (debugfs_dump_index >= hisi_sas_debugfs_dump_count) { + dev_warn(dev, "dump count exceeded!\n"); + return -EINVAL; + } + + if (debugfs_alloc_v3_hw(hisi_hba, debugfs_dump_index)) { + dev_warn(dev, "failed to alloc memory\n"); + return -ENOMEM; + } + + do_div(timestamp, NSEC_PER_MSEC); + hisi_hba->debugfs_timestamp[debugfs_dump_index] = timestamp; + + debugfs_snapshot_prepare_v3_hw(hisi_hba); + + debugfs_snapshot_global_reg_v3_hw(hisi_hba); + debugfs_snapshot_port_reg_v3_hw(hisi_hba); + debugfs_snapshot_axi_reg_v3_hw(hisi_hba); + debugfs_snapshot_ras_reg_v3_hw(hisi_hba); + debugfs_snapshot_cq_reg_v3_hw(hisi_hba); + debugfs_snapshot_dq_reg_v3_hw(hisi_hba); + debugfs_snapshot_itct_reg_v3_hw(hisi_hba); + debugfs_snapshot_iost_reg_v3_hw(hisi_hba); + + debugfs_create_files_v3_hw(hisi_hba); + + debugfs_snapshot_restore_v3_hw(hisi_hba); + hisi_hba->debugfs_dump_index++; + + return 0; +} + +static void debugfs_phy_down_cnt_init_v3_hw(struct hisi_hba *hisi_hba) +{ + struct dentry *dir = debugfs_create_dir("phy_down_cnt", + hisi_hba->debugfs_dir); + char name[16]; + int phy_no; + + for (phy_no = 0; phy_no < hisi_hba->n_phy; phy_no++) { + snprintf(name, 16, "%d", phy_no); + debugfs_create_file(name, 0600, dir, + &hisi_hba->phy[phy_no], + &debugfs_phy_down_cnt_v3_hw_fops); + } +} + +static void debugfs_bist_init_v3_hw(struct hisi_hba *hisi_hba) +{ + struct dentry *ports_dentry; + int phy_no; + + hisi_hba->debugfs_bist_dentry = + debugfs_create_dir("bist", hisi_hba->debugfs_dir); + debugfs_create_file("link_rate", 0600, + hisi_hba->debugfs_bist_dentry, hisi_hba, + &debugfs_bist_linkrate_v3_hw_fops); + + debugfs_create_file("code_mode", 0600, + hisi_hba->debugfs_bist_dentry, hisi_hba, + &debugfs_bist_code_mode_v3_hw_fops); + + debugfs_create_file("fixed_code", 0600, + hisi_hba->debugfs_bist_dentry, + &hisi_hba->debugfs_bist_fixed_code[0], + &debugfs_v3_hw_fops); + + debugfs_create_file("fixed_code_1", 0600, + hisi_hba->debugfs_bist_dentry, + &hisi_hba->debugfs_bist_fixed_code[1], + &debugfs_v3_hw_fops); + + debugfs_create_file("phy_id", 0600, hisi_hba->debugfs_bist_dentry, + hisi_hba, &debugfs_bist_phy_v3_hw_fops); + + debugfs_create_file("cnt", 0600, hisi_hba->debugfs_bist_dentry, + hisi_hba, &debugfs_bist_cnt_v3_hw_ops); + + debugfs_create_file("loopback_mode", 0600, + hisi_hba->debugfs_bist_dentry, + hisi_hba, &debugfs_bist_mode_v3_hw_fops); + + debugfs_create_file("enable", 0600, hisi_hba->debugfs_bist_dentry, + hisi_hba, &debugfs_bist_enable_v3_hw_fops); + + ports_dentry = debugfs_create_dir("port", hisi_hba->debugfs_bist_dentry); + + for (phy_no = 0; phy_no < hisi_hba->n_phy; phy_no++) { + struct dentry *port_dentry; + struct dentry *ffe_dentry; + char name[256]; + int i; + + snprintf(name, 256, "%d", phy_no); + port_dentry = debugfs_create_dir(name, ports_dentry); + ffe_dentry = debugfs_create_dir("ffe", port_dentry); + for (i = 0; i < FFE_CFG_MAX; i++) { + if (i == FFE_RESV) + continue; + debugfs_create_file(debugfs_ffe_name_v3_hw[i].name, + 0600, ffe_dentry, + &hisi_hba->debugfs_bist_ffe[phy_no][i], + &debugfs_v3_hw_fops); + } + } + + hisi_hba->debugfs_bist_linkrate = SAS_LINK_RATE_1_5_GBPS; +} + +static void debugfs_exit_v3_hw(struct hisi_hba *hisi_hba) +{ + debugfs_remove_recursive(hisi_hba->debugfs_dir); + hisi_hba->debugfs_dir = NULL; +} + +static void debugfs_init_v3_hw(struct hisi_hba *hisi_hba) +{ + struct device *dev = hisi_hba->dev; + + hisi_hba->debugfs_dir = debugfs_create_dir(dev_name(dev), + hisi_sas_debugfs_dir); + debugfs_create_file("trigger_dump", 0200, + hisi_hba->debugfs_dir, + hisi_hba, + &debugfs_trigger_dump_v3_hw_fops); + + /* create bist structures */ + debugfs_bist_init_v3_hw(hisi_hba); + + hisi_hba->debugfs_dump_dentry = + debugfs_create_dir("dump", hisi_hba->debugfs_dir); + + debugfs_phy_down_cnt_init_v3_hw(hisi_hba); + debugfs_fifo_init_v3_hw(hisi_hba); +} + +static int +hisi_sas_v3_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct Scsi_Host *shost; + struct hisi_hba *hisi_hba; + struct device *dev = &pdev->dev; + struct asd_sas_phy **arr_phy; + struct asd_sas_port **arr_port; + struct sas_ha_struct *sha; + int rc, phy_nr, port_nr, i; + + rc = pcim_enable_device(pdev); + if (rc) + goto err_out; + + pci_set_master(pdev); + + rc = pcim_iomap_regions(pdev, 1 << BAR_NO_V3_HW, DRV_NAME); + if (rc) + goto err_out; + + rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (rc) + rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (rc) { + dev_err(dev, "No usable DMA addressing method\n"); + rc = -ENODEV; + goto err_out; + } + + shost = hisi_sas_shost_alloc_pci(pdev); + if (!shost) { + rc = -ENOMEM; + goto err_out; + } + + sha = SHOST_TO_SAS_HA(shost); + hisi_hba = shost_priv(shost); + dev_set_drvdata(dev, sha); + + hisi_hba->regs = pcim_iomap_table(pdev)[BAR_NO_V3_HW]; + if (!hisi_hba->regs) { + dev_err(dev, "cannot map register\n"); + rc = -ENOMEM; + goto err_out_ha; + } + + phy_nr = port_nr = hisi_hba->n_phy; + + arr_phy = devm_kcalloc(dev, phy_nr, sizeof(void *), GFP_KERNEL); + arr_port = devm_kcalloc(dev, port_nr, sizeof(void *), GFP_KERNEL); + if (!arr_phy || !arr_port) { + rc = -ENOMEM; + goto err_out_ha; + } + + sha->sas_phy = arr_phy; + sha->sas_port = arr_port; + sha->core.shost = shost; + sha->lldd_ha = hisi_hba; + + shost->transportt = hisi_sas_stt; + shost->max_id = HISI_SAS_MAX_DEVICES; + shost->max_lun = ~0; + shost->max_channel = 1; + shost->max_cmd_len = 16; + shost->can_queue = HISI_SAS_UNRESERVED_IPTT; + shost->cmd_per_lun = HISI_SAS_UNRESERVED_IPTT; + + sha->sas_ha_name = DRV_NAME; + sha->dev = dev; + sha->lldd_module = THIS_MODULE; + sha->sas_addr = &hisi_hba->sas_addr[0]; + sha->num_phys = hisi_hba->n_phy; + + for (i = 0; i < hisi_hba->n_phy; i++) { + sha->sas_phy[i] = &hisi_hba->phy[i].sas_phy; + sha->sas_port[i] = &hisi_hba->port[i].sas_port; + } + + if (hisi_hba->prot_mask) { + dev_info(dev, "Registering for DIF/DIX prot_mask=0x%x\n", + prot_mask); + scsi_host_set_prot(hisi_hba->shost, prot_mask); + if (hisi_hba->prot_mask & HISI_SAS_DIX_PROT_MASK) + scsi_host_set_guard(hisi_hba->shost, + SHOST_DIX_GUARD_CRC); + } + + if (hisi_sas_debugfs_enable) + debugfs_init_v3_hw(hisi_hba); + + rc = scsi_add_host(shost, dev); + if (rc) + goto err_out_free_irq_vectors; + + rc = sas_register_ha(sha); + if (rc) + goto err_out_register_ha; + + rc = hisi_sas_v3_init(hisi_hba); + if (rc) + goto err_out_hw_init; + + scsi_scan_host(shost); + + pm_runtime_set_autosuspend_delay(dev, 5000); + pm_runtime_use_autosuspend(dev); + /* + * For the situation that there are ATA disks connected with SAS + * controller, it additionally creates ata_port which will affect the + * child_count of hisi_hba->dev. Even if suspended all the disks, + * ata_port is still and the child_count of hisi_hba->dev is not 0. + * So use pm_suspend_ignore_children() to ignore the effect to + * hisi_hba->dev. + */ + pm_suspend_ignore_children(dev, true); + pm_runtime_put_noidle(&pdev->dev); + + return 0; + +err_out_hw_init: + sas_unregister_ha(sha); +err_out_register_ha: scsi_remove_host(shost); +err_out_free_irq_vectors: + pci_free_irq_vectors(pdev); err_out_ha: - hisi_sas_debugfs_exit(hisi_hba); + hisi_sas_free(hisi_hba); scsi_host_put(shost); -err_out_regions: - pci_release_regions(pdev); -err_out_disable_device: - pci_disable_device(pdev); err_out: return rc; } @@ -3294,19 +5009,16 @@ static void hisi_sas_v3_remove(struct pci_dev *pdev) struct hisi_hba *hisi_hba = sha->lldd_ha; struct Scsi_Host *shost = sha->core.shost; - hisi_sas_debugfs_exit(hisi_hba); - - if (timer_pending(&hisi_hba->timer)) - del_timer(&hisi_hba->timer); + pm_runtime_get_noresume(dev); + del_timer_sync(&hisi_hba->timer); sas_unregister_ha(sha); + flush_workqueue(hisi_hba->wq); sas_remove_host(sha->core.shost); hisi_sas_v3_destroy_irqs(pdev, hisi_hba); - hisi_sas_kill_tasklets(hisi_hba); - pci_release_regions(pdev); - pci_disable_device(pdev); hisi_sas_free(hisi_hba); + debugfs_exit_v3_hw(hisi_hba); scsi_host_put(shost); } @@ -3318,9 +5030,11 @@ static void hisi_sas_reset_prepare_v3_hw(struct pci_dev *pdev) int rc; dev_info(dev, "FLR prepare\n"); + down(&hisi_hba->sem); set_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags); hisi_sas_controller_reset_prepare(hisi_hba); + interrupt_disable_v3_hw(hisi_hba); rc = disable_host_v3_hw(hisi_hba); if (rc) dev_err(dev, "FLR: disable host failed rc=%d\n", rc); @@ -3330,6 +5044,7 @@ static void hisi_sas_reset_done_v3_hw(struct pci_dev *pdev) { struct sas_ha_struct *sha = pci_get_drvdata(pdev); struct hisi_hba *hisi_hba = sha->lldd_ha; + struct Scsi_Host *shost = hisi_hba->shost; struct device *dev = hisi_hba->dev; int rc; @@ -3338,6 +5053,10 @@ static void hisi_sas_reset_done_v3_hw(struct pci_dev *pdev) rc = hw_init_v3_hw(hisi_hba); if (rc) { dev_err(dev, "FLR: hw init failed rc=%d\n", rc); + clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags); + scsi_unblock_requests(shost); + clear_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags); + up(&hisi_hba->sem); return; } @@ -3350,13 +5069,28 @@ enum { hip08, }; -static int hisi_sas_v3_suspend(struct pci_dev *pdev, pm_message_t state) +static void enable_host_v3_hw(struct hisi_hba *hisi_hba) +{ + u32 reg_val; + + hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, + (u32)((1ULL << hisi_hba->queue_count) - 1)); + + phys_init_v3_hw(hisi_hba); + reg_val = hisi_sas_read32(hisi_hba, AXI_MASTER_CFG_BASE + + AM_CTRL_GLOBAL); + reg_val &= ~AM_CTRL_SHUTDOWN_REQ_MSK; + hisi_sas_write32(hisi_hba, AXI_MASTER_CFG_BASE + + AM_CTRL_GLOBAL, reg_val); +} + +static int _suspend_v3_hw(struct device *device) { + struct pci_dev *pdev = to_pci_dev(device); struct sas_ha_struct *sha = pci_get_drvdata(pdev); struct hisi_hba *hisi_hba = sha->lldd_ha; struct device *dev = hisi_hba->dev; struct Scsi_Host *shost = hisi_hba->shost; - pci_power_t device_state; int rc; if (!pdev->pm_cap) { @@ -3365,38 +5099,54 @@ static int hisi_sas_v3_suspend(struct pci_dev *pdev, pm_message_t state) } if (test_and_set_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags)) - return -1; + return -EPERM; + + dev_warn(dev, "entering suspend state\n"); scsi_block_requests(shost); set_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags); flush_workqueue(hisi_hba->wq); + interrupt_disable_v3_hw(hisi_hba); + +#ifdef CONFIG_PM + if ((device->power.runtime_status == RPM_SUSPENDING) && + atomic_read(&device->power.usage_count)) { + dev_err(dev, "PM suspend: host status cannot be suspended\n"); + rc = -EBUSY; + goto err_out; + } +#endif rc = disable_host_v3_hw(hisi_hba); if (rc) { dev_err(dev, "PM suspend: disable host failed rc=%d\n", rc); - clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags); - clear_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags); - scsi_unblock_requests(shost); - return rc; + goto err_out_recover_host; } hisi_sas_init_mem(hisi_hba); - device_state = pci_choose_state(pdev, state); - dev_warn(dev, "entering operating state [D%d]\n", - device_state); - pci_save_state(pdev); - pci_disable_device(pdev); - pci_set_power_state(pdev, device_state); - hisi_sas_release_tasks(hisi_hba); sas_suspend_ha(sha); + + dev_warn(dev, "end of suspending controller\n"); return 0; + +err_out_recover_host: + enable_host_v3_hw(hisi_hba); +#ifdef CONFIG_PM +err_out: +#endif + interrupt_enable_v3_hw(hisi_hba); + clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags); + clear_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags); + scsi_unblock_requests(shost); + return rc; } -static int hisi_sas_v3_resume(struct pci_dev *pdev) +static int _resume_v3_hw(struct device *device) { + struct pci_dev *pdev = to_pci_dev(device); struct sas_ha_struct *sha = pci_get_drvdata(pdev); struct hisi_hba *hisi_hba = sha->lldd_ha; struct Scsi_Host *shost = hisi_hba->shost; @@ -3406,16 +5156,7 @@ static int hisi_sas_v3_resume(struct pci_dev *pdev) dev_warn(dev, "resuming from operating state [D%d]\n", device_state); - pci_set_power_state(pdev, PCI_D0); - pci_enable_wake(pdev, PCI_D0, 0); - pci_restore_state(pdev); - rc = pci_enable_device(pdev); - if (rc) { - dev_err(dev, "enable device failed during resume (%d)\n", rc); - return rc; - } - pci_set_master(pdev); scsi_unblock_requests(shost); clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags); @@ -3423,16 +5164,53 @@ static int hisi_sas_v3_resume(struct pci_dev *pdev) rc = hw_init_v3_hw(hisi_hba); if (rc) { scsi_remove_host(shost); - pci_disable_device(pdev); return rc; } - hisi_hba->hw->phys_init(hisi_hba); - sas_resume_ha(sha); + phys_init_v3_hw(hisi_hba); + + /* + * If a directly-attached disk is removed during suspend, a deadlock + * may occur, as the PHYE_RESUME_TIMEOUT processing will require the + * hisi_hba->device to be active, which can only happen when resume + * completes. So don't wait for the HA event workqueue to drain upon + * resume. + */ + sas_resume_ha_no_sync(sha); clear_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags); + dev_warn(dev, "end of resuming controller\n"); + return 0; } +static int __maybe_unused suspend_v3_hw(struct device *device) +{ + struct pci_dev *pdev = to_pci_dev(device); + struct sas_ha_struct *sha = pci_get_drvdata(pdev); + struct hisi_hba *hisi_hba = sha->lldd_ha; + int rc; + + set_bit(HISI_SAS_PM_BIT, &hisi_hba->flags); + + rc = _suspend_v3_hw(device); + if (rc) + clear_bit(HISI_SAS_PM_BIT, &hisi_hba->flags); + + return rc; +} + +static int __maybe_unused resume_v3_hw(struct device *device) +{ + struct pci_dev *pdev = to_pci_dev(device); + struct sas_ha_struct *sha = pci_get_drvdata(pdev); + struct hisi_hba *hisi_hba = sha->lldd_ha; + int rc = _resume_v3_hw(device); + + clear_bit(HISI_SAS_PM_BIT, &hisi_hba->flags); + + return rc; +} + static const struct pci_device_id sas_v3_pci_table[] = { { PCI_VDEVICE(HUAWEI, 0xa230), hip08 }, {} @@ -3444,14 +5222,18 @@ static const struct pci_error_handlers hisi_sas_err_handler = { .reset_done = hisi_sas_reset_done_v3_hw, }; +static UNIVERSAL_DEV_PM_OPS(hisi_sas_v3_pm_ops, + suspend_v3_hw, + resume_v3_hw, + NULL); + static struct pci_driver sas_v3_pci_driver = { .name = DRV_NAME, .id_table = sas_v3_pci_table, .probe = hisi_sas_v3_probe, .remove = hisi_sas_v3_remove, - .suspend = hisi_sas_v3_suspend, - .resume = hisi_sas_v3_resume, .err_handler = &hisi_sas_err_handler, + .driver.pm = &hisi_sas_v3_pm_ops, }; module_pci_driver(sas_v3_pci_driver); diff --git a/drivers/scsi/isci/port.c b/drivers/scsi/isci/port.c index 1df45f028ea7570841d1b4ae7303cd8cafeb2d20..448a8c31ba35977a4be03b164d2c4d119c38900a 100644 --- a/drivers/scsi/isci/port.c +++ b/drivers/scsi/isci/port.c @@ -164,7 +164,8 @@ static void isci_port_bc_change_received(struct isci_host *ihost, "%s: isci_phy = %p, sas_phy = %p\n", __func__, iphy, &iphy->sas_phy); - ihost->sas_ha.notify_port_event(&iphy->sas_phy, PORTE_BROADCAST_RCVD); + sas_notify_port_event(&iphy->sas_phy, + PORTE_BROADCAST_RCVD, GFP_ATOMIC); sci_port_bcn_enable(iport); } @@ -223,8 +224,8 @@ static void isci_port_link_up(struct isci_host *isci_host, /* Notify libsas that we have an address frame, if indeed * we've found an SSP, SMP, or STP target */ if (success) - isci_host->sas_ha.notify_port_event(&iphy->sas_phy, - PORTE_BYTES_DMAED); + sas_notify_port_event(&iphy->sas_phy, + PORTE_BYTES_DMAED, GFP_ATOMIC); } @@ -270,8 +271,8 @@ static void isci_port_link_down(struct isci_host *isci_host, * isci_port_deformed and isci_dev_gone functions. */ sas_phy_disconnected(&isci_phy->sas_phy); - isci_host->sas_ha.notify_phy_event(&isci_phy->sas_phy, - PHYE_LOSS_OF_SIGNAL); + sas_notify_phy_event(&isci_phy->sas_phy, + PHYE_LOSS_OF_SIGNAL, GFP_ATOMIC); dev_dbg(&isci_host->pdev->dev, "%s: isci_port = %p - Done\n", __func__, isci_port); diff --git a/drivers/scsi/libsas/sas_ata.c b/drivers/scsi/libsas/sas_ata.c index 5c801705c4700f39580c57761dafc29b378c0f7a..bb7a70bca962c357cfb3921c010109a04e293429 100644 --- a/drivers/scsi/libsas/sas_ata.c +++ b/drivers/scsi/libsas/sas_ata.c @@ -146,8 +146,8 @@ static void sas_ata_task_done(struct sas_task *task) qc->flags |= ATA_QCFLAG_FAILED; } - dev->sata_dev.fis[3] = 0x04; /* status err */ - dev->sata_dev.fis[2] = ATA_ERR; + dev->sata_dev.fis[2] = ATA_ERR | ATA_DRDY; /* tf status */ + dev->sata_dev.fis[3] = ATA_ABORTED; /* tf error */ } } @@ -299,6 +299,31 @@ static int sas_ata_clear_pending(struct domain_device *dev, struct ex_phy *phy) return 1; } +int smp_ata_check_ready_type(struct ata_link *link) +{ + struct domain_device *dev = link->ap->private_data; + struct sas_phy *phy = sas_get_local_phy(dev); + struct domain_device *ex_dev = dev->parent; + enum sas_device_type type = SAS_PHY_UNUSED; + u8 sas_addr[SAS_ADDR_SIZE]; + int res; + + res = sas_get_phy_attached_dev(ex_dev, phy->number, sas_addr, &type); + sas_put_local_phy(phy); + if (res) + return res; + + switch (type) { + case SAS_SATA_PENDING: + return 0; + case SAS_END_DEVICE: + return 1; + default: + return -ENODEV; + } +} +EXPORT_SYMBOL_GPL(smp_ata_check_ready_type); + static int smp_ata_check_ready(struct ata_link *link) { int res; @@ -370,22 +395,14 @@ static int sas_ata_printk(const char *level, const struct domain_device *ddev, return r; } -static int sas_ata_hard_reset(struct ata_link *link, unsigned int *class, - unsigned long deadline) +static int sas_ata_wait_after_reset(struct domain_device *dev, unsigned long deadline) { - int ret = 0, res; - struct sas_phy *phy; - struct ata_port *ap = link->ap; + struct sata_device *sata_dev = &dev->sata_dev; int (*check_ready)(struct ata_link *link); - struct domain_device *dev = ap->private_data; - struct sas_internal *i = dev_to_sas_internal(dev); - - res = i->dft->lldd_I_T_nexus_reset(dev); - if (res == -ENODEV) - return res; - - if (res != TMF_RESP_FUNC_COMPLETE) - sas_ata_printk(KERN_DEBUG, dev, "Unable to reset ata device?\n"); + struct ata_port *ap = sata_dev->ap; + struct ata_link *link = &ap->link; + struct sas_phy *phy; + int ret; phy = sas_get_local_phy(dev); if (scsi_is_sas_phy_local(phy)) @@ -398,6 +415,26 @@ static int sas_ata_hard_reset(struct ata_link *link, unsigned int *class, if (ret && ret != -EAGAIN) sas_ata_printk(KERN_ERR, dev, "reset failed (errno=%d)\n", ret); + return ret; +} + +static int sas_ata_hard_reset(struct ata_link *link, unsigned int *class, + unsigned long deadline) +{ + struct ata_port *ap = link->ap; + struct domain_device *dev = ap->private_data; + struct sas_internal *i = dev_to_sas_internal(dev); + int ret; + + ret = i->dft->lldd_I_T_nexus_reset(dev); + if (ret == -ENODEV) + return ret; + + if (ret != TMF_RESP_FUNC_COMPLETE) + sas_ata_printk(KERN_DEBUG, dev, "Unable to reset ata device?\n"); + + ret = sas_ata_wait_after_reset(dev, deadline); + *class = dev->sata_dev.class; ap->cbl = ATA_CBL_SATA; @@ -853,3 +890,21 @@ void sas_ata_wait_eh(struct domain_device *dev) ap = dev->sata_dev.ap; ata_port_wait_eh(ap); } + +void sas_ata_device_link_abort(struct domain_device *device, bool force_reset) +{ + struct ata_port *ap = device->sata_dev.ap; + struct ata_link *link = &ap->link; + unsigned long flags; + + spin_lock_irqsave(ap->lock, flags); + device->sata_dev.fis[2] = ATA_ERR | ATA_DRDY; /* tf status */ + device->sata_dev.fis[3] = ATA_ABORTED; /* tf error */ + + link->eh_info.err_mask |= AC_ERR_DEV; + if (force_reset) + link->eh_info.action |= ATA_EH_RESET; + ata_link_abort(link); + spin_unlock_irqrestore(ap->lock, flags); +} +EXPORT_SYMBOL_GPL(sas_ata_device_link_abort); diff --git a/drivers/scsi/libsas/sas_event.c b/drivers/scsi/libsas/sas_event.c index a1852f6c042b9c9400cb61501653a060ad28cbf8..3613b9b315bc70545064952644b7ca59a8f181f9 100644 --- a/drivers/scsi/libsas/sas_event.c +++ b/drivers/scsi/libsas/sas_event.c @@ -41,12 +41,25 @@ static int sas_queue_event(int event, struct sas_work *work, return rc; } - -void __sas_drain_work(struct sas_ha_struct *ha) +void sas_queue_deferred_work(struct sas_ha_struct *ha) { struct sas_work *sw, *_sw; int ret; + spin_lock_irq(&ha->lock); + list_for_each_entry_safe(sw, _sw, &ha->defer_q, drain_node) { + list_del_init(&sw->drain_node); + ret = sas_queue_work(ha, sw); + if (ret != 1) { + pm_runtime_put(ha->dev); + sas_free_event(to_asd_sas_event(&sw->work)); + } + } + spin_unlock_irq(&ha->lock); +} + +void __sas_drain_work(struct sas_ha_struct *ha) +{ set_bit(SAS_HA_DRAINING, &ha->state); /* flush submitters */ spin_lock_irq(&ha->lock); @@ -55,16 +68,8 @@ void __sas_drain_work(struct sas_ha_struct *ha) drain_workqueue(ha->event_q); drain_workqueue(ha->disco_q); - spin_lock_irq(&ha->lock); clear_bit(SAS_HA_DRAINING, &ha->state); - list_for_each_entry_safe(sw, _sw, &ha->defer_q, drain_node) { - list_del_init(&sw->drain_node); - ret = sas_queue_work(ha, sw); - if (ret != 1) - sas_free_event(to_asd_sas_event(&sw->work)); - - } - spin_unlock_irq(&ha->lock); + sas_queue_deferred_work(ha); } int sas_drain_work(struct sas_ha_struct *ha) @@ -104,12 +109,17 @@ void sas_enable_revalidation(struct sas_ha_struct *ha) if (!test_and_clear_bit(ev, &d->pending)) continue; - if (list_empty(&port->phy_list)) + spin_lock(&port->phy_list_lock); + if (list_empty(&port->phy_list)) { + spin_unlock(&port->phy_list_lock); continue; + } sas_phy = container_of(port->phy_list.next, struct asd_sas_phy, port_phy_el); - ha->notify_port_event(sas_phy, PORTE_BROADCAST_RCVD); + spin_unlock(&port->phy_list_lock); + sas_notify_port_event(sas_phy, + PORTE_BROADCAST_RCVD, GFP_KERNEL); } mutex_unlock(&ha->disco_mutex); } @@ -118,65 +128,101 @@ void sas_enable_revalidation(struct sas_ha_struct *ha) static void sas_port_event_worker(struct work_struct *work) { struct asd_sas_event *ev = to_asd_sas_event(work); + struct asd_sas_phy *phy = ev->phy; + struct sas_ha_struct *ha = phy->ha; sas_port_event_fns[ev->event](work); + pm_runtime_put(ha->dev); sas_free_event(ev); } static void sas_phy_event_worker(struct work_struct *work) { struct asd_sas_event *ev = to_asd_sas_event(work); + struct asd_sas_phy *phy = ev->phy; + struct sas_ha_struct *ha = phy->ha; sas_phy_event_fns[ev->event](work); + pm_runtime_put(ha->dev); sas_free_event(ev); } -static int sas_notify_port_event(struct asd_sas_phy *phy, enum port_event event) +/* defer works of new phys during suspend */ +static bool sas_defer_event(struct asd_sas_phy *phy, struct asd_sas_event *ev) +{ + struct sas_ha_struct *ha = phy->ha; + unsigned long flags; + bool deferred = false; + + spin_lock_irqsave(&ha->lock, flags); + if (test_bit(SAS_HA_RESUMING, &ha->state) && !phy->suspended) { + struct sas_work *sw = &ev->work; + + list_add_tail(&sw->drain_node, &ha->defer_q); + deferred = true; + } + spin_unlock_irqrestore(&ha->lock, flags); + return deferred; +} + +int sas_notify_port_event(struct asd_sas_phy *phy, enum port_event event, + gfp_t gfp_flags) { - struct asd_sas_event *ev; struct sas_ha_struct *ha = phy->ha; + struct asd_sas_event *ev; int ret; BUG_ON(event >= PORT_NUM_EVENTS); - ev = sas_alloc_event(phy); + ev = sas_alloc_event(phy, gfp_flags); if (!ev) return -ENOMEM; + /* Call pm_runtime_put() with pairs in sas_port_event_worker() */ + pm_runtime_get_noresume(ha->dev); + INIT_SAS_EVENT(ev, sas_port_event_worker, phy, event); + if (sas_defer_event(phy, ev)) + return 0; + ret = sas_queue_event(event, &ev->work, ha); - if (ret != 1) + if (ret != 1) { + pm_runtime_put(ha->dev); sas_free_event(ev); + } return ret; } +EXPORT_SYMBOL_GPL(sas_notify_port_event); -int sas_notify_phy_event(struct asd_sas_phy *phy, enum phy_event event) +int sas_notify_phy_event(struct asd_sas_phy *phy, enum phy_event event, + gfp_t gfp_flags) { - struct asd_sas_event *ev; struct sas_ha_struct *ha = phy->ha; + struct asd_sas_event *ev; int ret; BUG_ON(event >= PHY_NUM_EVENTS); - ev = sas_alloc_event(phy); + ev = sas_alloc_event(phy, gfp_flags); if (!ev) return -ENOMEM; + /* Call pm_runtime_put() with pairs in sas_phy_event_worker() */ + pm_runtime_get_noresume(ha->dev); + INIT_SAS_EVENT(ev, sas_phy_event_worker, phy, event); + if (sas_defer_event(phy, ev)) + return 0; + ret = sas_queue_event(event, &ev->work, ha); - if (ret != 1) + if (ret != 1) { + pm_runtime_put(ha->dev); sas_free_event(ev); + } return ret; } - -int sas_init_events(struct sas_ha_struct *sas_ha) -{ - sas_ha->notify_port_event = sas_notify_port_event; - sas_ha->notify_phy_event = sas_notify_phy_event; - - return 0; -} +EXPORT_SYMBOL_GPL(sas_notify_phy_event); diff --git a/drivers/scsi/libsas/sas_expander.c b/drivers/scsi/libsas/sas_expander.c index 173f91ae38f021b54ddd85864ae56b7814f277e5..08e6f5b90b5bc7058d37e0f2ca28690db89b0cbe 100644 --- a/drivers/scsi/libsas/sas_expander.c +++ b/drivers/scsi/libsas/sas_expander.c @@ -58,7 +58,9 @@ static int smp_execute_task_sg(struct domain_device *dev, struct sas_task *task = NULL; struct sas_internal *i = to_sas_internal(dev->port->ha->core.shost->transportt); + struct sas_ha_struct *ha = dev->port->ha; + pm_runtime_get_sync(ha->dev); mutex_lock(&dev->ex_dev.cmd_mutex); for (retry = 0; retry < 3; retry++) { if (test_bit(SAS_DEV_GONE, &dev->state)) { @@ -131,6 +133,7 @@ static int smp_execute_task_sg(struct domain_device *dev, } } mutex_unlock(&dev->ex_dev.cmd_mutex); + pm_runtime_put_sync(ha->dev); BUG_ON(retry == 3 && task != NULL); sas_free_task(task); @@ -1716,8 +1719,8 @@ static int sas_get_phy_change_count(struct domain_device *dev, return res; } -static int sas_get_phy_attached_dev(struct domain_device *dev, int phy_id, - u8 *sas_addr, enum sas_device_type *type) +int sas_get_phy_attached_dev(struct domain_device *dev, int phy_id, + u8 *sas_addr, enum sas_device_type *type) { int res; struct smp_resp *disc_resp; diff --git a/drivers/scsi/libsas/sas_init.c b/drivers/scsi/libsas/sas_init.c index 21c43b18d5d5b6bc93ad2298262631c0e78e4193..58ffcecf1a2f75372c2091475b4ab52ec64cb7f6 100644 --- a/drivers/scsi/libsas/sas_init.c +++ b/drivers/scsi/libsas/sas_init.c @@ -123,12 +123,6 @@ int sas_register_ha(struct sas_ha_struct *sas_ha) goto Undo_phys; } - error = sas_init_events(sas_ha); - if (error) { - pr_notice("couldn't start event thread:%d\n", error); - goto Undo_ports; - } - error = -ENOMEM; snprintf(name, sizeof(name), "%s_event_q", dev_name(sas_ha->dev)); sas_ha->event_q = create_singlethread_workqueue(name); @@ -258,7 +252,7 @@ static int transport_sas_phy_reset(struct sas_phy *phy, int hard_reset) } } -static int sas_phy_enable(struct sas_phy *phy, int enable) +int sas_phy_enable(struct sas_phy *phy, int enable) { int ret; enum phy_func cmd; @@ -290,6 +284,7 @@ static int sas_phy_enable(struct sas_phy *phy, int enable) } return ret; } +EXPORT_SYMBOL_GPL(sas_phy_enable); int sas_phy_reset(struct sas_phy *phy, int hard_reset) { @@ -364,6 +359,7 @@ void sas_prep_resume_ha(struct sas_ha_struct *ha) int i; set_bit(SAS_HA_REGISTERED, &ha->state); + set_bit(SAS_HA_RESUMING, &ha->state); /* clear out any stale link events/data from the suspension path */ for (i = 0; i < ha->num_phys; i++) { @@ -389,7 +385,31 @@ static int phys_suspended(struct sas_ha_struct *ha) return rc; } -void sas_resume_ha(struct sas_ha_struct *ha) +static void sas_resume_insert_broadcast_ha(struct sas_ha_struct *ha) +{ + int i; + + for (i = 0; i < ha->num_phys; i++) { + struct asd_sas_port *port = ha->sas_port[i]; + struct domain_device *dev = port->port_dev; + + if (dev && dev_is_expander(dev->dev_type)) { + struct asd_sas_phy *first_phy; + + spin_lock(&port->phy_list_lock); + first_phy = list_first_entry_or_null( + &port->phy_list, struct asd_sas_phy, + port_phy_el); + spin_unlock(&port->phy_list_lock); + + if (first_phy) + sas_notify_port_event(first_phy, + PORTE_BROADCAST_RCVD, GFP_KERNEL); + } + } +} + +static void _sas_resume_ha(struct sas_ha_struct *ha, bool drain) { const unsigned long tmo = msecs_to_jiffies(25000); int i; @@ -410,7 +430,8 @@ void sas_resume_ha(struct sas_ha_struct *ha) if (phy->suspended) { dev_warn(&phy->phy->dev, "resume timeout\n"); - sas_notify_phy_event(phy, PHYE_RESUME_TIMEOUT); + sas_notify_phy_event(phy, PHYE_RESUME_TIMEOUT, + GFP_KERNEL); } } @@ -418,10 +439,30 @@ void sas_resume_ha(struct sas_ha_struct *ha) * flush out disks that did not return */ scsi_unblock_requests(ha->core.shost); - sas_drain_work(ha); + if (drain) + sas_drain_work(ha); + clear_bit(SAS_HA_RESUMING, &ha->state); + + sas_queue_deferred_work(ha); + /* send event PORTE_BROADCAST_RCVD to identify some new inserted + * disks for expander + */ + sas_resume_insert_broadcast_ha(ha); +} + +void sas_resume_ha(struct sas_ha_struct *ha) +{ + _sas_resume_ha(ha, true); } EXPORT_SYMBOL(sas_resume_ha); +/* A no-sync variant, which does not call sas_drain_ha(). */ +void sas_resume_ha_no_sync(struct sas_ha_struct *ha) +{ + _sas_resume_ha(ha, false); +} +EXPORT_SYMBOL(sas_resume_ha_no_sync); + void sas_suspend_ha(struct sas_ha_struct *ha) { int i; @@ -487,6 +528,7 @@ static int queue_phy_reset(struct sas_phy *phy, int hard_reset) if (!d) return -ENOMEM; + pm_runtime_get_sync(ha->dev); /* libsas workqueue coordinates ata-eh reset with discovery */ mutex_lock(&d->event_lock); d->reset_result = 0; @@ -500,6 +542,7 @@ static int queue_phy_reset(struct sas_phy *phy, int hard_reset) if (rc == 0) rc = d->reset_result; mutex_unlock(&d->event_lock); + pm_runtime_put_sync(ha->dev); return rc; } @@ -514,6 +557,7 @@ static int queue_phy_enable(struct sas_phy *phy, int enable) if (!d) return -ENOMEM; + pm_runtime_get_sync(ha->dev); /* libsas workqueue coordinates ata-eh reset with discovery */ mutex_lock(&d->event_lock); d->enable_result = 0; @@ -527,6 +571,7 @@ static int queue_phy_enable(struct sas_phy *phy, int enable) if (rc == 0) rc = d->enable_result; mutex_unlock(&d->event_lock); + pm_runtime_put_sync(ha->dev); return rc; } @@ -590,16 +635,15 @@ sas_domain_attach_transport(struct sas_domain_function_template *dft) } EXPORT_SYMBOL_GPL(sas_domain_attach_transport); - -struct asd_sas_event *sas_alloc_event(struct asd_sas_phy *phy) +struct asd_sas_event *sas_alloc_event(struct asd_sas_phy *phy, + gfp_t gfp_flags) { struct asd_sas_event *event; - gfp_t flags = in_interrupt() ? GFP_ATOMIC : GFP_KERNEL; struct sas_ha_struct *sas_ha = phy->ha; struct sas_internal *i = to_sas_internal(sas_ha->core.shost->transportt); - event = kmem_cache_zalloc(sas_event_cache, flags); + event = kmem_cache_zalloc(sas_event_cache, gfp_flags); if (!event) return NULL; @@ -610,7 +654,8 @@ struct asd_sas_event *sas_alloc_event(struct asd_sas_phy *phy) if (cmpxchg(&phy->in_shutdown, 0, 1) == 0) { pr_notice("The phy%d bursting events, shut it down.\n", phy->id); - sas_notify_phy_event(phy, PHYE_SHUTDOWN); + sas_notify_phy_event(phy, PHYE_SHUTDOWN, + gfp_flags); } } else { /* Do not support PHY control, stop allocating events */ diff --git a/drivers/scsi/libsas/sas_internal.h b/drivers/scsi/libsas/sas_internal.h index 01f1738ce6dfff2f4dd8f356fb3aa2f9dff90aeb..e99a4c1c5e352a605400aa59939f88fbe83e9ef6 100644 --- a/drivers/scsi/libsas/sas_internal.h +++ b/drivers/scsi/libsas/sas_internal.h @@ -14,6 +14,7 @@ #include #include #include +#include #ifdef pr_fmt #undef pr_fmt @@ -48,15 +49,15 @@ int sas_show_oob_mode(enum sas_oob_mode oob_mode, char *buf); int sas_register_phys(struct sas_ha_struct *sas_ha); void sas_unregister_phys(struct sas_ha_struct *sas_ha); -struct asd_sas_event *sas_alloc_event(struct asd_sas_phy *phy); +struct asd_sas_event *sas_alloc_event(struct asd_sas_phy *phy, gfp_t gfp_flags); void sas_free_event(struct asd_sas_event *event); int sas_register_ports(struct sas_ha_struct *sas_ha); void sas_unregister_ports(struct sas_ha_struct *sas_ha); -int sas_init_events(struct sas_ha_struct *sas_ha); void sas_disable_revalidation(struct sas_ha_struct *ha); void sas_enable_revalidation(struct sas_ha_struct *ha); +void sas_queue_deferred_work(struct sas_ha_struct *ha); void __sas_drain_work(struct sas_ha_struct *ha); void sas_deform_port(struct asd_sas_phy *phy, int gone); @@ -77,13 +78,16 @@ int sas_smp_phy_control(struct domain_device *dev, int phy_id, enum phy_func phy_func, struct sas_phy_linkrates *); int sas_smp_get_phy_events(struct sas_phy *phy); -int sas_notify_phy_event(struct asd_sas_phy *phy, enum phy_event event); +int sas_notify_phy_event(struct asd_sas_phy *phy, enum phy_event event, + gfp_t flags); void sas_device_set_phy(struct domain_device *dev, struct sas_port *port); struct domain_device *sas_find_dev_by_rphy(struct sas_rphy *rphy); struct domain_device *sas_ex_to_ata(struct domain_device *ex_dev, int phy_id); int sas_ex_phy_discover(struct domain_device *dev, int single); int sas_get_report_phy_sata(struct domain_device *dev, int phy_id, struct smp_resp *rps_resp); +int sas_get_phy_attached_dev(struct domain_device *dev, int phy_id, + u8 *sas_addr, enum sas_device_type *type); int sas_try_ata_reset(struct asd_sas_phy *phy); void sas_hae_reset(struct work_struct *work); diff --git a/drivers/scsi/mvsas/mv_sas.c b/drivers/scsi/mvsas/mv_sas.c index 3e0b8ebe257ff93ba5c87f8bfba2ef289fd4fe74..fea74624c1dcce279cd7c9cb3e8fc470944ed46b 100644 --- a/drivers/scsi/mvsas/mv_sas.c +++ b/drivers/scsi/mvsas/mv_sas.c @@ -67,8 +67,10 @@ static struct mvs_info *mvs_find_dev_mvi(struct domain_device *dev) while (sha->sas_port[i]) { if (sha->sas_port[i] == dev->port) { + spin_lock(&sha->sas_port[i]->phy_list_lock); phy = container_of(sha->sas_port[i]->phy_list.next, struct asd_sas_phy, port_phy_el); + spin_unlock(&sha->sas_port[i]->phy_list_lock); j = 0; while (sha->sas_phy[j]) { if (sha->sas_phy[j] == phy) @@ -96,6 +98,8 @@ static int mvs_find_dev_phyno(struct domain_device *dev, int *phyno) while (sha->sas_port[i]) { if (sha->sas_port[i] == dev->port) { struct asd_sas_phy *phy; + + spin_lock(&sha->sas_port[i]->phy_list_lock); list_for_each_entry(phy, &sha->sas_port[i]->phy_list, port_phy_el) { j = 0; @@ -109,6 +113,7 @@ static int mvs_find_dev_phyno(struct domain_device *dev, int *phyno) num++; n++; } + spin_unlock(&sha->sas_port[i]->phy_list_lock); break; } i++; @@ -216,11 +221,11 @@ void mvs_set_sas_addr(struct mvs_info *mvi, int port_id, u32 off_lo, MVS_CHIP_DISP->write_port_cfg_data(mvi, port_id, hi); } -static void mvs_bytes_dmaed(struct mvs_info *mvi, int i) +static void mvs_bytes_dmaed(struct mvs_info *mvi, int i, gfp_t gfp_flags) { struct mvs_phy *phy = &mvi->phy[i]; struct asd_sas_phy *sas_phy = &phy->sas_phy; - struct sas_ha_struct *sas_ha; + if (!phy->phy_attached) return; @@ -229,8 +234,7 @@ static void mvs_bytes_dmaed(struct mvs_info *mvi, int i) return; } - sas_ha = mvi->sas; - sas_ha->notify_phy_event(sas_phy, PHYE_OOB_DONE); + sas_notify_phy_event(sas_phy, PHYE_OOB_DONE, gfp_flags); if (sas_phy->phy) { struct sas_phy *sphy = sas_phy->phy; @@ -262,8 +266,7 @@ static void mvs_bytes_dmaed(struct mvs_info *mvi, int i) sas_phy->frame_rcvd_size = phy->frame_rcvd_size; - mvi->sas->notify_port_event(sas_phy, - PORTE_BYTES_DMAED); + sas_notify_port_event(sas_phy, PORTE_BYTES_DMAED, gfp_flags); } void mvs_scan_start(struct Scsi_Host *shost) @@ -279,7 +282,7 @@ void mvs_scan_start(struct Scsi_Host *shost) for (j = 0; j < core_nr; j++) { mvi = ((struct mvs_prv_info *)sha->lldd_ha)->mvi[j]; for (i = 0; i < mvi->chip->n_phy; ++i) - mvs_bytes_dmaed(mvi, i); + mvs_bytes_dmaed(mvi, i, GFP_KERNEL); } mvs_prv->scan_finished = 1; } @@ -1880,7 +1883,6 @@ static void mvs_work_queue(struct work_struct *work) struct mvs_info *mvi = mwq->mvi; unsigned long flags; u32 phy_no = (unsigned long) mwq->data; - struct sas_ha_struct *sas_ha = mvi->sas; struct mvs_phy *phy = &mvi->phy[phy_no]; struct asd_sas_phy *sas_phy = &phy->sas_phy; @@ -1895,21 +1897,21 @@ static void mvs_work_queue(struct work_struct *work) if (!(tmp & PHY_READY_MASK)) { sas_phy_disconnected(sas_phy); mvs_phy_disconnected(phy); - sas_ha->notify_phy_event(sas_phy, - PHYE_LOSS_OF_SIGNAL); + sas_notify_phy_event(sas_phy, + PHYE_LOSS_OF_SIGNAL, GFP_ATOMIC); mv_dprintk("phy%d Removed Device\n", phy_no); } else { MVS_CHIP_DISP->detect_porttype(mvi, phy_no); mvs_update_phyinfo(mvi, phy_no, 1); - mvs_bytes_dmaed(mvi, phy_no); + mvs_bytes_dmaed(mvi, phy_no, GFP_ATOMIC); mvs_port_notify_formed(sas_phy, 0); mv_dprintk("phy%d Attached Device\n", phy_no); } } } else if (mwq->handler & EXP_BRCT_CHG) { phy->phy_event &= ~EXP_BRCT_CHG; - sas_ha->notify_port_event(sas_phy, - PORTE_BROADCAST_RCVD); + sas_notify_port_event(sas_phy, + PORTE_BROADCAST_RCVD, GFP_ATOMIC); mv_dprintk("phy%d Got Broadcast Change\n", phy_no); } list_del(&mwq->entry); @@ -2026,7 +2028,7 @@ void mvs_int_port(struct mvs_info *mvi, int phy_no, u32 events) mdelay(10); } - mvs_bytes_dmaed(mvi, phy_no); + mvs_bytes_dmaed(mvi, phy_no, GFP_ATOMIC); /* whether driver is going to handle hot plug */ if (phy->phy_event & PHY_PLUG_OUT) { mvs_port_notify_formed(&phy->sas_phy, 0); diff --git a/drivers/scsi/pm8001/pm8001_hwi.c b/drivers/scsi/pm8001/pm8001_hwi.c index cba30618e8fed571e39867460c6158d281664c12..9627bddc45fea62b93ed5f8970f55843d44b0b4c 100644 --- a/drivers/scsi/pm8001/pm8001_hwi.c +++ b/drivers/scsi/pm8001/pm8001_hwi.c @@ -3270,7 +3270,7 @@ void pm8001_bytes_dmaed(struct pm8001_hba_info *pm8001_ha, int i) PM8001_MSG_DBG(pm8001_ha, pm8001_printk("phy %d byte dmaded.\n", i)); sas_phy->frame_rcvd_size = phy->frame_rcvd_size; - pm8001_ha->sas->notify_port_event(sas_phy, PORTE_BYTES_DMAED); + sas_notify_port_event(sas_phy, PORTE_BYTES_DMAED, GFP_ATOMIC); } /* Get the link rate speed */ @@ -3383,7 +3383,6 @@ hw_event_sas_phy_up(struct pm8001_hba_info *pm8001_ha, void *piomb) u32 npip_portstate = le32_to_cpu(pPayload->npip_portstate); u8 portstate = (u8)(npip_portstate & 0x0000000F); struct pm8001_port *port = &pm8001_ha->port[port_id]; - struct sas_ha_struct *sas_ha = pm8001_ha->sas; struct pm8001_phy *phy = &pm8001_ha->phy[phy_id]; unsigned long flags; u8 deviceType = pPayload->sas_identify.dev_type; @@ -3430,7 +3429,7 @@ hw_event_sas_phy_up(struct pm8001_hba_info *pm8001_ha, void *piomb) else if (phy->identify.device_type != SAS_PHY_UNUSED) phy->identify.target_port_protocols = SAS_PROTOCOL_SMP; phy->sas_phy.oob_mode = SAS_OOB_MODE; - sas_ha->notify_phy_event(&phy->sas_phy, PHYE_OOB_DONE); + sas_notify_phy_event(&phy->sas_phy, PHYE_OOB_DONE, GFP_ATOMIC); spin_lock_irqsave(&phy->sas_phy.frame_rcvd_lock, flags); memcpy(phy->frame_rcvd, &pPayload->sas_identify, sizeof(struct sas_identify_frame)-4); @@ -3462,7 +3461,6 @@ hw_event_sata_phy_up(struct pm8001_hba_info *pm8001_ha, void *piomb) u32 npip_portstate = le32_to_cpu(pPayload->npip_portstate); u8 portstate = (u8)(npip_portstate & 0x0000000F); struct pm8001_port *port = &pm8001_ha->port[port_id]; - struct sas_ha_struct *sas_ha = pm8001_ha->sas; struct pm8001_phy *phy = &pm8001_ha->phy[phy_id]; unsigned long flags; PM8001_MSG_DBG(pm8001_ha, @@ -3475,7 +3473,7 @@ hw_event_sata_phy_up(struct pm8001_hba_info *pm8001_ha, void *piomb) phy->phy_type |= PORT_TYPE_SATA; phy->phy_attached = 1; phy->sas_phy.oob_mode = SATA_OOB_MODE; - sas_ha->notify_phy_event(&phy->sas_phy, PHYE_OOB_DONE); + sas_notify_phy_event(&phy->sas_phy, PHYE_OOB_DONE, GFP_ATOMIC); spin_lock_irqsave(&phy->sas_phy.frame_rcvd_lock, flags); memcpy(phy->frame_rcvd, ((u8 *)&pPayload->sata_fis - 4), sizeof(struct dev_to_host_fis)); @@ -3840,12 +3838,14 @@ static int mpi_hw_event(struct pm8001_hba_info *pm8001_ha, void* piomb) case HW_EVENT_SATA_SPINUP_HOLD: PM8001_MSG_DBG(pm8001_ha, pm8001_printk("HW_EVENT_SATA_SPINUP_HOLD\n")); - sas_ha->notify_phy_event(&phy->sas_phy, PHYE_SPINUP_HOLD); + sas_notify_phy_event(&phy->sas_phy, PHYE_SPINUP_HOLD, + GFP_ATOMIC); break; case HW_EVENT_PHY_DOWN: PM8001_MSG_DBG(pm8001_ha, pm8001_printk("HW_EVENT_PHY_DOWN\n")); - sas_ha->notify_phy_event(&phy->sas_phy, PHYE_LOSS_OF_SIGNAL); + sas_notify_phy_event(&phy->sas_phy, PHYE_LOSS_OF_SIGNAL, + GFP_ATOMIC); phy->phy_attached = 0; phy->phy_state = 0; hw_event_phy_down(pm8001_ha, piomb); @@ -3855,7 +3855,8 @@ static int mpi_hw_event(struct pm8001_hba_info *pm8001_ha, void* piomb) pm8001_printk("HW_EVENT_PORT_INVALID\n")); sas_phy_disconnected(sas_phy); phy->phy_attached = 0; - sas_ha->notify_port_event(sas_phy, PORTE_LINK_RESET_ERR); + sas_notify_port_event(sas_phy, PORTE_LINK_RESET_ERR, + GFP_ATOMIC); break; /* the broadcast change primitive received, tell the LIBSAS this event to revalidate the sas domain*/ @@ -3867,14 +3868,15 @@ static int mpi_hw_event(struct pm8001_hba_info *pm8001_ha, void* piomb) spin_lock_irqsave(&sas_phy->sas_prim_lock, flags); sas_phy->sas_prim = HW_EVENT_BROADCAST_CHANGE; spin_unlock_irqrestore(&sas_phy->sas_prim_lock, flags); - sas_ha->notify_port_event(sas_phy, PORTE_BROADCAST_RCVD); + sas_notify_port_event(sas_phy, PORTE_BROADCAST_RCVD, + GFP_ATOMIC); break; case HW_EVENT_PHY_ERROR: PM8001_MSG_DBG(pm8001_ha, pm8001_printk("HW_EVENT_PHY_ERROR\n")); sas_phy_disconnected(&phy->sas_phy); phy->phy_attached = 0; - sas_ha->notify_phy_event(&phy->sas_phy, PHYE_OOB_ERROR); + sas_notify_phy_event(&phy->sas_phy, PHYE_OOB_ERROR, GFP_ATOMIC); break; case HW_EVENT_BROADCAST_EXP: PM8001_MSG_DBG(pm8001_ha, @@ -3882,7 +3884,8 @@ static int mpi_hw_event(struct pm8001_hba_info *pm8001_ha, void* piomb) spin_lock_irqsave(&sas_phy->sas_prim_lock, flags); sas_phy->sas_prim = HW_EVENT_BROADCAST_EXP; spin_unlock_irqrestore(&sas_phy->sas_prim_lock, flags); - sas_ha->notify_port_event(sas_phy, PORTE_BROADCAST_RCVD); + sas_notify_port_event(sas_phy, PORTE_BROADCAST_RCVD, + GFP_ATOMIC); break; case HW_EVENT_LINK_ERR_INVALID_DWORD: PM8001_MSG_DBG(pm8001_ha, @@ -3891,7 +3894,8 @@ static int mpi_hw_event(struct pm8001_hba_info *pm8001_ha, void* piomb) HW_EVENT_LINK_ERR_INVALID_DWORD, port_id, phy_id, 0, 0); sas_phy_disconnected(sas_phy); phy->phy_attached = 0; - sas_ha->notify_port_event(sas_phy, PORTE_LINK_RESET_ERR); + sas_notify_port_event(sas_phy, PORTE_LINK_RESET_ERR, + GFP_ATOMIC); break; case HW_EVENT_LINK_ERR_DISPARITY_ERROR: PM8001_MSG_DBG(pm8001_ha, @@ -3901,7 +3905,8 @@ static int mpi_hw_event(struct pm8001_hba_info *pm8001_ha, void* piomb) port_id, phy_id, 0, 0); sas_phy_disconnected(sas_phy); phy->phy_attached = 0; - sas_ha->notify_port_event(sas_phy, PORTE_LINK_RESET_ERR); + sas_notify_port_event(sas_phy, PORTE_LINK_RESET_ERR, + GFP_ATOMIC); break; case HW_EVENT_LINK_ERR_CODE_VIOLATION: PM8001_MSG_DBG(pm8001_ha, @@ -3911,7 +3916,8 @@ static int mpi_hw_event(struct pm8001_hba_info *pm8001_ha, void* piomb) port_id, phy_id, 0, 0); sas_phy_disconnected(sas_phy); phy->phy_attached = 0; - sas_ha->notify_port_event(sas_phy, PORTE_LINK_RESET_ERR); + sas_notify_port_event(sas_phy, PORTE_LINK_RESET_ERR, + GFP_ATOMIC); break; case HW_EVENT_LINK_ERR_LOSS_OF_DWORD_SYNCH: PM8001_MSG_DBG(pm8001_ha, @@ -3921,7 +3927,8 @@ static int mpi_hw_event(struct pm8001_hba_info *pm8001_ha, void* piomb) port_id, phy_id, 0, 0); sas_phy_disconnected(sas_phy); phy->phy_attached = 0; - sas_ha->notify_port_event(sas_phy, PORTE_LINK_RESET_ERR); + sas_notify_port_event(sas_phy, PORTE_LINK_RESET_ERR, + GFP_ATOMIC); break; case HW_EVENT_MALFUNCTION: PM8001_MSG_DBG(pm8001_ha, @@ -3933,7 +3940,8 @@ static int mpi_hw_event(struct pm8001_hba_info *pm8001_ha, void* piomb) spin_lock_irqsave(&sas_phy->sas_prim_lock, flags); sas_phy->sas_prim = HW_EVENT_BROADCAST_SES; spin_unlock_irqrestore(&sas_phy->sas_prim_lock, flags); - sas_ha->notify_port_event(sas_phy, PORTE_BROADCAST_RCVD); + sas_notify_port_event(sas_phy, PORTE_BROADCAST_RCVD, + GFP_ATOMIC); break; case HW_EVENT_INBOUND_CRC_ERROR: PM8001_MSG_DBG(pm8001_ha, @@ -3945,14 +3953,15 @@ static int mpi_hw_event(struct pm8001_hba_info *pm8001_ha, void* piomb) case HW_EVENT_HARD_RESET_RECEIVED: PM8001_MSG_DBG(pm8001_ha, pm8001_printk("HW_EVENT_HARD_RESET_RECEIVED\n")); - sas_ha->notify_port_event(sas_phy, PORTE_HARD_RESET); + sas_notify_port_event(sas_phy, PORTE_HARD_RESET, GFP_ATOMIC); break; case HW_EVENT_ID_FRAME_TIMEOUT: PM8001_MSG_DBG(pm8001_ha, pm8001_printk("HW_EVENT_ID_FRAME_TIMEOUT\n")); sas_phy_disconnected(sas_phy); phy->phy_attached = 0; - sas_ha->notify_port_event(sas_phy, PORTE_LINK_RESET_ERR); + sas_notify_port_event(sas_phy, PORTE_LINK_RESET_ERR, + GFP_ATOMIC); break; case HW_EVENT_LINK_ERR_PHY_RESET_FAILED: PM8001_MSG_DBG(pm8001_ha, @@ -3962,21 +3971,24 @@ static int mpi_hw_event(struct pm8001_hba_info *pm8001_ha, void* piomb) port_id, phy_id, 0, 0); sas_phy_disconnected(sas_phy); phy->phy_attached = 0; - sas_ha->notify_port_event(sas_phy, PORTE_LINK_RESET_ERR); + sas_notify_port_event(sas_phy, PORTE_LINK_RESET_ERR, + GFP_ATOMIC); break; case HW_EVENT_PORT_RESET_TIMER_TMO: PM8001_MSG_DBG(pm8001_ha, pm8001_printk("HW_EVENT_PORT_RESET_TIMER_TMO\n")); sas_phy_disconnected(sas_phy); phy->phy_attached = 0; - sas_ha->notify_port_event(sas_phy, PORTE_LINK_RESET_ERR); + sas_notify_port_event(sas_phy, PORTE_LINK_RESET_ERR, + GFP_ATOMIC); break; case HW_EVENT_PORT_RECOVERY_TIMER_TMO: PM8001_MSG_DBG(pm8001_ha, pm8001_printk("HW_EVENT_PORT_RECOVERY_TIMER_TMO\n")); sas_phy_disconnected(sas_phy); phy->phy_attached = 0; - sas_ha->notify_port_event(sas_phy, PORTE_LINK_RESET_ERR); + sas_notify_port_event(sas_phy, PORTE_LINK_RESET_ERR, + GFP_ATOMIC); break; case HW_EVENT_PORT_RECOVER: PM8001_MSG_DBG(pm8001_ha, diff --git a/drivers/scsi/pm8001/pm8001_sas.c b/drivers/scsi/pm8001/pm8001_sas.c index 36f5bab09f73e0b15e94874cdf0525985fc9b773..d87009173b802d56f34f2c3516216abdc879c965 100644 --- a/drivers/scsi/pm8001/pm8001_sas.c +++ b/drivers/scsi/pm8001/pm8001_sas.c @@ -157,7 +157,6 @@ int pm8001_phy_control(struct asd_sas_phy *sas_phy, enum phy_func func, int rc = 0, phy_id = sas_phy->id; struct pm8001_hba_info *pm8001_ha = NULL; struct sas_phy_linkrates *rates; - struct sas_ha_struct *sas_ha; struct pm8001_phy *phy; DECLARE_COMPLETION_ONSTACK(completion); unsigned long flags; @@ -206,19 +205,17 @@ int pm8001_phy_control(struct asd_sas_phy *sas_phy, enum phy_func func, if (pm8001_ha->chip_id != chip_8001) { if (pm8001_ha->phy[phy_id].phy_state == PHY_STATE_LINK_UP_SPCV) { - sas_ha = pm8001_ha->sas; sas_phy_disconnected(&phy->sas_phy); - sas_ha->notify_phy_event(&phy->sas_phy, - PHYE_LOSS_OF_SIGNAL); + sas_notify_phy_event(&phy->sas_phy, + PHYE_LOSS_OF_SIGNAL, GFP_KERNEL); phy->phy_attached = 0; } } else { if (pm8001_ha->phy[phy_id].phy_state == PHY_STATE_LINK_UP_SPC) { - sas_ha = pm8001_ha->sas; sas_phy_disconnected(&phy->sas_phy); - sas_ha->notify_phy_event(&phy->sas_phy, - PHYE_LOSS_OF_SIGNAL); + sas_notify_phy_event(&phy->sas_phy, + PHYE_LOSS_OF_SIGNAL, GFP_KERNEL); phy->phy_attached = 0; } } @@ -1358,4 +1355,3 @@ int pm8001_clear_task_set(struct domain_device *dev, u8 *lun) tmf_task.tmf = TMF_CLEAR_TASK_SET; return pm8001_issue_ssp_tmf(dev, lun, &tmf_task); } - diff --git a/drivers/scsi/pm8001/pm80xx_hwi.c b/drivers/scsi/pm8001/pm80xx_hwi.c index bdc6812acf9847d7999bd959a325f5adae05255c..3f77863142aa50076d1446d39112114b66972513 100644 --- a/drivers/scsi/pm8001/pm80xx_hwi.c +++ b/drivers/scsi/pm8001/pm80xx_hwi.c @@ -2932,7 +2932,6 @@ hw_event_sas_phy_up(struct pm8001_hba_info *pm8001_ha, void *piomb) u8 portstate = (u8)(phyid_npip_portstate & 0x0000000F); struct pm8001_port *port = &pm8001_ha->port[port_id]; - struct sas_ha_struct *sas_ha = pm8001_ha->sas; struct pm8001_phy *phy = &pm8001_ha->phy[phy_id]; unsigned long flags; u8 deviceType = pPayload->sas_identify.dev_type; @@ -2981,7 +2980,7 @@ hw_event_sas_phy_up(struct pm8001_hba_info *pm8001_ha, void *piomb) else if (phy->identify.device_type != SAS_PHY_UNUSED) phy->identify.target_port_protocols = SAS_PROTOCOL_SMP; phy->sas_phy.oob_mode = SAS_OOB_MODE; - sas_ha->notify_phy_event(&phy->sas_phy, PHYE_OOB_DONE); + sas_notify_phy_event(&phy->sas_phy, PHYE_OOB_DONE, GFP_ATOMIC); spin_lock_irqsave(&phy->sas_phy.frame_rcvd_lock, flags); memcpy(phy->frame_rcvd, &pPayload->sas_identify, sizeof(struct sas_identify_frame)-4); @@ -3015,7 +3014,6 @@ hw_event_sata_phy_up(struct pm8001_hba_info *pm8001_ha, void *piomb) u8 portstate = (u8)(phyid_npip_portstate & 0x0000000F); struct pm8001_port *port = &pm8001_ha->port[port_id]; - struct sas_ha_struct *sas_ha = pm8001_ha->sas; struct pm8001_phy *phy = &pm8001_ha->phy[phy_id]; unsigned long flags; PM8001_MSG_DBG(pm8001_ha, pm8001_printk( @@ -3029,7 +3027,7 @@ hw_event_sata_phy_up(struct pm8001_hba_info *pm8001_ha, void *piomb) phy->phy_type |= PORT_TYPE_SATA; phy->phy_attached = 1; phy->sas_phy.oob_mode = SATA_OOB_MODE; - sas_ha->notify_phy_event(&phy->sas_phy, PHYE_OOB_DONE); + sas_notify_phy_event(&phy->sas_phy, PHYE_OOB_DONE, GFP_ATOMIC); spin_lock_irqsave(&phy->sas_phy.frame_rcvd_lock, flags); memcpy(phy->frame_rcvd, ((u8 *)&pPayload->sata_fis - 4), sizeof(struct dev_to_host_fis)); @@ -3112,11 +3110,9 @@ hw_event_phy_down(struct pm8001_hba_info *pm8001_ha, void *piomb) break; } - if (port_sata && (portstate != PORT_IN_RESET)) { - struct sas_ha_struct *sas_ha = pm8001_ha->sas; - - sas_ha->notify_phy_event(&phy->sas_phy, PHYE_LOSS_OF_SIGNAL); - } + if (port_sata && (portstate != PORT_IN_RESET)) + sas_notify_phy_event(&phy->sas_phy, PHYE_LOSS_OF_SIGNAL, + GFP_ATOMIC); } static int mpi_phy_start_resp(struct pm8001_hba_info *pm8001_ha, void *piomb) @@ -3215,7 +3211,8 @@ static int mpi_hw_event(struct pm8001_hba_info *pm8001_ha, void *piomb) case HW_EVENT_SATA_SPINUP_HOLD: PM8001_MSG_DBG(pm8001_ha, pm8001_printk("HW_EVENT_SATA_SPINUP_HOLD\n")); - sas_ha->notify_phy_event(&phy->sas_phy, PHYE_SPINUP_HOLD); + sas_notify_phy_event(&phy->sas_phy, PHYE_SPINUP_HOLD, + GFP_ATOMIC); break; case HW_EVENT_PHY_DOWN: PM8001_MSG_DBG(pm8001_ha, @@ -3234,7 +3231,8 @@ static int mpi_hw_event(struct pm8001_hba_info *pm8001_ha, void *piomb) pm8001_printk("HW_EVENT_PORT_INVALID\n")); sas_phy_disconnected(sas_phy); phy->phy_attached = 0; - sas_ha->notify_port_event(sas_phy, PORTE_LINK_RESET_ERR); + sas_notify_port_event(sas_phy, PORTE_LINK_RESET_ERR, + GFP_ATOMIC); break; /* the broadcast change primitive received, tell the LIBSAS this event to revalidate the sas domain*/ @@ -3246,14 +3244,15 @@ static int mpi_hw_event(struct pm8001_hba_info *pm8001_ha, void *piomb) spin_lock_irqsave(&sas_phy->sas_prim_lock, flags); sas_phy->sas_prim = HW_EVENT_BROADCAST_CHANGE; spin_unlock_irqrestore(&sas_phy->sas_prim_lock, flags); - sas_ha->notify_port_event(sas_phy, PORTE_BROADCAST_RCVD); + sas_notify_port_event(sas_phy, PORTE_BROADCAST_RCVD, + GFP_ATOMIC); break; case HW_EVENT_PHY_ERROR: PM8001_MSG_DBG(pm8001_ha, pm8001_printk("HW_EVENT_PHY_ERROR\n")); sas_phy_disconnected(&phy->sas_phy); phy->phy_attached = 0; - sas_ha->notify_phy_event(&phy->sas_phy, PHYE_OOB_ERROR); + sas_notify_phy_event(&phy->sas_phy, PHYE_OOB_ERROR, GFP_ATOMIC); break; case HW_EVENT_BROADCAST_EXP: PM8001_MSG_DBG(pm8001_ha, @@ -3261,7 +3260,8 @@ static int mpi_hw_event(struct pm8001_hba_info *pm8001_ha, void *piomb) spin_lock_irqsave(&sas_phy->sas_prim_lock, flags); sas_phy->sas_prim = HW_EVENT_BROADCAST_EXP; spin_unlock_irqrestore(&sas_phy->sas_prim_lock, flags); - sas_ha->notify_port_event(sas_phy, PORTE_BROADCAST_RCVD); + sas_notify_port_event(sas_phy, PORTE_BROADCAST_RCVD, + GFP_ATOMIC); break; case HW_EVENT_LINK_ERR_INVALID_DWORD: PM8001_MSG_DBG(pm8001_ha, @@ -3300,7 +3300,8 @@ static int mpi_hw_event(struct pm8001_hba_info *pm8001_ha, void *piomb) spin_lock_irqsave(&sas_phy->sas_prim_lock, flags); sas_phy->sas_prim = HW_EVENT_BROADCAST_SES; spin_unlock_irqrestore(&sas_phy->sas_prim_lock, flags); - sas_ha->notify_port_event(sas_phy, PORTE_BROADCAST_RCVD); + sas_notify_port_event(sas_phy, PORTE_BROADCAST_RCVD, + GFP_ATOMIC); break; case HW_EVENT_INBOUND_CRC_ERROR: PM8001_MSG_DBG(pm8001_ha, @@ -3312,14 +3313,15 @@ static int mpi_hw_event(struct pm8001_hba_info *pm8001_ha, void *piomb) case HW_EVENT_HARD_RESET_RECEIVED: PM8001_MSG_DBG(pm8001_ha, pm8001_printk("HW_EVENT_HARD_RESET_RECEIVED\n")); - sas_ha->notify_port_event(sas_phy, PORTE_HARD_RESET); + sas_notify_port_event(sas_phy, PORTE_HARD_RESET, GFP_ATOMIC); break; case HW_EVENT_ID_FRAME_TIMEOUT: PM8001_MSG_DBG(pm8001_ha, pm8001_printk("HW_EVENT_ID_FRAME_TIMEOUT\n")); sas_phy_disconnected(sas_phy); phy->phy_attached = 0; - sas_ha->notify_port_event(sas_phy, PORTE_LINK_RESET_ERR); + sas_notify_port_event(sas_phy, PORTE_LINK_RESET_ERR, + GFP_ATOMIC); break; case HW_EVENT_LINK_ERR_PHY_RESET_FAILED: PM8001_MSG_DBG(pm8001_ha, @@ -3329,7 +3331,8 @@ static int mpi_hw_event(struct pm8001_hba_info *pm8001_ha, void *piomb) port_id, phy_id, 0, 0); sas_phy_disconnected(sas_phy); phy->phy_attached = 0; - sas_ha->notify_port_event(sas_phy, PORTE_LINK_RESET_ERR); + sas_notify_port_event(sas_phy, PORTE_LINK_RESET_ERR, + GFP_ATOMIC); break; case HW_EVENT_PORT_RESET_TIMER_TMO: PM8001_MSG_DBG(pm8001_ha, @@ -3338,7 +3341,8 @@ static int mpi_hw_event(struct pm8001_hba_info *pm8001_ha, void *piomb) port_id, phy_id, 0, 0); sas_phy_disconnected(sas_phy); phy->phy_attached = 0; - sas_ha->notify_port_event(sas_phy, PORTE_LINK_RESET_ERR); + sas_notify_port_event(sas_phy, PORTE_LINK_RESET_ERR, + GFP_ATOMIC); if (pm8001_ha->phy[phy_id].reset_completion) { pm8001_ha->phy[phy_id].port_reset_status = PORT_RESET_TMO; @@ -3355,8 +3359,8 @@ static int mpi_hw_event(struct pm8001_hba_info *pm8001_ha, void *piomb) for (i = 0; i < pm8001_ha->chip->n_phy; i++) { if (port->wide_port_phymap & (1 << i)) { phy = &pm8001_ha->phy[i]; - sas_ha->notify_phy_event(&phy->sas_phy, - PHYE_LOSS_OF_SIGNAL); + sas_notify_phy_event(&phy->sas_phy, + PHYE_LOSS_OF_SIGNAL, GFP_ATOMIC); port->wide_port_phymap &= ~(1 << i); } } diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c index b5cfbdcde2fcc30199f005503ef2529553800023..8d006d95eeb4e3c9c8a070e641776350de963ae5 100644 --- a/drivers/scsi/sd.c +++ b/drivers/scsi/sd.c @@ -2201,40 +2201,45 @@ static int sd_read_protection_type(struct scsi_disk *sdkp, unsigned char *buffer { struct scsi_device *sdp = sdkp->device; u8 type; - int ret = 0; if (scsi_device_protection(sdp) == 0 || (buffer[12] & 1) == 0) { sdkp->protection_type = 0; - return ret; + return 0; } type = ((buffer[12] >> 1) & 7) + 1; /* P_TYPE 0 = Type 1 */ - if (type > T10_PI_TYPE3_PROTECTION) - ret = -ENODEV; - else if (scsi_host_dif_capable(sdp->host, type)) - ret = 1; - - if (sdkp->first_scan || type != sdkp->protection_type) - switch (ret) { - case -ENODEV: - sd_printk(KERN_ERR, sdkp, "formatted with unsupported" \ - " protection type %u. Disabling disk!\n", - type); - break; - case 1: - sd_printk(KERN_NOTICE, sdkp, - "Enabling DIF Type %u protection\n", type); - break; - case 0: - sd_printk(KERN_NOTICE, sdkp, - "Disabling DIF Type %u protection\n", type); - break; - } + if (type > T10_PI_TYPE3_PROTECTION) { + sd_printk(KERN_ERR, sdkp, "formatted with unsupported" \ + " protection type %u. Disabling disk!\n", + type); + sdkp->protection_type = 0; + return -ENODEV; + } sdkp->protection_type = type; - return ret; + return 0; +} + +static void sd_config_protection(struct scsi_disk *sdkp) +{ + struct scsi_device *sdp = sdkp->device; + + sd_dif_config_host(sdkp); + + if (!sdkp->protection_type) + return; + + if (!scsi_host_dif_capable(sdp->host, sdkp->protection_type)) { + sd_first_printk(KERN_NOTICE, sdkp, + "Disabling DIF Type %u protection\n", + sdkp->protection_type); + sdkp->protection_type = 0; + } + + sd_first_printk(KERN_NOTICE, sdkp, "Enabling DIF Type %u protection\n", + sdkp->protection_type); } static void read_capacity_error(struct scsi_disk *sdkp, struct scsi_device *sdp, @@ -3154,6 +3159,7 @@ static int sd_revalidate_disk(struct gendisk *disk) sd_read_app_tag_own(sdkp, buffer); sd_read_write_same(sdkp, buffer); sd_read_security(sdkp, buffer); + sd_config_protection(sdkp); } /* @@ -3386,10 +3392,6 @@ static int sd_probe(struct device *dev) blk_pm_runtime_init(sdp->request_queue, dev); device_add_disk(dev, gd, NULL); - if (sdkp->capacity) - sd_dif_config_host(sdkp); - - sd_revalidate_disk(gd); if (sdkp->security) { sdkp->opal_dev = init_opal_dev(sdp, &sd_sec_submit); @@ -3552,6 +3554,7 @@ static int sd_suspend_common(struct device *dev, bool ignore_stop_errors) { struct scsi_disk *sdkp = dev_get_drvdata(dev); struct scsi_sense_hdr sshdr; + int retries; int ret = 0; if (!sdkp) /* E.g.: runtime suspend following sd_remove() */ @@ -3582,9 +3585,15 @@ static int sd_suspend_common(struct device *dev, bool ignore_stop_errors) if (sdkp->device->manage_start_stop) { sd_printk(KERN_NOTICE, sdkp, "Stopping disk\n"); /* an error is not worth aborting a system sleep */ - ret = sd_start_stop_device(sdkp, 0); - if (ignore_stop_errors) - ret = 0; + for (retries = 3; retries > 0; --retries) { + ret = sd_start_stop_device(sdkp, 0); + if (!ret) + break; + if (ignore_stop_errors) { + ret = 0; + break; + } + } } return ret; @@ -3603,6 +3612,7 @@ static int sd_suspend_runtime(struct device *dev) static int sd_resume(struct device *dev) { struct scsi_disk *sdkp = dev_get_drvdata(dev); + int retries; int ret; if (!sdkp) /* E.g.: runtime resume at the start of sd_probe() */ @@ -3612,9 +3622,13 @@ static int sd_resume(struct device *dev) return 0; sd_printk(KERN_NOTICE, sdkp, "Starting disk\n"); - ret = sd_start_stop_device(sdkp, 1); - if (!ret) - opal_unlock_from_suspend(sdkp->opal_dev); + for (retries = 3; retries > 0; --retries) { + ret = sd_start_stop_device(sdkp, 1); + if (!ret) { + opal_unlock_from_suspend(sdkp->opal_dev); + break; + } + } return ret; } diff --git a/drivers/scsi/sd_dif.c b/drivers/scsi/sd_dif.c index 4cadb26070a8f5c816220cb1fd96ad0639c7c218..7dd57a5fd4eb63a607c6f48cf55878af20f4c7ea 100644 --- a/drivers/scsi/sd_dif.c +++ b/drivers/scsi/sd_dif.c @@ -39,8 +39,10 @@ void sd_dif_config_host(struct scsi_disk *sdkp) dif = 0; dix = 1; } - if (!dix) + if (!dix) { + blk_integrity_unregister(disk); return; + } memset(&bi, 0, sizeof(bi)); @@ -59,8 +61,6 @@ void sd_dif_config_host(struct scsi_disk *sdkp) bi.profile = &t10_pi_type1_crc; bi.tuple_size = sizeof(struct t10_pi_tuple); - sd_printk(KERN_NOTICE, sdkp, - "Enabling DIX %s protection\n", bi.profile->name); if (dif && type) { bi.flags |= BLK_INTEGRITY_DEVICE_CAPABLE; @@ -72,11 +72,11 @@ void sd_dif_config_host(struct scsi_disk *sdkp) bi.tag_size = sizeof(u16) + sizeof(u32); else bi.tag_size = sizeof(u16); - - sd_printk(KERN_NOTICE, sdkp, "DIF application tag size %u\n", - bi.tag_size); } + sd_first_printk(KERN_NOTICE, sdkp, + "Enabling DIX %s, application tag size %u bytes\n", + bi.profile->name, bi.tag_size); out: blk_integrity_register(disk, &bi); } diff --git a/drivers/soc/Kconfig b/drivers/soc/Kconfig index 833e04a7835c56b80f77af30db86af5f38d0e4c7..e77d4215916ba4336492e12cd1498039bd4e15de 100644 --- a/drivers/soc/Kconfig +++ b/drivers/soc/Kconfig @@ -7,6 +7,7 @@ source "drivers/soc/aspeed/Kconfig" source "drivers/soc/atmel/Kconfig" source "drivers/soc/bcm/Kconfig" source "drivers/soc/fsl/Kconfig" +source "drivers/soc/hisilicon/Kconfig" source "drivers/soc/imx/Kconfig" source "drivers/soc/ixp4xx/Kconfig" source "drivers/soc/mediatek/Kconfig" diff --git a/drivers/soc/Makefile b/drivers/soc/Makefile index 2ec3550035243791054161097517da45366c3a15..356946d707daebc3fa480adf343f3d30e12e641a 100644 --- a/drivers/soc/Makefile +++ b/drivers/soc/Makefile @@ -11,6 +11,7 @@ obj-$(CONFIG_ARCH_DOVE) += dove/ obj-$(CONFIG_MACH_DOVE) += dove/ obj-y += fsl/ obj-$(CONFIG_ARCH_GEMINI) += gemini/ +obj-y += hisilicon/ obj-$(CONFIG_ARCH_MXC) += imx/ obj-$(CONFIG_ARCH_IXP4XX) += ixp4xx/ obj-$(CONFIG_SOC_XWAY) += lantiq/ diff --git a/drivers/soc/hisilicon/Kconfig b/drivers/soc/hisilicon/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..0ab688af308fed625ec8c04723a87911b8da7373 --- /dev/null +++ b/drivers/soc/hisilicon/Kconfig @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: GPL-2.0-only + +menu "Hisilicon SoC drivers" + depends on ARCH_HISI || COMPILE_TEST + +config KUNPENG_HCCS + tristate "HCCS driver on Kunpeng SoC" + depends on ACPI + depends on MAILBOX + depends on ARM64 || COMPILE_TEST + help + The Huawei Cache Coherence System (HCCS) is a multi-chip + interconnection bus protocol. + The performance of application may be affected if some HCCS + ports are not in full lane status, have a large number of CRC + errors and so on. + + Say M here if you want to include support for querying the + health status and port information of HCCS on Kunpeng SoC. + +endmenu diff --git a/drivers/soc/hisilicon/Makefile b/drivers/soc/hisilicon/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..226e747e70d67511e1954010e143d39df6b73f55 --- /dev/null +++ b/drivers/soc/hisilicon/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_KUNPENG_HCCS) += kunpeng_hccs.o diff --git a/drivers/soc/hisilicon/kunpeng_hccs.c b/drivers/soc/hisilicon/kunpeng_hccs.c new file mode 100644 index 0000000000000000000000000000000000000000..3f2d4aa7b39971bb4f7732bb685125a794af9acb --- /dev/null +++ b/drivers/soc/hisilicon/kunpeng_hccs.c @@ -0,0 +1,1358 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * The Huawei Cache Coherence System (HCCS) is a multi-chip interconnection + * bus protocol. + * + * Copyright (c) 2023 Hisilicon Limited. + * Author: Huisong Li + * + * HCCS driver for Kunpeng SoC provides the following features: + * - Retrieve the following information about each port: + * - port type + * - lane mode + * - enable + * - current lane mode + * - link finite state machine + * - lane mask + * - CRC error count + * + * - Retrieve the following information about all the ports on the chip or + * the die: + * - if all enabled ports are in linked + * - if all linked ports are in full lane + * - CRC error count sum + */ +#include +#include +#include +#include + +#include + +#include "kunpeng_hccs.h" + +/* + * Arbitrary retries in case the remote processor is slow to respond + * to PCC commands + */ +#define HCCS_PCC_CMD_WAIT_RETRIES_NUM 500ULL +#define HCCS_POLL_STATUS_TIME_INTERVAL_US 3 + +static struct hccs_port_info *kobj_to_port_info(struct kobject *k) +{ + return container_of(k, struct hccs_port_info, kobj); +} + +static struct hccs_die_info *kobj_to_die_info(struct kobject *k) +{ + return container_of(k, struct hccs_die_info, kobj); +} + +static struct hccs_chip_info *kobj_to_chip_info(struct kobject *k) +{ + return container_of(k, struct hccs_chip_info, kobj); +} + +struct hccs_register_ctx { + struct device *dev; + u8 chan_id; + int err; +}; + +static acpi_status hccs_get_register_cb(struct acpi_resource *ares, + void *context) +{ + struct acpi_resource_generic_register *reg; + struct hccs_register_ctx *ctx = context; + + if (ares->type != ACPI_RESOURCE_TYPE_GENERIC_REGISTER) + return AE_OK; + + reg = &ares->data.generic_reg; + if (reg->space_id != ACPI_ADR_SPACE_PLATFORM_COMM) { + dev_err(ctx->dev, "Bad register resource.\n"); + ctx->err = -EINVAL; + return AE_ERROR; + } + ctx->chan_id = reg->access_size; + + return AE_OK; +} + +static int hccs_get_pcc_chan_id(struct hccs_dev *hdev) +{ + acpi_handle handle = ACPI_HANDLE(hdev->dev); + struct hccs_register_ctx ctx = {0}; + acpi_status status; + + if (!acpi_has_method(handle, METHOD_NAME__CRS)) { + dev_err(hdev->dev, "No _CRS method.\n"); + return -ENODEV; + } + + ctx.dev = hdev->dev; + status = acpi_walk_resources(handle, METHOD_NAME__CRS, + hccs_get_register_cb, &ctx); + if (ACPI_FAILURE(status)) + return ctx.err; + hdev->chan_id = ctx.chan_id; + + return 0; +} + +static void hccs_chan_tx_done(struct mbox_client *cl, void *msg, int ret) +{ + if (ret < 0) + pr_debug("TX did not complete: CMD sent:0x%x, ret:%d\n", + *(u8 *)msg, ret); + else + pr_debug("TX completed. CMD sent:0x%x, ret:%d\n", + *(u8 *)msg, ret); +} + +static void hccs_pcc_rx_callback(struct mbox_client *cl, void *mssg) +{ + struct hccs_mbox_client_info *cl_info = + container_of(cl, struct hccs_mbox_client_info, client); + + complete(&cl_info->done); +} + +static void hccs_unregister_pcc_channel(struct hccs_dev *hdev) +{ + struct hccs_mbox_client_info *cl_info = &hdev->cl_info; + + if (cl_info->pcc_comm_addr) + iounmap(cl_info->pcc_comm_addr); + pcc_mbox_free_channel(hdev->cl_info.pcc_chan); +} + +static int hccs_register_pcc_channel(struct hccs_dev *hdev) +{ + struct hccs_mbox_client_info *cl_info = &hdev->cl_info; + struct mbox_client *cl = &cl_info->client; + struct pcc_mbox_chan *pcc_chan; + struct device *dev = hdev->dev; + int rc; + + cl->dev = dev; + cl->tx_block = false; + cl->knows_txdone = true; + cl->tx_done = hccs_chan_tx_done; + cl->rx_callback = hdev->verspec_data->rx_callback; + init_completion(&cl_info->done); + + pcc_chan = pcc_mbox_request_channel(cl, hdev->chan_id); + if (IS_ERR(pcc_chan)) { + dev_err(dev, "PPC channel request failed.\n"); + rc = -ENODEV; + goto out; + } + cl_info->pcc_chan = pcc_chan; + cl_info->mbox_chan = pcc_chan->mchan; + + /* + * pcc_chan->latency is just a nominal value. In reality the remote + * processor could be much slower to reply. So add an arbitrary amount + * of wait on top of nominal. + */ + cl_info->deadline_us = + HCCS_PCC_CMD_WAIT_RETRIES_NUM * pcc_chan->latency; + if (!hdev->verspec_data->has_txdone_irq && + cl_info->mbox_chan->mbox->txdone_irq) { + dev_err(dev, "PCC IRQ in PCCT is enabled.\n"); + rc = -EINVAL; + goto err_mbx_channel_free; + } else if (hdev->verspec_data->has_txdone_irq && + !cl_info->mbox_chan->mbox->txdone_irq) { + dev_err(dev, "PCC IRQ in PCCT isn't supported.\n"); + rc = -EINVAL; + goto err_mbx_channel_free; + } + + if (pcc_chan->shmem_base_addr) { + cl_info->pcc_comm_addr = ioremap(pcc_chan->shmem_base_addr, + pcc_chan->shmem_size); + if (!cl_info->pcc_comm_addr) { + dev_err(dev, "Failed to ioremap PCC communication region for channel-%d.\n", + hdev->chan_id); + rc = -ENOMEM; + goto err_mbx_channel_free; + } + } + + return 0; + +err_mbx_channel_free: + pcc_mbox_free_channel(cl_info->pcc_chan); +out: + return rc; +} + +static int hccs_wait_cmd_complete_by_poll(struct hccs_dev *hdev) +{ + struct hccs_mbox_client_info *cl_info = &hdev->cl_info; + struct acpi_pcct_shared_memory __iomem *comm_base = + cl_info->pcc_comm_addr; + u16 status; + int ret; + + /* + * Poll PCC status register every 3us(delay_us) for maximum of + * deadline_us(timeout_us) until PCC command complete bit is set(cond) + */ + ret = readw_poll_timeout(&comm_base->status, status, + status & PCC_STATUS_CMD_COMPLETE, + HCCS_POLL_STATUS_TIME_INTERVAL_US, + cl_info->deadline_us); + if (unlikely(ret)) + dev_err(hdev->dev, "poll PCC status failed, ret = %d.\n", ret); + + return ret; +} + +static int hccs_wait_cmd_complete_by_irq(struct hccs_dev *hdev) +{ + struct hccs_mbox_client_info *cl_info = &hdev->cl_info; + + if (!wait_for_completion_timeout(&cl_info->done, + usecs_to_jiffies(cl_info->deadline_us))) { + dev_err(hdev->dev, "PCC command executed timeout!\n"); + return -ETIMEDOUT; + } + + return 0; +} + +static inline void hccs_fill_pcc_shared_mem_region(struct hccs_dev *hdev, + u8 cmd, + struct hccs_desc *desc, + void __iomem *comm_space, + u16 space_size) +{ + struct acpi_pcct_shared_memory tmp = { + .signature = PCC_SIGNATURE | hdev->chan_id, + .command = cmd, + .status = 0, + }; + + memcpy_toio(hdev->cl_info.pcc_comm_addr, (void *)&tmp, + sizeof(struct acpi_pcct_shared_memory)); + + /* Copy the message to the PCC comm space */ + memcpy_toio(comm_space, (void *)desc, space_size); +} + +static inline void hccs_fill_ext_pcc_shared_mem_region(struct hccs_dev *hdev, + u8 cmd, + struct hccs_desc *desc, + void __iomem *comm_space, + u16 space_size) +{ + struct acpi_pcct_ext_pcc_shared_memory tmp = { + .signature = PCC_SIGNATURE | hdev->chan_id, + .flags = PCC_CMD_COMPLETION_NOTIFY, + .length = HCCS_PCC_SHARE_MEM_BYTES, + .command = cmd, + }; + + memcpy_toio(hdev->cl_info.pcc_comm_addr, (void *)&tmp, + sizeof(struct acpi_pcct_ext_pcc_shared_memory)); + + /* Copy the message to the PCC comm space */ + memcpy_toio(comm_space, (void *)desc, space_size); +} + +static int hccs_pcc_cmd_send(struct hccs_dev *hdev, u8 cmd, + struct hccs_desc *desc) +{ + const struct hccs_verspecific_data *verspec_data = hdev->verspec_data; + struct hccs_mbox_client_info *cl_info = &hdev->cl_info; + struct hccs_fw_inner_head *fw_inner_head; + void __iomem *comm_space; + u16 space_size; + int ret; + + comm_space = cl_info->pcc_comm_addr + verspec_data->shared_mem_size; + space_size = HCCS_PCC_SHARE_MEM_BYTES - verspec_data->shared_mem_size; + verspec_data->fill_pcc_shared_mem(hdev, cmd, desc, + comm_space, space_size); + if (verspec_data->has_txdone_irq) + reinit_completion(&cl_info->done); + + /* Ring doorbell */ + ret = mbox_send_message(cl_info->mbox_chan, &cmd); + if (ret < 0) { + dev_err(hdev->dev, "Send PCC mbox message failed, ret = %d.\n", + ret); + goto end; + } + + ret = verspec_data->wait_cmd_complete(hdev); + if (ret) + goto end; + + /* Copy response data */ + memcpy_fromio((void *)desc, comm_space, space_size); + fw_inner_head = &desc->rsp.fw_inner_head; + if (fw_inner_head->retStatus) { + dev_err(hdev->dev, "Execute PCC command failed, error code = %u.\n", + fw_inner_head->retStatus); + ret = -EIO; + } + +end: + if (verspec_data->has_txdone_irq) + mbox_chan_txdone(cl_info->mbox_chan, ret); + else + mbox_client_txdone(cl_info->mbox_chan, ret); + return ret; +} + +static void hccs_init_req_desc(struct hccs_desc *desc) +{ + struct hccs_req_desc *req = &desc->req; + + memset(desc, 0, sizeof(*desc)); + req->req_head.module_code = HCCS_SERDES_MODULE_CODE; +} + +static int hccs_get_dev_caps(struct hccs_dev *hdev) +{ + struct hccs_desc desc; + int ret; + + hccs_init_req_desc(&desc); + ret = hccs_pcc_cmd_send(hdev, HCCS_GET_DEV_CAP, &desc); + if (ret) { + dev_err(hdev->dev, "Get device capabilities failed, ret = %d.\n", + ret); + return ret; + } + memcpy(&hdev->caps, desc.rsp.data, sizeof(hdev->caps)); + + return 0; +} + +static int hccs_query_chip_num_on_platform(struct hccs_dev *hdev) +{ + struct hccs_desc desc; + int ret; + + hccs_init_req_desc(&desc); + ret = hccs_pcc_cmd_send(hdev, HCCS_GET_CHIP_NUM, &desc); + if (ret) { + dev_err(hdev->dev, "query system chip number failed, ret = %d.\n", + ret); + return ret; + } + + hdev->chip_num = *((u8 *)&desc.rsp.data); + if (!hdev->chip_num) { + dev_err(hdev->dev, "chip num obtained from firmware is zero.\n"); + return -EINVAL; + } + + return 0; +} + +static int hccs_get_chip_info(struct hccs_dev *hdev, + struct hccs_chip_info *chip) +{ + struct hccs_die_num_req_param *req_param; + struct hccs_desc desc; + int ret; + + hccs_init_req_desc(&desc); + req_param = (struct hccs_die_num_req_param *)desc.req.data; + req_param->chip_id = chip->chip_id; + ret = hccs_pcc_cmd_send(hdev, HCCS_GET_DIE_NUM, &desc); + if (ret) + return ret; + + chip->die_num = *((u8 *)&desc.rsp.data); + + return 0; +} + +static int hccs_query_chip_info_on_platform(struct hccs_dev *hdev) +{ + struct hccs_chip_info *chip; + int ret; + u8 idx; + + ret = hccs_query_chip_num_on_platform(hdev); + if (ret) { + dev_err(hdev->dev, "query chip number on platform failed, ret = %d.\n", + ret); + return ret; + } + + hdev->chips = devm_kzalloc(hdev->dev, + hdev->chip_num * sizeof(struct hccs_chip_info), + GFP_KERNEL); + if (!hdev->chips) { + dev_err(hdev->dev, "allocate all chips memory failed.\n"); + return -ENOMEM; + } + + for (idx = 0; idx < hdev->chip_num; idx++) { + chip = &hdev->chips[idx]; + chip->chip_id = idx; + ret = hccs_get_chip_info(hdev, chip); + if (ret) { + dev_err(hdev->dev, "get chip%u info failed, ret = %d.\n", + idx, ret); + return ret; + } + chip->hdev = hdev; + } + + return 0; +} + +static int hccs_query_die_info_on_chip(struct hccs_dev *hdev, u8 chip_id, + u8 die_idx, struct hccs_die_info *die) +{ + struct hccs_die_info_req_param *req_param; + struct hccs_die_info_rsp_data *rsp_data; + struct hccs_desc desc; + int ret; + + hccs_init_req_desc(&desc); + req_param = (struct hccs_die_info_req_param *)desc.req.data; + req_param->chip_id = chip_id; + req_param->die_idx = die_idx; + ret = hccs_pcc_cmd_send(hdev, HCCS_GET_DIE_INFO, &desc); + if (ret) + return ret; + + rsp_data = (struct hccs_die_info_rsp_data *)desc.rsp.data; + die->die_id = rsp_data->die_id; + die->port_num = rsp_data->port_num; + die->min_port_id = rsp_data->min_port_id; + die->max_port_id = rsp_data->max_port_id; + if (die->min_port_id > die->max_port_id) { + dev_err(hdev->dev, "min port id(%u) > max port id(%u) on die_idx(%u).\n", + die->min_port_id, die->max_port_id, die_idx); + return -EINVAL; + } + if (die->max_port_id > HCCS_DIE_MAX_PORT_ID) { + dev_err(hdev->dev, "max port id(%u) on die_idx(%u) is too big.\n", + die->max_port_id, die_idx); + return -EINVAL; + } + + return 0; +} + +static int hccs_query_all_die_info_on_platform(struct hccs_dev *hdev) +{ + struct device *dev = hdev->dev; + struct hccs_chip_info *chip; + struct hccs_die_info *die; + u8 i, j; + int ret; + + for (i = 0; i < hdev->chip_num; i++) { + chip = &hdev->chips[i]; + if (!chip->die_num) + continue; + + chip->dies = devm_kzalloc(hdev->dev, + chip->die_num * sizeof(struct hccs_die_info), + GFP_KERNEL); + if (!chip->dies) { + dev_err(dev, "allocate all dies memory on chip%u failed.\n", + i); + return -ENOMEM; + } + + for (j = 0; j < chip->die_num; j++) { + die = &chip->dies[j]; + ret = hccs_query_die_info_on_chip(hdev, i, j, die); + if (ret) { + dev_err(dev, "get die idx (%u) info on chip%u failed, ret = %d.\n", + j, i, ret); + return ret; + } + die->chip = chip; + } + } + + return 0; +} + +static int hccs_get_bd_info(struct hccs_dev *hdev, u8 opcode, + struct hccs_desc *desc, + void *buf, size_t buf_len, + struct hccs_rsp_head *rsp_head) +{ + struct hccs_rsp_head *head; + struct hccs_rsp_desc *rsp; + int ret; + + ret = hccs_pcc_cmd_send(hdev, opcode, desc); + if (ret) + return ret; + + rsp = &desc->rsp; + head = &rsp->rsp_head; + if (head->data_len > buf_len) { + dev_err(hdev->dev, + "buffer overflow (buf_len = %zu, data_len = %u)!\n", + buf_len, head->data_len); + return -ENOMEM; + } + + memcpy(buf, rsp->data, head->data_len); + *rsp_head = *head; + + return 0; +} + +static int hccs_get_all_port_attr(struct hccs_dev *hdev, + struct hccs_die_info *die, + struct hccs_port_attr *attrs, u16 size) +{ + struct hccs_die_comm_req_param *req_param; + struct hccs_req_head *req_head; + struct hccs_rsp_head rsp_head; + struct hccs_desc desc; + size_t left_buf_len; + u32 data_len = 0; + u8 start_id; + u8 *buf; + int ret; + + buf = (u8 *)attrs; + left_buf_len = sizeof(struct hccs_port_attr) * size; + start_id = die->min_port_id; + while (start_id <= die->max_port_id) { + hccs_init_req_desc(&desc); + req_head = &desc.req.req_head; + req_head->start_id = start_id; + req_param = (struct hccs_die_comm_req_param *)desc.req.data; + req_param->chip_id = die->chip->chip_id; + req_param->die_id = die->die_id; + + ret = hccs_get_bd_info(hdev, HCCS_GET_DIE_PORT_INFO, &desc, + buf + data_len, left_buf_len, &rsp_head); + if (ret) { + dev_err(hdev->dev, + "get the information of port%u on die%u failed, ret = %d.\n", + start_id, die->die_id, ret); + return ret; + } + + data_len += rsp_head.data_len; + left_buf_len -= rsp_head.data_len; + if (unlikely(rsp_head.next_id <= start_id)) { + dev_err(hdev->dev, + "next port id (%u) is not greater than last start id (%u) on die%u.\n", + rsp_head.next_id, start_id, die->die_id); + return -EINVAL; + } + start_id = rsp_head.next_id; + } + + return 0; +} + +static int hccs_get_all_port_info_on_die(struct hccs_dev *hdev, + struct hccs_die_info *die) +{ + struct hccs_port_attr *attrs; + struct hccs_port_info *port; + int ret; + u8 i; + + attrs = kcalloc(die->port_num, sizeof(struct hccs_port_attr), + GFP_KERNEL); + if (!attrs) + return -ENOMEM; + + ret = hccs_get_all_port_attr(hdev, die, attrs, die->port_num); + if (ret) + goto out; + + for (i = 0; i < die->port_num; i++) { + port = &die->ports[i]; + port->port_id = attrs[i].port_id; + port->port_type = attrs[i].port_type; + port->lane_mode = attrs[i].lane_mode; + port->enable = attrs[i].enable; + port->die = die; + } + +out: + kfree(attrs); + return ret; +} + +static int hccs_query_all_port_info_on_platform(struct hccs_dev *hdev) +{ + struct device *dev = hdev->dev; + struct hccs_chip_info *chip; + struct hccs_die_info *die; + u8 i, j; + int ret; + + for (i = 0; i < hdev->chip_num; i++) { + chip = &hdev->chips[i]; + for (j = 0; j < chip->die_num; j++) { + die = &chip->dies[j]; + if (!die->port_num) + continue; + + die->ports = devm_kzalloc(dev, + die->port_num * sizeof(struct hccs_port_info), + GFP_KERNEL); + if (!die->ports) { + dev_err(dev, "allocate ports memory on chip%u/die%u failed.\n", + i, die->die_id); + return -ENOMEM; + } + + ret = hccs_get_all_port_info_on_die(hdev, die); + if (ret) { + dev_err(dev, "get all port info on chip%u/die%u failed, ret = %d.\n", + i, die->die_id, ret); + return ret; + } + } + } + + return 0; +} + +static int hccs_get_hw_info(struct hccs_dev *hdev) +{ + int ret; + + ret = hccs_query_chip_info_on_platform(hdev); + if (ret) { + dev_err(hdev->dev, "query chip info on platform failed, ret = %d.\n", + ret); + return ret; + } + + ret = hccs_query_all_die_info_on_platform(hdev); + if (ret) { + dev_err(hdev->dev, "query all die info on platform failed, ret = %d.\n", + ret); + return ret; + } + + ret = hccs_query_all_port_info_on_platform(hdev); + if (ret) { + dev_err(hdev->dev, "query all port info on platform failed, ret = %d.\n", + ret); + return ret; + } + + return 0; +} + +static int hccs_query_port_link_status(struct hccs_dev *hdev, + const struct hccs_port_info *port, + struct hccs_link_status *link_status) +{ + const struct hccs_die_info *die = port->die; + const struct hccs_chip_info *chip = die->chip; + struct hccs_port_comm_req_param *req_param; + struct hccs_desc desc; + int ret; + + hccs_init_req_desc(&desc); + req_param = (struct hccs_port_comm_req_param *)desc.req.data; + req_param->chip_id = chip->chip_id; + req_param->die_id = die->die_id; + req_param->port_id = port->port_id; + ret = hccs_pcc_cmd_send(hdev, HCCS_GET_PORT_LINK_STATUS, &desc); + if (ret) { + dev_err(hdev->dev, + "get port link status info failed, ret = %d.\n", ret); + return ret; + } + + *link_status = *((struct hccs_link_status *)desc.rsp.data); + + return 0; +} + +static int hccs_query_port_crc_err_cnt(struct hccs_dev *hdev, + const struct hccs_port_info *port, + u64 *crc_err_cnt) +{ + const struct hccs_die_info *die = port->die; + const struct hccs_chip_info *chip = die->chip; + struct hccs_port_comm_req_param *req_param; + struct hccs_desc desc; + int ret; + + hccs_init_req_desc(&desc); + req_param = (struct hccs_port_comm_req_param *)desc.req.data; + req_param->chip_id = chip->chip_id; + req_param->die_id = die->die_id; + req_param->port_id = port->port_id; + ret = hccs_pcc_cmd_send(hdev, HCCS_GET_PORT_CRC_ERR_CNT, &desc); + if (ret) { + dev_err(hdev->dev, + "get port crc error count failed, ret = %d.\n", ret); + return ret; + } + + memcpy(crc_err_cnt, &desc.rsp.data, sizeof(u64)); + + return 0; +} + +static int hccs_get_die_all_link_status(struct hccs_dev *hdev, + const struct hccs_die_info *die, + u8 *all_linked) +{ + struct hccs_die_comm_req_param *req_param; + struct hccs_desc desc; + int ret; + + if (die->port_num == 0) { + *all_linked = 1; + return 0; + } + + hccs_init_req_desc(&desc); + req_param = (struct hccs_die_comm_req_param *)desc.req.data; + req_param->chip_id = die->chip->chip_id; + req_param->die_id = die->die_id; + ret = hccs_pcc_cmd_send(hdev, HCCS_GET_DIE_PORTS_LINK_STA, &desc); + if (ret) { + dev_err(hdev->dev, + "get link status of all ports failed on die%u, ret = %d.\n", + die->die_id, ret); + return ret; + } + + *all_linked = *((u8 *)&desc.rsp.data); + + return 0; +} + +static int hccs_get_die_all_port_lane_status(struct hccs_dev *hdev, + const struct hccs_die_info *die, + u8 *full_lane) +{ + struct hccs_die_comm_req_param *req_param; + struct hccs_desc desc; + int ret; + + if (die->port_num == 0) { + *full_lane = 1; + return 0; + } + + hccs_init_req_desc(&desc); + req_param = (struct hccs_die_comm_req_param *)desc.req.data; + req_param->chip_id = die->chip->chip_id; + req_param->die_id = die->die_id; + ret = hccs_pcc_cmd_send(hdev, HCCS_GET_DIE_PORTS_LANE_STA, &desc); + if (ret) { + dev_err(hdev->dev, "get lane status of all ports failed on die%u, ret = %d.\n", + die->die_id, ret); + return ret; + } + + *full_lane = *((u8 *)&desc.rsp.data); + + return 0; +} + +static int hccs_get_die_total_crc_err_cnt(struct hccs_dev *hdev, + const struct hccs_die_info *die, + u64 *total_crc_err_cnt) +{ + struct hccs_die_comm_req_param *req_param; + struct hccs_desc desc; + int ret; + + if (die->port_num == 0) { + *total_crc_err_cnt = 0; + return 0; + } + + hccs_init_req_desc(&desc); + req_param = (struct hccs_die_comm_req_param *)desc.req.data; + req_param->chip_id = die->chip->chip_id; + req_param->die_id = die->die_id; + ret = hccs_pcc_cmd_send(hdev, HCCS_GET_DIE_PORTS_CRC_ERR_CNT, &desc); + if (ret) { + dev_err(hdev->dev, "get crc error count sum failed on die%u, ret = %d.\n", + die->die_id, ret); + return ret; + } + + memcpy(total_crc_err_cnt, &desc.rsp.data, sizeof(u64)); + + return 0; +} + +static ssize_t hccs_show(struct kobject *k, struct attribute *attr, char *buf) +{ + struct kobj_attribute *kobj_attr; + + kobj_attr = container_of(attr, struct kobj_attribute, attr); + + return kobj_attr->show(k, kobj_attr, buf); +} + +static const struct sysfs_ops hccs_comm_ops = { + .show = hccs_show, +}; + +static ssize_t type_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + const struct hccs_port_info *port = kobj_to_port_info(kobj); + + return sysfs_emit(buf, "HCCS-v%u\n", port->port_type); +} +static struct kobj_attribute hccs_type_attr = __ATTR_RO(type); + +static ssize_t lane_mode_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + const struct hccs_port_info *port = kobj_to_port_info(kobj); + + return sysfs_emit(buf, "x%u\n", port->lane_mode); +} +static struct kobj_attribute lane_mode_attr = __ATTR_RO(lane_mode); + +static ssize_t enable_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + const struct hccs_port_info *port = kobj_to_port_info(kobj); + + return sysfs_emit(buf, "%u\n", port->enable); +} +static struct kobj_attribute port_enable_attr = __ATTR_RO(enable); + +static ssize_t cur_lane_num_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + const struct hccs_port_info *port = kobj_to_port_info(kobj); + struct hccs_dev *hdev = port->die->chip->hdev; + struct hccs_link_status link_status = {0}; + int ret; + + mutex_lock(&hdev->lock); + ret = hccs_query_port_link_status(hdev, port, &link_status); + mutex_unlock(&hdev->lock); + if (ret) + return ret; + + return sysfs_emit(buf, "%u\n", link_status.lane_num); +} +static struct kobj_attribute cur_lane_num_attr = __ATTR_RO(cur_lane_num); + +static ssize_t link_fsm_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + const struct hccs_port_info *port = kobj_to_port_info(kobj); + struct hccs_dev *hdev = port->die->chip->hdev; + struct hccs_link_status link_status = {0}; + const struct { + u8 link_fsm; + char *str; + } link_fsm_map[] = { + {HCCS_PORT_RESET, "reset"}, + {HCCS_PORT_SETUP, "setup"}, + {HCCS_PORT_CONFIG, "config"}, + {HCCS_PORT_READY, "link-up"}, + }; + const char *link_fsm_str = "unknown"; + size_t i; + int ret; + + mutex_lock(&hdev->lock); + ret = hccs_query_port_link_status(hdev, port, &link_status); + mutex_unlock(&hdev->lock); + if (ret) + return ret; + + for (i = 0; i < ARRAY_SIZE(link_fsm_map); i++) { + if (link_fsm_map[i].link_fsm == link_status.link_fsm) { + link_fsm_str = link_fsm_map[i].str; + break; + } + } + + return sysfs_emit(buf, "%s\n", link_fsm_str); +} +static struct kobj_attribute link_fsm_attr = __ATTR_RO(link_fsm); + +static ssize_t lane_mask_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + const struct hccs_port_info *port = kobj_to_port_info(kobj); + struct hccs_dev *hdev = port->die->chip->hdev; + struct hccs_link_status link_status = {0}; + int ret; + + mutex_lock(&hdev->lock); + ret = hccs_query_port_link_status(hdev, port, &link_status); + mutex_unlock(&hdev->lock); + if (ret) + return ret; + + return sysfs_emit(buf, "0x%x\n", link_status.lane_mask); +} +static struct kobj_attribute lane_mask_attr = __ATTR_RO(lane_mask); + +static ssize_t crc_err_cnt_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + const struct hccs_port_info *port = kobj_to_port_info(kobj); + struct hccs_dev *hdev = port->die->chip->hdev; + u64 crc_err_cnt; + int ret; + + mutex_lock(&hdev->lock); + ret = hccs_query_port_crc_err_cnt(hdev, port, &crc_err_cnt); + mutex_unlock(&hdev->lock); + if (ret) + return ret; + + return sysfs_emit(buf, "%llu\n", crc_err_cnt); +} +static struct kobj_attribute crc_err_cnt_attr = __ATTR_RO(crc_err_cnt); + +static struct attribute *hccs_port_default_attrs[] = { + &hccs_type_attr.attr, + &lane_mode_attr.attr, + &port_enable_attr.attr, + &cur_lane_num_attr.attr, + &link_fsm_attr.attr, + &lane_mask_attr.attr, + &crc_err_cnt_attr.attr, + NULL, +}; +ATTRIBUTE_GROUPS(hccs_port_default); + +static struct kobj_type hccs_port_type = { + .sysfs_ops = &hccs_comm_ops, + .default_groups = hccs_port_default_groups, +}; + +static ssize_t all_linked_on_die_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + const struct hccs_die_info *die = kobj_to_die_info(kobj); + struct hccs_dev *hdev = die->chip->hdev; + u8 all_linked; + int ret; + + mutex_lock(&hdev->lock); + ret = hccs_get_die_all_link_status(hdev, die, &all_linked); + mutex_unlock(&hdev->lock); + if (ret) + return ret; + + return sysfs_emit(buf, "%u\n", all_linked); +} +static struct kobj_attribute all_linked_on_die_attr = + __ATTR(all_linked, 0444, all_linked_on_die_show, NULL); + +static ssize_t linked_full_lane_on_die_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + const struct hccs_die_info *die = kobj_to_die_info(kobj); + struct hccs_dev *hdev = die->chip->hdev; + u8 full_lane; + int ret; + + mutex_lock(&hdev->lock); + ret = hccs_get_die_all_port_lane_status(hdev, die, &full_lane); + mutex_unlock(&hdev->lock); + if (ret) + return ret; + + return sysfs_emit(buf, "%u\n", full_lane); +} +static struct kobj_attribute linked_full_lane_on_die_attr = + __ATTR(linked_full_lane, 0444, linked_full_lane_on_die_show, NULL); + +static ssize_t crc_err_cnt_sum_on_die_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + const struct hccs_die_info *die = kobj_to_die_info(kobj); + struct hccs_dev *hdev = die->chip->hdev; + u64 total_crc_err_cnt; + int ret; + + mutex_lock(&hdev->lock); + ret = hccs_get_die_total_crc_err_cnt(hdev, die, &total_crc_err_cnt); + mutex_unlock(&hdev->lock); + if (ret) + return ret; + + return sysfs_emit(buf, "%llu\n", total_crc_err_cnt); +} +static struct kobj_attribute crc_err_cnt_sum_on_die_attr = + __ATTR(crc_err_cnt, 0444, crc_err_cnt_sum_on_die_show, NULL); + +static struct attribute *hccs_die_default_attrs[] = { + &all_linked_on_die_attr.attr, + &linked_full_lane_on_die_attr.attr, + &crc_err_cnt_sum_on_die_attr.attr, + NULL, +}; +ATTRIBUTE_GROUPS(hccs_die_default); + +static struct kobj_type hccs_die_type = { + .sysfs_ops = &hccs_comm_ops, + .default_groups = hccs_die_default_groups, +}; + +static ssize_t all_linked_on_chip_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + const struct hccs_chip_info *chip = kobj_to_chip_info(kobj); + struct hccs_dev *hdev = chip->hdev; + const struct hccs_die_info *die; + u8 all_linked = 1; + u8 i, tmp; + int ret; + + mutex_lock(&hdev->lock); + for (i = 0; i < chip->die_num; i++) { + die = &chip->dies[i]; + ret = hccs_get_die_all_link_status(hdev, die, &tmp); + if (ret) { + mutex_unlock(&hdev->lock); + return ret; + } + if (tmp != all_linked) { + all_linked = 0; + break; + } + } + mutex_unlock(&hdev->lock); + + return sysfs_emit(buf, "%u\n", all_linked); +} +static struct kobj_attribute all_linked_on_chip_attr = + __ATTR(all_linked, 0444, all_linked_on_chip_show, NULL); + +static ssize_t linked_full_lane_on_chip_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + const struct hccs_chip_info *chip = kobj_to_chip_info(kobj); + struct hccs_dev *hdev = chip->hdev; + const struct hccs_die_info *die; + u8 full_lane = 1; + u8 i, tmp; + int ret; + + mutex_lock(&hdev->lock); + for (i = 0; i < chip->die_num; i++) { + die = &chip->dies[i]; + ret = hccs_get_die_all_port_lane_status(hdev, die, &tmp); + if (ret) { + mutex_unlock(&hdev->lock); + return ret; + } + if (tmp != full_lane) { + full_lane = 0; + break; + } + } + mutex_unlock(&hdev->lock); + + return sysfs_emit(buf, "%u\n", full_lane); +} +static struct kobj_attribute linked_full_lane_on_chip_attr = + __ATTR(linked_full_lane, 0444, linked_full_lane_on_chip_show, NULL); + +static ssize_t crc_err_cnt_sum_on_chip_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + const struct hccs_chip_info *chip = kobj_to_chip_info(kobj); + u64 crc_err_cnt, total_crc_err_cnt = 0; + struct hccs_dev *hdev = chip->hdev; + const struct hccs_die_info *die; + int ret; + u16 i; + + mutex_lock(&hdev->lock); + for (i = 0; i < chip->die_num; i++) { + die = &chip->dies[i]; + ret = hccs_get_die_total_crc_err_cnt(hdev, die, &crc_err_cnt); + if (ret) { + mutex_unlock(&hdev->lock); + return ret; + } + + total_crc_err_cnt += crc_err_cnt; + } + mutex_unlock(&hdev->lock); + + return sysfs_emit(buf, "%llu\n", total_crc_err_cnt); +} +static struct kobj_attribute crc_err_cnt_sum_on_chip_attr = + __ATTR(crc_err_cnt, 0444, crc_err_cnt_sum_on_chip_show, NULL); + +static struct attribute *hccs_chip_default_attrs[] = { + &all_linked_on_chip_attr.attr, + &linked_full_lane_on_chip_attr.attr, + &crc_err_cnt_sum_on_chip_attr.attr, + NULL, +}; +ATTRIBUTE_GROUPS(hccs_chip_default); + +static struct kobj_type hccs_chip_type = { + .sysfs_ops = &hccs_comm_ops, + .default_groups = hccs_chip_default_groups, +}; + +static void hccs_remove_die_dir(struct hccs_die_info *die) +{ + struct hccs_port_info *port; + u8 i; + + for (i = 0; i < die->port_num; i++) { + port = &die->ports[i]; + if (port->dir_created) + kobject_put(&port->kobj); + } + + kobject_put(&die->kobj); +} + +static void hccs_remove_chip_dir(struct hccs_chip_info *chip) +{ + struct hccs_die_info *die; + u8 i; + + for (i = 0; i < chip->die_num; i++) { + die = &chip->dies[i]; + if (die->dir_created) + hccs_remove_die_dir(die); + } + + kobject_put(&chip->kobj); +} + +static void hccs_remove_topo_dirs(struct hccs_dev *hdev) +{ + u8 i; + + for (i = 0; i < hdev->chip_num; i++) + hccs_remove_chip_dir(&hdev->chips[i]); +} + +static int hccs_create_hccs_dir(struct hccs_dev *hdev, + struct hccs_die_info *die, + struct hccs_port_info *port) +{ + int ret; + + ret = kobject_init_and_add(&port->kobj, &hccs_port_type, + &die->kobj, "hccs%d", port->port_id); + if (ret) { + kobject_put(&port->kobj); + return ret; + } + + return 0; +} + +static int hccs_create_die_dir(struct hccs_dev *hdev, + struct hccs_chip_info *chip, + struct hccs_die_info *die) +{ + struct hccs_port_info *port; + int ret; + u16 i; + + ret = kobject_init_and_add(&die->kobj, &hccs_die_type, + &chip->kobj, "die%d", die->die_id); + if (ret) { + kobject_put(&die->kobj); + return ret; + } + + for (i = 0; i < die->port_num; i++) { + port = &die->ports[i]; + ret = hccs_create_hccs_dir(hdev, die, port); + if (ret) { + dev_err(hdev->dev, "create hccs%d dir failed.\n", + port->port_id); + goto err; + } + port->dir_created = true; + } + + return 0; +err: + hccs_remove_die_dir(die); + + return ret; +} + +static int hccs_create_chip_dir(struct hccs_dev *hdev, + struct hccs_chip_info *chip) +{ + struct hccs_die_info *die; + int ret; + u16 id; + + ret = kobject_init_and_add(&chip->kobj, &hccs_chip_type, + &hdev->dev->kobj, "chip%d", chip->chip_id); + if (ret) { + kobject_put(&chip->kobj); + return ret; + } + + for (id = 0; id < chip->die_num; id++) { + die = &chip->dies[id]; + ret = hccs_create_die_dir(hdev, chip, die); + if (ret) + goto err; + die->dir_created = true; + } + + return 0; +err: + hccs_remove_chip_dir(chip); + + return ret; +} + +static int hccs_create_topo_dirs(struct hccs_dev *hdev) +{ + struct hccs_chip_info *chip; + u8 id, k; + int ret; + + for (id = 0; id < hdev->chip_num; id++) { + chip = &hdev->chips[id]; + ret = hccs_create_chip_dir(hdev, chip); + if (ret) { + dev_err(hdev->dev, "init chip%d dir failed!\n", id); + goto err; + } + } + + return 0; +err: + for (k = 0; k < id; k++) + hccs_remove_chip_dir(&hdev->chips[k]); + + return ret; +} + +static int hccs_probe(struct platform_device *pdev) +{ + struct acpi_device *acpi_dev; + struct hccs_dev *hdev; + int rc; + + if (acpi_disabled) { + dev_err(&pdev->dev, "acpi is disabled.\n"); + return -ENODEV; + } + acpi_dev = ACPI_COMPANION(&pdev->dev); + if (!acpi_dev) + return -ENODEV; + + hdev = devm_kzalloc(&pdev->dev, sizeof(*hdev), GFP_KERNEL); + if (!hdev) + return -ENOMEM; + hdev->acpi_dev = acpi_dev; + hdev->dev = &pdev->dev; + platform_set_drvdata(pdev, hdev); + + /* + * Here would never be failure as the driver and device has been matched. + */ + hdev->verspec_data = acpi_device_get_match_data(hdev->dev); + + mutex_init(&hdev->lock); + rc = hccs_get_pcc_chan_id(hdev); + if (rc) + return rc; + rc = hccs_register_pcc_channel(hdev); + if (rc) + return rc; + + rc = hccs_get_dev_caps(hdev); + if (rc) + goto unregister_pcc_chan; + + rc = hccs_get_hw_info(hdev); + if (rc) + goto unregister_pcc_chan; + + rc = hccs_create_topo_dirs(hdev); + if (rc) + goto unregister_pcc_chan; + + return 0; + +unregister_pcc_chan: + hccs_unregister_pcc_channel(hdev); + + return rc; +} + +static int hccs_remove(struct platform_device *pdev) +{ + struct hccs_dev *hdev = platform_get_drvdata(pdev); + + hccs_remove_topo_dirs(hdev); + hccs_unregister_pcc_channel(hdev); + + return 0; +} + +static const struct hccs_verspecific_data hisi04b1_verspec_data = { + .rx_callback = NULL, + .wait_cmd_complete = hccs_wait_cmd_complete_by_poll, + .fill_pcc_shared_mem = hccs_fill_pcc_shared_mem_region, + .shared_mem_size = sizeof(struct acpi_pcct_shared_memory), + .has_txdone_irq = false, +}; + +static const struct hccs_verspecific_data hisi04b2_verspec_data = { + .rx_callback = hccs_pcc_rx_callback, + .wait_cmd_complete = hccs_wait_cmd_complete_by_irq, + .fill_pcc_shared_mem = hccs_fill_ext_pcc_shared_mem_region, + .shared_mem_size = sizeof(struct acpi_pcct_ext_pcc_shared_memory), + .has_txdone_irq = true, +}; + +static const struct acpi_device_id hccs_acpi_match[] = { + { "HISI04B1", (unsigned long)&hisi04b1_verspec_data}, + { "HISI04B2", (unsigned long)&hisi04b2_verspec_data}, + { } +}; +MODULE_DEVICE_TABLE(acpi, hccs_acpi_match); + +static struct platform_driver hccs_driver = { + .probe = hccs_probe, + .remove = hccs_remove, + .driver = { + .name = "kunpeng_hccs", + .acpi_match_table = hccs_acpi_match, + }, +}; + +module_platform_driver(hccs_driver); + +MODULE_DESCRIPTION("Kunpeng SoC HCCS driver"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Huisong Li "); diff --git a/drivers/soc/hisilicon/kunpeng_hccs.h b/drivers/soc/hisilicon/kunpeng_hccs.h new file mode 100644 index 0000000000000000000000000000000000000000..c3adbc01b4719b7cb3ec866698c2ba1f2bbf079a --- /dev/null +++ b/drivers/soc/hisilicon/kunpeng_hccs.h @@ -0,0 +1,206 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright (c) 2023 Hisilicon Limited. */ + +#ifndef __KUNPENG_HCCS_H__ +#define __KUNPENG_HCCS_H__ + +/* + * |--------------- Chip0 ---------------|---------------- ChipN -------------| + * |--------Die0-------|--------DieN-------|--------Die0-------|-------DieN-------| + * | P0 | P1 | P2 | P3 | P0 | P1 | P2 | P3 | P0 | P1 | P2 | P3 |P0 | P1 | P2 | P3 | + */ + +/* + * This value cannot be 255, otherwise the loop of the multi-BD communication + * case cannot end. + */ +#define HCCS_DIE_MAX_PORT_ID 254 + +struct hccs_port_info { + u8 port_id; + u8 port_type; + u8 lane_mode; + bool enable; /* if the port is enabled */ + struct kobject kobj; + bool dir_created; + struct hccs_die_info *die; /* point to the die the port is located */ +}; + +struct hccs_die_info { + u8 die_id; + u8 port_num; + u8 min_port_id; + u8 max_port_id; + struct hccs_port_info *ports; + struct kobject kobj; + bool dir_created; + struct hccs_chip_info *chip; /* point to the chip the die is located */ +}; + +struct hccs_chip_info { + u8 chip_id; + u8 die_num; + struct hccs_die_info *dies; + struct kobject kobj; + struct hccs_dev *hdev; +}; + +struct hccs_mbox_client_info { + struct mbox_client client; + struct mbox_chan *mbox_chan; + struct pcc_mbox_chan *pcc_chan; + u64 deadline_us; + void __iomem *pcc_comm_addr; + struct completion done; +}; + +struct hccs_desc; + +struct hccs_verspecific_data { + void (*rx_callback)(struct mbox_client *cl, void *mssg); + int (*wait_cmd_complete)(struct hccs_dev *hdev); + void (*fill_pcc_shared_mem)(struct hccs_dev *hdev, + u8 cmd, struct hccs_desc *desc, + void __iomem *comm_space, + u16 space_size); + u16 shared_mem_size; + bool has_txdone_irq; +}; + +struct hccs_dev { + struct device *dev; + struct acpi_device *acpi_dev; + const struct hccs_verspecific_data *verspec_data; + u64 caps; + u8 chip_num; + struct hccs_chip_info *chips; + u8 chan_id; + struct mutex lock; + struct hccs_mbox_client_info cl_info; +}; + +#define HCCS_SERDES_MODULE_CODE 0x32 +enum hccs_subcmd_type { + HCCS_GET_CHIP_NUM = 0x1, + HCCS_GET_DIE_NUM, + HCCS_GET_DIE_INFO, + HCCS_GET_DIE_PORT_INFO, + HCCS_GET_DEV_CAP, + HCCS_GET_PORT_LINK_STATUS, + HCCS_GET_PORT_CRC_ERR_CNT, + HCCS_GET_DIE_PORTS_LANE_STA, + HCCS_GET_DIE_PORTS_LINK_STA, + HCCS_GET_DIE_PORTS_CRC_ERR_CNT, + HCCS_SUB_CMD_MAX = 255, +}; + +struct hccs_die_num_req_param { + u8 chip_id; +}; + +struct hccs_die_info_req_param { + u8 chip_id; + u8 die_idx; +}; + +struct hccs_die_info_rsp_data { + u8 die_id; + u8 port_num; + u8 min_port_id; + u8 max_port_id; +}; + +struct hccs_port_attr { + u8 port_id; + u8 port_type; + u8 lane_mode; + u8 enable : 1; /* if the port is enabled */ + u16 rsv[2]; +}; + +/* + * The common command request for getting the information of all HCCS port on + * specified DIE. + */ +struct hccs_die_comm_req_param { + u8 chip_id; + u8 die_id; /* id in hardware */ +}; + +/* The common command request for getting the information of a specific port */ +struct hccs_port_comm_req_param { + u8 chip_id; + u8 die_id; + u8 port_id; +}; + +#define HCCS_PORT_RESET 1 +#define HCCS_PORT_SETUP 2 +#define HCCS_PORT_CONFIG 3 +#define HCCS_PORT_READY 4 +struct hccs_link_status { + u8 lane_mask; /* indicate which lanes are used. */ + u8 link_fsm : 3; /* link fsm, 1: reset 2: setup 3: config 4: link-up */ + u8 lane_num : 5; /* current lane number */ +}; + +struct hccs_req_head { + u8 module_code; /* set to 0x32 for serdes */ + u8 start_id; + u8 rsv[2]; +}; + +struct hccs_rsp_head { + u8 data_len; + u8 next_id; + u8 rsv[2]; +}; + +struct hccs_fw_inner_head { + u8 retStatus; /* 0: success, other: failure */ + u8 rsv[7]; +}; + +#define HCCS_PCC_SHARE_MEM_BYTES 64 +#define HCCS_FW_INNER_HEAD_BYTES 8 +#define HCCS_RSP_HEAD_BYTES 4 + +#define HCCS_MAX_RSP_DATA_BYTES (HCCS_PCC_SHARE_MEM_BYTES - \ + HCCS_FW_INNER_HEAD_BYTES - \ + HCCS_RSP_HEAD_BYTES) +#define HCCS_MAX_RSP_DATA_SIZE_MAX (HCCS_MAX_RSP_DATA_BYTES / 4) + +/* + * Note: Actual available size of data field also depands on the PCC header + * bytes of the specific type. Driver needs to copy the response data in the + * communication space based on the real length. + */ +struct hccs_rsp_desc { + struct hccs_fw_inner_head fw_inner_head; /* 8 Bytes */ + struct hccs_rsp_head rsp_head; /* 4 Bytes */ + u32 data[HCCS_MAX_RSP_DATA_SIZE_MAX]; +}; + +#define HCCS_REQ_HEAD_BYTES 4 +#define HCCS_MAX_REQ_DATA_BYTES (HCCS_PCC_SHARE_MEM_BYTES - \ + HCCS_REQ_HEAD_BYTES) +#define HCCS_MAX_REQ_DATA_SIZE_MAX (HCCS_MAX_REQ_DATA_BYTES / 4) + +/* + * Note: Actual available size of data field also depands on the PCC header + * bytes of the specific type. Driver needs to copy the request data to the + * communication space based on the real length. + */ +struct hccs_req_desc { + struct hccs_req_head req_head; /* 4 Bytes */ + u32 data[HCCS_MAX_REQ_DATA_SIZE_MAX]; +}; + +struct hccs_desc { + union { + struct hccs_req_desc req; + struct hccs_rsp_desc rsp; + }; +}; + +#endif /* __KUNPENG_HCCS_H__ */ diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig index 5bf7542087776410ac16a4f53e5c53cec1b2b37c..27a0c642a03294c2b588dec6f057a6fa9161b3e0 100644 --- a/drivers/spi/Kconfig +++ b/drivers/spi/Kconfig @@ -280,6 +280,25 @@ config SPI_FSL_QUADSPI This controller does not support generic SPI messages. It only supports the high-level SPI memory interface. +config SPI_HISI_KUNPENG + tristate "HiSilicon SPI Controller for Kunpeng SoCs" + depends on (ARM64 && ACPI) || COMPILE_TEST + help + This enables support for HiSilicon SPI controller found on + Kunpeng SoCs. + + This driver can also be built as a module. If so, the module + will be called hisi-kunpeng-spi. + +config SPI_HISI_SFC_V3XX + tristate "HiSilicon SPI-NOR Flash Controller for Hi16XX chipsets" + depends on (ARM64 && ACPI) || COMPILE_TEST + depends on HAS_IOMEM + select MTD_SPI_NOR + help + This enables support for HiSilicon v3xx SPI-NOR flash controller + found in hi16xx chipsets. + config SPI_NXP_FLEXSPI tristate "NXP Flex SPI controller" depends on ARCH_LAYERSCAPE || HAS_IOMEM diff --git a/drivers/spi/Makefile b/drivers/spi/Makefile index bb49c9e6d0a0c00cddf2275a84cb74fdc08ddf74..51c1bbdb132c8288268b062e55c210a724f689b3 100644 --- a/drivers/spi/Makefile +++ b/drivers/spi/Makefile @@ -48,6 +48,8 @@ obj-$(CONFIG_SPI_FSL_LPSPI) += spi-fsl-lpspi.o obj-$(CONFIG_SPI_FSL_QUADSPI) += spi-fsl-qspi.o obj-$(CONFIG_SPI_FSL_SPI) += spi-fsl-spi.o obj-$(CONFIG_SPI_GPIO) += spi-gpio.o +obj-$(CONFIG_SPI_HISI_KUNPENG) += spi-hisi-kunpeng.o +obj-$(CONFIG_SPI_HISI_SFC_V3XX) += spi-hisi-sfc-v3xx.o obj-$(CONFIG_SPI_IMG_SPFI) += spi-img-spfi.o obj-$(CONFIG_SPI_IMX) += spi-imx.o obj-$(CONFIG_SPI_LANTIQ_SSC) += spi-lantiq-ssc.o diff --git a/drivers/spi/spi-hisi-kunpeng.c b/drivers/spi/spi-hisi-kunpeng.c new file mode 100644 index 0000000000000000000000000000000000000000..525cc0143a3050ec7a592aea4ea1eb75097277ad --- /dev/null +++ b/drivers/spi/spi-hisi-kunpeng.c @@ -0,0 +1,555 @@ +// SPDX-License-Identifier: GPL-2.0-only +// +// HiSilicon SPI Controller Driver for Kunpeng SoCs +// +// Copyright (c) 2021 HiSilicon Technologies Co., Ltd. +// Author: Jay Fang +// +// This code is based on spi-dw-core.c. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Register offsets */ +#define HISI_SPI_CSCR 0x00 /* cs control register */ +#define HISI_SPI_CR 0x04 /* spi common control register */ +#define HISI_SPI_ENR 0x08 /* spi enable register */ +#define HISI_SPI_FIFOC 0x0c /* fifo level control register */ +#define HISI_SPI_IMR 0x10 /* interrupt mask register */ +#define HISI_SPI_DIN 0x14 /* data in register */ +#define HISI_SPI_DOUT 0x18 /* data out register */ +#define HISI_SPI_SR 0x1c /* status register */ +#define HISI_SPI_RISR 0x20 /* raw interrupt status register */ +#define HISI_SPI_ISR 0x24 /* interrupt status register */ +#define HISI_SPI_ICR 0x28 /* interrupt clear register */ +#define HISI_SPI_VERSION 0xe0 /* version register */ + +/* Bit fields in HISI_SPI_CR */ +#define CR_LOOP_MASK GENMASK(1, 1) +#define CR_CPOL_MASK GENMASK(2, 2) +#define CR_CPHA_MASK GENMASK(3, 3) +#define CR_DIV_PRE_MASK GENMASK(11, 4) +#define CR_DIV_POST_MASK GENMASK(19, 12) +#define CR_BPW_MASK GENMASK(24, 20) +#define CR_SPD_MODE_MASK GENMASK(25, 25) + +/* Bit fields in HISI_SPI_FIFOC */ +#define FIFOC_TX_MASK GENMASK(5, 3) +#define FIFOC_RX_MASK GENMASK(11, 9) + +/* Bit fields in HISI_SPI_IMR, 4 bits */ +#define IMR_RXOF BIT(0) /* Receive Overflow */ +#define IMR_RXTO BIT(1) /* Receive Timeout */ +#define IMR_RX BIT(2) /* Receive */ +#define IMR_TX BIT(3) /* Transmit */ +#define IMR_MASK (IMR_RXOF | IMR_RXTO | IMR_RX | IMR_TX) + +/* Bit fields in HISI_SPI_SR, 5 bits */ +#define SR_TXE BIT(0) /* Transmit FIFO empty */ +#define SR_TXNF BIT(1) /* Transmit FIFO not full */ +#define SR_RXNE BIT(2) /* Receive FIFO not empty */ +#define SR_RXF BIT(3) /* Receive FIFO full */ +#define SR_BUSY BIT(4) /* Busy Flag */ + +/* Bit fields in HISI_SPI_ISR, 4 bits */ +#define ISR_RXOF BIT(0) /* Receive Overflow */ +#define ISR_RXTO BIT(1) /* Receive Timeout */ +#define ISR_RX BIT(2) /* Receive */ +#define ISR_TX BIT(3) /* Transmit */ +#define ISR_MASK (ISR_RXOF | ISR_RXTO | ISR_RX | ISR_TX) + +/* Bit fields in HISI_SPI_ICR, 2 bits */ +#define ICR_RXOF BIT(0) /* Receive Overflow */ +#define ICR_RXTO BIT(1) /* Receive Timeout */ +#define ICR_MASK (ICR_RXOF | ICR_RXTO) + +#define DIV_POST_MAX 0xFF +#define DIV_POST_MIN 0x00 +#define DIV_PRE_MAX 0xFE +#define DIV_PRE_MIN 0x02 +#define CLK_DIV_MAX ((1 + DIV_POST_MAX) * DIV_PRE_MAX) +#define CLK_DIV_MIN ((1 + DIV_POST_MIN) * DIV_PRE_MIN) + +#define DEFAULT_NUM_CS 1 + +#define HISI_SPI_WAIT_TIMEOUT_MS 10UL + +enum hisi_spi_rx_level_trig { + HISI_SPI_RX_1, + HISI_SPI_RX_4, + HISI_SPI_RX_8, + HISI_SPI_RX_16, + HISI_SPI_RX_32, + HISI_SPI_RX_64, + HISI_SPI_RX_128 +}; + +enum hisi_spi_tx_level_trig { + HISI_SPI_TX_1_OR_LESS, + HISI_SPI_TX_4_OR_LESS, + HISI_SPI_TX_8_OR_LESS, + HISI_SPI_TX_16_OR_LESS, + HISI_SPI_TX_32_OR_LESS, + HISI_SPI_TX_64_OR_LESS, + HISI_SPI_TX_128_OR_LESS +}; + +enum hisi_spi_frame_n_bytes { + HISI_SPI_N_BYTES_NULL, + HISI_SPI_N_BYTES_U8, + HISI_SPI_N_BYTES_U16, + HISI_SPI_N_BYTES_U32 = 4 +}; + +/* Slave spi_dev related */ +struct hisi_chip_data { + u32 cr; + u32 speed_hz; /* baud rate */ + u16 clk_div; /* baud rate divider */ + + /* clk_div = (1 + div_post) * div_pre */ + u8 div_post; /* value from 0 to 255 */ + u8 div_pre; /* value from 2 to 254 (even only!) */ +}; + +struct hisi_spi { + struct device *dev; + + void __iomem *regs; + int irq; + u32 fifo_len; /* depth of the FIFO buffer */ + + /* Current message transfer state info */ + const void *tx; + unsigned int tx_len; + void *rx; + unsigned int rx_len; + u8 n_bytes; /* current is a 1/2/4 bytes op */ + + struct dentry *debugfs; + struct debugfs_regset32 regset; +}; + +#define HISI_SPI_DBGFS_REG(_name, _off) \ +{ \ + .name = _name, \ + .offset = _off, \ +} + +static const struct debugfs_reg32 hisi_spi_regs[] = { + HISI_SPI_DBGFS_REG("CSCR", HISI_SPI_CSCR), + HISI_SPI_DBGFS_REG("CR", HISI_SPI_CR), + HISI_SPI_DBGFS_REG("ENR", HISI_SPI_ENR), + HISI_SPI_DBGFS_REG("FIFOC", HISI_SPI_FIFOC), + HISI_SPI_DBGFS_REG("IMR", HISI_SPI_IMR), + HISI_SPI_DBGFS_REG("DIN", HISI_SPI_DIN), + HISI_SPI_DBGFS_REG("DOUT", HISI_SPI_DOUT), + HISI_SPI_DBGFS_REG("SR", HISI_SPI_SR), + HISI_SPI_DBGFS_REG("RISR", HISI_SPI_RISR), + HISI_SPI_DBGFS_REG("ISR", HISI_SPI_ISR), + HISI_SPI_DBGFS_REG("ICR", HISI_SPI_ICR), + HISI_SPI_DBGFS_REG("VERSION", HISI_SPI_VERSION), +}; + +static int hisi_spi_debugfs_init(struct hisi_spi *hs) +{ + char name[32]; + + struct spi_controller *master; + + master = container_of(hs->dev, struct spi_controller, dev); + snprintf(name, 32, "hisi_spi%d", master->bus_num); + hs->debugfs = debugfs_create_dir(name, NULL); + if (!hs->debugfs) + return -ENOMEM; + + hs->regset.regs = hisi_spi_regs; + hs->regset.nregs = ARRAY_SIZE(hisi_spi_regs); + hs->regset.base = hs->regs; + debugfs_create_regset32("registers", 0400, hs->debugfs, &hs->regset); + + return 0; +} + +static u32 hisi_spi_busy(struct hisi_spi *hs) +{ + return readl(hs->regs + HISI_SPI_SR) & SR_BUSY; +} + +static u32 hisi_spi_rx_not_empty(struct hisi_spi *hs) +{ + return readl(hs->regs + HISI_SPI_SR) & SR_RXNE; +} + +static u32 hisi_spi_tx_not_full(struct hisi_spi *hs) +{ + return readl(hs->regs + HISI_SPI_SR) & SR_TXNF; +} + +static void hisi_spi_flush_fifo(struct hisi_spi *hs) +{ + unsigned long limit = loops_per_jiffy << 1; + + do { + while (hisi_spi_rx_not_empty(hs)) + readl(hs->regs + HISI_SPI_DOUT); + } while (hisi_spi_busy(hs) && limit--); +} + +/* Disable the controller and all interrupts */ +static void hisi_spi_disable(struct hisi_spi *hs) +{ + writel(0, hs->regs + HISI_SPI_ENR); + writel(IMR_MASK, hs->regs + HISI_SPI_IMR); + writel(ICR_MASK, hs->regs + HISI_SPI_ICR); +} + +static u8 hisi_spi_n_bytes(struct spi_transfer *transfer) +{ + if (transfer->bits_per_word <= 8) + return HISI_SPI_N_BYTES_U8; + else if (transfer->bits_per_word <= 16) + return HISI_SPI_N_BYTES_U16; + else + return HISI_SPI_N_BYTES_U32; +} + +static void hisi_spi_reader(struct hisi_spi *hs) +{ + u32 max = min_t(u32, hs->rx_len, hs->fifo_len); + u32 rxw; + + while (hisi_spi_rx_not_empty(hs) && max--) { + rxw = readl(hs->regs + HISI_SPI_DOUT); + /* Check the transfer's original "rx" is not null */ + if (hs->rx) { + switch (hs->n_bytes) { + case HISI_SPI_N_BYTES_U8: + *(u8 *)(hs->rx) = rxw; + break; + case HISI_SPI_N_BYTES_U16: + *(u16 *)(hs->rx) = rxw; + break; + case HISI_SPI_N_BYTES_U32: + *(u32 *)(hs->rx) = rxw; + break; + } + hs->rx += hs->n_bytes; + } + --hs->rx_len; + } +} + +static void hisi_spi_writer(struct hisi_spi *hs) +{ + u32 max = min_t(u32, hs->tx_len, hs->fifo_len); + u32 txw = 0; + + while (hisi_spi_tx_not_full(hs) && max--) { + /* Check the transfer's original "tx" is not null */ + if (hs->tx) { + switch (hs->n_bytes) { + case HISI_SPI_N_BYTES_U8: + txw = *(u8 *)(hs->tx); + break; + case HISI_SPI_N_BYTES_U16: + txw = *(u16 *)(hs->tx); + break; + case HISI_SPI_N_BYTES_U32: + txw = *(u32 *)(hs->tx); + break; + } + hs->tx += hs->n_bytes; + } + writel(txw, hs->regs + HISI_SPI_DIN); + --hs->tx_len; + } +} + +static void __hisi_calc_div_reg(struct hisi_chip_data *chip) +{ + chip->div_pre = DIV_PRE_MAX; + while (chip->div_pre >= DIV_PRE_MIN) { + if (chip->clk_div % chip->div_pre == 0) + break; + + chip->div_pre -= 2; + } + + if (chip->div_pre > chip->clk_div) + chip->div_pre = chip->clk_div; + + chip->div_post = (chip->clk_div / chip->div_pre) - 1; +} + +static u32 hisi_calc_effective_speed(struct spi_controller *master, + struct hisi_chip_data *chip, u32 speed_hz) +{ + u32 effective_speed; + + /* Note clock divider doesn't support odd numbers */ + chip->clk_div = DIV_ROUND_UP(master->max_speed_hz, speed_hz) + 1; + chip->clk_div &= 0xfffe; + if (chip->clk_div > CLK_DIV_MAX) + chip->clk_div = CLK_DIV_MAX; + + effective_speed = master->max_speed_hz / chip->clk_div; + if (chip->speed_hz != effective_speed) { + __hisi_calc_div_reg(chip); + chip->speed_hz = effective_speed; + } + + return effective_speed; +} + +static u32 hisi_spi_prepare_cr(struct spi_device *spi) +{ + u32 cr = FIELD_PREP(CR_SPD_MODE_MASK, 1); + + cr |= FIELD_PREP(CR_CPHA_MASK, (spi->mode & SPI_CPHA) ? 1 : 0); + cr |= FIELD_PREP(CR_CPOL_MASK, (spi->mode & SPI_CPOL) ? 1 : 0); + cr |= FIELD_PREP(CR_LOOP_MASK, (spi->mode & SPI_LOOP) ? 1 : 0); + + return cr; +} + +static void hisi_spi_hw_init(struct hisi_spi *hs) +{ + hisi_spi_disable(hs); + + /* FIFO default config */ + writel(FIELD_PREP(FIFOC_TX_MASK, HISI_SPI_TX_64_OR_LESS) | + FIELD_PREP(FIFOC_RX_MASK, HISI_SPI_RX_16), + hs->regs + HISI_SPI_FIFOC); + + hs->fifo_len = 256; +} + +static irqreturn_t hisi_spi_irq(int irq, void *dev_id) +{ + struct spi_controller *master = dev_id; + struct hisi_spi *hs = spi_controller_get_devdata(master); + u32 irq_status = readl(hs->regs + HISI_SPI_ISR) & ISR_MASK; + + if (!irq_status) + return IRQ_NONE; + + if (!master->cur_msg) + return IRQ_HANDLED; + + /* Error handling */ + if (irq_status & ISR_RXOF) { + dev_err(hs->dev, "interrupt_transfer: fifo overflow\n"); + master->cur_msg->status = -EIO; + goto finalize_transfer; + } + + /* + * Read data from the Rx FIFO every time. If there is + * nothing left to receive, finalize the transfer. + */ + hisi_spi_reader(hs); + if (!hs->rx_len) + goto finalize_transfer; + + /* Send data out when Tx FIFO IRQ triggered */ + if (irq_status & ISR_TX) + hisi_spi_writer(hs); + + return IRQ_HANDLED; + +finalize_transfer: + hisi_spi_disable(hs); + spi_finalize_current_transfer(master); + return IRQ_HANDLED; +} + +static int hisi_spi_transfer_one(struct spi_controller *master, + struct spi_device *spi, struct spi_transfer *transfer) +{ + struct hisi_spi *hs = spi_controller_get_devdata(master); + struct hisi_chip_data *chip = spi_get_ctldata(spi); + u32 cr = chip->cr; + + /* Update per transfer options for speed and bpw */ + transfer->effective_speed_hz = + hisi_calc_effective_speed(master, chip, transfer->speed_hz); + cr |= FIELD_PREP(CR_DIV_PRE_MASK, chip->div_pre); + cr |= FIELD_PREP(CR_DIV_POST_MASK, chip->div_post); + cr |= FIELD_PREP(CR_BPW_MASK, transfer->bits_per_word - 1); + writel(cr, hs->regs + HISI_SPI_CR); + + hisi_spi_flush_fifo(hs); + + hs->n_bytes = hisi_spi_n_bytes(transfer); + hs->tx = transfer->tx_buf; + hs->tx_len = transfer->len / hs->n_bytes; + hs->rx = transfer->rx_buf; + hs->rx_len = hs->tx_len; + + /* + * Ensure that the transfer data above has been updated + * before the interrupt to start. + */ + smp_mb(); + + /* Enable all interrupts and the controller */ + writel(~(u32)IMR_MASK, hs->regs + HISI_SPI_IMR); + writel(1, hs->regs + HISI_SPI_ENR); + + return 1; +} + +static void hisi_spi_handle_err(struct spi_controller *master, + struct spi_message *msg) +{ + struct hisi_spi *hs = spi_controller_get_devdata(master); + + hisi_spi_disable(hs); + + /* + * Wait for interrupt handler that is + * already in timeout to complete. + */ + msleep(HISI_SPI_WAIT_TIMEOUT_MS); +} + +static int hisi_spi_setup(struct spi_device *spi) +{ + struct hisi_chip_data *chip; + + /* Only alloc on first setup */ + chip = spi_get_ctldata(spi); + if (!chip) { + chip = kzalloc(sizeof(*chip), GFP_KERNEL); + if (!chip) + return -ENOMEM; + spi_set_ctldata(spi, chip); + } + + chip->cr = hisi_spi_prepare_cr(spi); + + return 0; +} + +static void hisi_spi_cleanup(struct spi_device *spi) +{ + struct hisi_chip_data *chip = spi_get_ctldata(spi); + + kfree(chip); + spi_set_ctldata(spi, NULL); +} + +static int hisi_spi_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct spi_controller *master; + struct hisi_spi *hs; + int ret, irq; + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + master = devm_spi_alloc_master(dev, sizeof(*hs)); + if (!master) + return -ENOMEM; + + platform_set_drvdata(pdev, master); + + hs = spi_controller_get_devdata(master); + hs->dev = dev; + hs->irq = irq; + + hs->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(hs->regs)) + return PTR_ERR(hs->regs); + + /* Specify maximum SPI clocking speed (master only) by firmware */ + ret = device_property_read_u32(dev, "spi-max-frequency", + &master->max_speed_hz); + if (ret) { + dev_err(dev, "failed to get max SPI clocking speed, ret=%d\n", + ret); + return -EINVAL; + } + + ret = device_property_read_u16(dev, "num-cs", + &master->num_chipselect); + if (ret) + master->num_chipselect = DEFAULT_NUM_CS; + + master->use_gpio_descriptors = true; + master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH | SPI_LOOP; + master->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 32); + master->bus_num = pdev->id; + master->setup = hisi_spi_setup; + master->cleanup = hisi_spi_cleanup; + master->transfer_one = hisi_spi_transfer_one; + master->handle_err = hisi_spi_handle_err; + master->dev.fwnode = dev->fwnode; + + hisi_spi_hw_init(hs); + + ret = devm_request_irq(dev, hs->irq, hisi_spi_irq, 0, dev_name(dev), + master); + if (ret < 0) { + dev_err(dev, "failed to get IRQ=%d, ret=%d\n", hs->irq, ret); + return ret; + } + + ret = spi_register_controller(master); + if (ret) { + dev_err(dev, "failed to register spi master, ret=%d\n", ret); + return ret; + } + + if (hisi_spi_debugfs_init(hs)) + dev_info(dev, "failed to create debugfs dir\n"); + + dev_info(dev, "hw version:0x%x max-freq:%u kHz\n", + readl(hs->regs + HISI_SPI_VERSION), + master->max_speed_hz / 1000); + + return 0; +} + +static int hisi_spi_remove(struct platform_device *pdev) +{ + struct spi_controller *master = platform_get_drvdata(pdev); + struct hisi_spi *hs = spi_controller_get_devdata(master); + + debugfs_remove_recursive(hs->debugfs); + spi_unregister_controller(master); + + return 0; +} + +static const struct acpi_device_id hisi_spi_acpi_match[] = { + {"HISI03E1", 0}, + {} +}; +MODULE_DEVICE_TABLE(acpi, hisi_spi_acpi_match); + +static struct platform_driver hisi_spi_driver = { + .probe = hisi_spi_probe, + .remove = hisi_spi_remove, + .driver = { + .name = "hisi-kunpeng-spi", + .acpi_match_table = hisi_spi_acpi_match, + }, +}; +module_platform_driver(hisi_spi_driver); + +MODULE_AUTHOR("Jay Fang "); +MODULE_DESCRIPTION("HiSilicon SPI Controller Driver for Kunpeng SoCs"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/spi/spi-hisi-sfc-v3xx.c b/drivers/spi/spi-hisi-sfc-v3xx.c new file mode 100644 index 0000000000000000000000000000000000000000..61bf00dfe9c33ee7b4e98a5c7850096a45de326a --- /dev/null +++ b/drivers/spi/spi-hisi-sfc-v3xx.c @@ -0,0 +1,547 @@ +// SPDX-License-Identifier: GPL-2.0-only +// +// HiSilicon SPI NOR V3XX Flash Controller Driver for hi16xx chipsets +// +// Copyright (c) 2019 HiSilicon Technologies Co., Ltd. +// Author: John Garry + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define HISI_SFC_V3XX_VERSION (0x1f8) + +#define HISI_SFC_V3XX_GLB_CFG (0x100) +#define HISI_SFC_V3XX_GLB_CFG_CS0_ADDR_MODE BIT(2) +#define HISI_SFC_V3XX_RAW_INT_STAT (0x120) +#define HISI_SFC_V3XX_INT_STAT (0x124) +#define HISI_SFC_V3XX_INT_MASK (0x128) +#define HISI_SFC_V3XX_INT_CLR (0x12c) +#define HISI_SFC_V3XX_CMD_CFG (0x300) +#define HISI_SFC_V3XX_CMD_CFG_DATA_CNT_OFF 9 +#define HISI_SFC_V3XX_CMD_CFG_RW_MSK BIT(8) +#define HISI_SFC_V3XX_CMD_CFG_DATA_EN_MSK BIT(7) +#define HISI_SFC_V3XX_CMD_CFG_DUMMY_CNT_OFF 4 +#define HISI_SFC_V3XX_CMD_CFG_ADDR_EN_MSK BIT(3) +#define HISI_SFC_V3XX_CMD_CFG_CS_SEL_OFF 1 +#define HISI_SFC_V3XX_CMD_CFG_START_MSK BIT(0) +#define HISI_SFC_V3XX_CMD_INS (0x308) +#define HISI_SFC_V3XX_CMD_ADDR (0x30c) +#define HISI_SFC_V3XX_CMD_DATABUF0 (0x400) + +/* Common definition of interrupt bit masks */ +#define HISI_SFC_V3XX_INT_MASK_ALL (0x1ff) /* all the masks */ +#define HISI_SFC_V3XX_INT_MASK_CPLT BIT(0) /* command execution complete */ +#define HISI_SFC_V3XX_INT_MASK_PP_ERR BIT(2) /* page progrom error */ +#define HISI_SFC_V3XX_INT_MASK_IACCES BIT(5) /* error visiting inaccessible/ + * protected address + */ + +/* IO Mode definition in HISI_SFC_V3XX_CMD_CFG */ +#define HISI_SFC_V3XX_STD (0 << 17) +#define HISI_SFC_V3XX_DIDO (1 << 17) +#define HISI_SFC_V3XX_DIO (2 << 17) +#define HISI_SFC_V3XX_FULL_DIO (3 << 17) +#define HISI_SFC_V3XX_QIQO (5 << 17) +#define HISI_SFC_V3XX_QIO (6 << 17) +#define HISI_SFC_V3XX_FULL_QIO (7 << 17) + +/* + * The IO modes lookup table. hisi_sfc_v3xx_io_modes[(z - 1) / 2][y / 2][x / 2] + * stands for x-y-z mode, as described in SFDP terminology. -EIO indicates + * an invalid mode. + */ +static const int hisi_sfc_v3xx_io_modes[2][3][3] = { + { + { HISI_SFC_V3XX_DIDO, HISI_SFC_V3XX_DIDO, HISI_SFC_V3XX_DIDO }, + { HISI_SFC_V3XX_DIO, HISI_SFC_V3XX_FULL_DIO, -EIO }, + { -EIO, -EIO, -EIO }, + }, + { + { HISI_SFC_V3XX_QIQO, HISI_SFC_V3XX_QIQO, HISI_SFC_V3XX_QIQO }, + { -EIO, -EIO, -EIO }, + { HISI_SFC_V3XX_QIO, -EIO, HISI_SFC_V3XX_FULL_QIO }, + }, +}; + +struct hisi_sfc_v3xx_host { + struct device *dev; + void __iomem *regbase; + int max_cmd_dword; + struct completion *completion; + u8 address_mode; + int irq; +}; + +static void hisi_sfc_v3xx_disable_int(struct hisi_sfc_v3xx_host *host) +{ + writel(0, host->regbase + HISI_SFC_V3XX_INT_MASK); +} + +static void hisi_sfc_v3xx_enable_int(struct hisi_sfc_v3xx_host *host) +{ + writel(HISI_SFC_V3XX_INT_MASK_ALL, host->regbase + HISI_SFC_V3XX_INT_MASK); +} + +static void hisi_sfc_v3xx_clear_int(struct hisi_sfc_v3xx_host *host) +{ + writel(HISI_SFC_V3XX_INT_MASK_ALL, host->regbase + HISI_SFC_V3XX_INT_CLR); +} + +/* + * The interrupt status register indicates whether an error occurs + * after per operation. Check it, and clear the interrupts for + * next time judgement. + */ +static int hisi_sfc_v3xx_handle_completion(struct hisi_sfc_v3xx_host *host) +{ + u32 reg; + + reg = readl(host->regbase + HISI_SFC_V3XX_RAW_INT_STAT); + hisi_sfc_v3xx_clear_int(host); + + if (reg & HISI_SFC_V3XX_INT_MASK_IACCES) { + dev_err(host->dev, "fail to access protected address\n"); + return -EIO; + } + + if (reg & HISI_SFC_V3XX_INT_MASK_PP_ERR) { + dev_err(host->dev, "page program operation failed\n"); + return -EIO; + } + + /* + * The other bits of the interrupt registers is not currently + * used and probably not be triggered in this driver. When it + * happens, we regard it as an unsupported error here. + */ + if (!(reg & HISI_SFC_V3XX_INT_MASK_CPLT)) { + dev_err(host->dev, "unsupported error occurred, status=0x%x\n", reg); + return -EIO; + } + + return 0; +} + +#define HISI_SFC_V3XX_WAIT_TIMEOUT_US 1000000 +#define HISI_SFC_V3XX_WAIT_POLL_INTERVAL_US 10 + +static int hisi_sfc_v3xx_wait_cmd_idle(struct hisi_sfc_v3xx_host *host) +{ + u32 reg; + + return readl_poll_timeout(host->regbase + HISI_SFC_V3XX_CMD_CFG, reg, + !(reg & HISI_SFC_V3XX_CMD_CFG_START_MSK), + HISI_SFC_V3XX_WAIT_POLL_INTERVAL_US, + HISI_SFC_V3XX_WAIT_TIMEOUT_US); +} + +static int hisi_sfc_v3xx_adjust_op_size(struct spi_mem *mem, + struct spi_mem_op *op) +{ + struct spi_device *spi = mem->spi; + struct hisi_sfc_v3xx_host *host; + uintptr_t addr = (uintptr_t)op->data.buf.in; + int max_byte_count; + + host = spi_controller_get_devdata(spi->master); + + max_byte_count = host->max_cmd_dword * 4; + + if (!IS_ALIGNED(addr, 4) && op->data.nbytes >= 4) + op->data.nbytes = 4 - (addr % 4); + else if (op->data.nbytes > max_byte_count) + op->data.nbytes = max_byte_count; + + return 0; +} + +/* + * The controller only supports Standard SPI mode, Duall mode and + * Quad mode. Double sanitize the ops here to avoid OOB access. + */ +static bool hisi_sfc_v3xx_supports_op(struct spi_mem *mem, + const struct spi_mem_op *op) +{ + struct spi_device *spi = mem->spi; + struct hisi_sfc_v3xx_host *host; + + host = spi_controller_get_devdata(spi->master); + + if (op->data.buswidth > 4 || op->dummy.buswidth > 4 || + op->addr.buswidth > 4 || op->cmd.buswidth > 4) + return false; + + if (op->addr.nbytes != host->address_mode && op->addr.nbytes) + return false; + + return spi_mem_default_supports_op(mem, op); +} + +/* + * memcpy_{to,from}io doesn't gurantee 32b accesses - which we require for the + * DATABUF registers -so use __io{read,write}32_copy when possible. For + * trailing bytes, copy them byte-by-byte from the DATABUF register, as we + * can't clobber outside the source/dest buffer. + * + * For efficient data read/write, we try to put any start 32b unaligned data + * into a separate transaction in hisi_sfc_v3xx_adjust_op_size(). + */ +static void hisi_sfc_v3xx_read_databuf(struct hisi_sfc_v3xx_host *host, + u8 *to, unsigned int len) +{ + void __iomem *from; + int i; + + from = host->regbase + HISI_SFC_V3XX_CMD_DATABUF0; + + if (IS_ALIGNED((uintptr_t)to, 4)) { + int words = len / 4; + + __ioread32_copy(to, from, words); + + len -= words * 4; + if (len) { + u32 val; + + to += words * 4; + from += words * 4; + + val = __raw_readl(from); + + for (i = 0; i < len; i++, val >>= 8, to++) + *to = (u8)val; + } + } else { + for (i = 0; i < DIV_ROUND_UP(len, 4); i++, from += 4) { + u32 val = __raw_readl(from); + int j; + + for (j = 0; j < 4 && (j + (i * 4) < len); + to++, val >>= 8, j++) + *to = (u8)val; + } + } +} + +static void hisi_sfc_v3xx_write_databuf(struct hisi_sfc_v3xx_host *host, + const u8 *from, unsigned int len) +{ + void __iomem *to; + int i; + + to = host->regbase + HISI_SFC_V3XX_CMD_DATABUF0; + + if (IS_ALIGNED((uintptr_t)from, 4)) { + int words = len / 4; + + __iowrite32_copy(to, from, words); + + len -= words * 4; + if (len) { + u32 val = 0; + + to += words * 4; + from += words * 4; + + for (i = 0; i < len; i++, from++) + val |= *from << i * 8; + __raw_writel(val, to); + } + + } else { + for (i = 0; i < DIV_ROUND_UP(len, 4); i++, to += 4) { + u32 val = 0; + int j; + + for (j = 0; j < 4 && (j + (i * 4) < len); + from++, j++) + val |= *from << j * 8; + __raw_writel(val, to); + } + } +} + +static int hisi_sfc_v3xx_start_bus(struct hisi_sfc_v3xx_host *host, + const struct spi_mem_op *op, + u8 chip_select) +{ + int len = op->data.nbytes, buswidth_mode; + u32 config = 0; + + if (op->addr.nbytes) + config |= HISI_SFC_V3XX_CMD_CFG_ADDR_EN_MSK; + + if (op->data.buswidth == 0 || op->data.buswidth == 1) { + buswidth_mode = HISI_SFC_V3XX_STD; + } else { + int data_idx, addr_idx, cmd_idx; + + data_idx = (op->data.buswidth - 1) / 2; + addr_idx = op->addr.buswidth / 2; + cmd_idx = op->cmd.buswidth / 2; + buswidth_mode = hisi_sfc_v3xx_io_modes[data_idx][addr_idx][cmd_idx]; + } + if (buswidth_mode < 0) + return buswidth_mode; + config |= buswidth_mode; + + if (op->data.dir != SPI_MEM_NO_DATA) { + config |= (len - 1) << HISI_SFC_V3XX_CMD_CFG_DATA_CNT_OFF; + config |= HISI_SFC_V3XX_CMD_CFG_DATA_EN_MSK; + } + + if (op->data.dir == SPI_MEM_DATA_IN) + config |= HISI_SFC_V3XX_CMD_CFG_RW_MSK; + + config |= op->dummy.nbytes << HISI_SFC_V3XX_CMD_CFG_DUMMY_CNT_OFF | + chip_select << HISI_SFC_V3XX_CMD_CFG_CS_SEL_OFF | + HISI_SFC_V3XX_CMD_CFG_START_MSK; + + writel(op->addr.val, host->regbase + HISI_SFC_V3XX_CMD_ADDR); + writel(op->cmd.opcode, host->regbase + HISI_SFC_V3XX_CMD_INS); + + writel(config, host->regbase + HISI_SFC_V3XX_CMD_CFG); + + return 0; +} + +static int hisi_sfc_v3xx_generic_exec_op(struct hisi_sfc_v3xx_host *host, + const struct spi_mem_op *op, + u8 chip_select) +{ + DECLARE_COMPLETION_ONSTACK(done); + int ret; + + if (host->irq) { + host->completion = &done; + hisi_sfc_v3xx_enable_int(host); + } + + if (op->data.dir == SPI_MEM_DATA_OUT) + hisi_sfc_v3xx_write_databuf(host, op->data.buf.out, op->data.nbytes); + + ret = hisi_sfc_v3xx_start_bus(host, op, chip_select); + if (ret) + return ret; + + if (host->irq) { + ret = wait_for_completion_timeout(host->completion, + usecs_to_jiffies(HISI_SFC_V3XX_WAIT_TIMEOUT_US)); + if (!ret) + ret = -ETIMEDOUT; + else + ret = 0; + + hisi_sfc_v3xx_disable_int(host); + synchronize_irq(host->irq); + host->completion = NULL; + } else { + ret = hisi_sfc_v3xx_wait_cmd_idle(host); + } + if (hisi_sfc_v3xx_handle_completion(host) || ret) + return -EIO; + + if (op->data.dir == SPI_MEM_DATA_IN) + hisi_sfc_v3xx_read_databuf(host, op->data.buf.in, op->data.nbytes); + + return 0; +} + +static int hisi_sfc_v3xx_exec_op(struct spi_mem *mem, + const struct spi_mem_op *op) +{ + struct hisi_sfc_v3xx_host *host; + struct spi_device *spi = mem->spi; + u8 chip_select = spi->chip_select; + + host = spi_controller_get_devdata(spi->master); + + return hisi_sfc_v3xx_generic_exec_op(host, op, chip_select); +} + +static const struct spi_controller_mem_ops hisi_sfc_v3xx_mem_ops = { + .adjust_op_size = hisi_sfc_v3xx_adjust_op_size, + .supports_op = hisi_sfc_v3xx_supports_op, + .exec_op = hisi_sfc_v3xx_exec_op, +}; + +static irqreturn_t hisi_sfc_v3xx_isr(int irq, void *data) +{ + struct hisi_sfc_v3xx_host *host = data; + u32 reg; + + reg = readl(host->regbase + HISI_SFC_V3XX_INT_STAT); + if (!reg) + return IRQ_NONE; + + hisi_sfc_v3xx_disable_int(host); + + complete(host->completion); + + return IRQ_HANDLED; +} + +static int hisi_sfc_v3xx_buswidth_override_bits; + +/* + * ACPI FW does not allow us to currently set the device buswidth, so quirk it + * depending on the board. + */ +static int __init hisi_sfc_v3xx_dmi_quirk(const struct dmi_system_id *d) +{ + hisi_sfc_v3xx_buswidth_override_bits = SPI_RX_QUAD | SPI_TX_QUAD; + + return 0; +} + +static const struct dmi_system_id hisi_sfc_v3xx_dmi_quirk_table[] = { + { + .callback = hisi_sfc_v3xx_dmi_quirk, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Huawei"), + DMI_MATCH(DMI_PRODUCT_NAME, "D06"), + }, + }, + { + .callback = hisi_sfc_v3xx_dmi_quirk, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Huawei"), + DMI_MATCH(DMI_PRODUCT_NAME, "TaiShan 2280 V2"), + }, + }, + { + .callback = hisi_sfc_v3xx_dmi_quirk, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Huawei"), + DMI_MATCH(DMI_PRODUCT_NAME, "TaiShan 200 (Model 2280)"), + }, + }, + {} +}; + +static int hisi_sfc_v3xx_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct hisi_sfc_v3xx_host *host; + struct spi_controller *ctlr; + u32 version, glb_config; + int ret; + + ctlr = spi_alloc_master(&pdev->dev, sizeof(*host)); + if (!ctlr) + return -ENOMEM; + + ctlr->mode_bits = SPI_RX_DUAL | SPI_RX_QUAD | + SPI_TX_DUAL | SPI_TX_QUAD; + + ctlr->buswidth_override_bits = hisi_sfc_v3xx_buswidth_override_bits; + + host = spi_controller_get_devdata(ctlr); + host->dev = dev; + + platform_set_drvdata(pdev, host); + + host->regbase = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(host->regbase)) { + ret = PTR_ERR(host->regbase); + goto err_put_master; + } + + host->irq = platform_get_irq_optional(pdev, 0); + if (host->irq == -EPROBE_DEFER) { + ret = -EPROBE_DEFER; + goto err_put_master; + } + + hisi_sfc_v3xx_disable_int(host); + + if (host->irq > 0) { + ret = devm_request_irq(dev, host->irq, hisi_sfc_v3xx_isr, 0, + "hisi-sfc-v3xx", host); + + if (ret) { + dev_err(dev, "failed to request irq%d, ret = %d\n", host->irq, ret); + host->irq = 0; + } + } else { + host->irq = 0; + } + + ctlr->bus_num = -1; + ctlr->num_chipselect = 1; + ctlr->mem_ops = &hisi_sfc_v3xx_mem_ops; + + /* + * The address mode of the controller is either 3 or 4, + * which is indicated by the address mode bit in + * the global config register. The register is read only + * for the OS driver. + */ + glb_config = readl(host->regbase + HISI_SFC_V3XX_GLB_CFG); + if (glb_config & HISI_SFC_V3XX_GLB_CFG_CS0_ADDR_MODE) + host->address_mode = 4; + else + host->address_mode = 3; + + version = readl(host->regbase + HISI_SFC_V3XX_VERSION); + + if (version >= 0x351) + host->max_cmd_dword = 64; + else + host->max_cmd_dword = 16; + + ret = devm_spi_register_controller(dev, ctlr); + if (ret) + goto err_put_master; + + dev_info(&pdev->dev, "hw version 0x%x, %s mode.\n", + version, host->irq ? "irq" : "polling"); + + return 0; + +err_put_master: + spi_master_put(ctlr); + return ret; +} + +static const struct acpi_device_id hisi_sfc_v3xx_acpi_ids[] = { + {"HISI0341", 0}, + {} +}; +MODULE_DEVICE_TABLE(acpi, hisi_sfc_v3xx_acpi_ids); + +static struct platform_driver hisi_sfc_v3xx_spi_driver = { + .driver = { + .name = "hisi-sfc-v3xx", + .acpi_match_table = hisi_sfc_v3xx_acpi_ids, + }, + .probe = hisi_sfc_v3xx_probe, +}; + +static int __init hisi_sfc_v3xx_spi_init(void) +{ + dmi_check_system(hisi_sfc_v3xx_dmi_quirk_table); + + return platform_driver_register(&hisi_sfc_v3xx_spi_driver); +} + +static void __exit hisi_sfc_v3xx_spi_exit(void) +{ + platform_driver_unregister(&hisi_sfc_v3xx_spi_driver); +} + +module_init(hisi_sfc_v3xx_spi_init); +module_exit(hisi_sfc_v3xx_spi_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("John Garry "); +MODULE_DESCRIPTION("HiSilicon SPI NOR V3XX Flash Controller Driver for hi16xx chipsets"); diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index b18ae50db1f5297a3f2aceaa14cb37346ed1e7d7..808d309046c7073e97b3ff8f466c4983493728d1 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -513,6 +513,7 @@ struct spi_device *spi_alloc_device(struct spi_controller *ctlr) spi->dev.bus = &spi_bus_type; spi->dev.release = spidev_release; spi->cs_gpio = -ENOENT; + spi->mode = ctlr->buswidth_override_bits; spin_lock_init(&spi->statistics.lock); @@ -2037,9 +2038,10 @@ static acpi_status acpi_register_spi_device(struct spi_controller *ctlr, return AE_NO_MEMORY; } + ACPI_COMPANION_SET(&spi->dev, adev); spi->max_speed_hz = lookup.max_speed_hz; - spi->mode = lookup.mode; + spi->mode |= lookup.mode; spi->irq = lookup.irq; spi->bits_per_word = lookup.bits_per_word; spi->chip_select = lookup.chip_select; diff --git a/drivers/usb/host/xhci-pci.c b/drivers/usb/host/xhci-pci.c index aa4c21a12de719f6066976ae110324fb92c65df6..73d41b30ed53a056fb15a37dff4cc7c59d63d976 100644 --- a/drivers/usb/host/xhci-pci.c +++ b/drivers/usb/host/xhci-pci.c @@ -325,6 +325,10 @@ static void xhci_pci_quirks(struct device *dev, struct xhci_hcd *xhci) if (xhci->quirks & XHCI_RESET_ON_RESUME) xhci_dbg_trace(xhci, trace_xhci_dbg_quirks, "QUIRK: Resetting on resume"); + + if (pdev->vendor == PCI_VENDOR_ID_HUAWEI && + (pdev->device == 0xa23c || pdev->device == 0xa23d)) + xhci->quirks |= XHCI_USB3_NOOP; } #ifdef CONFIG_ACPI diff --git a/drivers/usb/host/xhci-ring.c b/drivers/usb/host/xhci-ring.c index 35372f3e3b5723b9fd1bd3c876df27a4b4ad3fd2..52bd8907290e1a62299b3a58a14a575c3d73abf8 100644 --- a/drivers/usb/host/xhci-ring.c +++ b/drivers/usb/host/xhci-ring.c @@ -1455,14 +1455,16 @@ static void handle_cmd_completion(struct xhci_hcd *xhci, cmd_trb = xhci->cmd_ring->dequeue; trace_xhci_handle_command(xhci->cmd_ring, &cmd_trb->generic); - + cmd_comp_code = GET_COMP_CODE(le32_to_cpu(event->status)); cmd_dequeue_dma = xhci_trb_virt_to_dma(xhci->cmd_ring->deq_seg, cmd_trb); /* * Check whether the completion event is for our internal kept * command. */ - if (!cmd_dequeue_dma || cmd_dma != (u64)cmd_dequeue_dma) { + if (!cmd_dequeue_dma || ((cmd_dma != (u64)cmd_dequeue_dma) && + !((xhci->quirks & XHCI_USB3_NOOP) && (cmd_comp_code == + COMP_COMMAND_RING_STOPPED)))) { xhci_warn(xhci, "ERROR mismatched command completion event\n"); return; @@ -1472,8 +1474,6 @@ static void handle_cmd_completion(struct xhci_hcd *xhci, cancel_delayed_work(&xhci->cmd_timer); - cmd_comp_code = GET_COMP_CODE(le32_to_cpu(event->status)); - /* If CMD ring stopped we own the trbs between enqueue and dequeue */ if (cmd_comp_code == COMP_COMMAND_RING_STOPPED) { complete_all(&xhci->cmd_ring_stop_completion); @@ -1495,6 +1495,8 @@ static void handle_cmd_completion(struct xhci_hcd *xhci, if (cmd_comp_code == COMP_COMMAND_ABORTED) { xhci->cmd_ring_state = CMD_RING_STATE_STOPPED; if (cmd->status == COMP_COMMAND_ABORTED) { + if (xhci->quirks & XHCI_USB3_NOOP) + trb_to_noop(cmd->command_trb, TRB_CMD_NOOP); if (xhci->current_cmd == cmd) xhci->current_cmd = NULL; goto event_handled; diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h index e696f1508318e9841d94595457cbd140f7f55eee..b1913f883f70d3a6dc26ed9119e20711837ada5c 100644 --- a/drivers/usb/host/xhci.h +++ b/drivers/usb/host/xhci.h @@ -1834,6 +1834,7 @@ struct xhci_hcd { #define XHCI_STATE_HALTED (1 << 1) #define XHCI_STATE_REMOVING (1 << 2) unsigned long long quirks; +#define XHCI_USB3_NOOP BIT_ULL(63) #define XHCI_LINK_TRB_QUIRK BIT_ULL(0) #define XHCI_RESET_EP_QUIRK BIT_ULL(1) #define XHCI_NEC_HOST BIT_ULL(2) diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index b4c71ae955cba7f9d60c043a808cbcfa0888a5ed..87e7cd9a702202ce3313b72f64593a8e818d6854 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -1086,8 +1086,7 @@ static int ext4_block_write_begin(struct page *page, loff_t pos, unsigned len, block_end = block_start + blocksize; if (block_end <= from || block_start >= to) { if (PageUptodate(page)) { - if (!buffer_uptodate(bh)) - set_buffer_uptodate(bh); + set_buffer_uptodate(bh); } continue; } @@ -1112,8 +1111,7 @@ static int ext4_block_write_begin(struct page *page, loff_t pos, unsigned len, } } if (PageUptodate(page)) { - if (!buffer_uptodate(bh)) - set_buffer_uptodate(bh); + set_buffer_uptodate(bh); continue; } if (!buffer_uptodate(bh) && !buffer_delay(bh) && diff --git a/include/acpi/pcc.h b/include/acpi/pcc.h index 4dec4ed138cd7fdc8a7f5fb74a55b6c708e29528..9b373d172a77604cb65c4b43eb35acaa5a18344f 100644 --- a/include/acpi/pcc.h +++ b/include/acpi/pcc.h @@ -9,18 +9,40 @@ #include #include +struct pcc_mbox_chan { + struct mbox_chan *mchan; + u64 shmem_base_addr; + u64 shmem_size; + u32 latency; + u32 max_access_rate; + u16 min_turnaround_time; +}; + +/* Generic Communications Channel Shared Memory Region */ +#define PCC_SIGNATURE 0x50434300 +/* Generic Communications Channel Command Field */ +#define PCC_CMD_GENERATE_DB_INTR BIT(15) +/* Generic Communications Channel Status Field */ +#define PCC_STATUS_CMD_COMPLETE BIT(0) +#define PCC_STATUS_SCI_DOORBELL BIT(1) +#define PCC_STATUS_ERROR BIT(2) +#define PCC_STATUS_PLATFORM_NOTIFY BIT(3) +/* Initiator Responder Communications Channel Flags */ +#define PCC_CMD_COMPLETION_NOTIFY BIT(0) + #define MAX_PCC_SUBSPACES 256 + #ifdef CONFIG_PCC -extern struct mbox_chan *pcc_mbox_request_channel(struct mbox_client *cl, - int subspace_id); -extern void pcc_mbox_free_channel(struct mbox_chan *chan); +extern struct pcc_mbox_chan * +pcc_mbox_request_channel(struct mbox_client *cl, int subspace_id); +extern void pcc_mbox_free_channel(struct pcc_mbox_chan *chan); #else -static inline struct mbox_chan *pcc_mbox_request_channel(struct mbox_client *cl, - int subspace_id) +static inline struct pcc_mbox_chan * +pcc_mbox_request_channel(struct mbox_client *cl, int subspace_id) { return ERR_PTR(-ENODEV); } -static inline void pcc_mbox_free_channel(struct mbox_chan *chan) { } +static inline void pcc_mbox_free_channel(struct pcc_mbox_chan *chan) { } #endif #endif /* _PCC_H */ diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h index 0575ad84cc555edc62150f680b45c1cb1751c6d1..b56cc825f64d9811ee3c81e8c989b17216377d84 100644 --- a/include/linux/buffer_head.h +++ b/include/linux/buffer_head.h @@ -117,6 +117,7 @@ static __always_inline int test_clear_buffer_##name(struct buffer_head *bh) \ * of the form "mark_buffer_foo()". These are higher-level functions which * do something in addition to setting a b_state bit. */ +BUFFER_FNS(Uptodate, uptodate) BUFFER_FNS(Dirty, dirty) TAS_BUFFER_FNS(Dirty, dirty) BUFFER_FNS(Lock, locked) @@ -134,41 +135,6 @@ BUFFER_FNS(Meta, meta) BUFFER_FNS(Prio, prio) BUFFER_FNS(Defer_Completion, defer_completion) -static __always_inline void set_buffer_uptodate(struct buffer_head *bh) -{ - /* - * If somebody else already set this uptodate, they will - * have done the memory barrier, and a reader will thus - * see *some* valid buffer state. - * - * Any other serialization (with IO errors or whatever that - * might clear the bit) has to come from other state (eg BH_Lock). - */ - if (test_bit(BH_Uptodate, &bh->b_state)) - return; - - /* - * make it consistent with folio_mark_uptodate - * pairs with smp_load_acquire in buffer_uptodate - */ - smp_mb__before_atomic(); - set_bit(BH_Uptodate, &bh->b_state); -} - -static __always_inline void clear_buffer_uptodate(struct buffer_head *bh) -{ - clear_bit(BH_Uptodate, &bh->b_state); -} - -static __always_inline int buffer_uptodate(const struct buffer_head *bh) -{ - /* - * make it consistent with folio_test_uptodate - * pairs with smp_mb__before_atomic in set_buffer_uptodate - */ - return (smp_load_acquire(&bh->b_state) & (1UL << BH_Uptodate)) != 0; -} - #define bh_offset(bh) ((unsigned long)(bh)->b_data & ~PAGE_MASK) /* If we *know* page->private refers to buffer_heads */ diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index 018dce868de630f0f6b2168ee8d3d095916ae121..81b9a6be2e02ddd3117f146c6442aaab45b843d6 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -365,7 +365,7 @@ struct cpufreq_driver { /* platform specific boost support code */ bool boost_enabled; - int (*set_boost)(int state); + int (*set_boost)(struct cpufreq_policy *policy, int state); }; /* flags */ diff --git a/include/linux/device.h b/include/linux/device.h index a8cb163cdf42511ddb914029a4b94a4f2ff32806..219449177e930a6304b41e05badc858e2a1d3649 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -1083,6 +1083,7 @@ enum device_link_state { * AUTOREMOVE_SUPPLIER: Remove the link automatically on supplier driver unbind. * AUTOPROBE_CONSUMER: Probe consumer driver automatically after supplier binds. * MANAGED: The core tracks presence of supplier/consumer drivers (internal). + * SYNC_STATE_ONLY: Link only affects sync_state() behavior. */ #define DL_FLAG_STATELESS BIT(0) #define DL_FLAG_AUTOREMOVE_CONSUMER BIT(1) @@ -1091,34 +1092,7 @@ enum device_link_state { #define DL_FLAG_AUTOREMOVE_SUPPLIER BIT(4) #define DL_FLAG_AUTOPROBE_CONSUMER BIT(5) #define DL_FLAG_MANAGED BIT(6) - -/** - * struct device_link - Device link representation. - * @supplier: The device on the supplier end of the link. - * @s_node: Hook to the supplier device's list of links to consumers. - * @consumer: The device on the consumer end of the link. - * @c_node: Hook to the consumer device's list of links to suppliers. - * @status: The state of the link (with respect to the presence of drivers). - * @flags: Link flags. - * @rpm_active: Whether or not the consumer device is runtime-PM-active. - * @kref: Count repeated addition of the same link. - * @rcu_head: An RCU head to use for deferred execution of SRCU callbacks. - * @supplier_preactivated: Supplier has been made active before consumer probe. - */ -struct device_link { - struct device *supplier; - struct list_head s_node; - struct device *consumer; - struct list_head c_node; - enum device_link_state status; - u32 flags; - refcount_t rpm_active; - struct kref kref; -#ifdef CONFIG_SRCU - struct rcu_head rcu_head; -#endif - bool supplier_preactivated; /* Owned by consumer probe. */ -}; +#define DL_FLAG_SYNC_STATE_ONLY BIT(7) /** * enum dl_dev_state - Device driver presence tracking information. @@ -1322,6 +1296,34 @@ struct device { #endif }; +/** + * struct device_link - Device link representation. + * @supplier: The device on the supplier end of the link. + * @s_node: Hook to the supplier device's list of links to consumers. + * @consumer: The device on the consumer end of the link. + * @c_node: Hook to the consumer device's list of links to suppliers. + * @link_dev: device used to expose link details in sysfs + * @status: The state of the link (with respect to the presence of drivers). + * @flags: Link flags. + * @rpm_active: Whether or not the consumer device is runtime-PM-active. + * @kref: Count repeated addition of the same link. + * @rm_work: Work structure used for removing the link. + * @supplier_preactivated: Supplier has been made active before consumer probe. + */ +struct device_link { + struct device *supplier; + struct list_head s_node; + struct device *consumer; + struct list_head c_node; + struct device link_dev; + enum device_link_state status; + u32 flags; + refcount_t rpm_active; + struct kref kref; + struct work_struct rm_work; + bool supplier_preactivated; /* Owned by consumer probe. */ +}; + static inline struct device *kobj_to_dev(struct kobject *kobj) { return container_of(kobj, struct device, kobj); diff --git a/include/linux/gpio/driver.h b/include/linux/gpio/driver.h index 5dd9c982e2cbee6d10aa42fbcd888d702e6751cc..30a5b8b177d80367d008e7746e32845a189a93da 100644 --- a/include/linux/gpio/driver.h +++ b/include/linux/gpio/driver.h @@ -567,6 +567,7 @@ int bgpio_init(struct gpio_chip *gc, struct device *dev, #define BGPIOF_BIG_ENDIAN_BYTE_ORDER BIT(3) #define BGPIOF_READ_OUTPUT_REG_SET BIT(4) /* reg_set stores output value */ #define BGPIOF_NO_OUTPUT BIT(5) /* only input */ +#define BGPIOF_NO_SET_ON_INPUT BIT(6) int gpiochip_irq_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hwirq); diff --git a/include/linux/i2c.h b/include/linux/i2c.h index af2b799d7a665d5c1eb813d790c2965b9a268e37..0a61f93514b6043ef5ac2905e269ae004d6af822 100644 --- a/include/linux/i2c.h +++ b/include/linux/i2c.h @@ -39,10 +39,21 @@ enum i2c_slave_event; typedef int (*i2c_slave_cb_t)(struct i2c_client *client, enum i2c_slave_event event, u8 *val); +/* I2C Frequency Modes */ +#define I2C_MAX_STANDARD_MODE_FREQ 100000 +#define I2C_MAX_FAST_MODE_FREQ 400000 +#define I2C_MAX_FAST_MODE_PLUS_FREQ 1000000 +#define I2C_MAX_TURBO_MODE_FREQ 1400000 +#define I2C_MAX_HIGH_SPEED_MODE_FREQ 3400000 +#define I2C_MAX_ULTRA_FAST_MODE_FREQ 5000000 + struct module; struct property_entry; #if IS_ENABLED(CONFIG_I2C) +/* Return the Frequency mode string based on the bus frequency */ +const char *i2c_freq_mode_string(u32 bus_freq_hz); + /* * The master routines are the ones normally used to transmit data to devices * on a bus (or read from them). Apart from two basic transfer functions to @@ -575,6 +586,10 @@ struct i2c_lock_operations { * @scl_int_delay_ns: time IP core additionally needs to setup SCL in ns * @sda_fall_ns: time SDA signal takes to fall in ns; t(f) in the I2C specification * @sda_hold_ns: time IP core additionally needs to hold SDA in ns + * @digital_filter_width_ns: width in ns of spikes on i2c lines that the IP core + * digital filter can filter out + * @analog_filter_cutoff_freq_hz: threshold frequency for the low pass IP core + * analog filter */ struct i2c_timings { u32 bus_freq_hz; @@ -583,6 +598,8 @@ struct i2c_timings { u32 scl_int_delay_ns; u32 sda_fall_ns; u32 sda_hold_ns; + u32 digital_filter_width_ns; + u32 analog_filter_cutoff_freq_hz; }; /** @@ -835,9 +852,10 @@ static inline void i2c_mark_adapter_resumed(struct i2c_adapter *adap) /* administration... */ #if IS_ENABLED(CONFIG_I2C) -extern int i2c_add_adapter(struct i2c_adapter *adap); -extern void i2c_del_adapter(struct i2c_adapter *adap); -extern int i2c_add_numbered_adapter(struct i2c_adapter *adap); +int i2c_add_adapter(struct i2c_adapter *adap); +int devm_i2c_add_adapter(struct device *dev, struct i2c_adapter *adapter); +void i2c_del_adapter(struct i2c_adapter *adap); +int i2c_add_numbered_adapter(struct i2c_adapter *adap); extern int i2c_register_driver(struct module *owner, struct i2c_driver *driver); extern void i2c_del_driver(struct i2c_driver *driver); diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h index 5e9ab5ba39b00da4b9b8ec1a6a7d2349637726de..151c2a7baa26e117e5670ad27f3656c02b054cbf 100644 --- a/include/linux/interrupt.h +++ b/include/linux/interrupt.h @@ -305,39 +305,8 @@ struct irq_affinity_desc { extern cpumask_var_t irq_default_affinity; -/* Internal implementation. Use the helpers below */ -extern int __irq_set_affinity(unsigned int irq, const struct cpumask *cpumask, - bool force); - -/** - * irq_set_affinity - Set the irq affinity of a given irq - * @irq: Interrupt to set affinity - * @cpumask: cpumask - * - * Fails if cpumask does not contain an online CPU - */ -static inline int -irq_set_affinity(unsigned int irq, const struct cpumask *cpumask) -{ - return __irq_set_affinity(irq, cpumask, false); -} - -/** - * irq_force_affinity - Force the irq affinity of a given irq - * @irq: Interrupt to set affinity - * @cpumask: cpumask - * - * Same as irq_set_affinity, but without checking the mask against - * online cpus. - * - * Solely for low level cpu hotplug code, where we need to make per - * cpu interrupts affine before the cpu becomes online. - */ -static inline int -irq_force_affinity(unsigned int irq, const struct cpumask *cpumask) -{ - return __irq_set_affinity(irq, cpumask, true); -} +extern int irq_set_affinity(unsigned int irq, const struct cpumask *cpumask); +extern int irq_force_affinity(unsigned int irq, const struct cpumask *cpumask); extern int irq_can_set_affinity(unsigned int irq); extern int irq_select_affinity(unsigned int irq); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index cd373f048c2ffb521bf4ae6121592cf8123bd843..3f080779918bfa9189e7fe9980e98c9a4e779ae5 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -307,6 +307,7 @@ struct iommu_ops { struct iommu_fault_event *evt, struct iommu_page_response *msg); + int (*def_domain_type)(struct device *dev); unsigned long pgsize_bitmap; #ifdef CONFIG_SMMU_BYPASS_DEV diff --git a/include/linux/iova.h b/include/linux/iova.h index 6c19b09e96634eed9d558077a17816b38fced79e..de446cb4b53b078c6e552abe6e2bfc50f1742447 100644 --- a/include/linux/iova.h +++ b/include/linux/iova.h @@ -26,13 +26,14 @@ struct iova_magazine; struct iova_cpu_rcache; #define IOVA_RANGE_CACHE_MAX_SIZE 6 /* log of max cached IOVA range size (in pages) */ -#define MAX_GLOBAL_MAGS 32 /* magazines per bin */ struct iova_rcache { spinlock_t lock; - unsigned long depot_size; - struct iova_magazine *depot[MAX_GLOBAL_MAGS]; + unsigned int depot_size; + struct iova_magazine *depot; struct iova_cpu_rcache __percpu *cpu_rcaches; + struct iova_domain *iovad; + struct delayed_work work; }; struct iova_domain; diff --git a/include/linux/sched/isolation.h b/include/linux/sched/isolation.h index 6c8512d3be88e4d1d34cbfa861fdbc3094ee925b..0fbcbacd1b2900bae38fd90f7aea73c04f06572d 100644 --- a/include/linux/sched/isolation.h +++ b/include/linux/sched/isolation.h @@ -13,6 +13,7 @@ enum hk_flags { HK_FLAG_TICK = (1 << 4), HK_FLAG_DOMAIN = (1 << 5), HK_FLAG_WQ = (1 << 6), + HK_FLAG_MANAGED_IRQ = (1 << 7), }; #ifdef CONFIG_CPU_ISOLATION diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index 7067f85cef0bf8bbe7e0c798a9ef3cb90ba4e8eb..6d5158c29a9d92209bebb31aa7dd669d4a501e94 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -319,6 +319,7 @@ static inline void spi_unregister_driver(struct spi_driver *sdrv) * @max_speed_hz: Highest supported transfer speed * @flags: other constraints relevant to this driver * @slave: indicates that this is an SPI slave controller + * @devm_allocated: whether the allocation of this struct is devres-managed * @max_transfer_size: function that returns the max transfer size for * a &spi_device; may be %NULL, so the default %SIZE_MAX will be used. * @max_message_size: function that returns the max message size for @@ -447,6 +448,9 @@ struct spi_controller { /* spi_device.mode flags understood by this controller driver */ u32 mode_bits; + /* spi_device.mode flags override flags for this controller */ + u32 buswidth_override_bits; + /* bitmask of supported bits_per_word for transfers */ u32 bits_per_word_mask; #define SPI_BPW_MASK(bits) BIT((bits) - 1) @@ -466,7 +470,7 @@ struct spi_controller { #define SPI_MASTER_GPIO_SS BIT(5) /* GPIO CS must select slave */ - /* flag indicating this is a non-devres managed controller */ + /* flag indicating if the allocation of this struct is devres-managed */ bool devm_allocated; /* flag indicating this is an SPI slave controller */ diff --git a/include/scsi/libsas.h b/include/scsi/libsas.h index 4e2d61e8fb1ed6e1b9dbad130522de8e2542b9b0..080b08a4ba72d2b26c7e50a80131bdcd6eccb71c 100644 --- a/include/scsi/libsas.h +++ b/include/scsi/libsas.h @@ -356,6 +356,7 @@ enum sas_ha_state { SAS_HA_DRAINING, SAS_HA_ATA_EH_ACTIVE, SAS_HA_FROZEN, + SAS_HA_RESUMING, }; struct sas_ha_struct { @@ -391,10 +392,6 @@ struct sas_ha_struct { int strict_wide_ports; /* both sas_addr and attached_sas_addr must match * their siblings when forming wide ports */ - /* LLDD calls these to notify the class of an event. */ - int (*notify_port_event)(struct asd_sas_phy *, enum port_event); - int (*notify_phy_event)(struct asd_sas_phy *, enum phy_event); - void *lldd_ha; /* not touched by sas class code */ struct list_head eh_done_q; /* complete via scsi_eh_flush_done_q */ @@ -658,10 +655,12 @@ extern int sas_register_ha(struct sas_ha_struct *); extern int sas_unregister_ha(struct sas_ha_struct *); extern void sas_prep_resume_ha(struct sas_ha_struct *sas_ha); extern void sas_resume_ha(struct sas_ha_struct *sas_ha); +extern void sas_resume_ha_no_sync(struct sas_ha_struct *sas_ha); extern void sas_suspend_ha(struct sas_ha_struct *sas_ha); int sas_set_phy_speed(struct sas_phy *phy, struct sas_phy_linkrates *rates); int sas_phy_reset(struct sas_phy *phy, int hard_reset); +int sas_phy_enable(struct sas_phy *phy, int enable); extern int sas_queuecommand(struct Scsi_Host *, struct scsi_cmnd *); extern int sas_target_alloc(struct scsi_target *); extern int sas_slave_configure(struct scsi_device *); @@ -706,4 +705,9 @@ struct sas_phy *sas_get_local_phy(struct domain_device *dev); int sas_request_addr(struct Scsi_Host *shost, u8 *addr); +int sas_notify_port_event(struct asd_sas_phy *phy, enum port_event event, + gfp_t gfp_flags); +int sas_notify_phy_event(struct asd_sas_phy *phy, enum phy_event event, + gfp_t gfp_flags); + #endif /* _SASLIB_H_ */ diff --git a/include/scsi/sas_ata.h b/include/scsi/sas_ata.h index 416c9c47d0e72f3a39812e3ad8a1f8e1eed10b68..49d8cff7badb4d0ffcf7cfdbf73df7a99b783ddc 100644 --- a/include/scsi/sas_ata.h +++ b/include/scsi/sas_ata.h @@ -33,6 +33,8 @@ void sas_probe_sata(struct asd_sas_port *port); void sas_suspend_sata(struct asd_sas_port *port); void sas_resume_sata(struct asd_sas_port *port); void sas_ata_end_eh(struct ata_port *ap); +void sas_ata_device_link_abort(struct domain_device *dev, bool force_reset); +int smp_ata_check_ready_type(struct ata_link *link); #else @@ -85,6 +87,16 @@ static inline int sas_get_ata_info(struct domain_device *dev, struct ex_phy *phy static inline void sas_ata_end_eh(struct ata_port *ap) { } + +static inline void sas_ata_device_link_abort(struct domain_device *dev, + bool force_reset) +{ +} + +static inline int smp_ata_check_ready_type(struct ata_link *link) +{ + return 0; +} #endif #endif /* _SAS_ATA_H_ */ diff --git a/kernel/irq/cpuhotplug.c b/kernel/irq/cpuhotplug.c index 6c7ca2e983a595ff561396e3d3b9400072426c56..02236b13b359974e95a2e3ca4ec3f0cadfd0962a 100644 --- a/kernel/irq/cpuhotplug.c +++ b/kernel/irq/cpuhotplug.c @@ -12,6 +12,7 @@ #include #include #include +#include #include "internals.h" @@ -171,6 +172,20 @@ void irq_migrate_all_off_this_cpu(void) } } +static bool hk_should_isolate(struct irq_data *data, unsigned int cpu) +{ + const struct cpumask *hk_mask; + + if (!housekeeping_enabled(HK_FLAG_MANAGED_IRQ)) + return false; + + hk_mask = housekeeping_cpumask(HK_FLAG_MANAGED_IRQ); + if (cpumask_subset(irq_data_get_effective_affinity_mask(data), hk_mask)) + return false; + + return cpumask_test_cpu(cpu, hk_mask); +} + static void irq_restore_affinity_of_irq(struct irq_desc *desc, unsigned int cpu) { struct irq_data *data = irq_desc_get_irq_data(desc); @@ -188,9 +203,11 @@ static void irq_restore_affinity_of_irq(struct irq_desc *desc, unsigned int cpu) /* * If the interrupt can only be directed to a single target * CPU then it is already assigned to a CPU in the affinity - * mask. No point in trying to move it around. + * mask. No point in trying to move it around unless the + * isolation mechanism requests to move it to an upcoming + * housekeeping CPU. */ - if (!irqd_is_single_target(data)) + if (!irqd_is_single_target(data) || hk_should_isolate(data, cpu)) irq_set_affinity_locked(data, affinity, false); } diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h index c76429ef15907ffda5285cd9666f791f13ef158f..5973c32c54d52b3d1dcadeb9abdd5be796a78de0 100644 --- a/kernel/irq/internals.h +++ b/kernel/irq/internals.h @@ -12,7 +12,7 @@ #include #ifdef CONFIG_SPARSE_IRQ -# define IRQ_BITMAP_BITS (NR_IRQS + 8196) +# define IRQ_BITMAP_BITS (1 << 19) #else # define IRQ_BITMAP_BITS NR_IRQS #endif diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index 2a8a5e1779c9c9fcb8a94432c3b95b401eef139d..a549b5a7440405e0e265783baa4c004007d78f52 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -222,12 +223,59 @@ int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask, { struct irq_desc *desc = irq_data_to_desc(data); struct irq_chip *chip = irq_data_get_irq_chip(data); + const struct cpumask *prog_mask; int ret; + static DEFINE_RAW_SPINLOCK(tmp_mask_lock); + static struct cpumask tmp_mask; + if (!chip || !chip->irq_set_affinity) return -EINVAL; - ret = chip->irq_set_affinity(data, mask, force); + raw_spin_lock(&tmp_mask_lock); + /* + * If this is a managed interrupt and housekeeping is enabled on + * it check whether the requested affinity mask intersects with + * a housekeeping CPU. If so, then remove the isolated CPUs from + * the mask and just keep the housekeeping CPU(s). This prevents + * the affinity setter from routing the interrupt to an isolated + * CPU to avoid that I/O submitted from a housekeeping CPU causes + * interrupts on an isolated one. + * + * If the masks do not intersect or include online CPU(s) then + * keep the requested mask. The isolated target CPUs are only + * receiving interrupts when the I/O operation was submitted + * directly from them. + * + * If all housekeeping CPUs in the affinity mask are offline, the + * interrupt will be migrated by the CPU hotplug code once a + * housekeeping CPU which belongs to the affinity mask comes + * online. + */ + if (irqd_affinity_is_managed(data) && + housekeeping_enabled(HK_FLAG_MANAGED_IRQ)) { + const struct cpumask *hk_mask; + + hk_mask = housekeeping_cpumask(HK_FLAG_MANAGED_IRQ); + + cpumask_and(&tmp_mask, mask, hk_mask); + if (!cpumask_intersects(&tmp_mask, cpu_online_mask)) + prog_mask = mask; + else + prog_mask = &tmp_mask; + } else { + prog_mask = mask; + } + + /* Make sure we only provide online CPUs to the irqchip */ + cpumask_and(&tmp_mask, prog_mask, cpu_online_mask); + if (!cpumask_empty(&tmp_mask)) + ret = chip->irq_set_affinity(data, &tmp_mask, force); + else + ret = -EINVAL; + + raw_spin_unlock(&tmp_mask_lock); + switch (ret) { case IRQ_SET_MASK_OK: case IRQ_SET_MASK_OK_DONE: @@ -332,7 +380,78 @@ int irq_set_affinity_locked(struct irq_data *data, const struct cpumask *mask, return ret; } -int __irq_set_affinity(unsigned int irq, const struct cpumask *mask, bool force) +/** + * irq_update_affinity_desc - Update affinity management for an interrupt + * @irq: The interrupt number to update + * @affinity: Pointer to the affinity descriptor + * + * This interface can be used to configure the affinity management of + * interrupts which have been allocated already. + * + * There are certain limitations on when it may be used - attempts to use it + * for when the kernel is configured for generic IRQ reservation mode (in + * config GENERIC_IRQ_RESERVATION_MODE) will fail, as it may conflict with + * managed/non-managed interrupt accounting. In addition, attempts to use it on + * an interrupt which is already started or which has already been configured + * as managed will also fail, as these mean invalid init state or double init. + */ +int irq_update_affinity_desc(unsigned int irq, + struct irq_affinity_desc *affinity) +{ + struct irq_desc *desc; + unsigned long flags; + bool activated; + int ret = 0; + + /* + * Supporting this with the reservation scheme used by x86 needs + * some more thought. Fail it for now. + */ + if (IS_ENABLED(CONFIG_GENERIC_IRQ_RESERVATION_MODE)) + return -EOPNOTSUPP; + + desc = irq_get_desc_buslock(irq, &flags, 0); + if (!desc) + return -EINVAL; + + /* Requires the interrupt to be shut down */ + if (irqd_is_started(&desc->irq_data)) { + ret = -EBUSY; + goto out_unlock; + } + + /* Interrupts which are already managed cannot be modified */ + if (irqd_affinity_is_managed(&desc->irq_data)) { + ret = -EBUSY; + goto out_unlock; + } + + /* + * Deactivate the interrupt. That's required to undo + * anything an earlier activation has established. + */ + activated = irqd_is_activated(&desc->irq_data); + if (activated) + irq_domain_deactivate_irq(&desc->irq_data); + + if (affinity->is_managed) { + irqd_set(&desc->irq_data, IRQD_AFFINITY_MANAGED); + irqd_set(&desc->irq_data, IRQD_MANAGED_SHUTDOWN); + } + + cpumask_copy(desc->irq_common_data.affinity, &affinity->mask); + + /* Restore the activation state */ + if (activated) + irq_domain_activate_irq(&desc->irq_data, false); + +out_unlock: + irq_put_desc_busunlock(desc, flags); + return ret; +} + +static int __irq_set_affinity(unsigned int irq, const struct cpumask *mask, + bool force) { struct irq_desc *desc = irq_to_desc(irq); unsigned long flags; @@ -347,6 +466,36 @@ int __irq_set_affinity(unsigned int irq, const struct cpumask *mask, bool force) return ret; } +/** + * irq_set_affinity - Set the irq affinity of a given irq + * @irq: Interrupt to set affinity + * @cpumask: cpumask + * + * Fails if cpumask does not contain an online CPU + */ +int irq_set_affinity(unsigned int irq, const struct cpumask *cpumask) +{ + return __irq_set_affinity(irq, cpumask, false); +} +EXPORT_SYMBOL_GPL(irq_set_affinity); + +/** + * irq_force_affinity - Force the irq affinity of a given irq + * @irq: Interrupt to set affinity + * @cpumask: cpumask + * + * Same as irq_set_affinity, but without checking the mask against + * online cpus. + * + * Solely for low level cpu hotplug code, where we need to make per + * cpu interrupts affine before the cpu becomes online. + */ +int irq_force_affinity(unsigned int irq, const struct cpumask *cpumask) +{ + return __irq_set_affinity(irq, cpumask, true); +} +EXPORT_SYMBOL_GPL(irq_force_affinity); + int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m) { unsigned long flags; diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c index b7e4c5999cc8031fcb93b76b9a4914513bb5ff87..a6b73c36d91c53d8716e9a4bee8f14f95c9bc2d7 100644 --- a/kernel/irq/msi.c +++ b/kernel/irq/msi.c @@ -457,6 +457,21 @@ int msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, irqd_clr_can_reserve(irq_data); if (domain->flags & IRQ_DOMAIN_MSI_NOMASK_QUIRK) irqd_set_msi_nomask_quirk(irq_data); + + /* + * If the interrupt is managed but no CPU is available to + * service it, shut it down until better times. Note that + * we only do this on the !RESERVE path as x86 (the only + * architecture using this flag) deals with this in a + * different way by using a catch-all vector. + */ + if ((info->flags & MSI_FLAG_ACTIVATE_EARLY) && + irqd_affinity_is_managed(irq_data) && + !cpumask_intersects(irq_data_get_affinity_mask(irq_data), + cpu_online_mask)) { + irqd_set_managed_shutdown(irq_data); + return 0; + } } ret = irq_domain_activate_irq(irq_data, can_reserve); if (ret) diff --git a/kernel/sched/isolation.c b/kernel/sched/isolation.c index 9fcb2a695a41289fe490c64a0656300a74fefb99..008d6ac2342b7536fff293b2ce7054cf9d9cfc98 100644 --- a/kernel/sched/isolation.c +++ b/kernel/sched/isolation.c @@ -163,6 +163,12 @@ static int __init housekeeping_isolcpus_setup(char *str) continue; } + if (!strncmp(str, "managed_irq,", 12)) { + str += 12; + flags |= HK_FLAG_MANAGED_IRQ; + continue; + } + pr_warn("isolcpus: Error, unknown flag\n"); return 0; } diff --git a/tools/perf/Documentation/itrace.txt b/tools/perf/Documentation/itrace.txt index 82ff7dad40c2744d9619715689e060615c1e4439..467659bfdeec9c4d87cd4bdbe4d9ad0e5a039c70 100644 --- a/tools/perf/Documentation/itrace.txt +++ b/tools/perf/Documentation/itrace.txt @@ -1,5 +1,5 @@ i synthesize instructions events - b synthesize branches events + b synthesize branches events (branch misses for Arm SPE) c synthesize branches events (calls only) r synthesize branches events (returns only) x synthesize transactions events @@ -9,6 +9,10 @@ of aux-output (refer to perf record) e synthesize error events d create a debug log + f synthesize first level cache events + m synthesize last level cache events + t synthesize TLB events + a synthesize remote access events g synthesize a call chain (use with i or x) l synthesize last branch entries (use with i or x) s skip initial number of events diff --git a/tools/perf/arch/arm/util/auxtrace.c b/tools/perf/arch/arm/util/auxtrace.c index 28a5d0c18b1d2b89098af90f141079f00ca99df4..63097cb6c43e35a53937bfd3e764a704c6332e1b 100644 --- a/tools/perf/arch/arm/util/auxtrace.c +++ b/tools/perf/arch/arm/util/auxtrace.c @@ -4,16 +4,19 @@ * Author: Mathieu Poirier */ +#include #include #include #include +#include -#include "../../util/auxtrace.h" -#include "../../util/debug.h" +#include "../../../util/auxtrace.h" +#include "../../../util/debug.h" #include "../../util/evlist.h" -#include "../../util/pmu.h" +#include "../../../util/pmu.h" #include "cs-etm.h" #include "arm-spe.h" +#include "hisi-ptt.h" static struct perf_pmu **find_all_arm_spe_pmus(int *nr_spes, int *err) { @@ -50,43 +53,114 @@ static struct perf_pmu **find_all_arm_spe_pmus(int *nr_spes, int *err) return arm_spe_pmus; } +static struct perf_pmu **find_all_hisi_ptt_pmus(int *nr_ptts, int *err) +{ + const char *sysfs = sysfs__mountpoint(); + struct perf_pmu **hisi_ptt_pmus = NULL; + struct dirent *dent; + char path[PATH_MAX]; + DIR *dir = NULL; + int idx = 0; + + snprintf(path, PATH_MAX, "%s" EVENT_SOURCE_DEVICE_PATH, sysfs); + dir = opendir(path); + if (!dir) { + pr_err("can't read directory '%s'\n", EVENT_SOURCE_DEVICE_PATH); + *err = -EINVAL; + return NULL; + } + + while ((dent = readdir(dir))) { + if (strstr(dent->d_name, HISI_PTT_PMU_NAME)) + (*nr_ptts)++; + } + + if (!(*nr_ptts)) + goto out; + + hisi_ptt_pmus = zalloc(sizeof(struct perf_pmu *) * (*nr_ptts)); + if (!hisi_ptt_pmus) { + pr_err("hisi_ptt alloc failed\n"); + *err = -ENOMEM; + goto out; + } + + rewinddir(dir); + while ((dent = readdir(dir))) { + if (strstr(dent->d_name, HISI_PTT_PMU_NAME) && idx < *nr_ptts) { + hisi_ptt_pmus[idx] = perf_pmu__find(dent->d_name); + if (hisi_ptt_pmus[idx]) + idx++; + } + } + +out: + closedir(dir); + return hisi_ptt_pmus; +} + +static struct perf_pmu *find_pmu_for_event(struct perf_pmu **pmus, + int pmu_nr, struct evsel *evsel) +{ + int i; + + if (!pmus) + return NULL; + + for (i = 0; i < pmu_nr; i++) { + if (evsel->core.attr.type == pmus[i]->type) + return pmus[i]; + } + + return NULL; +} + struct auxtrace_record *auxtrace_record__init(struct evlist *evlist, int *err) { - struct perf_pmu *cs_etm_pmu; + struct perf_pmu *cs_etm_pmu = NULL; + struct perf_pmu **arm_spe_pmus = NULL; + struct perf_pmu **hisi_ptt_pmus = NULL; struct evsel *evsel; - bool found_etm = false; + struct perf_pmu *found_etm = NULL; struct perf_pmu *found_spe = NULL; - static struct perf_pmu **arm_spe_pmus = NULL; - static int nr_spes = 0; - int i = 0; + struct perf_pmu *found_ptt = NULL; + int auxtrace_event_cnt = 0; + int nr_spes = 0; + int nr_ptts = 0; if (!evlist) return NULL; cs_etm_pmu = perf_pmu__find(CORESIGHT_ETM_PMU_NAME); - - if (!arm_spe_pmus) - arm_spe_pmus = find_all_arm_spe_pmus(&nr_spes, err); + arm_spe_pmus = find_all_arm_spe_pmus(&nr_spes, err); + hisi_ptt_pmus = find_all_hisi_ptt_pmus(&nr_ptts, err); evlist__for_each_entry(evlist, evsel) { - if (cs_etm_pmu && - evsel->core.attr.type == cs_etm_pmu->type) - found_etm = true; - - if (!nr_spes || found_spe) - continue; - - for (i = 0; i < nr_spes; i++) { - if (evsel->core.attr.type == arm_spe_pmus[i]->type) { - found_spe = arm_spe_pmus[i]; - break; - } - } + if (cs_etm_pmu && !found_etm) + found_etm = find_pmu_for_event(&cs_etm_pmu, 1, evsel); + + if (arm_spe_pmus && !found_spe) + found_spe = find_pmu_for_event(arm_spe_pmus, nr_spes, evsel); + + if (hisi_ptt_pmus && !found_ptt) + found_ptt = find_pmu_for_event(hisi_ptt_pmus, nr_ptts, evsel); } - if (found_etm && found_spe) { - pr_err("Concurrent ARM Coresight ETM and SPE operation not currently supported\n"); + free(arm_spe_pmus); + free(hisi_ptt_pmus); + + if (found_etm) + auxtrace_event_cnt++; + + if (found_spe) + auxtrace_event_cnt++; + + if (found_ptt) + auxtrace_event_cnt++; + + if (auxtrace_event_cnt > 1) { + pr_err("Concurrent AUX trace operation not currently supported\n"); *err = -EOPNOTSUPP; return NULL; } @@ -97,6 +171,9 @@ struct auxtrace_record #if defined(__aarch64__) if (found_spe) return arm_spe_recording_init(err, found_spe); + + if (found_ptt) + return hisi_ptt_recording_init(err, found_ptt); #endif /* diff --git a/tools/perf/arch/arm/util/pmu.c b/tools/perf/arch/arm/util/pmu.c index bbc297a7e2e3517059f9c3b33816deb69074ac20..fd2b1df4a877ee9b3111a1f318d51f1a84e4b828 100644 --- a/tools/perf/arch/arm/util/pmu.c +++ b/tools/perf/arch/arm/util/pmu.c @@ -10,7 +10,9 @@ #include #include "arm-spe.h" -#include "../../util/pmu.h" +#include "hisi-ptt.h" +#include "../../../util/pmu.h" + struct perf_event_attr *perf_pmu__get_default_config(struct perf_pmu *pmu __maybe_unused) @@ -22,6 +24,8 @@ struct perf_event_attr #if defined(__aarch64__) } else if (strstarts(pmu->name, ARM_SPE_PMU_NAME)) { return arm_spe_pmu_default_config(pmu); + } else if (strstarts(pmu->name, HISI_PTT_PMU_NAME)) { + pmu->selectable = true; #endif } diff --git a/tools/perf/arch/arm64/util/Build b/tools/perf/arch/arm64/util/Build index 3cde540d2fcf3c99be3f936e4efef02a10547a4d..40874d0e11cc26b8a30561672e5e5946bc726a33 100644 --- a/tools/perf/arch/arm64/util/Build +++ b/tools/perf/arch/arm64/util/Build @@ -7,4 +7,4 @@ perf-$(CONFIG_LIBDW_DWARF_UNWIND) += unwind-libdw.o perf-$(CONFIG_AUXTRACE) += ../../arm/util/pmu.o \ ../../arm/util/auxtrace.o \ ../../arm/util/cs-etm.o \ - arm-spe.o + arm-spe.o mem-events.o hisi-ptt.o diff --git a/tools/perf/arch/arm64/util/hisi-ptt.c b/tools/perf/arch/arm64/util/hisi-ptt.c new file mode 100644 index 0000000000000000000000000000000000000000..110b2edf3e6b60916831d15722ef2cb1797a5db2 --- /dev/null +++ b/tools/perf/arch/arm64/util/hisi-ptt.c @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * HiSilicon PCIe Trace and Tuning (PTT) support + * Copyright (c) 2022 HiSilicon Technologies Co., Ltd. + */ + +#include +#include +#include +#include +#include +#include + +#include // page_size +#include "../../../util/auxtrace.h" +#include "../../../util/cpumap.h" +#include "../../../util/debug.h" +#include "../../../util/event.h" +#include "../../../util/evlist.h" +#include "../../../util/evsel.h" +#include "../../../util/hisi-ptt.h" +#include "../../../util/pmu.h" +#include "../../../util/record.h" +#include "../../../util/session.h" +#include "../../../util/tsc.h" + +#define KiB(x) ((x) * 1024) +#define MiB(x) ((x) * 1024 * 1024) + +struct hisi_ptt_recording { + struct auxtrace_record itr; + struct perf_pmu *hisi_ptt_pmu; + struct evlist *evlist; +}; + +static size_t +hisi_ptt_info_priv_size(struct auxtrace_record *itr __maybe_unused, + struct evlist *evlist __maybe_unused) +{ + return HISI_PTT_AUXTRACE_PRIV_SIZE; +} + +static int hisi_ptt_info_fill(struct auxtrace_record *itr, + struct perf_session *session, + struct perf_record_auxtrace_info *auxtrace_info, + size_t priv_size) +{ + struct hisi_ptt_recording *pttr = + container_of(itr, struct hisi_ptt_recording, itr); + struct perf_pmu *hisi_ptt_pmu = pttr->hisi_ptt_pmu; + + if (priv_size != HISI_PTT_AUXTRACE_PRIV_SIZE) + return -EINVAL; + + if (!session->evlist->core.nr_mmaps) + return -EINVAL; + + auxtrace_info->type = PERF_AUXTRACE_HISI_PTT; + auxtrace_info->priv[0] = hisi_ptt_pmu->type; + + return 0; +} + +static int hisi_ptt_set_auxtrace_mmap_page(struct record_opts *opts) +{ + bool privileged = perf_event_paranoid_check(-1); + + if (!opts->full_auxtrace) + return 0; + + if (opts->full_auxtrace && !opts->auxtrace_mmap_pages) { + if (privileged) { + opts->auxtrace_mmap_pages = MiB(16) / page_size; + } else { + opts->auxtrace_mmap_pages = KiB(128) / page_size; + if (opts->mmap_pages == UINT_MAX) + opts->mmap_pages = KiB(256) / page_size; + } + } + + /* Validate auxtrace_mmap_pages */ + if (opts->auxtrace_mmap_pages) { + size_t sz = opts->auxtrace_mmap_pages * (size_t)page_size; + size_t min_sz = KiB(8); + + if (sz < min_sz || !is_power_of_2(sz)) { + pr_err("Invalid mmap size for HISI PTT: must be at least %zuKiB and a power of 2\n", + min_sz / 1024); + return -EINVAL; + } + } + + return 0; +} + +static int hisi_ptt_recording_options(struct auxtrace_record *itr, + struct evlist *evlist, + struct record_opts *opts) +{ + struct hisi_ptt_recording *pttr = + container_of(itr, struct hisi_ptt_recording, itr); + struct perf_pmu *hisi_ptt_pmu = pttr->hisi_ptt_pmu; + struct evsel *evsel, *hisi_ptt_evsel = NULL; + struct evsel *tracking_evsel; + int err; + + pttr->evlist = evlist; + evlist__for_each_entry(evlist, evsel) { + if (evsel->core.attr.type == hisi_ptt_pmu->type) { + if (hisi_ptt_evsel) { + pr_err("There may be only one " HISI_PTT_PMU_NAME "x event\n"); + return -EINVAL; + } + evsel->core.attr.freq = 0; + evsel->core.attr.sample_period = 1; + hisi_ptt_evsel = evsel; + opts->full_auxtrace = true; + } + } + + err = hisi_ptt_set_auxtrace_mmap_page(opts); + if (err) + return err; + /* + * To obtain the auxtrace buffer file descriptor, the auxtrace event + * must come first. + */ + perf_evlist__to_front(evlist, hisi_ptt_evsel); + evsel__set_sample_bit(hisi_ptt_evsel, TIME); + + /* Add dummy event to keep tracking */ + err = parse_events(evlist, "dummy:u", NULL); + if (err) + return err; + + tracking_evsel = evlist__last(evlist); + perf_evlist__set_tracking_event(evlist, tracking_evsel); + + tracking_evsel->core.attr.freq = 0; + tracking_evsel->core.attr.sample_period = 1; + evsel__set_sample_bit(tracking_evsel, TIME); + + return 0; +} + +static u64 hisi_ptt_reference(struct auxtrace_record *itr __maybe_unused) +{ + return rdtsc(); +} + +static void hisi_ptt_recording_free(struct auxtrace_record *itr) +{ + struct hisi_ptt_recording *pttr = + container_of(itr, struct hisi_ptt_recording, itr); + + free(pttr); +} + +struct auxtrace_record *hisi_ptt_recording_init(int *err, + struct perf_pmu *hisi_ptt_pmu) +{ + struct hisi_ptt_recording *pttr; + + if (!hisi_ptt_pmu) { + *err = -ENODEV; + return NULL; + } + + pttr = zalloc(sizeof(*pttr)); + if (!pttr) { + *err = -ENOMEM; + return NULL; + } + + pttr->hisi_ptt_pmu = hisi_ptt_pmu; + pttr->itr.pmu = hisi_ptt_pmu; + pttr->itr.recording_options = hisi_ptt_recording_options; + pttr->itr.info_priv_size = hisi_ptt_info_priv_size; + pttr->itr.info_fill = hisi_ptt_info_fill; + pttr->itr.free = hisi_ptt_recording_free; + pttr->itr.reference = hisi_ptt_reference; + pttr->itr.read_finish = auxtrace_record__read_finish; + pttr->itr.alignment = 0; + + *err = 0; + return &pttr->itr; +} diff --git a/tools/perf/util/Build b/tools/perf/util/Build index 8dcfca1a882f0e9abb341fa45992d281f2e438b5..5e51fe6e4e6e0d86afbf6484fdb29daec12f6e06 100644 --- a/tools/perf/util/Build +++ b/tools/perf/util/Build @@ -101,6 +101,10 @@ perf-$(CONFIG_AUXTRACE) += intel-pt.o perf-$(CONFIG_AUXTRACE) += intel-bts.o perf-$(CONFIG_AUXTRACE) += arm-spe.o perf-$(CONFIG_AUXTRACE) += arm-spe-pkt-decoder.o +perf-$(CONFIG_AUXTRACE) += arm-spe-decoder/ +perf-$(CONFIG_AUXTRACE) += hisi-ptt.o +perf-$(CONFIG_AUXTRACE) += hisi-ptt-decoder/ +perf-$(CONFIG_AUXTRACE) += arm-spe-decoder/ perf-$(CONFIG_AUXTRACE) += s390-cpumsf.o ifdef CONFIG_LIBOPENCSD diff --git a/tools/perf/util/arm-spe-decoder/Build b/tools/perf/util/arm-spe-decoder/Build new file mode 100644 index 0000000000000000000000000000000000000000..f8dae13fc876c0c157997dd3c95a5f3442631a39 --- /dev/null +++ b/tools/perf/util/arm-spe-decoder/Build @@ -0,0 +1 @@ +perf-$(CONFIG_AUXTRACE) += arm-spe-pkt-decoder.o arm-spe-decoder.o diff --git a/tools/perf/util/arm-spe-decoder/arm-spe-decoder.c b/tools/perf/util/arm-spe-decoder/arm-spe-decoder.c new file mode 100644 index 0000000000000000000000000000000000000000..90d575cee1b90f07578f1eb6813a441babfc9d2e --- /dev/null +++ b/tools/perf/util/arm-spe-decoder/arm-spe-decoder.c @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * arm_spe_decoder.c: ARM SPE support + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../auxtrace.h" +#include "../debug.h" +#include "../util.h" + +#include "arm-spe-decoder.h" + +static u64 arm_spe_calc_ip(int index, u64 payload) +{ + u64 ns, el, val; + + /* Instruction virtual address or Branch target address */ + if (index == SPE_ADDR_PKT_HDR_INDEX_INS || + index == SPE_ADDR_PKT_HDR_INDEX_BRANCH) { + ns = SPE_ADDR_PKT_GET_NS(payload); + el = SPE_ADDR_PKT_GET_EL(payload); + + /* Clean highest byte */ + payload = SPE_ADDR_PKT_ADDR_GET_BYTES_0_6(payload); + + /* Fill highest byte for EL1 or EL2 (VHE) mode */ + if (ns && (el == SPE_ADDR_PKT_EL1 || el == SPE_ADDR_PKT_EL2)) + payload |= 0xffULL << SPE_ADDR_PKT_ADDR_BYTE7_SHIFT; + + /* Data access virtual address */ + } else if (index == SPE_ADDR_PKT_HDR_INDEX_DATA_VIRT) { + + /* Clean tags */ + payload = SPE_ADDR_PKT_ADDR_GET_BYTES_0_6(payload); + + /* + * Armv8 ARM (ARM DDI 0487F.c), chapter "D10.2.1 Address packet" + * defines the data virtual address payload format, the top byte + * (bits [63:56]) is assigned as top-byte tag; so we only can + * retrieve address value from bits [55:0]. + * + * According to Documentation/arm64/memory.rst, if detects the + * specific pattern in bits [55:52] of payload which falls in + * the kernel space, should fixup the top byte and this allows + * perf tool to parse DSO symbol for data address correctly. + * + * For this reason, if detects the bits [55:52] is 0xf, will + * fill 0xff into the top byte. + */ + val = SPE_ADDR_PKT_ADDR_GET_BYTE_6(payload); + if ((val & 0xf0ULL) == 0xf0ULL) + payload |= 0xffULL << SPE_ADDR_PKT_ADDR_BYTE7_SHIFT; + + /* Data access physical address */ + } else if (index == SPE_ADDR_PKT_HDR_INDEX_DATA_PHYS) { + /* Clean highest byte */ + payload = SPE_ADDR_PKT_ADDR_GET_BYTES_0_6(payload); + } else { + pr_err("unsupported address packet index: 0x%x\n", index); + } + + return payload; +} + +struct arm_spe_decoder *arm_spe_decoder_new(struct arm_spe_params *params) +{ + struct arm_spe_decoder *decoder; + + if (!params->get_trace) + return NULL; + + decoder = zalloc(sizeof(struct arm_spe_decoder)); + if (!decoder) + return NULL; + + decoder->get_trace = params->get_trace; + decoder->data = params->data; + + return decoder; +} + +void arm_spe_decoder_free(struct arm_spe_decoder *decoder) +{ + free(decoder); +} + +static int arm_spe_get_data(struct arm_spe_decoder *decoder) +{ + struct arm_spe_buffer buffer = { .buf = 0, }; + int ret; + + pr_debug("Getting more data\n"); + ret = decoder->get_trace(&buffer, decoder->data); + if (ret < 0) + return ret; + + decoder->buf = buffer.buf; + decoder->len = buffer.len; + + if (!decoder->len) + pr_debug("No more data\n"); + + return decoder->len; +} + +static int arm_spe_get_next_packet(struct arm_spe_decoder *decoder) +{ + int ret; + + do { + if (!decoder->len) { + ret = arm_spe_get_data(decoder); + + /* Failed to read out trace data */ + if (ret <= 0) + return ret; + } + + ret = arm_spe_get_packet(decoder->buf, decoder->len, + &decoder->packet); + if (ret <= 0) { + /* Move forward for 1 byte */ + decoder->buf += 1; + decoder->len -= 1; + return -EBADMSG; + } + + decoder->buf += ret; + decoder->len -= ret; + } while (decoder->packet.type == ARM_SPE_PAD); + + return 1; +} + +static int arm_spe_read_record(struct arm_spe_decoder *decoder) +{ + int err; + int idx; + u64 payload, ip; + + memset(&decoder->record, 0x0, sizeof(decoder->record)); + + while (1) { + err = arm_spe_get_next_packet(decoder); + if (err <= 0) + return err; + + idx = decoder->packet.index; + payload = decoder->packet.payload; + + switch (decoder->packet.type) { + case ARM_SPE_TIMESTAMP: + decoder->record.timestamp = payload; + return 1; + case ARM_SPE_END: + return 1; + case ARM_SPE_ADDRESS: + ip = arm_spe_calc_ip(idx, payload); + if (idx == SPE_ADDR_PKT_HDR_INDEX_INS) + decoder->record.from_ip = ip; + else if (idx == SPE_ADDR_PKT_HDR_INDEX_BRANCH) + decoder->record.to_ip = ip; + break; + case ARM_SPE_COUNTER: + break; + case ARM_SPE_CONTEXT: + break; + case ARM_SPE_OP_TYPE: + break; + case ARM_SPE_EVENTS: + if (payload & BIT(EV_L1D_REFILL)) + decoder->record.type |= ARM_SPE_L1D_MISS; + + if (payload & BIT(EV_L1D_ACCESS)) + decoder->record.type |= ARM_SPE_L1D_ACCESS; + + if (payload & BIT(EV_TLB_WALK)) + decoder->record.type |= ARM_SPE_TLB_MISS; + + if (payload & BIT(EV_TLB_ACCESS)) + decoder->record.type |= ARM_SPE_TLB_ACCESS; + + if (payload & BIT(EV_LLC_MISS)) + decoder->record.type |= ARM_SPE_LLC_MISS; + + if (payload & BIT(EV_LLC_ACCESS)) + decoder->record.type |= ARM_SPE_LLC_ACCESS; + + if (payload & BIT(EV_REMOTE_ACCESS)) + decoder->record.type |= ARM_SPE_REMOTE_ACCESS; + + if (payload & BIT(EV_MISPRED)) + decoder->record.type |= ARM_SPE_BRANCH_MISS; + + break; + case ARM_SPE_DATA_SOURCE: + break; + case ARM_SPE_BAD: + break; + case ARM_SPE_PAD: + break; + default: + pr_err("Get packet error!\n"); + return -1; + } + } + + return 0; +} + +int arm_spe_decode(struct arm_spe_decoder *decoder) +{ + return arm_spe_read_record(decoder); +} diff --git a/tools/perf/util/arm-spe-decoder/arm-spe-decoder.h b/tools/perf/util/arm-spe-decoder/arm-spe-decoder.h new file mode 100644 index 0000000000000000000000000000000000000000..24727b8ca7ffa2ab7b33a7243c358c51ea1a2302 --- /dev/null +++ b/tools/perf/util/arm-spe-decoder/arm-spe-decoder.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * arm_spe_decoder.h: Arm Statistical Profiling Extensions support + * Copyright (c) 2019-2020, Arm Ltd. + */ + +#ifndef INCLUDE__ARM_SPE_DECODER_H__ +#define INCLUDE__ARM_SPE_DECODER_H__ + +#include +#include +#include + +#include "arm-spe-pkt-decoder.h" + +enum arm_spe_sample_type { + ARM_SPE_L1D_ACCESS = 1 << 0, + ARM_SPE_L1D_MISS = 1 << 1, + ARM_SPE_LLC_ACCESS = 1 << 2, + ARM_SPE_LLC_MISS = 1 << 3, + ARM_SPE_TLB_ACCESS = 1 << 4, + ARM_SPE_TLB_MISS = 1 << 5, + ARM_SPE_BRANCH_MISS = 1 << 6, + ARM_SPE_REMOTE_ACCESS = 1 << 7, +}; + +struct arm_spe_record { + enum arm_spe_sample_type type; + int err; + u64 from_ip; + u64 to_ip; + u64 timestamp; +}; + +struct arm_spe_insn; + +struct arm_spe_buffer { + const unsigned char *buf; + size_t len; + u64 offset; + u64 trace_nr; +}; + +struct arm_spe_params { + int (*get_trace)(struct arm_spe_buffer *buffer, void *data); + void *data; +}; + +struct arm_spe_decoder { + int (*get_trace)(struct arm_spe_buffer *buffer, void *data); + void *data; + struct arm_spe_record record; + + const unsigned char *buf; + size_t len; + + struct arm_spe_pkt packet; +}; + +struct arm_spe_decoder *arm_spe_decoder_new(struct arm_spe_params *params); +void arm_spe_decoder_free(struct arm_spe_decoder *decoder); + +int arm_spe_decode(struct arm_spe_decoder *decoder); + +#endif diff --git a/tools/perf/util/arm-spe-pkt-decoder.c b/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.c similarity index 30% rename from tools/perf/util/arm-spe-pkt-decoder.c rename to tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.c index b94001b756c7918bc68787d782536555b226667d..f3ac9d40cebf4cfa0b8cfef7dcc7e23a9449131e 100644 --- a/tools/perf/util/arm-spe-pkt-decoder.c +++ b/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.c @@ -8,36 +8,11 @@ #include #include #include +#include +#include #include "arm-spe-pkt-decoder.h" -#define BIT(n) (1ULL << (n)) - -#define NS_FLAG BIT(63) -#define EL_FLAG (BIT(62) | BIT(61)) - -#define SPE_HEADER0_PAD 0x0 -#define SPE_HEADER0_END 0x1 -#define SPE_HEADER0_ADDRESS 0x30 /* address packet (short) */ -#define SPE_HEADER0_ADDRESS_MASK 0x38 -#define SPE_HEADER0_COUNTER 0x18 /* counter packet (short) */ -#define SPE_HEADER0_COUNTER_MASK 0x38 -#define SPE_HEADER0_TIMESTAMP 0x71 -#define SPE_HEADER0_TIMESTAMP 0x71 -#define SPE_HEADER0_EVENTS 0x2 -#define SPE_HEADER0_EVENTS_MASK 0xf -#define SPE_HEADER0_SOURCE 0x3 -#define SPE_HEADER0_SOURCE_MASK 0xf -#define SPE_HEADER0_CONTEXT 0x24 -#define SPE_HEADER0_CONTEXT_MASK 0x3c -#define SPE_HEADER0_OP_TYPE 0x8 -#define SPE_HEADER0_OP_TYPE_MASK 0x3c -#define SPE_HEADER1_ALIGNMENT 0x0 -#define SPE_HEADER1_ADDRESS 0xb0 /* address packet (extended) */ -#define SPE_HEADER1_ADDRESS_MASK 0xf8 -#define SPE_HEADER1_COUNTER 0x98 /* counter packet (extended) */ -#define SPE_HEADER1_COUNTER_MASK 0xf8 - #if __BYTE_ORDER == __BIG_ENDIAN #define le16_to_cpu bswap_16 #define le32_to_cpu bswap_32 @@ -70,27 +45,28 @@ const char *arm_spe_pkt_name(enum arm_spe_pkt_type type) return arm_spe_packet_name[type]; } -/* return ARM SPE payload size from its encoding, - * which is in bits 5:4 of the byte. - * 00 : byte - * 01 : halfword (2) - * 10 : word (4) - * 11 : doubleword (8) +/* + * Extracts the field "sz" from header bits and converts to bytes: + * 00 : byte (1) + * 01 : halfword (2) + * 10 : word (4) + * 11 : doubleword (8) */ -static int payloadlen(unsigned char byte) +static unsigned int arm_spe_payload_len(unsigned char hdr) { - return 1 << ((byte & 0x30) >> 4); + return 1U << ((hdr & GENMASK_ULL(5, 4)) >> 4); } static int arm_spe_get_payload(const unsigned char *buf, size_t len, + unsigned char ext_hdr, struct arm_spe_pkt *packet) { - size_t payload_len = payloadlen(buf[0]); + size_t payload_len = arm_spe_payload_len(buf[ext_hdr]); - if (len < 1 + payload_len) + if (len < 1 + ext_hdr + payload_len) return ARM_SPE_NEED_MORE_BYTES; - buf++; + buf += 1 + ext_hdr; switch (payload_len) { case 1: packet->payload = *(uint8_t *)buf; break; @@ -100,7 +76,7 @@ static int arm_spe_get_payload(const unsigned char *buf, size_t len, default: return ARM_SPE_BAD_PACKET; } - return 1 + payload_len; + return 1 + ext_hdr + payload_len; } static int arm_spe_get_pad(struct arm_spe_pkt *packet) @@ -131,127 +107,128 @@ static int arm_spe_get_timestamp(const unsigned char *buf, size_t len, struct arm_spe_pkt *packet) { packet->type = ARM_SPE_TIMESTAMP; - return arm_spe_get_payload(buf, len, packet); + return arm_spe_get_payload(buf, len, 0, packet); } static int arm_spe_get_events(const unsigned char *buf, size_t len, struct arm_spe_pkt *packet) { - int ret = arm_spe_get_payload(buf, len, packet); - packet->type = ARM_SPE_EVENTS; /* we use index to identify Events with a less number of * comparisons in arm_spe_pkt_desc(): E.g., the LLC-ACCESS, - * LLC-REFILL, and REMOTE-ACCESS events are identified iff + * LLC-REFILL, and REMOTE-ACCESS events are identified if * index > 1. */ - packet->index = ret - 1; + packet->index = arm_spe_payload_len(buf[0]); - return ret; + return arm_spe_get_payload(buf, len, 0, packet); } static int arm_spe_get_data_source(const unsigned char *buf, size_t len, struct arm_spe_pkt *packet) { packet->type = ARM_SPE_DATA_SOURCE; - return arm_spe_get_payload(buf, len, packet); + return arm_spe_get_payload(buf, len, 0, packet); } static int arm_spe_get_context(const unsigned char *buf, size_t len, struct arm_spe_pkt *packet) { packet->type = ARM_SPE_CONTEXT; - packet->index = buf[0] & 0x3; - - return arm_spe_get_payload(buf, len, packet); + packet->index = SPE_CTX_PKT_HDR_INDEX(buf[0]); + return arm_spe_get_payload(buf, len, 0, packet); } static int arm_spe_get_op_type(const unsigned char *buf, size_t len, struct arm_spe_pkt *packet) { packet->type = ARM_SPE_OP_TYPE; - packet->index = buf[0] & 0x3; - return arm_spe_get_payload(buf, len, packet); + packet->index = SPE_OP_PKT_HDR_CLASS(buf[0]); + return arm_spe_get_payload(buf, len, 0, packet); } static int arm_spe_get_counter(const unsigned char *buf, size_t len, const unsigned char ext_hdr, struct arm_spe_pkt *packet) { - if (len < 2) - return ARM_SPE_NEED_MORE_BYTES; - packet->type = ARM_SPE_COUNTER; + if (ext_hdr) - packet->index = ((buf[0] & 0x3) << 3) | (buf[1] & 0x7); + packet->index = SPE_HDR_EXTENDED_INDEX(buf[0], buf[1]); else - packet->index = buf[0] & 0x7; - - packet->payload = le16_to_cpu(*(uint16_t *)(buf + 1)); + packet->index = SPE_HDR_SHORT_INDEX(buf[0]); - return 1 + ext_hdr + 2; + return arm_spe_get_payload(buf, len, ext_hdr, packet); } static int arm_spe_get_addr(const unsigned char *buf, size_t len, const unsigned char ext_hdr, struct arm_spe_pkt *packet) { - if (len < 8) - return ARM_SPE_NEED_MORE_BYTES; - packet->type = ARM_SPE_ADDRESS; + if (ext_hdr) - packet->index = ((buf[0] & 0x3) << 3) | (buf[1] & 0x7); + packet->index = SPE_HDR_EXTENDED_INDEX(buf[0], buf[1]); else - packet->index = buf[0] & 0x7; - - memcpy_le64(&packet->payload, buf + 1, 8); + packet->index = SPE_HDR_SHORT_INDEX(buf[0]); - return 1 + ext_hdr + 8; + return arm_spe_get_payload(buf, len, ext_hdr, packet); } static int arm_spe_do_get_packet(const unsigned char *buf, size_t len, struct arm_spe_pkt *packet) { - unsigned int byte; + unsigned int hdr; + unsigned char ext_hdr = 0; memset(packet, 0, sizeof(struct arm_spe_pkt)); if (!len) return ARM_SPE_NEED_MORE_BYTES; - byte = buf[0]; - if (byte == SPE_HEADER0_PAD) + hdr = buf[0]; + + if (hdr == SPE_HEADER0_PAD) return arm_spe_get_pad(packet); - else if (byte == SPE_HEADER0_END) /* no timestamp at end of record */ + + if (hdr == SPE_HEADER0_END) /* no timestamp at end of record */ return arm_spe_get_end(packet); - else if (byte & 0xc0 /* 0y11xxxxxx */) { - if (byte & 0x80) { - if ((byte & SPE_HEADER0_ADDRESS_MASK) == SPE_HEADER0_ADDRESS) - return arm_spe_get_addr(buf, len, 0, packet); - if ((byte & SPE_HEADER0_COUNTER_MASK) == SPE_HEADER0_COUNTER) - return arm_spe_get_counter(buf, len, 0, packet); - } else - if (byte == SPE_HEADER0_TIMESTAMP) - return arm_spe_get_timestamp(buf, len, packet); - else if ((byte & SPE_HEADER0_EVENTS_MASK) == SPE_HEADER0_EVENTS) - return arm_spe_get_events(buf, len, packet); - else if ((byte & SPE_HEADER0_SOURCE_MASK) == SPE_HEADER0_SOURCE) - return arm_spe_get_data_source(buf, len, packet); - else if ((byte & SPE_HEADER0_CONTEXT_MASK) == SPE_HEADER0_CONTEXT) - return arm_spe_get_context(buf, len, packet); - else if ((byte & SPE_HEADER0_OP_TYPE_MASK) == SPE_HEADER0_OP_TYPE) - return arm_spe_get_op_type(buf, len, packet); - } else if ((byte & 0xe0) == 0x20 /* 0y001xxxxx */) { - /* 16-bit header */ - byte = buf[1]; - if (byte == SPE_HEADER1_ALIGNMENT) + + if (hdr == SPE_HEADER0_TIMESTAMP) + return arm_spe_get_timestamp(buf, len, packet); + + if ((hdr & SPE_HEADER0_MASK1) == SPE_HEADER0_EVENTS) + return arm_spe_get_events(buf, len, packet); + + if ((hdr & SPE_HEADER0_MASK1) == SPE_HEADER0_SOURCE) + return arm_spe_get_data_source(buf, len, packet); + + if ((hdr & SPE_HEADER0_MASK2) == SPE_HEADER0_CONTEXT) + return arm_spe_get_context(buf, len, packet); + + if ((hdr & SPE_HEADER0_MASK2) == SPE_HEADER0_OP_TYPE) + return arm_spe_get_op_type(buf, len, packet); + + if ((hdr & SPE_HEADER0_MASK2) == SPE_HEADER0_EXTENDED) { + /* 16-bit extended format header */ + ext_hdr = 1; + + hdr = buf[1]; + if (hdr == SPE_HEADER1_ALIGNMENT) return arm_spe_get_alignment(buf, len, packet); - else if ((byte & SPE_HEADER1_ADDRESS_MASK) == SPE_HEADER1_ADDRESS) - return arm_spe_get_addr(buf, len, 1, packet); - else if ((byte & SPE_HEADER1_COUNTER_MASK) == SPE_HEADER1_COUNTER) - return arm_spe_get_counter(buf, len, 1, packet); } + /* + * The short format header's byte 0 or the extended format header's + * byte 1 has been assigned to 'hdr', which uses the same encoding for + * address packet and counter packet, so don't need to distinguish if + * it's short format or extended format and handle in once. + */ + if ((hdr & SPE_HEADER0_MASK3) == SPE_HEADER0_ADDRESS) + return arm_spe_get_addr(buf, len, ext_hdr, packet); + + if ((hdr & SPE_HEADER0_MASK3) == SPE_HEADER0_COUNTER) + return arm_spe_get_counter(buf, len, ext_hdr, packet); + return ARM_SPE_BAD_PACKET; } @@ -271,192 +248,286 @@ int arm_spe_get_packet(const unsigned char *buf, size_t len, return ret; } +static int arm_spe_pkt_out_string(int *err, char **buf_p, size_t *blen, + const char *fmt, ...) +{ + va_list ap; + int ret; + + /* Bail out if any error occurred */ + if (err && *err) + return *err; + + va_start(ap, fmt); + ret = vsnprintf(*buf_p, *blen, fmt, ap); + va_end(ap); + + if (ret < 0) { + if (err && !*err) + *err = ret; + + /* + * A return value of *blen or more means that the output was + * truncated and the buffer is overrun. + */ + } else if ((size_t)ret >= *blen) { + (*buf_p)[*blen - 1] = '\0'; + + /* + * Set *err to 'ret' to avoid overflow if tries to + * fill this buffer sequentially. + */ + if (err && !*err) + *err = ret; + } else { + *buf_p += ret; + *blen -= ret; + } + + return ret; +} + +static int arm_spe_pkt_desc_event(const struct arm_spe_pkt *packet, + char *buf, size_t buf_len) +{ + u64 payload = packet->payload; + int err = 0; + + arm_spe_pkt_out_string(&err, &buf, &buf_len, "EV"); + + if (payload & BIT(EV_EXCEPTION_GEN)) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " EXCEPTION-GEN"); + if (payload & BIT(EV_RETIRED)) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " RETIRED"); + if (payload & BIT(EV_L1D_ACCESS)) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " L1D-ACCESS"); + if (payload & BIT(EV_L1D_REFILL)) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " L1D-REFILL"); + if (payload & BIT(EV_TLB_ACCESS)) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " TLB-ACCESS"); + if (payload & BIT(EV_TLB_WALK)) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " TLB-REFILL"); + if (payload & BIT(EV_NOT_TAKEN)) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " NOT-TAKEN"); + if (payload & BIT(EV_MISPRED)) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " MISPRED"); + if (payload & BIT(EV_LLC_ACCESS)) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " LLC-ACCESS"); + if (payload & BIT(EV_LLC_MISS)) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " LLC-REFILL"); + if (payload & BIT(EV_REMOTE_ACCESS)) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " REMOTE-ACCESS"); + if (payload & BIT(EV_ALIGNMENT)) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " ALIGNMENT"); + if (payload & BIT(EV_PARTIAL_PREDICATE)) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " SVE-PARTIAL-PRED"); + if (payload & BIT(EV_EMPTY_PREDICATE)) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " SVE-EMPTY-PRED"); + + return err; +} + +static int arm_spe_pkt_desc_op_type(const struct arm_spe_pkt *packet, + char *buf, size_t buf_len) +{ + u64 payload = packet->payload; + int err = 0; + + switch (packet->index) { + case SPE_OP_PKT_HDR_CLASS_OTHER: + if (SPE_OP_PKT_IS_OTHER_SVE_OP(payload)) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, "SVE-OTHER"); + + /* SVE effective vector length */ + arm_spe_pkt_out_string(&err, &buf, &buf_len, " EVLEN %d", + SPE_OP_PKG_SVE_EVL(payload)); + + if (payload & SPE_OP_PKT_SVE_FP) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " FP"); + if (payload & SPE_OP_PKT_SVE_PRED) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " PRED"); + } else { + arm_spe_pkt_out_string(&err, &buf, &buf_len, "OTHER"); + arm_spe_pkt_out_string(&err, &buf, &buf_len, " %s", + payload & SPE_OP_PKT_COND ? + "COND-SELECT" : "INSN-OTHER"); + } + break; + case SPE_OP_PKT_HDR_CLASS_LD_ST_ATOMIC: + arm_spe_pkt_out_string(&err, &buf, &buf_len, + payload & 0x1 ? "ST" : "LD"); + + if (SPE_OP_PKT_IS_LDST_ATOMIC(payload)) { + if (payload & SPE_OP_PKT_AT) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " AT"); + if (payload & SPE_OP_PKT_EXCL) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " EXCL"); + if (payload & SPE_OP_PKT_AR) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " AR"); + } + + switch (SPE_OP_PKT_LDST_SUBCLASS_GET(payload)) { + case SPE_OP_PKT_LDST_SUBCLASS_SIMD_FP: + arm_spe_pkt_out_string(&err, &buf, &buf_len, " SIMD-FP"); + break; + case SPE_OP_PKT_LDST_SUBCLASS_GP_REG: + arm_spe_pkt_out_string(&err, &buf, &buf_len, " GP-REG"); + break; + case SPE_OP_PKT_LDST_SUBCLASS_UNSPEC_REG: + arm_spe_pkt_out_string(&err, &buf, &buf_len, " UNSPEC-REG"); + break; + case SPE_OP_PKT_LDST_SUBCLASS_NV_SYSREG: + arm_spe_pkt_out_string(&err, &buf, &buf_len, " NV-SYSREG"); + break; + default: + break; + } + + if (SPE_OP_PKT_IS_LDST_SVE(payload)) { + /* SVE effective vector length */ + arm_spe_pkt_out_string(&err, &buf, &buf_len, " EVLEN %d", + SPE_OP_PKG_SVE_EVL(payload)); + + if (payload & SPE_OP_PKT_SVE_PRED) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " PRED"); + if (payload & SPE_OP_PKT_SVE_SG) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " SG"); + } + break; + case SPE_OP_PKT_HDR_CLASS_BR_ERET: + arm_spe_pkt_out_string(&err, &buf, &buf_len, "B"); + + if (payload & SPE_OP_PKT_COND) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " COND"); + + if (SPE_OP_PKT_IS_INDIRECT_BRANCH(payload)) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " IND"); + + break; + default: + /* Unknown index */ + err = -1; + break; + } + + return err; +} + +static int arm_spe_pkt_desc_addr(const struct arm_spe_pkt *packet, + char *buf, size_t buf_len) +{ + int ns, el, idx = packet->index; + int ch, pat; + u64 payload = packet->payload; + int err = 0; + + switch (idx) { + case SPE_ADDR_PKT_HDR_INDEX_INS: + case SPE_ADDR_PKT_HDR_INDEX_BRANCH: + ns = !!SPE_ADDR_PKT_GET_NS(payload); + el = SPE_ADDR_PKT_GET_EL(payload); + payload = SPE_ADDR_PKT_ADDR_GET_BYTES_0_6(payload); + arm_spe_pkt_out_string(&err, &buf, &buf_len, + "%s 0x%llx el%d ns=%d", + (idx == 1) ? "TGT" : "PC", payload, el, ns); + break; + case SPE_ADDR_PKT_HDR_INDEX_DATA_VIRT: + arm_spe_pkt_out_string(&err, &buf, &buf_len, + "VA 0x%llx", payload); + break; + case SPE_ADDR_PKT_HDR_INDEX_DATA_PHYS: + ns = !!SPE_ADDR_PKT_GET_NS(payload); + ch = !!SPE_ADDR_PKT_GET_CH(payload); + pat = SPE_ADDR_PKT_GET_PAT(payload); + payload = SPE_ADDR_PKT_ADDR_GET_BYTES_0_6(payload); + arm_spe_pkt_out_string(&err, &buf, &buf_len, + "PA 0x%llx ns=%d ch=%d pat=%x", + payload, ns, ch, pat); + break; + default: + /* Unknown index */ + err = -1; + break; + } + + return err; +} + +static int arm_spe_pkt_desc_counter(const struct arm_spe_pkt *packet, + char *buf, size_t buf_len) +{ + u64 payload = packet->payload; + const char *name = arm_spe_pkt_name(packet->type); + int err = 0; + + arm_spe_pkt_out_string(&err, &buf, &buf_len, "%s %d ", name, + (unsigned short)payload); + + switch (packet->index) { + case SPE_CNT_PKT_HDR_INDEX_TOTAL_LAT: + arm_spe_pkt_out_string(&err, &buf, &buf_len, "TOT"); + break; + case SPE_CNT_PKT_HDR_INDEX_ISSUE_LAT: + arm_spe_pkt_out_string(&err, &buf, &buf_len, "ISSUE"); + break; + case SPE_CNT_PKT_HDR_INDEX_TRANS_LAT: + arm_spe_pkt_out_string(&err, &buf, &buf_len, "XLAT"); + break; + default: + break; + } + + return err; +} + int arm_spe_pkt_desc(const struct arm_spe_pkt *packet, char *buf, size_t buf_len) { - int ret, ns, el, idx = packet->index; + int idx = packet->index; unsigned long long payload = packet->payload; const char *name = arm_spe_pkt_name(packet->type); + char *buf_orig = buf; + size_t blen = buf_len; + int err = 0; switch (packet->type) { case ARM_SPE_BAD: case ARM_SPE_PAD: case ARM_SPE_END: - return snprintf(buf, buf_len, "%s", name); - case ARM_SPE_EVENTS: { - size_t blen = buf_len; - - ret = 0; - ret = snprintf(buf, buf_len, "EV"); - buf += ret; - blen -= ret; - if (payload & 0x1) { - ret = snprintf(buf, buf_len, " EXCEPTION-GEN"); - buf += ret; - blen -= ret; - } - if (payload & 0x2) { - ret = snprintf(buf, buf_len, " RETIRED"); - buf += ret; - blen -= ret; - } - if (payload & 0x4) { - ret = snprintf(buf, buf_len, " L1D-ACCESS"); - buf += ret; - blen -= ret; - } - if (payload & 0x8) { - ret = snprintf(buf, buf_len, " L1D-REFILL"); - buf += ret; - blen -= ret; - } - if (payload & 0x10) { - ret = snprintf(buf, buf_len, " TLB-ACCESS"); - buf += ret; - blen -= ret; - } - if (payload & 0x20) { - ret = snprintf(buf, buf_len, " TLB-REFILL"); - buf += ret; - blen -= ret; - } - if (payload & 0x40) { - ret = snprintf(buf, buf_len, " NOT-TAKEN"); - buf += ret; - blen -= ret; - } - if (payload & 0x80) { - ret = snprintf(buf, buf_len, " MISPRED"); - buf += ret; - blen -= ret; - } - if (idx > 1) { - if (payload & 0x100) { - ret = snprintf(buf, buf_len, " LLC-ACCESS"); - buf += ret; - blen -= ret; - } - if (payload & 0x200) { - ret = snprintf(buf, buf_len, " LLC-REFILL"); - buf += ret; - blen -= ret; - } - if (payload & 0x400) { - ret = snprintf(buf, buf_len, " REMOTE-ACCESS"); - buf += ret; - blen -= ret; - } - } - if (ret < 0) - return ret; - blen -= ret; - return buf_len - blen; - } + arm_spe_pkt_out_string(&err, &buf, &blen, "%s", name); + break; + case ARM_SPE_EVENTS: + err = arm_spe_pkt_desc_event(packet, buf, buf_len); + break; case ARM_SPE_OP_TYPE: - switch (idx) { - case 0: return snprintf(buf, buf_len, "%s", payload & 0x1 ? - "COND-SELECT" : "INSN-OTHER"); - case 1: { - size_t blen = buf_len; - - if (payload & 0x1) - ret = snprintf(buf, buf_len, "ST"); - else - ret = snprintf(buf, buf_len, "LD"); - buf += ret; - blen -= ret; - if (payload & 0x2) { - if (payload & 0x4) { - ret = snprintf(buf, buf_len, " AT"); - buf += ret; - blen -= ret; - } - if (payload & 0x8) { - ret = snprintf(buf, buf_len, " EXCL"); - buf += ret; - blen -= ret; - } - if (payload & 0x10) { - ret = snprintf(buf, buf_len, " AR"); - buf += ret; - blen -= ret; - } - } else if (payload & 0x4) { - ret = snprintf(buf, buf_len, " SIMD-FP"); - buf += ret; - blen -= ret; - } - if (ret < 0) - return ret; - blen -= ret; - return buf_len - blen; - } - case 2: { - size_t blen = buf_len; - - ret = snprintf(buf, buf_len, "B"); - buf += ret; - blen -= ret; - if (payload & 0x1) { - ret = snprintf(buf, buf_len, " COND"); - buf += ret; - blen -= ret; - } - if (payload & 0x2) { - ret = snprintf(buf, buf_len, " IND"); - buf += ret; - blen -= ret; - } - if (ret < 0) - return ret; - blen -= ret; - return buf_len - blen; - } - default: return 0; - } + err = arm_spe_pkt_desc_op_type(packet, buf, buf_len); + break; case ARM_SPE_DATA_SOURCE: case ARM_SPE_TIMESTAMP: - return snprintf(buf, buf_len, "%s %lld", name, payload); + arm_spe_pkt_out_string(&err, &buf, &blen, "%s %lld", name, payload); + break; case ARM_SPE_ADDRESS: - switch (idx) { - case 0: - case 1: ns = !!(packet->payload & NS_FLAG); - el = (packet->payload & EL_FLAG) >> 61; - payload &= ~(0xffULL << 56); - return snprintf(buf, buf_len, "%s 0x%llx el%d ns=%d", - (idx == 1) ? "TGT" : "PC", payload, el, ns); - case 2: return snprintf(buf, buf_len, "VA 0x%llx", payload); - case 3: ns = !!(packet->payload & NS_FLAG); - payload &= ~(0xffULL << 56); - return snprintf(buf, buf_len, "PA 0x%llx ns=%d", - payload, ns); - default: return 0; - } + err = arm_spe_pkt_desc_addr(packet, buf, buf_len); + break; case ARM_SPE_CONTEXT: - return snprintf(buf, buf_len, "%s 0x%lx el%d", name, - (unsigned long)payload, idx + 1); - case ARM_SPE_COUNTER: { - size_t blen = buf_len; - - ret = snprintf(buf, buf_len, "%s %d ", name, - (unsigned short)payload); - buf += ret; - blen -= ret; - switch (idx) { - case 0: ret = snprintf(buf, buf_len, "TOT"); break; - case 1: ret = snprintf(buf, buf_len, "ISSUE"); break; - case 2: ret = snprintf(buf, buf_len, "XLAT"); break; - default: ret = 0; - } - if (ret < 0) - return ret; - blen -= ret; - return buf_len - blen; - } + arm_spe_pkt_out_string(&err, &buf, &blen, "%s 0x%lx el%d", + name, (unsigned long)payload, idx + 1); + break; + case ARM_SPE_COUNTER: + err = arm_spe_pkt_desc_counter(packet, buf, buf_len); + break; default: + /* Unknown packet type */ + err = -1; break; } - return snprintf(buf, buf_len, "%s 0x%llx (%d)", - name, payload, packet->index); + /* Output raw data if detect any error */ + if (err) { + err = 0; + arm_spe_pkt_out_string(&err, &buf_orig, &buf_len, "%s 0x%llx (%d)", + name, payload, packet->index); + } + + return err; } diff --git a/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.h b/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.h new file mode 100644 index 0000000000000000000000000000000000000000..9b970e7bf1e21965c82cb36202a92eb719c8a553 --- /dev/null +++ b/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.h @@ -0,0 +1,155 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Arm Statistical Profiling Extensions (SPE) support + * Copyright (c) 2017-2018, Arm Ltd. + */ + +#ifndef INCLUDE__ARM_SPE_PKT_DECODER_H__ +#define INCLUDE__ARM_SPE_PKT_DECODER_H__ + +#include +#include + +#define ARM_SPE_PKT_DESC_MAX 256 + +#define ARM_SPE_NEED_MORE_BYTES -1 +#define ARM_SPE_BAD_PACKET -2 + +#define ARM_SPE_PKT_MAX_SZ 16 + +enum arm_spe_pkt_type { + ARM_SPE_BAD, + ARM_SPE_PAD, + ARM_SPE_END, + ARM_SPE_TIMESTAMP, + ARM_SPE_ADDRESS, + ARM_SPE_COUNTER, + ARM_SPE_CONTEXT, + ARM_SPE_OP_TYPE, + ARM_SPE_EVENTS, + ARM_SPE_DATA_SOURCE, +}; + +struct arm_spe_pkt { + enum arm_spe_pkt_type type; + unsigned char index; + uint64_t payload; +}; + +/* Short header (HEADER0) and extended header (HEADER1) */ +#define SPE_HEADER0_PAD 0x0 +#define SPE_HEADER0_END 0x1 +#define SPE_HEADER0_TIMESTAMP 0x71 +/* Mask for event & data source */ +#define SPE_HEADER0_MASK1 (GENMASK_ULL(7, 6) | GENMASK_ULL(3, 0)) +#define SPE_HEADER0_EVENTS 0x42 +#define SPE_HEADER0_SOURCE 0x43 +/* Mask for context & operation */ +#define SPE_HEADER0_MASK2 GENMASK_ULL(7, 2) +#define SPE_HEADER0_CONTEXT 0x64 +#define SPE_HEADER0_OP_TYPE 0x48 +/* Mask for extended format */ +#define SPE_HEADER0_EXTENDED 0x20 +/* Mask for address & counter */ +#define SPE_HEADER0_MASK3 GENMASK_ULL(7, 3) +#define SPE_HEADER0_ADDRESS 0xb0 +#define SPE_HEADER0_COUNTER 0x98 +#define SPE_HEADER1_ALIGNMENT 0x0 + +#define SPE_HDR_SHORT_INDEX(h) ((h) & GENMASK_ULL(2, 0)) +#define SPE_HDR_EXTENDED_INDEX(h0, h1) (((h0) & GENMASK_ULL(1, 0)) << 3 | \ + SPE_HDR_SHORT_INDEX(h1)) + +/* Address packet header */ +#define SPE_ADDR_PKT_HDR_INDEX_INS 0x0 +#define SPE_ADDR_PKT_HDR_INDEX_BRANCH 0x1 +#define SPE_ADDR_PKT_HDR_INDEX_DATA_VIRT 0x2 +#define SPE_ADDR_PKT_HDR_INDEX_DATA_PHYS 0x3 + +/* Address packet payload */ +#define SPE_ADDR_PKT_ADDR_BYTE7_SHIFT 56 +#define SPE_ADDR_PKT_ADDR_GET_BYTES_0_6(v) ((v) & GENMASK_ULL(55, 0)) +#define SPE_ADDR_PKT_ADDR_GET_BYTE_6(v) (((v) & GENMASK_ULL(55, 48)) >> 48) + +#define SPE_ADDR_PKT_GET_NS(v) (((v) & BIT_ULL(63)) >> 63) +#define SPE_ADDR_PKT_GET_EL(v) (((v) & GENMASK_ULL(62, 61)) >> 61) +#define SPE_ADDR_PKT_GET_CH(v) (((v) & BIT_ULL(62)) >> 62) +#define SPE_ADDR_PKT_GET_PAT(v) (((v) & GENMASK_ULL(59, 56)) >> 56) + +#define SPE_ADDR_PKT_EL0 0 +#define SPE_ADDR_PKT_EL1 1 +#define SPE_ADDR_PKT_EL2 2 +#define SPE_ADDR_PKT_EL3 3 + +/* Context packet header */ +#define SPE_CTX_PKT_HDR_INDEX(h) ((h) & GENMASK_ULL(1, 0)) + +/* Counter packet header */ +#define SPE_CNT_PKT_HDR_INDEX_TOTAL_LAT 0x0 +#define SPE_CNT_PKT_HDR_INDEX_ISSUE_LAT 0x1 +#define SPE_CNT_PKT_HDR_INDEX_TRANS_LAT 0x2 + +/* Event packet payload */ +enum arm_spe_events { + EV_EXCEPTION_GEN = 0, + EV_RETIRED = 1, + EV_L1D_ACCESS = 2, + EV_L1D_REFILL = 3, + EV_TLB_ACCESS = 4, + EV_TLB_WALK = 5, + EV_NOT_TAKEN = 6, + EV_MISPRED = 7, + EV_LLC_ACCESS = 8, + EV_LLC_MISS = 9, + EV_REMOTE_ACCESS = 10, + EV_ALIGNMENT = 11, + EV_PARTIAL_PREDICATE = 17, + EV_EMPTY_PREDICATE = 18, +}; + +/* Operation packet header */ +#define SPE_OP_PKT_HDR_CLASS(h) ((h) & GENMASK_ULL(1, 0)) +#define SPE_OP_PKT_HDR_CLASS_OTHER 0x0 +#define SPE_OP_PKT_HDR_CLASS_LD_ST_ATOMIC 0x1 +#define SPE_OP_PKT_HDR_CLASS_BR_ERET 0x2 + +#define SPE_OP_PKT_IS_OTHER_SVE_OP(v) (((v) & (BIT(7) | BIT(3) | BIT(0))) == 0x8) + +#define SPE_OP_PKT_COND BIT(0) + +#define SPE_OP_PKT_LDST_SUBCLASS_GET(v) ((v) & GENMASK_ULL(7, 1)) +#define SPE_OP_PKT_LDST_SUBCLASS_GP_REG 0x0 +#define SPE_OP_PKT_LDST_SUBCLASS_SIMD_FP 0x4 +#define SPE_OP_PKT_LDST_SUBCLASS_UNSPEC_REG 0x10 +#define SPE_OP_PKT_LDST_SUBCLASS_NV_SYSREG 0x30 + +#define SPE_OP_PKT_IS_LDST_ATOMIC(v) (((v) & (GENMASK_ULL(7, 5) | BIT(1))) == 0x2) + +#define SPE_OP_PKT_AR BIT(4) +#define SPE_OP_PKT_EXCL BIT(3) +#define SPE_OP_PKT_AT BIT(2) +#define SPE_OP_PKT_ST BIT(0) + +#define SPE_OP_PKT_IS_LDST_SVE(v) (((v) & (BIT(3) | BIT(1))) == 0x8) + +#define SPE_OP_PKT_SVE_SG BIT(7) +/* + * SVE effective vector length (EVL) is stored in byte 0 bits [6:4]; + * the length is rounded up to a power of two and use 32 as one step, + * so EVL calculation is: + * + * 32 * (2 ^ bits [6:4]) = 32 << (bits [6:4]) + */ +#define SPE_OP_PKG_SVE_EVL(v) (32 << (((v) & GENMASK_ULL(6, 4)) >> 4)) +#define SPE_OP_PKT_SVE_PRED BIT(2) +#define SPE_OP_PKT_SVE_FP BIT(1) + +#define SPE_OP_PKT_IS_INDIRECT_BRANCH(v) (((v) & GENMASK_ULL(7, 1)) == 0x2) + +const char *arm_spe_pkt_name(enum arm_spe_pkt_type); + +int arm_spe_get_packet(const unsigned char *buf, size_t len, + struct arm_spe_pkt *packet); + +int arm_spe_pkt_desc(const struct arm_spe_pkt *packet, char *buf, size_t len); +#endif diff --git a/tools/perf/util/arm-spe-pkt-decoder.h b/tools/perf/util/arm-spe-pkt-decoder.h deleted file mode 100644 index d786ef65113faeadce2984bf95f08cfb417621e8..0000000000000000000000000000000000000000 --- a/tools/perf/util/arm-spe-pkt-decoder.h +++ /dev/null @@ -1,43 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Arm Statistical Profiling Extensions (SPE) support - * Copyright (c) 2017-2018, Arm Ltd. - */ - -#ifndef INCLUDE__ARM_SPE_PKT_DECODER_H__ -#define INCLUDE__ARM_SPE_PKT_DECODER_H__ - -#include -#include - -#define ARM_SPE_PKT_DESC_MAX 256 - -#define ARM_SPE_NEED_MORE_BYTES -1 -#define ARM_SPE_BAD_PACKET -2 - -enum arm_spe_pkt_type { - ARM_SPE_BAD, - ARM_SPE_PAD, - ARM_SPE_END, - ARM_SPE_TIMESTAMP, - ARM_SPE_ADDRESS, - ARM_SPE_COUNTER, - ARM_SPE_CONTEXT, - ARM_SPE_OP_TYPE, - ARM_SPE_EVENTS, - ARM_SPE_DATA_SOURCE, -}; - -struct arm_spe_pkt { - enum arm_spe_pkt_type type; - unsigned char index; - uint64_t payload; -}; - -const char *arm_spe_pkt_name(enum arm_spe_pkt_type); - -int arm_spe_get_packet(const unsigned char *buf, size_t len, - struct arm_spe_pkt *packet); - -int arm_spe_pkt_desc(const struct arm_spe_pkt *packet, char *buf, size_t len); -#endif diff --git a/tools/perf/util/arm-spe.c b/tools/perf/util/arm-spe.c index 53be12b23ff406cc74d67e36f9f311e3b84f732b..71308e4830a3ee67534b2d4594e93ecc9b05186d 100644 --- a/tools/perf/util/arm-spe.c +++ b/tools/perf/util/arm-spe.c @@ -4,46 +4,85 @@ * Copyright (c) 2017-2018, Arm Ltd. */ +#include #include #include -#include #include -#include -#include -#include -#include #include +#include #include +#include #include +#include +#include +#include "auxtrace.h" #include "color.h" +#include "debug.h" +#include "evlist.h" #include "evsel.h" #include "machine.h" #include "session.h" -#include "debug.h" -#include "auxtrace.h" +#include "symbol.h" +#include "thread.h" +#include "thread-stack.h" +#include "tool.h" +#include "util/synthetic-events.h" + #include "arm-spe.h" -#include "arm-spe-pkt-decoder.h" +#include "arm-spe-decoder/arm-spe-decoder.h" +#include "arm-spe-decoder/arm-spe-pkt-decoder.h" + +#define MAX_TIMESTAMP (~0ULL) struct arm_spe { struct auxtrace auxtrace; struct auxtrace_queues queues; struct auxtrace_heap heap; + struct itrace_synth_opts synth_opts; u32 auxtrace_type; struct perf_session *session; struct machine *machine; u32 pmu_type; + + u8 timeless_decoding; + u8 data_queued; + + u8 sample_flc; + u8 sample_llc; + u8 sample_tlb; + u8 sample_branch; + u8 sample_remote_access; + + u64 l1d_miss_id; + u64 l1d_access_id; + u64 llc_miss_id; + u64 llc_access_id; + u64 tlb_miss_id; + u64 tlb_access_id; + u64 branch_miss_id; + u64 remote_access_id; + + u64 kernel_start; + + unsigned long num_events; }; struct arm_spe_queue { - struct arm_spe *spe; - unsigned int queue_nr; - struct auxtrace_buffer *buffer; - bool on_heap; - bool done; - pid_t pid; - pid_t tid; - int cpu; + struct arm_spe *spe; + unsigned int queue_nr; + struct auxtrace_buffer *buffer; + struct auxtrace_buffer *old_buffer; + union perf_event *event_buf; + bool on_heap; + bool done; + pid_t pid; + pid_t tid; + int cpu; + struct arm_spe_decoder *decoder; + u64 time; + u64 timestamp; + struct thread *thread; }; static void arm_spe_dump(struct arm_spe *spe __maybe_unused, @@ -74,7 +113,7 @@ static void arm_spe_dump(struct arm_spe *spe __maybe_unused, if (ret > 0) { ret = arm_spe_pkt_desc(&packet, desc, ARM_SPE_PKT_DESC_MAX); - if (ret > 0) + if (!ret) color_fprintf(stdout, color, " %s\n", desc); } else { color_fprintf(stdout, color, " Bad packet!\n"); @@ -92,44 +131,520 @@ static void arm_spe_dump_event(struct arm_spe *spe, unsigned char *buf, arm_spe_dump(spe, buf, len); } -static int arm_spe_process_event(struct perf_session *session __maybe_unused, - union perf_event *event __maybe_unused, - struct perf_sample *sample __maybe_unused, - struct perf_tool *tool __maybe_unused) +static int arm_spe_get_trace(struct arm_spe_buffer *b, void *data) +{ + struct arm_spe_queue *speq = data; + struct auxtrace_buffer *buffer = speq->buffer; + struct auxtrace_buffer *old_buffer = speq->old_buffer; + struct auxtrace_queue *queue; + + queue = &speq->spe->queues.queue_array[speq->queue_nr]; + + buffer = auxtrace_buffer__next(queue, buffer); + /* If no more data, drop the previous auxtrace_buffer and return */ + if (!buffer) { + if (old_buffer) + auxtrace_buffer__drop_data(old_buffer); + b->len = 0; + return 0; + } + + speq->buffer = buffer; + + /* If the aux_buffer doesn't have data associated, try to load it */ + if (!buffer->data) { + /* get the file desc associated with the perf data file */ + int fd = perf_data__fd(speq->spe->session->data); + + buffer->data = auxtrace_buffer__get_data(buffer, fd); + if (!buffer->data) + return -ENOMEM; + } + + b->len = buffer->size; + b->buf = buffer->data; + + if (b->len) { + if (old_buffer) + auxtrace_buffer__drop_data(old_buffer); + speq->old_buffer = buffer; + } else { + auxtrace_buffer__drop_data(buffer); + return arm_spe_get_trace(b, data); + } + + return 0; +} + +static struct arm_spe_queue *arm_spe__alloc_queue(struct arm_spe *spe, + unsigned int queue_nr) +{ + struct arm_spe_params params = { .get_trace = 0, }; + struct arm_spe_queue *speq; + + speq = zalloc(sizeof(*speq)); + if (!speq) + return NULL; + + speq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE); + if (!speq->event_buf) + goto out_free; + + speq->spe = spe; + speq->queue_nr = queue_nr; + speq->pid = -1; + speq->tid = -1; + speq->cpu = -1; + + /* params set */ + params.get_trace = arm_spe_get_trace; + params.data = speq; + + /* create new decoder */ + speq->decoder = arm_spe_decoder_new(¶ms); + if (!speq->decoder) + goto out_free; + + return speq; + +out_free: + zfree(&speq->event_buf); + free(speq); + + return NULL; +} + +static inline u8 arm_spe_cpumode(struct arm_spe *spe, u64 ip) +{ + return ip >= spe->kernel_start ? + PERF_RECORD_MISC_KERNEL : + PERF_RECORD_MISC_USER; +} + +static void arm_spe_prep_sample(struct arm_spe *spe, + struct arm_spe_queue *speq, + union perf_event *event, + struct perf_sample *sample) +{ + struct arm_spe_record *record = &speq->decoder->record; + + if (!spe->timeless_decoding) + sample->time = speq->timestamp; + + sample->ip = record->from_ip; + sample->cpumode = arm_spe_cpumode(spe, sample->ip); + sample->pid = speq->pid; + sample->tid = speq->tid; + sample->addr = record->to_ip; + sample->period = 1; + sample->cpu = speq->cpu; + + event->sample.header.type = PERF_RECORD_SAMPLE; + event->sample.header.misc = sample->cpumode; + event->sample.header.size = sizeof(struct perf_event_header); +} + +static inline int +arm_spe_deliver_synth_event(struct arm_spe *spe, + struct arm_spe_queue *speq __maybe_unused, + union perf_event *event, + struct perf_sample *sample) +{ + int ret; + + ret = perf_session__deliver_synth_event(spe->session, event, sample); + if (ret) + pr_err("ARM SPE: failed to deliver event, error %d\n", ret); + + return ret; +} + +static int +arm_spe_synth_spe_events_sample(struct arm_spe_queue *speq, + u64 spe_events_id) +{ + struct arm_spe *spe = speq->spe; + union perf_event *event = speq->event_buf; + struct perf_sample sample = { .ip = 0, }; + + arm_spe_prep_sample(spe, speq, event, &sample); + + sample.id = spe_events_id; + sample.stream_id = spe_events_id; + + return arm_spe_deliver_synth_event(spe, speq, event, &sample); +} + +static int arm_spe_sample(struct arm_spe_queue *speq) +{ + const struct arm_spe_record *record = &speq->decoder->record; + struct arm_spe *spe = speq->spe; + int err; + + if (spe->sample_flc) { + if (record->type & ARM_SPE_L1D_MISS) { + err = arm_spe_synth_spe_events_sample( + speq, spe->l1d_miss_id); + if (err) + return err; + } + + if (record->type & ARM_SPE_L1D_ACCESS) { + err = arm_spe_synth_spe_events_sample( + speq, spe->l1d_access_id); + if (err) + return err; + } + } + + if (spe->sample_llc) { + if (record->type & ARM_SPE_LLC_MISS) { + err = arm_spe_synth_spe_events_sample( + speq, spe->llc_miss_id); + if (err) + return err; + } + + if (record->type & ARM_SPE_LLC_ACCESS) { + err = arm_spe_synth_spe_events_sample( + speq, spe->llc_access_id); + if (err) + return err; + } + } + + if (spe->sample_tlb) { + if (record->type & ARM_SPE_TLB_MISS) { + err = arm_spe_synth_spe_events_sample( + speq, spe->tlb_miss_id); + if (err) + return err; + } + + if (record->type & ARM_SPE_TLB_ACCESS) { + err = arm_spe_synth_spe_events_sample( + speq, spe->tlb_access_id); + if (err) + return err; + } + } + + if (spe->sample_branch && (record->type & ARM_SPE_BRANCH_MISS)) { + err = arm_spe_synth_spe_events_sample(speq, + spe->branch_miss_id); + if (err) + return err; + } + + if (spe->sample_remote_access && + (record->type & ARM_SPE_REMOTE_ACCESS)) { + err = arm_spe_synth_spe_events_sample(speq, + spe->remote_access_id); + if (err) + return err; + } + + return 0; +} + +static int arm_spe_run_decoder(struct arm_spe_queue *speq, u64 *timestamp) +{ + struct arm_spe *spe = speq->spe; + int ret; + + if (!spe->kernel_start) + spe->kernel_start = machine__kernel_start(spe->machine); + + while (1) { + ret = arm_spe_decode(speq->decoder); + if (!ret) { + pr_debug("No data or all data has been processed.\n"); + return 1; + } + + /* + * Error is detected when decode SPE trace data, continue to + * the next trace data and find out more records. + */ + if (ret < 0) + continue; + + ret = arm_spe_sample(speq); + if (ret) + return ret; + + if (!spe->timeless_decoding && speq->timestamp >= *timestamp) { + *timestamp = speq->timestamp; + return 0; + } + } + + return 0; +} + +static int arm_spe__setup_queue(struct arm_spe *spe, + struct auxtrace_queue *queue, + unsigned int queue_nr) +{ + struct arm_spe_queue *speq = queue->priv; + struct arm_spe_record *record; + + if (list_empty(&queue->head) || speq) + return 0; + + speq = arm_spe__alloc_queue(spe, queue_nr); + + if (!speq) + return -ENOMEM; + + queue->priv = speq; + + if (queue->cpu != -1) + speq->cpu = queue->cpu; + + if (!speq->on_heap) { + int ret; + + if (spe->timeless_decoding) + return 0; + +retry: + ret = arm_spe_decode(speq->decoder); + + if (!ret) + return 0; + + if (ret < 0) + goto retry; + + record = &speq->decoder->record; + + speq->timestamp = record->timestamp; + ret = auxtrace_heap__add(&spe->heap, queue_nr, speq->timestamp); + if (ret) + return ret; + speq->on_heap = true; + } + + return 0; +} + +static int arm_spe__setup_queues(struct arm_spe *spe) +{ + unsigned int i; + int ret; + + for (i = 0; i < spe->queues.nr_queues; i++) { + ret = arm_spe__setup_queue(spe, &spe->queues.queue_array[i], i); + if (ret) + return ret; + } + + return 0; +} + +static int arm_spe__update_queues(struct arm_spe *spe) { + if (spe->queues.new_data) { + spe->queues.new_data = false; + return arm_spe__setup_queues(spe); + } + return 0; } +static bool arm_spe__is_timeless_decoding(struct arm_spe *spe) +{ + struct evsel *evsel; + struct evlist *evlist = spe->session->evlist; + bool timeless_decoding = true; + + /* + * Circle through the list of event and complain if we find one + * with the time bit set. + */ + evlist__for_each_entry(evlist, evsel) { + if ((evsel->core.attr.sample_type & PERF_SAMPLE_TIME)) + timeless_decoding = false; + } + + return timeless_decoding; +} + +static void arm_spe_set_pid_tid_cpu(struct arm_spe *spe, + struct auxtrace_queue *queue) +{ + struct arm_spe_queue *speq = queue->priv; + pid_t tid; + + tid = machine__get_current_tid(spe->machine, speq->cpu); + if (tid != -1) { + speq->tid = tid; + thread__zput(speq->thread); + } else + speq->tid = queue->tid; + + if ((!speq->thread) && (speq->tid != -1)) { + speq->thread = machine__find_thread(spe->machine, -1, + speq->tid); + } + + if (speq->thread) { + speq->pid = speq->thread->pid_; + if (queue->cpu == -1) + speq->cpu = speq->thread->cpu; + } +} + +static int arm_spe_process_queues(struct arm_spe *spe, u64 timestamp) +{ + unsigned int queue_nr; + u64 ts; + int ret; + + while (1) { + struct auxtrace_queue *queue; + struct arm_spe_queue *speq; + + if (!spe->heap.heap_cnt) + return 0; + + if (spe->heap.heap_array[0].ordinal >= timestamp) + return 0; + + queue_nr = spe->heap.heap_array[0].queue_nr; + queue = &spe->queues.queue_array[queue_nr]; + speq = queue->priv; + + auxtrace_heap__pop(&spe->heap); + + if (spe->heap.heap_cnt) { + ts = spe->heap.heap_array[0].ordinal + 1; + if (ts > timestamp) + ts = timestamp; + } else { + ts = timestamp; + } + + arm_spe_set_pid_tid_cpu(spe, queue); + + ret = arm_spe_run_decoder(speq, &ts); + if (ret < 0) { + auxtrace_heap__add(&spe->heap, queue_nr, ts); + return ret; + } + + if (!ret) { + ret = auxtrace_heap__add(&spe->heap, queue_nr, ts); + if (ret < 0) + return ret; + } else { + speq->on_heap = false; + } + } + + return 0; +} + +static int arm_spe_process_timeless_queues(struct arm_spe *spe, pid_t tid, + u64 time_) +{ + struct auxtrace_queues *queues = &spe->queues; + unsigned int i; + u64 ts = 0; + + for (i = 0; i < queues->nr_queues; i++) { + struct auxtrace_queue *queue = &spe->queues.queue_array[i]; + struct arm_spe_queue *speq = queue->priv; + + if (speq && (tid == -1 || speq->tid == tid)) { + speq->time = time_; + arm_spe_set_pid_tid_cpu(spe, queue); + arm_spe_run_decoder(speq, &ts); + } + } + return 0; +} + +static int arm_spe_process_event(struct perf_session *session, + union perf_event *event, + struct perf_sample *sample, + struct perf_tool *tool) +{ + int err = 0; + u64 timestamp; + struct arm_spe *spe = container_of(session->auxtrace, + struct arm_spe, auxtrace); + + if (dump_trace) + return 0; + + if (!tool->ordered_events) { + pr_err("SPE trace requires ordered events\n"); + return -EINVAL; + } + + if (sample->time && (sample->time != (u64) -1)) + timestamp = sample->time; + else + timestamp = 0; + + if (timestamp || spe->timeless_decoding) { + err = arm_spe__update_queues(spe); + if (err) + return err; + } + + if (spe->timeless_decoding) { + if (event->header.type == PERF_RECORD_EXIT) { + err = arm_spe_process_timeless_queues(spe, + event->fork.tid, + sample->time); + } + } else if (timestamp) { + if (event->header.type == PERF_RECORD_EXIT) { + err = arm_spe_process_queues(spe, timestamp); + if (err) + return err; + } + } + + return err; +} + static int arm_spe_process_auxtrace_event(struct perf_session *session, union perf_event *event, struct perf_tool *tool __maybe_unused) { struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe, auxtrace); - struct auxtrace_buffer *buffer; - off_t data_offset; - int fd = perf_data__fd(session->data); - int err; - if (perf_data__is_pipe(session->data)) { - data_offset = 0; - } else { - data_offset = lseek(fd, 0, SEEK_CUR); - if (data_offset == -1) - return -errno; - } + if (!spe->data_queued) { + struct auxtrace_buffer *buffer; + off_t data_offset; + int fd = perf_data__fd(session->data); + int err; - err = auxtrace_queues__add_event(&spe->queues, session, event, - data_offset, &buffer); - if (err) - return err; + if (perf_data__is_pipe(session->data)) { + data_offset = 0; + } else { + data_offset = lseek(fd, 0, SEEK_CUR); + if (data_offset == -1) + return -errno; + } - /* Dump here now we have copied a piped trace out of the pipe */ - if (dump_trace) { - if (auxtrace_buffer__get_data(buffer, fd)) { - arm_spe_dump_event(spe, buffer->data, - buffer->size); - auxtrace_buffer__put_data(buffer); + err = auxtrace_queues__add_event(&spe->queues, session, event, + data_offset, &buffer); + if (err) + return err; + + /* Dump here now we have copied a piped trace out of the pipe */ + if (dump_trace) { + if (auxtrace_buffer__get_data(buffer, fd)) { + arm_spe_dump_event(spe, buffer->data, + buffer->size); + auxtrace_buffer__put_data(buffer); + } } } @@ -139,7 +654,25 @@ static int arm_spe_process_auxtrace_event(struct perf_session *session, static int arm_spe_flush(struct perf_session *session __maybe_unused, struct perf_tool *tool __maybe_unused) { - return 0; + struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe, + auxtrace); + int ret; + + if (dump_trace) + return 0; + + if (!tool->ordered_events) + return -EINVAL; + + ret = arm_spe__update_queues(spe); + if (ret < 0) + return ret; + + if (spe->timeless_decoding) + return arm_spe_process_timeless_queues(spe, -1, + MAX_TIMESTAMP - 1); + + return arm_spe_process_queues(spe, MAX_TIMESTAMP); } static void arm_spe_free_queue(void *priv) @@ -148,6 +681,9 @@ static void arm_spe_free_queue(void *priv) if (!speq) return; + thread__zput(speq->thread); + arm_spe_decoder_free(speq->decoder); + zfree(&speq->event_buf); free(speq); } @@ -188,11 +724,189 @@ static void arm_spe_print_info(__u64 *arr) fprintf(stdout, arm_spe_info_fmts[ARM_SPE_PMU_TYPE], arr[ARM_SPE_PMU_TYPE]); } +struct arm_spe_synth { + struct perf_tool dummy_tool; + struct perf_session *session; +}; + +static int arm_spe_event_synth(struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample __maybe_unused, + struct machine *machine __maybe_unused) +{ + struct arm_spe_synth *arm_spe_synth = + container_of(tool, struct arm_spe_synth, dummy_tool); + + return perf_session__deliver_synth_event(arm_spe_synth->session, + event, NULL); +} + +static int arm_spe_synth_event(struct perf_session *session, + struct perf_event_attr *attr, u64 id) +{ + struct arm_spe_synth arm_spe_synth; + + memset(&arm_spe_synth, 0, sizeof(struct arm_spe_synth)); + arm_spe_synth.session = session; + + return perf_event__synthesize_attr(&arm_spe_synth.dummy_tool, attr, 1, + &id, arm_spe_event_synth); +} + +static void arm_spe_set_event_name(struct evlist *evlist, u64 id, + const char *name) +{ + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) { + if (evsel->core.id && evsel->core.id[0] == id) { + if (evsel->name) + zfree(&evsel->name); + evsel->name = strdup(name); + break; + } + } +} + +static int +arm_spe_synth_events(struct arm_spe *spe, struct perf_session *session) +{ + struct evlist *evlist = session->evlist; + struct evsel *evsel; + struct perf_event_attr attr; + bool found = false; + u64 id; + int err; + + evlist__for_each_entry(evlist, evsel) { + if (evsel->core.attr.type == spe->pmu_type) { + found = true; + break; + } + } + + if (!found) { + pr_debug("No selected events with SPE trace data\n"); + return 0; + } + + memset(&attr, 0, sizeof(struct perf_event_attr)); + attr.size = sizeof(struct perf_event_attr); + attr.type = PERF_TYPE_HARDWARE; + attr.sample_type = evsel->core.attr.sample_type & PERF_SAMPLE_MASK; + attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID | + PERF_SAMPLE_PERIOD; + if (spe->timeless_decoding) + attr.sample_type &= ~(u64)PERF_SAMPLE_TIME; + else + attr.sample_type |= PERF_SAMPLE_TIME; + + attr.exclude_user = evsel->core.attr.exclude_user; + attr.exclude_kernel = evsel->core.attr.exclude_kernel; + attr.exclude_hv = evsel->core.attr.exclude_hv; + attr.exclude_host = evsel->core.attr.exclude_host; + attr.exclude_guest = evsel->core.attr.exclude_guest; + attr.sample_id_all = evsel->core.attr.sample_id_all; + attr.read_format = evsel->core.attr.read_format; + + /* create new id val to be a fixed offset from evsel id */ + id = evsel->core.id[0] + 1000000000; + + if (!id) + id = 1; + + if (spe->synth_opts.flc) { + spe->sample_flc = true; + + /* Level 1 data cache miss */ + err = arm_spe_synth_event(session, &attr, id); + if (err) + return err; + spe->l1d_miss_id = id; + arm_spe_set_event_name(evlist, id, "l1d-miss"); + id += 1; + + /* Level 1 data cache access */ + err = arm_spe_synth_event(session, &attr, id); + if (err) + return err; + spe->l1d_access_id = id; + arm_spe_set_event_name(evlist, id, "l1d-access"); + id += 1; + } + + if (spe->synth_opts.llc) { + spe->sample_llc = true; + + /* Last level cache miss */ + err = arm_spe_synth_event(session, &attr, id); + if (err) + return err; + spe->llc_miss_id = id; + arm_spe_set_event_name(evlist, id, "llc-miss"); + id += 1; + + /* Last level cache access */ + err = arm_spe_synth_event(session, &attr, id); + if (err) + return err; + spe->llc_access_id = id; + arm_spe_set_event_name(evlist, id, "llc-access"); + id += 1; + } + + if (spe->synth_opts.tlb) { + spe->sample_tlb = true; + + /* TLB miss */ + err = arm_spe_synth_event(session, &attr, id); + if (err) + return err; + spe->tlb_miss_id = id; + arm_spe_set_event_name(evlist, id, "tlb-miss"); + id += 1; + + /* TLB access */ + err = arm_spe_synth_event(session, &attr, id); + if (err) + return err; + spe->tlb_access_id = id; + arm_spe_set_event_name(evlist, id, "tlb-access"); + id += 1; + } + + if (spe->synth_opts.branches) { + spe->sample_branch = true; + + /* Branch miss */ + err = arm_spe_synth_event(session, &attr, id); + if (err) + return err; + spe->branch_miss_id = id; + arm_spe_set_event_name(evlist, id, "branch-miss"); + id += 1; + } + + if (spe->synth_opts.remote_access) { + spe->sample_remote_access = true; + + /* Remote access */ + err = arm_spe_synth_event(session, &attr, id); + if (err) + return err; + spe->remote_access_id = id; + arm_spe_set_event_name(evlist, id, "remote-access"); + id += 1; + } + + return 0; +} + int arm_spe_process_auxtrace_info(union perf_event *event, struct perf_session *session) { struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info; - size_t min_sz = sizeof(u64) * ARM_SPE_PMU_TYPE; + size_t min_sz = sizeof(u64) * ARM_SPE_AUXTRACE_PRIV_MAX; struct arm_spe *spe; int err; @@ -213,6 +927,7 @@ int arm_spe_process_auxtrace_info(union perf_event *event, spe->auxtrace_type = auxtrace_info->type; spe->pmu_type = auxtrace_info->priv[ARM_SPE_PMU_TYPE]; + spe->timeless_decoding = arm_spe__is_timeless_decoding(spe); spe->auxtrace.process_event = arm_spe_process_event; spe->auxtrace.process_auxtrace_event = arm_spe_process_auxtrace_event; spe->auxtrace.flush_events = arm_spe_flush; @@ -222,8 +937,30 @@ int arm_spe_process_auxtrace_info(union perf_event *event, arm_spe_print_info(&auxtrace_info->priv[0]); + if (dump_trace) + return 0; + + if (session->itrace_synth_opts && session->itrace_synth_opts->set) + spe->synth_opts = *session->itrace_synth_opts; + else + itrace_synth_opts__set_default(&spe->synth_opts, false); + + err = arm_spe_synth_events(spe, session); + if (err) + goto err_free_queues; + + err = auxtrace_queues__process_index(&spe->queues, session); + if (err) + goto err_free_queues; + + if (spe->queues.populated) + spe->data_queued = true; + return 0; +err_free_queues: + auxtrace_queues__free(&spe->queues); + session->auxtrace = NULL; err_free: free(spe); return err; diff --git a/tools/perf/util/auxtrace.c b/tools/perf/util/auxtrace.c index 3d5cd16ca4de1d56f3d58540246b7d3d4f0ecacd..4942daaee6224e866902100b94721d5fc4f96fe9 100644 --- a/tools/perf/util/auxtrace.c +++ b/tools/perf/util/auxtrace.c @@ -49,6 +49,7 @@ #include "intel-pt.h" #include "intel-bts.h" #include "arm-spe.h" +#include "hisi-ptt.h" #include "s390-cpumsf.h" #include "util/mmap.h" @@ -928,7 +929,11 @@ int perf_event__process_auxtrace_info(struct perf_session *session, case PERF_AUXTRACE_CS_ETM: return cs_etm__process_auxtrace_info(event, session); case PERF_AUXTRACE_S390_CPUMSF: - return s390_cpumsf_process_auxtrace_info(event, session); + err = s390_cpumsf_process_auxtrace_info(event, session); + break; + case PERF_AUXTRACE_HISI_PTT: + err = hisi_ptt_process_auxtrace_info(event, session); + break; case PERF_AUXTRACE_UNKNOWN: default: return -EINVAL; @@ -975,6 +980,11 @@ void itrace_synth_opts__set_default(struct itrace_synth_opts *synth_opts, synth_opts->pwr_events = true; synth_opts->other_events = true; synth_opts->errors = true; + synth_opts->flc = true; + synth_opts->llc = true; + synth_opts->tlb = true; + synth_opts->remote_access = true; + if (no_sample) { synth_opts->period_type = PERF_ITRACE_PERIOD_INSTRUCTIONS; synth_opts->period = 1; @@ -1127,6 +1137,18 @@ int itrace_parse_synth_opts(const struct option *opt, const char *str, goto out_err; p = endptr; break; + case 'f': + synth_opts->flc = true; + break; + case 'm': + synth_opts->llc = true; + break; + case 't': + synth_opts->tlb = true; + break; + case 'a': + synth_opts->remote_access = true; + break; case ' ': case ',': break; diff --git a/tools/perf/util/auxtrace.h b/tools/perf/util/auxtrace.h index f201f36bc35fba78c71708aa01d712359d99a175..2959190956fc95fdbdce8fdefa3db87e440d914c 100644 --- a/tools/perf/util/auxtrace.h +++ b/tools/perf/util/auxtrace.h @@ -45,6 +45,7 @@ enum auxtrace_type { PERF_AUXTRACE_CS_ETM, PERF_AUXTRACE_ARM_SPE, PERF_AUXTRACE_S390_CPUMSF, + PERF_AUXTRACE_HISI_PTT, }; enum itrace_period_type { @@ -61,6 +62,7 @@ enum itrace_period_type { * because 'perf inject' will write it out * @instructions: whether to synthesize 'instructions' events * @branches: whether to synthesize 'branches' events + * (branch misses only for Arm SPE) * @transactions: whether to synthesize events for transactions * @ptwrites: whether to synthesize events for ptwrites * @pwr_events: whether to synthesize power events @@ -74,6 +76,10 @@ enum itrace_period_type { * @callchain: add callchain to 'instructions' events * @thread_stack: feed branches to the thread_stack * @last_branch: add branch context to 'instruction' events + * @flc: whether to synthesize first level cache events + * @llc: whether to synthesize last level cache events + * @tlb: whether to synthesize TLB events + * @remote_access: whether to synthesize remote access events * @callchain_sz: maximum callchain size * @last_branch_sz: branch context size * @period: 'instructions' events period @@ -101,6 +107,10 @@ struct itrace_synth_opts { bool callchain; bool thread_stack; bool last_branch; + bool flc; + bool llc; + bool tlb; + bool remote_access; unsigned int callchain_sz; unsigned int last_branch_sz; unsigned long long period; @@ -555,7 +565,7 @@ void auxtrace__free(struct perf_session *session); #define ITRACE_HELP \ " i: synthesize instructions events\n" \ -" b: synthesize branches events\n" \ +" b: synthesize branches events (branch misses for Arm SPE)\n" \ " c: synthesize branches events (calls only)\n" \ " r: synthesize branches events (returns only)\n" \ " x: synthesize transactions events\n" \ @@ -563,6 +573,10 @@ void auxtrace__free(struct perf_session *session); " p: synthesize power events\n" \ " e: synthesize error events\n" \ " d: create a debug log\n" \ +" f: synthesize first level cache events\n" \ +" m: synthesize last level cache events\n" \ +" t: synthesize TLB events\n" \ +" a: synthesize remote access events\n" \ " g[len]: synthesize a call chain (use with i or x)\n" \ " l[len]: synthesize last branch entries (use with i or x)\n" \ " sNUMBER: skip initial number of events\n" \ diff --git a/tools/perf/util/hisi-ptt-decoder/Build b/tools/perf/util/hisi-ptt-decoder/Build new file mode 100644 index 0000000000000000000000000000000000000000..db3db8b7503322965046204cf2bcd1800572a2c8 --- /dev/null +++ b/tools/perf/util/hisi-ptt-decoder/Build @@ -0,0 +1 @@ +perf-$(CONFIG_AUXTRACE) += hisi-ptt-pkt-decoder.o diff --git a/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.c b/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.c new file mode 100644 index 0000000000000000000000000000000000000000..a17c423a526ddf798ee0cbc60c6030540e780510 --- /dev/null +++ b/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.c @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * HiSilicon PCIe Trace and Tuning (PTT) support + * Copyright (c) 2022 HiSilicon Technologies Co., Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "../color.h" +#include "hisi-ptt-pkt-decoder.h" + +/* + * For 8DW format, the bit[31:11] of DW0 is always 0x1fffff, which can be + * used to distinguish the data format. + * 8DW format is like: + * bits [ 31:11 ][ 10:0 ] + * |---------------------------------------|-------------------| + * DW0 [ 0x1fffff ][ Reserved (0x7ff) ] + * DW1 [ Prefix ] + * DW2 [ Header DW0 ] + * DW3 [ Header DW1 ] + * DW4 [ Header DW2 ] + * DW5 [ Header DW3 ] + * DW6 [ Reserved (0x0) ] + * DW7 [ Time ] + * + * 4DW format is like: + * bits [31:30] [ 29:25 ][24][23][22][21][ 20:11 ][ 10:0 ] + * |-----|---------|---|---|---|---|-------------|-------------| + * DW0 [ Fmt ][ Type ][T9][T8][TH][SO][ Length ][ Time ] + * DW1 [ Header DW1 ] + * DW2 [ Header DW2 ] + * DW3 [ Header DW3 ] + */ + +enum hisi_ptt_8dw_pkt_field_type { + HISI_PTT_8DW_CHK_AND_RSV0, + HISI_PTT_8DW_PREFIX, + HISI_PTT_8DW_HEAD0, + HISI_PTT_8DW_HEAD1, + HISI_PTT_8DW_HEAD2, + HISI_PTT_8DW_HEAD3, + HISI_PTT_8DW_RSV1, + HISI_PTT_8DW_TIME, + HISI_PTT_8DW_TYPE_MAX +}; + +enum hisi_ptt_4dw_pkt_field_type { + HISI_PTT_4DW_HEAD1, + HISI_PTT_4DW_HEAD2, + HISI_PTT_4DW_HEAD3, + HISI_PTT_4DW_TYPE_MAX +}; + +static const char * const hisi_ptt_8dw_pkt_field_name[] = { + [HISI_PTT_8DW_PREFIX] = "Prefix", + [HISI_PTT_8DW_HEAD0] = "Header DW0", + [HISI_PTT_8DW_HEAD1] = "Header DW1", + [HISI_PTT_8DW_HEAD2] = "Header DW2", + [HISI_PTT_8DW_HEAD3] = "Header DW3", + [HISI_PTT_8DW_TIME] = "Time" +}; + +static const char * const hisi_ptt_4dw_pkt_field_name[] = { + [HISI_PTT_4DW_HEAD1] = "Header DW1", + [HISI_PTT_4DW_HEAD2] = "Header DW2", + [HISI_PTT_4DW_HEAD3] = "Header DW3", +}; + +union hisi_ptt_4dw { + struct { + uint32_t format : 2; + uint32_t type : 5; + uint32_t t9 : 1; + uint32_t t8 : 1; + uint32_t th : 1; + uint32_t so : 1; + uint32_t len : 10; + uint32_t time : 11; + }; + uint32_t value; +}; + +static void hisi_ptt_print_pkt(const unsigned char *buf, int pos, const char *desc) +{ + const char *color = PERF_COLOR_BLUE; + int i; + + printf("."); + color_fprintf(stdout, color, " %08x: ", pos); + for (i = 0; i < HISI_PTT_FIELD_LENTH; i++) + color_fprintf(stdout, color, "%02x ", buf[pos + i]); + for (i = 0; i < HISI_PTT_MAX_SPACE_LEN; i++) + color_fprintf(stdout, color, " "); + color_fprintf(stdout, color, " %s\n", desc); +} + +static int hisi_ptt_8dw_kpt_desc(const unsigned char *buf, int pos) +{ + int i; + + for (i = 0; i < HISI_PTT_8DW_TYPE_MAX; i++) { + /* Do not show 8DW check field and reserved fields */ + if (i == HISI_PTT_8DW_CHK_AND_RSV0 || i == HISI_PTT_8DW_RSV1) { + pos += HISI_PTT_FIELD_LENTH; + continue; + } + + hisi_ptt_print_pkt(buf, pos, hisi_ptt_8dw_pkt_field_name[i]); + pos += HISI_PTT_FIELD_LENTH; + } + + return hisi_ptt_pkt_size[HISI_PTT_8DW_PKT]; +} + +static void hisi_ptt_4dw_print_dw0(const unsigned char *buf, int pos) +{ + const char *color = PERF_COLOR_BLUE; + union hisi_ptt_4dw dw0; + int i; + + dw0.value = *(uint32_t *)(buf + pos); + printf("."); + color_fprintf(stdout, color, " %08x: ", pos); + for (i = 0; i < HISI_PTT_FIELD_LENTH; i++) + color_fprintf(stdout, color, "%02x ", buf[pos + i]); + for (i = 0; i < HISI_PTT_MAX_SPACE_LEN; i++) + color_fprintf(stdout, color, " "); + + color_fprintf(stdout, color, + " %s %x %s %x %s %x %s %x %s %x %s %x %s %x %s %x\n", + "Format", dw0.format, "Type", dw0.type, "T9", dw0.t9, + "T8", dw0.t8, "TH", dw0.th, "SO", dw0.so, "Length", + dw0.len, "Time", dw0.time); +} + +static int hisi_ptt_4dw_kpt_desc(const unsigned char *buf, int pos) +{ + int i; + + hisi_ptt_4dw_print_dw0(buf, pos); + pos += HISI_PTT_FIELD_LENTH; + + for (i = 0; i < HISI_PTT_4DW_TYPE_MAX; i++) { + hisi_ptt_print_pkt(buf, pos, hisi_ptt_4dw_pkt_field_name[i]); + pos += HISI_PTT_FIELD_LENTH; + } + + return hisi_ptt_pkt_size[HISI_PTT_4DW_PKT]; +} + +int hisi_ptt_pkt_desc(const unsigned char *buf, int pos, enum hisi_ptt_pkt_type type) +{ + if (type == HISI_PTT_8DW_PKT) + return hisi_ptt_8dw_kpt_desc(buf, pos); + + return hisi_ptt_4dw_kpt_desc(buf, pos); +} diff --git a/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.h b/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.h new file mode 100644 index 0000000000000000000000000000000000000000..e78f1b5bc836e91b861f65ad9c5f90c9270a2fb6 --- /dev/null +++ b/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * HiSilicon PCIe Trace and Tuning (PTT) support + * Copyright (c) 2022 HiSilicon Technologies Co., Ltd. + */ + +#ifndef INCLUDE__HISI_PTT_PKT_DECODER_H__ +#define INCLUDE__HISI_PTT_PKT_DECODER_H__ + +#include +#include + +#define HISI_PTT_8DW_CHECK_MASK GENMASK(31, 11) +#define HISI_PTT_IS_8DW_PKT GENMASK(31, 11) +#define HISI_PTT_MAX_SPACE_LEN 10 +#define HISI_PTT_FIELD_LENTH 4 + +enum hisi_ptt_pkt_type { + HISI_PTT_4DW_PKT, + HISI_PTT_8DW_PKT, + HISI_PTT_PKT_MAX +}; + +static int hisi_ptt_pkt_size[] = { + [HISI_PTT_4DW_PKT] = 16, + [HISI_PTT_8DW_PKT] = 32, +}; + +int hisi_ptt_pkt_desc(const unsigned char *buf, int pos, enum hisi_ptt_pkt_type type); + +#endif diff --git a/tools/perf/util/hisi-ptt.c b/tools/perf/util/hisi-ptt.c new file mode 100644 index 0000000000000000000000000000000000000000..52d0ce302ca042ed0bae720273443d3f2794affb --- /dev/null +++ b/tools/perf/util/hisi-ptt.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * HiSilicon PCIe Trace and Tuning (PTT) support + * Copyright (c) 2022 HiSilicon Technologies Co., Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "auxtrace.h" +#include "color.h" +#include "debug.h" +#include "evsel.h" +#include "hisi-ptt.h" +#include "hisi-ptt-decoder/hisi-ptt-pkt-decoder.h" +#include "machine.h" +#include "session.h" +#include "tool.h" +#include + +struct hisi_ptt { + struct auxtrace auxtrace; + u32 auxtrace_type; + struct perf_session *session; + struct machine *machine; + u32 pmu_type; +}; + +struct hisi_ptt_queue { + struct hisi_ptt *ptt; + struct auxtrace_buffer *buffer; +}; + +static enum hisi_ptt_pkt_type hisi_ptt_check_packet_type(unsigned char *buf) +{ + uint32_t head = *(uint32_t *)buf; + + if ((HISI_PTT_8DW_CHECK_MASK & head) == HISI_PTT_IS_8DW_PKT) + return HISI_PTT_8DW_PKT; + + return HISI_PTT_4DW_PKT; +} + +static void hisi_ptt_dump(struct hisi_ptt *ptt __maybe_unused, + unsigned char *buf, size_t len) +{ + const char *color = PERF_COLOR_BLUE; + enum hisi_ptt_pkt_type type; + size_t pos = 0; + int pkt_len; + + type = hisi_ptt_check_packet_type(buf); + len = round_down(len, hisi_ptt_pkt_size[type]); + color_fprintf(stdout, color, ". ... HISI PTT data: size %zu bytes\n", + len); + + while (len > 0) { + pkt_len = hisi_ptt_pkt_desc(buf, pos, type); + if (!pkt_len) + color_fprintf(stdout, color, " Bad packet!\n"); + + pos += pkt_len; + len -= pkt_len; + } +} + +static void hisi_ptt_dump_event(struct hisi_ptt *ptt, unsigned char *buf, + size_t len) +{ + printf(".\n"); + + hisi_ptt_dump(ptt, buf, len); +} + +static int hisi_ptt_process_event(struct perf_session *session __maybe_unused, + union perf_event *event __maybe_unused, + struct perf_sample *sample __maybe_unused, + struct perf_tool *tool __maybe_unused) +{ + return 0; +} + +static int hisi_ptt_process_auxtrace_event(struct perf_session *session, + union perf_event *event, + struct perf_tool *tool __maybe_unused) +{ + struct hisi_ptt *ptt = container_of(session->auxtrace, struct hisi_ptt, + auxtrace); + int fd = perf_data__fd(session->data); + int size = event->auxtrace.size; + void *data = malloc(size); + off_t data_offset; + int err; + + if (!data) + return -errno; + + if (perf_data__is_pipe(session->data)) { + data_offset = 0; + } else { + data_offset = lseek(fd, 0, SEEK_CUR); + if (data_offset == -1) { + free(data); + return -errno; + } + } + + err = readn(fd, data, size); + if (err != (ssize_t)size) { + free(data); + return -errno; + } + + if (dump_trace) + hisi_ptt_dump_event(ptt, data, size); + + free(data); + return 0; +} + +static int hisi_ptt_flush(struct perf_session *session __maybe_unused, + struct perf_tool *tool __maybe_unused) +{ + return 0; +} + +static void hisi_ptt_free_events(struct perf_session *session __maybe_unused) +{ +} + +static void hisi_ptt_free(struct perf_session *session) +{ + struct hisi_ptt *ptt = container_of(session->auxtrace, struct hisi_ptt, + auxtrace); + + session->auxtrace = NULL; + free(ptt); +} + +static bool hisi_ptt_evsel_is_auxtrace(struct perf_session *session, + struct evsel *evsel) +{ + struct hisi_ptt *ptt = container_of(session->auxtrace, struct hisi_ptt, auxtrace); + + return evsel->core.attr.type == ptt->pmu_type; +} + +static void hisi_ptt_print_info(__u64 type) +{ + if (!dump_trace) + return; + + fprintf(stdout, " PMU Type %" PRId64 "\n", (s64) type); +} + +int hisi_ptt_process_auxtrace_info(union perf_event *event, + struct perf_session *session) +{ + struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info; + struct hisi_ptt *ptt; + + if (auxtrace_info->header.size < HISI_PTT_AUXTRACE_PRIV_SIZE + + sizeof(struct perf_record_auxtrace_info)) + return -EINVAL; + + ptt = zalloc(sizeof(*ptt)); + if (!ptt) + return -ENOMEM; + + ptt->session = session; + ptt->machine = &session->machines.host; /* No kvm support */ + ptt->auxtrace_type = auxtrace_info->type; + ptt->pmu_type = auxtrace_info->priv[0]; + + ptt->auxtrace.process_event = hisi_ptt_process_event; + ptt->auxtrace.process_auxtrace_event = hisi_ptt_process_auxtrace_event; + ptt->auxtrace.flush_events = hisi_ptt_flush; + ptt->auxtrace.free_events = hisi_ptt_free_events; + ptt->auxtrace.free = hisi_ptt_free; + ptt->auxtrace.evsel_is_auxtrace = hisi_ptt_evsel_is_auxtrace; + session->auxtrace = &ptt->auxtrace; + + hisi_ptt_print_info(auxtrace_info->priv[0]); + + return 0; +} diff --git a/tools/perf/util/hisi-ptt.h b/tools/perf/util/hisi-ptt.h new file mode 100644 index 0000000000000000000000000000000000000000..2db9b405621483dc6faebbbb49643049e3f33e60 --- /dev/null +++ b/tools/perf/util/hisi-ptt.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * HiSilicon PCIe Trace and Tuning (PTT) support + * Copyright (c) 2022 HiSilicon Technologies Co., Ltd. + */ + +#ifndef INCLUDE__PERF_HISI_PTT_H__ +#define INCLUDE__PERF_HISI_PTT_H__ + +#define HISI_PTT_PMU_NAME "hisi_ptt" +#define HISI_PTT_AUXTRACE_PRIV_SIZE sizeof(u64) + +struct auxtrace_record *hisi_ptt_recording_init(int *err, + struct perf_pmu *hisi_ptt_pmu); + +int hisi_ptt_process_auxtrace_info(union perf_event *event, + struct perf_session *session); + +#endif