If a helper fails to be enabled, unwind any helpers that were already enabled earlier in the loop. This avoids leaving partially enabled helpers behind.

Fixes: 6148652807ba ("coresight: Enable and disable helper devices adjacent to the path") Signed-off-by: Leo Yan leo.yan@arm.com --- drivers/hwtracing/coresight/coresight-core.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c index 46f247f73cf64a97b9353b84ba5b76b991676f5f..ed17c36ad46ac4270abdfdf2a1a742faa6273097 100644 --- a/drivers/hwtracing/coresight/coresight-core.c +++ b/drivers/hwtracing/coresight/coresight-core.c @@ -499,10 +499,19 @@ static int coresight_enable_helpers(struct coresight_device *csdev,

ret = coresight_enable_helper(helper, mode, path); if (ret) - return ret; + goto err; }

return 0; + +err: + while (i--) { + helper = csdev->pdata->out_conns[i]->dest_dev; + if (helper && coresight_is_helper(helper)) + coresight_disable_helper(helper, path); + } + + return ret; }

int coresight_enable_path(struct coresight_path *path, enum cs_mode mode)

Add a new flag CORESIGHT_DESC_CPU_BOUND to indicate components that are CPU bound. Populate CPU ID into the coresight_device structure; otherwise, set CPU ID to -1 for non CPU bound devices.

Use the {0} initializer to clear coresight_desc structures to avoid uninitialized values.

Tested-by: Jie Gan jie.gan@oss.qualcomm.com Reviewed-by: Yeoreum Yun yeoreum.yun@arm.com Reviewed-by: James Clark james.clark@linaro.org Tested-by: James Clark james.clark@linaro.org Signed-off-by: Leo Yan leo.yan@arm.com --- drivers/hwtracing/coresight/coresight-catu.c | 2 +- drivers/hwtracing/coresight/coresight-core.c | 13 +++++++++++++ drivers/hwtracing/coresight/coresight-cti-core.c | 9 ++++++--- drivers/hwtracing/coresight/coresight-etm3x-core.c | 2 ++ drivers/hwtracing/coresight/coresight-etm4x-core.c | 2 ++ drivers/hwtracing/coresight/coresight-trbe.c | 2 ++ include/linux/coresight.h | 8 ++++++++ 7 files changed, 34 insertions(+), 4 deletions(-)

diff --git a/drivers/hwtracing/coresight/coresight-catu.c b/drivers/hwtracing/coresight/coresight-catu.c index ce71dcddfca2558eddd625de58a709b151f2e07e..43abe13995cf3c96e70dcf97856872d70f71345a 100644 --- a/drivers/hwtracing/coresight/coresight-catu.c +++ b/drivers/hwtracing/coresight/coresight-catu.c @@ -514,7 +514,7 @@ static int __catu_probe(struct device *dev, struct resource *res) int ret = 0; u32 dma_mask; struct catu_drvdata *drvdata; - struct coresight_desc catu_desc; + struct coresight_desc catu_desc = { 0 }; struct coresight_platform_data *pdata = NULL; void __iomem *base;

diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c index 48ea882bd10870a6be253ecf0b58860d7a4a0bb0..f8c0c2b05888403318372dc76278910c8a5c7e15 100644 --- a/drivers/hwtracing/coresight/coresight-core.c +++ b/drivers/hwtracing/coresight/coresight-core.c @@ -1347,6 +1347,19 @@ coresight_init_device(struct coresight_desc *desc) csdev->access = desc->access; csdev->orphan = true;

+ if (desc->flags & CORESIGHT_DESC_CPU_BOUND) { + csdev->cpu = desc->cpu; + } else { + /* A per-CPU source or sink must set CPU_BOUND flag */ + if (coresight_is_percpu_source(csdev) || + coresight_is_percpu_sink(csdev)) { + kfree(csdev); + return ERR_PTR(-EINVAL); + } + + csdev->cpu = -1; + } + csdev->dev.type = &coresight_dev_type[desc->type]; csdev->dev.groups = desc->groups; csdev->dev.parent = desc->dev; diff --git a/drivers/hwtracing/coresight/coresight-cti-core.c b/drivers/hwtracing/coresight/coresight-cti-core.c index 2f4c9362709a90b12a1aeb5016905b7d4474b912..b2c9a4db13b4e5554bca565c17ed299fdfdb30ff 100644 --- a/drivers/hwtracing/coresight/coresight-cti-core.c +++ b/drivers/hwtracing/coresight/coresight-cti-core.c @@ -659,7 +659,7 @@ static int cti_probe(struct amba_device *adev, const struct amba_id *id) void __iomem *base; struct device *dev = &adev->dev; struct cti_drvdata *drvdata = NULL; - struct coresight_desc cti_desc; + struct coresight_desc cti_desc = { 0 }; struct coresight_platform_data *pdata = NULL; struct resource *res = &adev->res;

@@ -702,11 +702,14 @@ static int cti_probe(struct amba_device *adev, const struct amba_id *id) * eCPU ID. System CTIs will have the name cti_sys<I> where I is an * index allocated by order of discovery. */ - if (drvdata->ctidev.cpu >= 0) + if (drvdata->ctidev.cpu >= 0) { + cti_desc.cpu = drvdata->ctidev.cpu; + cti_desc.flags = CORESIGHT_DESC_CPU_BOUND; cti_desc.name = devm_kasprintf(dev, GFP_KERNEL, "cti_cpu%d", drvdata->ctidev.cpu); - else + } else { cti_desc.name = coresight_alloc_device_name("cti_sys", dev); + } if (!cti_desc.name) return -ENOMEM;

diff --git a/drivers/hwtracing/coresight/coresight-etm3x-core.c b/drivers/hwtracing/coresight/coresight-etm3x-core.c index a547a6d2e0bde84748f753e5529d316c4f5e82e2..eb665db1a37d9970f7f55395c0aa23b98a7f3118 100644 --- a/drivers/hwtracing/coresight/coresight-etm3x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm3x-core.c @@ -891,6 +891,8 @@ static int etm_probe(struct amba_device *adev, const struct amba_id *id) desc.pdata = pdata; desc.dev = dev; desc.groups = coresight_etm_groups; + desc.cpu = drvdata->cpu; + desc.flags = CORESIGHT_DESC_CPU_BOUND; drvdata->csdev = coresight_register(&desc); if (IS_ERR(drvdata->csdev)) return PTR_ERR(drvdata->csdev); diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index d565a73f0042e3e0b21fcf9cb94009cc25834d3d..b1e0254a534027d7ede8591e56be28745d0b9974 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -2260,6 +2260,8 @@ static int etm4_add_coresight_dev(struct etm4_init_arg *init_arg) desc.pdata = pdata; desc.dev = dev; desc.groups = coresight_etmv4_groups; + desc.cpu = drvdata->cpu; + desc.flags = CORESIGHT_DESC_CPU_BOUND; drvdata->csdev = coresight_register(&desc); if (IS_ERR(drvdata->csdev)) return PTR_ERR(drvdata->csdev); diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index 1511f8eb95afb5b4610b8fbdacc8b174b6b08530..14e35b9660d76e47619cc6026b94929b3bb3e02b 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -1289,6 +1289,8 @@ static void arm_trbe_register_coresight_cpu(struct trbe_drvdata *drvdata, int cp desc.ops = &arm_trbe_cs_ops; desc.groups = arm_trbe_groups; desc.dev = dev; + desc.cpu = cpu; + desc.flags = CORESIGHT_DESC_CPU_BOUND; trbe_csdev = coresight_register(&desc); if (IS_ERR(trbe_csdev)) goto cpu_clear; diff --git a/include/linux/coresight.h b/include/linux/coresight.h index 2131febebee93d609df1aea8534a10898b600be2..687190ca11ddeaa83193caa3903a480bac3060d1 100644 --- a/include/linux/coresight.h +++ b/include/linux/coresight.h @@ -141,6 +141,8 @@ struct csdev_access { .base = (_addr), \ })

+#define CORESIGHT_DESC_CPU_BOUND BIT(0) + /** * struct coresight_desc - description of a component required from drivers * @type: as defined by @coresight_dev_type. @@ -153,6 +155,8 @@ struct csdev_access { * in the component's sysfs sub-directory. * @name: name for the coresight device, also shown under sysfs. * @access: Describe access to the device + * @flags: The descritpion flags. + * @cpu: The CPU this component is affined to. */ struct coresight_desc { enum coresight_dev_type type; @@ -163,6 +167,8 @@ struct coresight_desc { const struct attribute_group **groups; const char *name; struct csdev_access access; + u32 flags; + int cpu; };

/** @@ -260,6 +266,7 @@ struct coresight_trace_id_map { * device's spinlock when the coresight_mutex held and mode == * CS_MODE_SYSFS. Otherwise it must be accessed from inside the * spinlock. + * @cpu: The CPU this component is affined to (-1 for not CPU bound). * @orphan: true if the component has connections that haven't been linked. * @sysfs_sink_activated: 'true' when a sink has been selected for use via sysfs * by writing a 1 to the 'enable_sink' file. A sink can be @@ -286,6 +293,7 @@ struct coresight_device { struct device dev; atomic_t mode; int refcnt; + int cpu; bool orphan; /* sink specific fields */ bool sysfs_sink_activated;

The hotplug lock is acquired and released in etm{3|4}_disable_sysfs(), which are low-level functions. This prevents us from a new solution for hotplug.

Firstly, hotplug callbacks cannot invoke etm{3|4}_disable_sysfs() to disable the source; otherwise, a deadlock issue occurs. The reason is that, in the hotplug flow, the kernel acquires the hotplug lock before calling callbacks. Subsequently, if coresight_disable_source() is invoked and it calls etm{3|4}_disable_sysfs(), the hotplug lock will be acquired twice, leading to a double lock issue.

Secondly, when hotplugging a CPU on or off, if we want to manipulate all components on a path attached to the CPU, we need to maintain atomicity for the entire path. Otherwise, a race condition may occur with users setting the same path via the Sysfs knobs, ultimately causing mess states in CoreSight components.

This patch moves the hotplug locking from etm{3|4}_disable_sysfs() into enable_source_store(). As a result, when users control the Sysfs knobs, the whole flow is protected by hotplug locking, ensuring it is mutual exclusive with hotplug callbacks.

Note, the paired function etm{3|4}_enable_sysfs() does not use hotplug locking, which is why this patch does not modify it.

Reviewed-by: Mike Leach mike.leach@linaro.org Tested-by: James Clark james.clark@linaro.org Reviewed-by: Yeoreum Yun yeoreum.yun@arm.com Reviewed-by: James Clark james.clark@linaro.org Tested-by: Jie Gan jie.gan@oss.qualcomm.com Signed-off-by: Leo Yan leo.yan@arm.com --- drivers/hwtracing/coresight/coresight-etm3x-core.c | 8 -------- drivers/hwtracing/coresight/coresight-etm4x-core.c | 9 --------- drivers/hwtracing/coresight/coresight-sysfs.c | 7 +++++++ 3 files changed, 7 insertions(+), 17 deletions(-)

diff --git a/drivers/hwtracing/coresight/coresight-etm3x-core.c b/drivers/hwtracing/coresight/coresight-etm3x-core.c index ab47f69e923fb191b48b82367dce465c79b3a93d..aeeb284abdbe4b6a0960da45baa1138e203f3e3c 100644 --- a/drivers/hwtracing/coresight/coresight-etm3x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm3x-core.c @@ -620,13 +620,6 @@ static void etm_disable_sysfs(struct coresight_device *csdev) { struct etm_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);

- /* - * Taking hotplug lock here protects from clocks getting disabled - * with tracing being left on (crash scenario) if user disable occurs - * after cpu online mask indicates the cpu is offline but before the - * DYING hotplug callback is serviced by the ETM driver. - */ - cpus_read_lock(); spin_lock(&drvdata->spinlock);

/* @@ -637,7 +630,6 @@ static void etm_disable_sysfs(struct coresight_device *csdev) drvdata, 1);

spin_unlock(&drvdata->spinlock); - cpus_read_unlock();

/* * we only release trace IDs when resetting sysfs. diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index 66a8058098376264d3f8c5815763a75ebffb352e..a776ebb3b2b0360c99a8dadacfde4c2303dd59d6 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -1106,13 +1106,6 @@ static void etm4_disable_sysfs(struct coresight_device *csdev) { struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);

- /* - * Taking hotplug lock here protects from clocks getting disabled - * with tracing being left on (crash scenario) if user disable occurs - * after cpu online mask indicates the cpu is offline but before the - * DYING hotplug callback is serviced by the ETM driver. - */ - cpus_read_lock(); raw_spin_lock(&drvdata->spinlock);

/* @@ -1126,8 +1119,6 @@ static void etm4_disable_sysfs(struct coresight_device *csdev)

cscfg_csdev_disable_active_config(csdev);

- cpus_read_unlock(); - /* * we only release trace IDs when resetting sysfs. * This permits sysfs users to read the trace ID after the trace diff --git a/drivers/hwtracing/coresight/coresight-sysfs.c b/drivers/hwtracing/coresight/coresight-sysfs.c index 3b1e7152dd108408d837c404ce607ba511ca14a6..f398dbfbcb8de0c5a873837c19b3fdcf97b64abe 100644 --- a/drivers/hwtracing/coresight/coresight-sysfs.c +++ b/drivers/hwtracing/coresight/coresight-sysfs.c @@ -360,6 +360,13 @@ static ssize_t enable_source_store(struct device *dev, if (ret) return ret;

+ /* + * CoreSight hotplug callbacks in core layer control a activated path + * from its source to sink. Taking hotplug lock here protects a race + * condition with hotplug callbacks. + */ + guard(cpus_read_lock)(); + if (val) { ret = coresight_enable_sysfs(csdev); if (ret)

coresight_get_ref() currently pins the provider module and takes a reference on the parent device. That does not pin the CoreSight device itself.

A driver can still be unbound while the CoreSight device is part of an active trace path. In that case coresight_unregister() may release the coresight_device while the path still holds its csdev pointer.

Take a reference on &csdev->dev when grabbing a CoreSight device and drop it in coresight_put_ref().

Signed-off-by: Leo Yan leo.yan@arm.com --- drivers/hwtracing/coresight/coresight-core.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c index 5519d323f21f38d719b9030c7d77a9a61948ba1d..1a389f63ed006e054dc6bc9764f8be8c1def9da1 100644 --- a/drivers/hwtracing/coresight/coresight-core.c +++ b/drivers/hwtracing/coresight/coresight-core.c @@ -651,14 +651,18 @@ struct coresight_device *coresight_get_sink_by_id(u32 id) */ static bool coresight_get_ref(struct coresight_device *csdev) { - struct device *dev = csdev->dev.parent; + struct device *parent = csdev->dev.parent;

/* Make sure the driver can't be removed */ - if (!try_module_get(dev->driver->owner)) + if (!try_module_get(parent->driver->owner)) return false; - /* Make sure the device can't go away */ - get_device(dev); - pm_runtime_get_sync(dev); + + /* Make sure parent device can't go away and is powered on */ + get_device(parent); + pm_runtime_get_sync(parent); + + /* Make sure csdev can't go away */ + get_device(&csdev->dev); return true; }

@@ -670,11 +674,12 @@ static bool coresight_get_ref(struct coresight_device *csdev) */ static void coresight_put_ref(struct coresight_device *csdev) { - struct device *dev = csdev->dev.parent; + struct device *parent = csdev->dev.parent;

- pm_runtime_put(dev); - put_device(dev); - module_put(dev->driver->owner); + put_device(&csdev->dev); + pm_runtime_put(parent); + put_device(parent); + module_put(parent->driver->owner); }

/*

etm_setup_aux() may access a per-CPU source device while ETM module is being unloaded, leading to a potential use-after-free.

Therefore, this commit takes references to the device, which is sufficient to prevent the csdev from being released. This ensures that csdev can be safely accessed.

Refactor the perf path build code into etm_event_build_path(), making it easier to use the coresight_{get|put}_percpu_source_ref() pairs. Update the comments accordingly to reflect the new flow.

Signed-off-by: Leo Yan leo.yan@arm.com --- drivers/hwtracing/coresight/coresight-core.c | 29 ++++- drivers/hwtracing/coresight/coresight-etm-perf.c | 157 +++++++++++++---------- drivers/hwtracing/coresight/coresight-priv.h | 3 +- 3 files changed, 120 insertions(+), 69 deletions(-)

diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c index 6da15f2ef9dc9770e7aa79cc94a7ed3d2f3ad871..b01e63fbbb4b990d4f5ec61c8fa4da63dd59a4b9 100644 --- a/drivers/hwtracing/coresight/coresight-core.c +++ b/drivers/hwtracing/coresight/coresight-core.c @@ -109,13 +109,38 @@ static void coresight_clear_percpu_source(struct coresight_device *csdev) per_cpu(csdev_source, csdev->cpu) = NULL; }

-struct coresight_device *coresight_get_percpu_source(int cpu) +struct coresight_device *coresight_get_percpu_source_ref(int cpu) { + struct coresight_device *csdev; + if (WARN_ON(cpu < 0)) return NULL;

guard(raw_spinlock_irqsave)(&coresight_dev_lock); - return per_cpu(csdev_source, cpu); + + csdev = per_cpu(csdev_source, cpu); + if (!csdev) + return NULL; + + /* Make sure csdev is safe to access */ + get_device(&csdev->dev); + + return csdev; +} + +void coresight_put_percpu_source_ref(struct coresight_device *csdev) +{ + if (!csdev || !coresight_is_percpu_source(csdev)) + return; + + guard(raw_spinlock_irqsave)(&coresight_dev_lock); + + /* + * When the device's refcount reaches zero, coresight_device_release() + * is invoked. This is safe even in atomic context, as the release + * function does not sleep. + */ + put_device(&csdev->dev); }

struct coresight_device *coresight_get_source(struct coresight_path *path) diff --git a/drivers/hwtracing/coresight/coresight-etm-perf.c b/drivers/hwtracing/coresight/coresight-etm-perf.c index d3b13ef130439fd501f88395d0de9dd21b84b827..9950ad481f29eed3bfac8fe4ae3a593b53830617 100644 --- a/drivers/hwtracing/coresight/coresight-etm-perf.c +++ b/drivers/hwtracing/coresight/coresight-etm-perf.c @@ -338,6 +338,85 @@ static struct coresight_path *etm_event_get_ctxt_path(struct etm_ctxt *ctxt) return path; }

+static struct coresight_path * +etm_event_build_path(struct perf_event *event, int cpu, + struct coresight_device *user_sink, + struct coresight_device *match_sink) +{ + struct coresight_path *path = NULL; + struct coresight_device *source, *sink; + + source = coresight_get_percpu_source_ref(cpu); + + /* + * If there is no ETM associated with this CPU or ever we try to trace + * on this CPU, we handle it accordingly. + */ + if (!source) + return NULL; + + /* + * If AUX pause feature is enabled but the ETM driver does not + * support the operations, skip for this source. + */ + if (event->attr.aux_start_paused && + (!source_ops(source)->pause_perf || + !source_ops(source)->resume_perf)) { + dev_err_once(&source->dev, "AUX pause is not supported.\n"); + goto out; + } + + /* If sink has been specified by user, directly use it */ + if (user_sink) { + sink = user_sink; + } else { + /* + * No sink provided - look for a default sink for all the ETMs, + * where this event can be scheduled. + * + * We allocate the sink specific buffers only once for this + * event. If the ETMs have different default sink devices, we + * can only use a single "type" of sink as the event can carry + * only one sink specific buffer. Thus we have to make sure + * that the sinks are of the same type and driven by the same + * driver, as the one we allocate the buffer for. We don't + * trace on a CPU if the sink is not compatible. + */ + + /* Find the default sink for this ETM */ + sink = coresight_find_default_sink(source); + if (!sink) + goto out; + + /* Check if this sink compatible with the last sink */ + if (match_sink && !sinks_compatible(match_sink, sink)) + goto out; + } + + /* + * Building a path doesn't enable it, it simply builds a + * list of devices from source to sink that can be + * referenced later when the path is actually needed. + */ + path = coresight_build_path(source, sink); + if (IS_ERR(path)) + goto out; + + /* ensure we can allocate a trace ID for this CPU */ + coresight_path_assign_trace_id(path, CS_MODE_PERF); + if (!IS_VALID_CS_TRACE_ID(path->trace_id)) { + coresight_release_path(path); + path = NULL; + goto out; + } + + coresight_trace_id_perf_start(&sink->perf_sink_id_map); + +out: + coresight_put_percpu_source_ref(source); + return IS_ERR_OR_NULL(path) ? NULL : path; +} + static void *etm_setup_aux(struct perf_event *event, void **pages, int nr_pages, bool overwrite) { @@ -345,7 +424,7 @@ static void *etm_setup_aux(struct perf_event *event, void **pages, int cpu = event->cpu; cpumask_t *mask; struct coresight_device *sink = NULL; - struct coresight_device *user_sink = NULL, *last_sink = NULL; + struct coresight_device *user_sink = NULL; struct etm_event_data *event_data = NULL;

event_data = alloc_event_data(cpu); @@ -377,80 +456,26 @@ static void *etm_setup_aux(struct perf_event *event, void **pages, */ for_each_cpu(cpu, mask) { struct coresight_path *path; - struct coresight_device *csdev;

- csdev = coresight_get_percpu_source(cpu); - /* - * If there is no ETM associated with this CPU clear it from - * the mask and continue with the rest. If ever we try to trace - * on this CPU, we handle it accordingly. - */ - if (!csdev) { + path = etm_event_build_path(event, cpu, user_sink, sink); + if (!path) { + /* + * Failed to create a path for the CPU, clear it from + * the mask and continue to next one. + */ cpumask_clear_cpu(cpu, mask); continue; }

- /* - * If AUX pause feature is enabled but the ETM driver does not - * support the operations, clear this CPU from the mask and - * continue to next one. - */ - if (event->attr.aux_start_paused && - (!source_ops(csdev)->pause_perf || !source_ops(csdev)->resume_perf)) { - dev_err_once(&csdev->dev, "AUX pause is not supported.\n"); - cpumask_clear_cpu(cpu, mask); - continue; - }

/* - * No sink provided - look for a default sink for all the ETMs, - * where this event can be scheduled. - * We allocate the sink specific buffers only once for this - * event. If the ETMs have different default sink devices, we - * can only use a single "type" of sink as the event can carry - * only one sink specific buffer. Thus we have to make sure - * that the sinks are of the same type and driven by the same - * driver, as the one we allocate the buffer for. As such - * we choose the first sink and check if the remaining ETMs - * have a compatible default sink. We don't trace on a CPU - * if the sink is not compatible. - */ - if (!user_sink) { - /* Find the default sink for this ETM */ - sink = coresight_find_default_sink(csdev); - if (!sink) { - cpumask_clear_cpu(cpu, mask); - continue; - } - - /* Check if this sink compatible with the last sink */ - if (last_sink && !sinks_compatible(last_sink, sink)) { - cpumask_clear_cpu(cpu, mask); - continue; - } - last_sink = sink; - } - - /* - * Building a path doesn't enable it, it simply builds a - * list of devices from source to sink that can be - * referenced later when the path is actually needed. + * The first found sink is saved here and passed to + * etm_event_build_path() to check whether the remaining ETMs + * have a compatible default sink. */ - path = coresight_build_path(csdev, sink); - if (IS_ERR(path)) { - cpumask_clear_cpu(cpu, mask); - continue; - } - - /* ensure we can allocate a trace ID for this CPU */ - coresight_path_assign_trace_id(path, CS_MODE_PERF); - if (!IS_VALID_CS_TRACE_ID(path->trace_id)) { - cpumask_clear_cpu(cpu, mask); - coresight_release_path(path); - continue; - } + if (!user_sink && !sink) + sink = coresight_get_sink(path);

- coresight_trace_id_perf_start(&sink->perf_sink_id_map); *etm_event_cpu_path_ptr(event_data, cpu) = path; }

diff --git a/drivers/hwtracing/coresight/coresight-priv.h b/drivers/hwtracing/coresight/coresight-priv.h index 7ce79fa36232bb1b0af768423777bab27cacee95..a1aab67e23db7fdea5139100312b3eb7cd31df51 100644 --- a/drivers/hwtracing/coresight/coresight-priv.h +++ b/drivers/hwtracing/coresight/coresight-priv.h @@ -249,7 +249,8 @@ void coresight_add_helper(struct coresight_device *csdev, void coresight_set_percpu_sink(int cpu, struct coresight_device *csdev); struct coresight_device *coresight_get_percpu_sink(int cpu); struct coresight_device *coresight_get_source(struct coresight_path *path); -struct coresight_device *coresight_get_percpu_source(int cpu); +struct coresight_device *coresight_get_percpu_source_ref(int cpu); +void coresight_put_percpu_source_ref(struct coresight_device *csdev); void coresight_disable_source(struct coresight_device *csdev, void *data); void coresight_pause_source(struct coresight_device *csdev); int coresight_resume_source(struct coresight_device *csdev);

The current implementation only saves and restores the context for ETM sources while ignoring the context of links. However, if funnels or replicators on a linked path resides in a CPU or cluster power domain, the hardware context for the link will be lost after resuming from low power states.

To support context management for links during CPU low power modes, a better way is to implement CPU PM callbacks in the Arm CoreSight core layer. As the core layer has sufficient information for linked paths, from tracers to links, which can be used for power management.

As a first step, this patch registers CPU PM notifier in the core layer. If a source device provides callbacks for saving and restoring context, these callbacks will be invoked in CPU suspend and resume.

Reviewed-by: James Clark james.clark@linaro.org Tested-by: James Clark james.clark@linaro.org Reviewed-by: Yeoreum Yun yeoreum.yun@arm.com Tested-by: Jie Gan jie.gan@oss.qualcomm.com Signed-off-by: Leo Yan leo.yan@arm.com --- drivers/hwtracing/coresight/coresight-core.c | 93 ++++++++++++++++++++++++++++ include/linux/coresight.h | 2 + 2 files changed, 95 insertions(+)

diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c index b01e63fbbb4b990d4f5ec61c8fa4da63dd59a4b9..0c11e7fff1bd5cf504b17241595d83a1d11bcf40 100644 --- a/drivers/hwtracing/coresight/coresight-core.c +++ b/drivers/hwtracing/coresight/coresight-core.c @@ -6,6 +6,7 @@ #include <linux/acpi.h> #include <linux/bitfield.h> #include <linux/build_bug.h> +#include <linux/cpu_pm.h> #include <linux/kernel.h> #include <linux/init.h> #include <linux/types.h> @@ -1716,6 +1717,91 @@ static void coresight_release_device_list(void) } }

+static struct coresight_device *coresight_cpu_get_active_source(void) +{ + struct coresight_device *source; + bool is_active = false; + + source = coresight_get_percpu_source_ref(smp_processor_id()); + if (!source) + return NULL; + + if (coresight_get_mode(source) != CS_MODE_DISABLED) + is_active = true; + + coresight_put_percpu_source_ref(source); + + /* + * It is expected to run in atomic context, so it cannot be preempted + * to disable the source. Here returns the active source pointer + * without concern that its state may change. Since the build path has + * taken a reference on the component, the source can be safely used + * by the caller. + */ + return is_active ? source : NULL; +} + +static int coresight_pm_is_needed(struct coresight_device *csdev) +{ + if (!csdev) + return 0; + + /* pm_save_disable() and pm_restore_enable() must be paired */ + if (coresight_ops(csdev)->pm_save_disable && + coresight_ops(csdev)->pm_restore_enable) + return 1; + + return 0; +} + +static int coresight_pm_device_save(struct coresight_device *csdev) +{ + return coresight_ops(csdev)->pm_save_disable(csdev); +} + +static void coresight_pm_device_restore(struct coresight_device *csdev) +{ + coresight_ops(csdev)->pm_restore_enable(csdev); +} + +static int coresight_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd, + void *v) +{ + struct coresight_device *csdev = coresight_cpu_get_active_source(); + + if (!coresight_pm_is_needed(csdev)) + return NOTIFY_DONE; + + switch (cmd) { + case CPU_PM_ENTER: + if (coresight_pm_device_save(csdev)) + return NOTIFY_BAD; + break; + case CPU_PM_EXIT: + case CPU_PM_ENTER_FAILED: + coresight_pm_device_restore(csdev); + break; + default: + return NOTIFY_DONE; + } + + return NOTIFY_OK; +} + +static struct notifier_block coresight_cpu_pm_nb = { + .notifier_call = coresight_cpu_pm_notify, +}; + +static int __init coresight_pm_setup(void) +{ + return cpu_pm_register_notifier(&coresight_cpu_pm_nb); +} + +static void coresight_pm_cleanup(void) +{ + cpu_pm_unregister_notifier(&coresight_cpu_pm_nb); +} + const struct bus_type coresight_bustype = { .name = "coresight", }; @@ -1770,9 +1856,15 @@ static int __init coresight_init(void)

/* initialise the coresight syscfg API */ ret = cscfg_init(); + if (ret) + goto exit_notifier; + + ret = coresight_pm_setup(); if (!ret) return 0;

+ cscfg_exit(); +exit_notifier: atomic_notifier_chain_unregister(&panic_notifier_list, &coresight_notifier); exit_perf: @@ -1784,6 +1876,7 @@ static int __init coresight_init(void)

static void __exit coresight_exit(void) { + coresight_pm_cleanup(); cscfg_exit(); atomic_notifier_chain_unregister(&panic_notifier_list, &coresight_notifier); diff --git a/include/linux/coresight.h b/include/linux/coresight.h index e9c20ceb9016fa3db256b8c1147c1fd2027b7b0d..5f9d7ea9f5941ab01eb6a084ca558a9417c7727f 100644 --- a/include/linux/coresight.h +++ b/include/linux/coresight.h @@ -438,6 +438,8 @@ struct coresight_ops_panic { struct coresight_ops { int (*trace_id)(struct coresight_device *csdev, enum cs_mode mode, struct coresight_device *sink); + int (*pm_save_disable)(struct coresight_device *csdev); + void (*pm_restore_enable)(struct coresight_device *csdev); const struct coresight_ops_sink *sink_ops; const struct coresight_ops_link *link_ops; const struct coresight_ops_source *source_ops;

ETMv4 driver save/restore callbacks still re-check conditions that are already validated by the core layer before the callbacks are invoked.

Remove the duplicated checks and fold the two-level functions into direct callback implementations. The obsolete WARN_ON() checks are removed as well.

Reviewed-by: Mike Leach mike.leach@linaro.org Tested-by: James Clark james.clark@linaro.org Reviewed-by: Yeoreum Yun yeoreum.yun@arm.com Reviewed-by: James Clark james.clark@linaro.org Tested-by: Jie Gan jie.gan@oss.qualcomm.com Signed-off-by: Leo Yan leo.yan@arm.com --- drivers/hwtracing/coresight/coresight-etm4x-core.c | 41 +++------------------- 1 file changed, 4 insertions(+), 37 deletions(-)

diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index 64bdc1b18b916b83fc063eb541193e0acfa741ee..bf29ecf487a6c57c67a31004a9911fffeba929f8 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -1849,17 +1849,14 @@ static inline bool etm4_pm_save_needed(struct etmv4_drvdata *drvdata) return !!drvdata->save_state; }

-static int __etm4_cpu_save(struct etmv4_drvdata *drvdata) +static int etm4_cpu_save(struct coresight_device *csdev) { int i, ret = 0; + struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); struct etmv4_save_state *state; - struct coresight_device *csdev = drvdata->csdev; struct csdev_access *csa; struct device *etm_dev;

- if (WARN_ON(!csdev)) - return -ENODEV; - etm_dev = &csdev->dev; csa = &csdev->access;

@@ -1991,32 +1988,13 @@ static int __etm4_cpu_save(struct etmv4_drvdata *drvdata) return ret; }

-static int etm4_cpu_save(struct coresight_device *csdev) -{ - struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); - int ret = 0; - - if (!etm4_pm_save_needed(drvdata)) - return 0; - - /* - * Save and restore the ETM Trace registers only if - * the ETM is active. - */ - if (coresight_get_mode(drvdata->csdev)) - ret = __etm4_cpu_save(drvdata); - return ret; -} - -static void __etm4_cpu_restore(struct etmv4_drvdata *drvdata) +static void etm4_cpu_restore(struct coresight_device *csdev) { int i; + struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); struct etmv4_save_state *state = drvdata->save_state; struct csdev_access *csa = &drvdata->csdev->access;

- if (WARN_ON(!drvdata->csdev)) - return; - etm4_cs_unlock(drvdata, csa); etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET);

@@ -2109,17 +2087,6 @@ static void __etm4_cpu_restore(struct etmv4_drvdata *drvdata) etm4_cs_lock(drvdata, csa); }

-static void etm4_cpu_restore(struct coresight_device *csdev) -{ - struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); - - if (!etm4_pm_save_needed(drvdata)) - return; - - if (coresight_get_mode(drvdata->csdev)) - __etm4_cpu_restore(drvdata); -} - static const struct coresight_ops etm4_cs_ops = { .trace_id = coresight_etm_get_trace_id, .source_ops = &etm4_source_ops,

Move source helper disabling to coresight_disable_path() to pair it with coresight_enable_path().

In coresight_enable_path(), if enabling a node fails, iterate to the next node (which is the last successfully enabled node) and pass it to coresight_disable_path() for rollback. If the failed node is the last node on the path, no device is actually enabled, so bail out directly.

As a result, coresight_disable_source() only controls the source, leaving the associated helpers to be managed as part of path. Update the comment to reflect this change.

Tested-by: Jie Gan jie.gan@oss.qualcomm.com Reviewed-by: Yeoreum Yun yeoreum.yun@arm.com Reviewed-by: James Clark james.clark@linaro.org Tested-by: James Clark james.clark@linaro.org Signed-off-by: Leo Yan leo.yan@arm.com --- drivers/hwtracing/coresight/coresight-core.c | 38 +++++++++++++--------------- 1 file changed, 17 insertions(+), 21 deletions(-)

diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c index 0c11e7fff1bd5cf504b17241595d83a1d11bcf40..6ab80ae7c389e9031d4ab065d83bb676fd12160b 100644 --- a/drivers/hwtracing/coresight/coresight-core.c +++ b/drivers/hwtracing/coresight/coresight-core.c @@ -447,19 +447,12 @@ static void coresight_disable_helpers(struct coresight_device *csdev, }

/* - * Helper function to call source_ops(csdev)->disable and also disable the - * helpers. - * - * There is an imbalance between coresight_enable_path() and - * coresight_disable_path(). Enabling also enables the source's helpers as part - * of the path, but disabling always skips the first item in the path (which is - * the source), so sources and their helpers don't get disabled as part of that - * function and we need the extra step here. + * coresight_disable_source() only disables the source, but do nothing for + * the associated helpers, which are controlled as part of the path. */ void coresight_disable_source(struct coresight_device *csdev, void *data) { source_ops(csdev)->disable(csdev, data); - coresight_disable_helpers(csdev, NULL); } EXPORT_SYMBOL_GPL(coresight_disable_source);

@@ -486,9 +479,9 @@ int coresight_resume_source(struct coresight_device *csdev) EXPORT_SYMBOL_GPL(coresight_resume_source);

/* - * coresight_disable_path_from : Disable components in the given path beyond - * @nd in the list. If @nd is NULL, all the components, except the SOURCE are - * disabled. + * coresight_disable_path_from : Disable components in the given path starting + * from @nd in the list. If @nd is NULL, all the components, except the SOURCE + * are disabled. */ static void coresight_disable_path_from(struct coresight_path *path, struct coresight_node *nd) @@ -499,7 +492,7 @@ static void coresight_disable_path_from(struct coresight_path *path, if (!nd) nd = list_first_entry(&path->path_list, struct coresight_node, link);

- list_for_each_entry_continue(nd, &path->path_list, link) { + list_for_each_entry_from(nd, &path->path_list, link) { csdev = nd->csdev; type = csdev->type;

@@ -519,12 +512,6 @@ static void coresight_disable_path_from(struct coresight_path *path, coresight_disable_sink(csdev); break; case CORESIGHT_DEV_TYPE_SOURCE: - /* - * We skip the first node in the path assuming that it - * is the source. So we don't expect a source device in - * the middle of a path. - */ - WARN_ON(1); break; case CORESIGHT_DEV_TYPE_LINK: parent = list_prev_entry(nd, link)->csdev; @@ -580,12 +567,16 @@ int coresight_enable_path(struct coresight_path *path, enum cs_mode mode) { int ret = 0; u32 type; - struct coresight_node *nd; + struct coresight_node *nd, *last; struct coresight_device *csdev, *parent, *child; struct coresight_device *source;

source = coresight_get_source(path); - list_for_each_entry_reverse(nd, &path->path_list, link) { + + last = list_last_entry(&path->path_list, struct coresight_node, link); + + nd = last; + list_for_each_entry_from_reverse(nd, &path->path_list, link) { csdev = nd->csdev; type = csdev->type;

@@ -639,6 +630,11 @@ int coresight_enable_path(struct coresight_path *path, enum cs_mode mode) err_disable_helpers: coresight_disable_helpers(csdev, path); err_disable_path: + /* No device is actually enabled */ + if (nd == last) + goto out; + + nd = list_next_entry(nd, link); coresight_disable_path_from(path, nd); goto out; }

Replace open code with coresight_path_first_node() and coresight_path_last_node() for fetching the nodes.

Check that the node is not NULL before accessing csdev field.

Tested-by: Jie Gan jie.gan@oss.qualcomm.com Reviewed-by: Yeoreum Yun yeoreum.yun@arm.com Reviewed-by: James Clark james.clark@linaro.org Tested-by: James Clark james.clark@linaro.org Signed-off-by: Leo Yan leo.yan@arm.com --- drivers/hwtracing/coresight/coresight-core.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c index 46b6447ce527b4c30d3f0d57801257f90580cc12..7ca142b77186143aeaf5b9903475e21559b7b2fa 100644 --- a/drivers/hwtracing/coresight/coresight-core.c +++ b/drivers/hwtracing/coresight/coresight-core.c @@ -165,11 +165,16 @@ void coresight_put_percpu_source_ref(struct coresight_device *csdev) struct coresight_device *coresight_get_source(struct coresight_path *path) { struct coresight_device *csdev; + struct coresight_node *nd;

if (!path) return NULL;

- csdev = list_first_entry(&path->path_list, struct coresight_node, link)->csdev; + nd = coresight_path_first_node(path); + if (!nd) + return NULL; + + csdev = nd->csdev; if (!coresight_is_device_source(csdev)) return NULL;

@@ -706,11 +711,16 @@ int coresight_enable_path(struct coresight_path *path, enum cs_mode mode) struct coresight_device *coresight_get_sink(struct coresight_path *path) { struct coresight_device *csdev; + struct coresight_node *nd;

if (!path) return NULL;

- csdev = list_last_entry(&path->path_list, struct coresight_node, link)->csdev; + nd = coresight_path_last_node(path); + if (!nd) + return NULL; + + csdev = nd->csdev; if (csdev->type != CORESIGHT_DEV_TYPE_SINK && csdev->type != CORESIGHT_DEV_TYPE_LINKSINK) return NULL;

This commit only set the path pointer for system tracers (e.g. STM) in coresight_{enable|disable}_source().

Later changes will set the path pointer locally for per-CPU sources. This is because the mode and path pointer must be set together, so that they are observed atomically by the CPU PM notifier.

Signed-off-by: Leo Yan leo.yan@arm.com --- drivers/hwtracing/coresight/coresight-core.c | 23 ++++++++++++++++++++++- include/linux/coresight.h | 2 ++ 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c index 308282d2d4611997834bd39dc2de271bfa568ee0..b45fb38eb9d652e44802c23ea4ee3b051d635207 100644 --- a/drivers/hwtracing/coresight/coresight-core.c +++ b/drivers/hwtracing/coresight/coresight-core.c @@ -478,10 +478,28 @@ int coresight_enable_source(struct coresight_device *csdev, struct perf_event *event, enum cs_mode mode, struct coresight_path *path) { + int ret; + if (!coresight_is_device_source(csdev)) return -EINVAL;

- return source_ops(csdev)->enable(csdev, event, mode, path); + ret = source_ops(csdev)->enable(csdev, event, mode, path); + if (ret) + return ret; + + /* + * Update the path pointer until after the source is enabled to avoid + * races where multiple paths attempt to enable the same source. + * + * Do not set the path pointer here for per-CPU sources; set it locally + * on the CPU instead. Otherwise, there is a window where the path is + * enabled but the pointer is not yet set, causing CPU PM notifiers to + * miss PM operations due to reading a NULL pointer. + */ + if (!coresight_is_percpu_source(csdev)) + csdev->path = path; + + return 0; }

void coresight_disable_source(struct coresight_device *csdev, void *data) @@ -489,6 +507,9 @@ void coresight_disable_source(struct coresight_device *csdev, void *data) if (!coresight_is_device_source(csdev)) return;

+ if (!coresight_is_percpu_source(csdev)) + csdev->path = NULL; + source_ops(csdev)->disable(csdev, data); } EXPORT_SYMBOL_GPL(coresight_disable_source); diff --git a/include/linux/coresight.h b/include/linux/coresight.h index 5f9d7ea9f5941ab01eb6a084ca558a9417c7727f..58d474b269806d32cad6ed87da96550b06f1f30f 100644 --- a/include/linux/coresight.h +++ b/include/linux/coresight.h @@ -257,6 +257,7 @@ struct coresight_trace_id_map { * by @coresight_ops. * @access: Device i/o access abstraction for this device. * @dev: The device entity associated to this component. + * @path: Activated path pointer (only used for source device). * @mode: The device mode, i.e sysFS, Perf or disabled. This is actually * an 'enum cs_mode' but stored in an atomic type. Access is always * through atomic APIs, ensuring SMP-safe synchronisation between @@ -291,6 +292,7 @@ struct coresight_device { const struct coresight_ops *ops; struct csdev_access access; struct device dev; + struct coresight_path *path; atomic_t mode; int refcnt; int cpu;

Set the path pointer on the target CPU during ETM enable and disable.

This ensures the device mode and path pointer are updated together and observed atomically by the CPU PM notifier.

Tested-by: James Clark james.clark@linaro.org Reviewed-by: Yeoreum Yun yeoreum.yun@arm.com Reviewed-by: James Clark james.clark@linaro.org Signed-off-by: Leo Yan leo.yan@arm.com --- drivers/hwtracing/coresight/coresight-etm3x-core.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/drivers/hwtracing/coresight/coresight-etm3x-core.c b/drivers/hwtracing/coresight/coresight-etm3x-core.c index aeeb284abdbe4b6a0960da45baa1138e203f3e3c..c6fe8b25b855a4119110fee4162f55c0154c3d05 100644 --- a/drivers/hwtracing/coresight/coresight-etm3x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm3x-core.c @@ -441,6 +441,7 @@ static int etm_enable_hw(struct etm_drvdata *drvdata)

struct etm_enable_arg { struct etm_drvdata *drvdata; + struct coresight_path *path; int rc; };

@@ -462,8 +463,12 @@ static void etm_enable_sysfs_smp_call(void *info) arg->rc = etm_enable_hw(arg->drvdata);

/* The tracer didn't start */ - if (arg->rc) + if (arg->rc) { coresight_set_mode(csdev, CS_MODE_DISABLED); + return; + } + + csdev->path = arg->path; }

void etm_release_trace_id(struct etm_drvdata *drvdata) @@ -492,10 +497,13 @@ static int etm_enable_perf(struct coresight_device *csdev, ret = etm_enable_hw(drvdata);

/* Failed to start tracer; roll back to DISABLED mode */ - if (ret) + if (ret) { coresight_set_mode(csdev, CS_MODE_DISABLED); + return ret; + }

- return ret; + csdev->path = path; + return 0; }

static int etm_enable_sysfs(struct coresight_device *csdev, struct coresight_path *path) @@ -514,6 +522,7 @@ static int etm_enable_sysfs(struct coresight_device *csdev, struct coresight_pat */ if (cpu_online(drvdata->cpu)) { arg.drvdata = drvdata; + arg.path = path; ret = smp_call_function_single(drvdata->cpu, etm_enable_sysfs_smp_call, &arg, 1); if (!ret) @@ -583,6 +592,7 @@ static void etm_disable_sysfs_smp_call(void *info)

etm_disable_hw(drvdata);

+ drvdata->csdev->path = NULL; coresight_set_mode(drvdata->csdev, CS_MODE_DISABLED); }

@@ -607,6 +617,7 @@ static void etm_disable_perf(struct coresight_device *csdev)

CS_LOCK(drvdata->csa.base);

+ drvdata->csdev->path = NULL; coresight_set_mode(drvdata->csdev, CS_MODE_DISABLED);

/*

Extend the CPU PM flow to control the path: disable from source up to the node before the sink, then re-enable the same range on restore. To avoid latency, control it up to the node before the sink.

Track per-CPU PM restore failures using percpu_pm_failed. Once a CPU hits a restore failure, set the percpu_pm_failed and return NOTIFY_BAD on subsequent notifications to avoid repeating half-completed transitions.

Setting percpu_pm_failed permanently blocks CPU PM on that CPU. Such failures are typically seen during development; disabling PM operations simplifies the implementation, and a warning highlights the issue.

Reviewed-by: James Clark james.clark@linaro.org Tested-by: Jie Gan jie.gan@oss.qualcomm.com Reviewed-by: Yeoreum Yun yeoreum.yun@arm.com Tested-by: James Clark james.clark@linaro.org Signed-off-by: Leo Yan leo.yan@arm.com --- drivers/hwtracing/coresight/coresight-core.c | 90 +++++++++++++++++++++++----- 1 file changed, 75 insertions(+), 15 deletions(-)

diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c index b45fb38eb9d652e44802c23ea4ee3b051d635207..d65bb2ea7b0560d44abf302a3939b3744ca0c880 100644 --- a/drivers/hwtracing/coresight/coresight-core.c +++ b/drivers/hwtracing/coresight/coresight-core.c @@ -38,6 +38,7 @@ static DEFINE_PER_CPU(struct coresight_device *, csdev_sink);

static DEFINE_RAW_SPINLOCK(coresight_dev_lock); static DEFINE_PER_CPU(struct coresight_device *, csdev_source); +static DEFINE_PER_CPU(bool, percpu_pm_failed);

/** * struct coresight_node - elements of a path, from source to sink @@ -1822,7 +1823,7 @@ static void coresight_release_device_list(void) } }

-static struct coresight_device *coresight_cpu_get_active_source(void) +static struct coresight_path *coresight_cpu_get_active_path(void) { struct coresight_device *source; bool is_active = false; @@ -1838,22 +1839,32 @@ static struct coresight_device *coresight_cpu_get_active_source(void)

/* * It is expected to run in atomic context, so it cannot be preempted - * to disable the source. Here returns the active source pointer - * without concern that its state may change. Since the build path has - * taken a reference on the component, the source can be safely used - * by the caller. + * to disable the path. Here returns the active path pointer without + * concern that its state may change. Since the build path has taken + * a reference on the component, the path can be safely used by the + * caller. */ - return is_active ? source : NULL; + return is_active ? source->path : NULL; }

-static int coresight_pm_is_needed(struct coresight_device *csdev) +/* Return: 1 if PM is required, 0 if skip, or a negative error */ +static int coresight_pm_is_needed(struct coresight_path *path) { - if (!csdev) + struct coresight_device *source; + + if (this_cpu_read(percpu_pm_failed)) + return -EIO; + + if (!path) + return 0; + + source = coresight_get_source(path); + if (!source) return 0;

/* pm_save_disable() and pm_restore_enable() must be paired */ - if (coresight_ops(csdev)->pm_save_disable && - coresight_ops(csdev)->pm_restore_enable) + if (coresight_ops(source)->pm_save_disable && + coresight_ops(source)->pm_restore_enable) return 1;

return 0; @@ -1869,22 +1880,71 @@ static void coresight_pm_device_restore(struct coresight_device *csdev) coresight_ops(csdev)->pm_restore_enable(csdev); }

+static int coresight_pm_save(struct coresight_path *path) +{ + struct coresight_device *source = coresight_get_source(path); + struct coresight_node *from, *to; + int ret; + + ret = coresight_pm_device_save(source); + if (ret) + return ret; + + from = coresight_path_first_node(path); + /* Disable up to the node before sink */ + to = list_prev_entry(coresight_path_last_node(path), link); + coresight_disable_path_from_to(path, from, to); + + return 0; +} + +static void coresight_pm_restore(struct coresight_path *path) +{ + struct coresight_device *source = coresight_get_source(path); + struct coresight_node *from, *to; + int ret; + + from = coresight_path_first_node(path); + /* Enable up to the node before sink */ + to = list_prev_entry(coresight_path_last_node(path), link); + ret = coresight_enable_path_from_to(path, coresight_get_mode(source), + from, to); + if (ret) + goto path_failed; + + coresight_pm_device_restore(source); + return; + +path_failed: + pr_err("Failed in coresight PM restore on CPU%d: %d\n", + smp_processor_id(), ret); + + /* + * Once PM fails on a CPU, set percpu_pm_failed and leave it set until + * reboot. This prevents repeated partial transitions during idle + * entry and exit. + */ + this_cpu_write(percpu_pm_failed, true); +} + static int coresight_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd, void *v) { - struct coresight_device *csdev = coresight_cpu_get_active_source(); + struct coresight_path *path = coresight_cpu_get_active_path(); + int ret;

- if (!coresight_pm_is_needed(csdev)) - return NOTIFY_DONE; + ret = coresight_pm_is_needed(path); + if (ret <= 0) + return ret ? NOTIFY_BAD : NOTIFY_DONE;

switch (cmd) { case CPU_PM_ENTER: - if (coresight_pm_device_save(csdev)) + if (coresight_pm_save(path)) return NOTIFY_BAD; break; case CPU_PM_EXIT: case CPU_PM_ENTER_FAILED: - coresight_pm_device_restore(csdev); + coresight_pm_restore(path); break; default: return NOTIFY_DONE;

From: Yabin Cui yabinc@google.com

TRBE context can be lost when a CPU enters low power states. If a trace source is restored while TRBE is not, tracing may run without an active sink, which can lead to hangs on some devices (e.g., Pixel 9).

The save and restore flows are described in the section K5.5 "Context switching" of Arm ARM (ARM DDI 0487 L.a). This commit adds save and restore callbacks with following the software usages defined in the architecture manual.

During the restore flow, since TRBLIMITR_EL1.E resets to 0 on a warm reset, the trace buffer unit is disabled when idle resume, it is safe to restore base/pointer/status registers first and program TRBLIMITR_EL1 last.

Signed-off-by: Yabin Cui yabinc@google.com Tested-by: James Clark james.clark@linaro.org Reviewed-by: Yeoreum Yun yeoreum.yun@arm.com Reviewed-by: James Clark james.clark@linaro.org Co-developed-by: Leo Yan leo.yan@arm.com Tested-by: Jie Gan jie.gan@oss.qualcomm.com Signed-off-by: Leo Yan leo.yan@arm.com --- drivers/hwtracing/coresight/coresight-trbe.c | 59 +++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-)

diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index 14e35b9660d76e47619cc6026b94929b3bb3e02b..c7cbca45f2debd4047b93283ea9fe5dd9e1f2ebf 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -116,6 +116,20 @@ static int trbe_errata_cpucaps[] = { */ #define TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES 256

+/* + * struct trbe_save_state: Register values representing TRBE state + * @trblimitr - Trace Buffer Limit Address Register value + * @trbbaser - Trace Buffer Base Register value + * @trbptr - Trace Buffer Write Pointer Register value + * @trbsr - Trace Buffer Status Register value + */ +struct trbe_save_state { + u64 trblimitr; + u64 trbbaser; + u64 trbptr; + u64 trbsr; +}; + /* * struct trbe_cpudata: TRBE instance specific data * @trbe_flag - TRBE dirty/access flag support @@ -134,6 +148,7 @@ struct trbe_cpudata { enum cs_mode mode; struct trbe_buf *buf; struct trbe_drvdata *drvdata; + struct trbe_save_state save_state; DECLARE_BITMAP(errata, TRBE_ERRATA_MAX); };

@@ -1189,6 +1204,46 @@ static irqreturn_t arm_trbe_irq_handler(int irq, void *dev) return IRQ_HANDLED; }

+static int arm_trbe_save(struct coresight_device *csdev) +{ + struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev); + struct trbe_save_state *state = &cpudata->save_state; + + state->trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1); + + /* Disable the unit, ensure the writes to memory are complete */ + if (state->trblimitr & TRBLIMITR_EL1_E) + trbe_drain_and_disable_local(cpudata); + + state->trbbaser = read_sysreg_s(SYS_TRBBASER_EL1); + state->trbptr = read_sysreg_s(SYS_TRBPTR_EL1); + state->trbsr = read_sysreg_s(SYS_TRBSR_EL1); + return 0; +} + +static void arm_trbe_restore(struct coresight_device *csdev) +{ + struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev); + struct trbe_save_state *state = &cpudata->save_state; + + write_sysreg_s(state->trbbaser, SYS_TRBBASER_EL1); + write_sysreg_s(state->trbptr, SYS_TRBPTR_EL1); + write_sysreg_s(state->trbsr, SYS_TRBSR_EL1); + + if (!(state->trblimitr & TRBLIMITR_EL1_E)) { + write_sysreg_s(state->trblimitr, SYS_TRBLIMITR_EL1); + } else { + /* + * The section K5.5 Context switching, Arm ARM (ARM DDI 0487 + * L.a), S_PKLXF requires a Context synchronization event to + * guarantee the Trace Buffer Unit will observe the new values + * of the system registers. + */ + isb(); + set_trbe_enabled(cpudata, state->trblimitr); + } +} + static const struct coresight_ops_sink arm_trbe_sink_ops = { .enable = arm_trbe_enable, .disable = arm_trbe_disable, @@ -1198,7 +1253,9 @@ static const struct coresight_ops_sink arm_trbe_sink_ops = { };

static const struct coresight_ops arm_trbe_cs_ops = { - .sink_ops = &arm_trbe_sink_ops, + .pm_save_disable = arm_trbe_save, + .pm_restore_enable = arm_trbe_restore, + .sink_ops = &arm_trbe_sink_ops, };

static ssize_t align_show(struct device *dev, struct device_attribute *attr, char *buf)

This commit moves CPU hotplug callbacks from ETMv4 driver to core layer. The motivation is the core layer can control all components on an activated path rather but not only managing tracer in ETMv4 driver.

The perf event layer will disable CoreSight PMU event 'cs_etm' when hotplug off a CPU. That means a perf mode will be always converted to disabled mode in CPU hotplug. Arm CoreSight CPU hotplug callbacks only need to handle the Sysfs mode and ignore the perf mode.

Add a 'mode' argument to coresight_pm_get_active_path() so it only returns active paths for the relevant mode.

Change CPUHP_AP_ARM_CORESIGHT_STARTING to CPUHP_AP_ARM_CORESIGHT_ONLINE so that the CPU hotplug callback runs in the online state and thread context, allowing coresight_disable_sysfs() to be called directly to disable the path.

Tested-by: James Clark james.clark@linaro.org Reviewed-by: Yeoreum Yun yeoreum.yun@arm.com Reviewed-by: James Clark james.clark@linaro.org Tested-by: Jie Gan jie.gan@oss.qualcomm.com Signed-off-by: Leo Yan leo.yan@arm.com --- drivers/hwtracing/coresight/coresight-core.c | 48 ++++++++++++++++++---- drivers/hwtracing/coresight/coresight-etm3x-core.c | 40 ------------------ drivers/hwtracing/coresight/coresight-etm4x-core.c | 37 ----------------- include/linux/cpuhotplug.h | 2 +- 4 files changed, 40 insertions(+), 87 deletions(-)

diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c index b2301ae0fd358242c59b9f4da99faae0db5ff894..015363da12fa8cb46e265bda9eb86c5e9abf0a63 100644 --- a/drivers/hwtracing/coresight/coresight-core.c +++ b/drivers/hwtracing/coresight/coresight-core.c @@ -1823,7 +1823,7 @@ static void coresight_release_device_list(void) } }

-static struct coresight_path *coresight_cpu_get_active_path(void) +static struct coresight_path *coresight_cpu_get_active_path(enum cs_mode mode) { struct coresight_device *source; bool is_active = false; @@ -1832,17 +1832,17 @@ static struct coresight_path *coresight_cpu_get_active_path(void) if (!source) return NULL;

- if (coresight_get_mode(source) != CS_MODE_DISABLED) + if (coresight_get_mode(source) & mode) is_active = true;

coresight_put_percpu_source_ref(source);

/* - * It is expected to run in atomic context, so it cannot be preempted - * to disable the path. Here returns the active path pointer without - * concern that its state may change. Since the build path has taken - * a reference on the component, the path can be safely used by the - * caller. + * It is expected to run in atomic context or with the CPU lock held for + * sysfs mode, so it cannot be preempted to disable the path. Here + * returns the active path pointer without concern that its state may + * change. Since the build path has taken a reference on the component, + * the path can be safely used by the caller. */ return is_active ? source->path : NULL; } @@ -1961,7 +1961,8 @@ static void coresight_pm_restore(struct coresight_path *path) static int coresight_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd, void *v) { - struct coresight_path *path = coresight_cpu_get_active_path(); + struct coresight_path *path = + coresight_cpu_get_active_path(CS_MODE_SYSFS | CS_MODE_PERF); int ret;

ret = coresight_pm_is_needed(path); @@ -1988,13 +1989,42 @@ static struct notifier_block coresight_cpu_pm_nb = { .notifier_call = coresight_cpu_pm_notify, };

+static int coresight_dying_cpu(unsigned int cpu) +{ + struct coresight_path *path; + + /* + * The perf event layer will disable PMU events in the CPU + * hotplug. Here only handles SYSFS case. + */ + path = coresight_cpu_get_active_path(CS_MODE_SYSFS); + if (!path) + return 0; + + coresight_disable_sysfs(coresight_get_source(path)); + return 0; +} + static int __init coresight_pm_setup(void) { - return cpu_pm_register_notifier(&coresight_cpu_pm_nb); + int ret; + + ret = cpu_pm_register_notifier(&coresight_cpu_pm_nb); + if (ret) + return ret; + + ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CORESIGHT_ONLINE, + "arm/coresight-core:dying", + NULL, coresight_dying_cpu); + if (ret) + cpu_pm_unregister_notifier(&coresight_cpu_pm_nb); + + return ret; }

static void coresight_pm_cleanup(void) { + cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_ONLINE); cpu_pm_unregister_notifier(&coresight_cpu_pm_nb); }

diff --git a/drivers/hwtracing/coresight/coresight-etm3x-core.c b/drivers/hwtracing/coresight/coresight-etm3x-core.c index c6fe8b25b855a4119110fee4162f55c0154c3d05..862ad0786699c41433eae8683406b3c1340a6cb6 100644 --- a/drivers/hwtracing/coresight/coresight-etm3x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm3x-core.c @@ -699,35 +699,6 @@ static int etm_online_cpu(unsigned int cpu) return 0; }

-static int etm_starting_cpu(unsigned int cpu) -{ - if (!etmdrvdata[cpu]) - return 0; - - spin_lock(&etmdrvdata[cpu]->spinlock); - if (!etmdrvdata[cpu]->os_unlock) { - etm_os_unlock(etmdrvdata[cpu]); - etmdrvdata[cpu]->os_unlock = true; - } - - if (coresight_get_mode(etmdrvdata[cpu]->csdev)) - etm_enable_hw(etmdrvdata[cpu]); - spin_unlock(&etmdrvdata[cpu]->spinlock); - return 0; -} - -static int etm_dying_cpu(unsigned int cpu) -{ - if (!etmdrvdata[cpu]) - return 0; - - spin_lock(&etmdrvdata[cpu]->spinlock); - if (coresight_get_mode(etmdrvdata[cpu]->csdev)) - etm_disable_hw(etmdrvdata[cpu]); - spin_unlock(&etmdrvdata[cpu]->spinlock); - return 0; -} - static bool etm_arch_supported(u8 arch) { switch (arch) { @@ -795,13 +766,6 @@ static int __init etm_hp_setup(void) { int ret;

- ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING, - "arm/coresight:starting", - etm_starting_cpu, etm_dying_cpu); - - if (ret) - return ret; - ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "arm/coresight:online", etm_online_cpu, NULL); @@ -812,15 +776,11 @@ static int __init etm_hp_setup(void) return 0; }

- /* failed dyn state - remove others */ - cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING); - return ret; }

static void etm_hp_clear(void) { - cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING); if (hp_online) { cpuhp_remove_state_nocalls(hp_online); hp_online = 0; diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index 7c9e0af2f24556e27f6c1bcbb22044c52f13de12..266dec109232ed2e8761485a9ee58b80ba7b5141 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -1829,33 +1829,6 @@ static int etm4_online_cpu(unsigned int cpu) return 0; }

-static int etm4_starting_cpu(unsigned int cpu) -{ - if (!etmdrvdata[cpu]) - return 0; - - raw_spin_lock(&etmdrvdata[cpu]->spinlock); - if (!etmdrvdata[cpu]->os_unlock) - etm4_os_unlock(etmdrvdata[cpu]); - - if (coresight_get_mode(etmdrvdata[cpu]->csdev)) - etm4_enable_hw(etmdrvdata[cpu]); - raw_spin_unlock(&etmdrvdata[cpu]->spinlock); - return 0; -} - -static int etm4_dying_cpu(unsigned int cpu) -{ - if (!etmdrvdata[cpu]) - return 0; - - raw_spin_lock(&etmdrvdata[cpu]->spinlock); - if (coresight_get_mode(etmdrvdata[cpu]->csdev)) - etm4_disable_hw(etmdrvdata[cpu]); - raw_spin_unlock(&etmdrvdata[cpu]->spinlock); - return 0; -} - static inline bool etm4_pm_save_needed(struct etmv4_drvdata *drvdata) { return !!drvdata->save_state; @@ -2116,13 +2089,6 @@ static int __init etm4_pm_setup(void) { int ret;

- ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING, - "arm/coresight4:starting", - etm4_starting_cpu, etm4_dying_cpu); - - if (ret) - return ret; - ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "arm/coresight4:online", etm4_online_cpu, NULL); @@ -2133,14 +2099,11 @@ static int __init etm4_pm_setup(void) return 0; }

- /* failed dyn state - remove others */ - cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING); return ret; }

static void etm4_pm_clear(void) { - cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING); if (hp_online) { cpuhp_remove_state_nocalls(hp_online); hp_online = 0; diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 62cd7b35a29c94f214fec3b55f9462eceffad694..b9196ef6d83471400bb8bc5a8c4fd486c0acd4cc 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -181,7 +181,6 @@ enum cpuhp_state { CPUHP_AP_DUMMY_TIMER_STARTING, CPUHP_AP_ARM_XEN_STARTING, CPUHP_AP_ARM_XEN_RUNSTATE_STARTING, - CPUHP_AP_ARM_CORESIGHT_STARTING, CPUHP_AP_ARM_CORESIGHT_CTI_STARTING, CPUHP_AP_ARM64_ISNDEP_STARTING, CPUHP_AP_SMPCFD_DYING, @@ -201,6 +200,7 @@ enum cpuhp_state { CPUHP_AP_IRQ_AFFINITY_ONLINE, CPUHP_AP_BLK_MQ_ONLINE, CPUHP_AP_ARM_MVEBU_SYNC_CLOCKS, + CPUHP_AP_ARM_CORESIGHT_ONLINE, CPUHP_AP_X86_INTEL_EPB_ONLINE, CPUHP_AP_PERF_ONLINE, CPUHP_AP_PERF_X86_ONLINE,