@@ -95,7 +95,7 @@ struct rpmh_ctrlr {
* @num_tcs: Number of TCSes in this DRV.
* @rsc_pm: CPU PM notifier for controller.
* Used when solver mode is not present.
- * @cpus_entered_pm: CPU mask for cpus in idle power collapse.
+ * @cpus_in_pm: Number of CPUs not in idle power collapse.
* Used when solver mode is not present.
* @tcs: TCS groups.
* @tcs_in_use: S/W state of the TCS; only set for ACTIVE_ONLY
@@ -103,9 +103,9 @@ struct rpmh_ctrlr {
* it was borrowed for an active_only transfer. You
* must hold the lock in this struct (AKA drv->lock) in
* order to update this.
- * @lock: Synchronize state of the controller.
- * @pm_lock: Synchronize during PM notifications.
- * Used when solver mode is not present.
+ * @lock: Synchronize state of the controller. If RPMH's cache
+ * lock will also be held, the order is: drv->lock then
+ * cache_lock.
* @client: Handle to the DRV's client.
*/
struct rsc_drv {
@@ -114,11 +114,10 @@ struct rsc_drv {
int id;
int num_tcs;
struct notifier_block rsc_pm;
- struct cpumask cpus_entered_pm;
+ atomic_t cpus_in_pm;
struct tcs_group tcs[TCS_TYPE_NR];
DECLARE_BITMAP(tcs_in_use, MAX_TCS_NR);
spinlock_t lock;
- spinlock_t pm_lock;
struct rpmh_ctrlr client;
};
@@ -737,6 +737,8 @@ int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, const struct tcs_request *msg)
* SLEEP and WAKE sets. If AMCs are busy, controller can not enter
* power collapse, so deny from the last cpu's pm notification.
*
+ * Context: Must be called with the drv->lock held.
+ *
* Return:
* * False - AMCs are idle
* * True - AMCs are busy
@@ -751,9 +753,6 @@ static bool rpmh_rsc_ctrlr_is_busy(struct rsc_drv *drv)
* dedicated TCS for active state use, then re-purposed wake TCSes
* should be checked for not busy, because we used wake TCSes for
* active requests in this case.
- *
- * Since this is called from the last cpu, need not take drv->lock
- * before checking tcs_is_free().
*/
if (!tcs->num_tcs)
tcs = &drv->tcs[WAKE_TCS];
@@ -788,42 +787,62 @@ static int rpmh_rsc_cpu_pm_callback(struct notifier_block *nfb,
{
struct rsc_drv *drv = container_of(nfb, struct rsc_drv, rsc_pm);
int ret = NOTIFY_OK;
-
- spin_lock(&drv->pm_lock);
+ int cpus_in_pm;
switch (action) {
case CPU_PM_ENTER:
- cpumask_set_cpu(smp_processor_id(), &drv->cpus_entered_pm);
-
- if (!cpumask_equal(&drv->cpus_entered_pm, cpu_online_mask))
- goto exit;
+ cpus_in_pm = atomic_inc_return(&drv->cpus_in_pm);
+ /*
+ * NOTE: comments for num_online_cpus() point out that it's
+ * only a snapshot so we need to be careful. It should be OK
+ * for us to use, though. It's important for us not to miss
+ * if we're the last CPU going down so it would only be a
+ * problem if a CPU went offline right after we did the check
+ * AND that CPU was not idle AND that CPU was the last non-idle
+ * CPU. That can't happen. CPUs would have to come out of idle
+ * before the CPU could go offline.
+ */
+ if (cpus_in_pm < num_online_cpus())
+ return NOTIFY_OK;
break;
case CPU_PM_ENTER_FAILED:
case CPU_PM_EXIT:
- cpumask_clear_cpu(smp_processor_id(), &drv->cpus_entered_pm);
- goto exit;
+ atomic_dec(&drv->cpus_in_pm);
+ return NOTIFY_OK;
default:
return NOTIFY_DONE;
}
- ret = rpmh_rsc_ctrlr_is_busy(drv);
- if (ret) {
- ret = NOTIFY_BAD;
- goto exit;
+ /*
+ * It's likely we're on the last CPU. Grab the drv->lock and write
+ * out the sleep/wake commands to RPMH hardware. Grabbing the lock
+ * means that if we race with another CPU coming up we are still
+ * guaranteed to be safe. If another CPU came up just after we checked
+ * and has grabbed the lock or started an active transfer then we'll
+ * notice we're busy and abort. If another CPU comes up after we start
+ * flushing it will be blocked from starting an active transfer until
+ * we're done flushing. If another CPU starts an active transfer after
+ * we release the lock we're still OK because we're no longer the last
+ * CPU.
+ */
+ if (spin_trylock(&drv->lock)) {
+ if (rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(&drv->client))
+ ret = NOTIFY_BAD;
+ spin_unlock(&drv->lock);
+ } else {
+ /* Another CPU must be up */
+ return NOTIFY_OK;
}
- ret = rpmh_flush(&drv->client);
- if (ret)
- ret = NOTIFY_BAD;
- else
- ret = NOTIFY_OK;
-
-exit:
- if (ret == NOTIFY_BAD)
- /* We won't be called w/ CPU_PM_ENTER_FAILED */
- cpumask_clear_cpu(smp_processor_id(), &drv->cpus_entered_pm);
+ if (ret == NOTIFY_BAD) {
+ /* Double-check if we're here because someone else is up */
+ if (cpus_in_pm < num_online_cpus())
+ ret = NOTIFY_OK;
+ else
+ /* We won't be called w/ CPU_PM_ENTER_FAILED */
+ atomic_dec(&drv->cpus_in_pm);
+ }
- spin_unlock(&drv->pm_lock);
return ret;
}
@@ -966,7 +985,6 @@ static int rpmh_rsc_probe(struct platform_device *pdev)
solver_config = solver_config >> DRV_HW_SOLVER_SHIFT;
if (!solver_config) {
drv->rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback;
- spin_lock_init(&drv->pm_lock);
cpu_pm_register_notifier(&drv->rsc_pm);
}
@@ -435,9 +435,6 @@ static int send_single(struct rpmh_ctrlr *ctrlr, enum rpmh_state state,
*
* @ctrlr: Controller making request to flush cached data
*
- * This function is called from sleep code on the last CPU
- * (thus no spinlock needed).
- *
* Return:
* * 0 - Success
* * Error code - Otherwise
@@ -445,13 +442,21 @@ static int send_single(struct rpmh_ctrlr *ctrlr, enum rpmh_state state,
int rpmh_flush(struct rpmh_ctrlr *ctrlr)
{
struct cache_req *p;
- int ret;
+ int ret = 0;
lockdep_assert_irqs_disabled();
+ /*
+ * Currently rpmh_flush() is only called when we think we're running
+ * on the last processor. If the lock is busy it means another
+ * processor is up and it's better to abort than spin.
+ */
+ if (!spin_trylock(&ctrlr->cache_lock))
+ return -EBUSY;
+
if (!ctrlr->dirty) {
pr_debug("Skipping flush, TCS has latest data.\n");
- return 0;
+ goto exit;
}
/* Invalidate the TCSes first to avoid stale data */
@@ -460,7 +465,7 @@ int rpmh_flush(struct rpmh_ctrlr *ctrlr)
/* First flush the cached batch requests */
ret = flush_batch(ctrlr);
if (ret)
- return ret;
+ goto exit;
list_for_each_entry(p, &ctrlr->cache, list) {
if (!is_req_valid(p)) {
@@ -471,16 +476,18 @@ int rpmh_flush(struct rpmh_ctrlr *ctrlr)
ret = send_single(ctrlr, RPMH_SLEEP_STATE, p->addr,
p->sleep_val);
if (ret)
- return ret;
+ goto exit;
ret = send_single(ctrlr, RPMH_WAKE_ONLY_STATE, p->addr,
p->wake_val);
if (ret)
- return ret;
+ goto exit;
}
ctrlr->dirty = false;
- return 0;
+exit:
+ spin_unlock(&ctrlr->cache_lock);
+ return ret;
}
/**