| Message ID | 20191023095000.3782-1-andi.shyti@intel.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | [v2,1/2] drm/i915: Extract GT render power state management | expand |
Hi, Sorry, I messed up with the e-mail (and with the in-reply-to, as well, and I forgot to add a note to this v2), please take this as 1/2 because it has the correct e-mail. Andi On Wed, Oct 23, 2019 at 12:50:00PM +0300, Andi Shyti wrote: > i915_irq.c is large. One reason for this is that has a large chunk of > the GT render power management stashed away in it. Extract that logic > out of i915_irq.c and intel_pm.c and put it under one roof. > > Based on a patch by Chris Wilson. > > Signed-off-by: Andi Shyti <andi.shyti@intel.com> > Cc: Chris Wilson <chris@chris-wilson.co.uk> > --- > drivers/gpu/drm/i915/Makefile | 3 +- > drivers/gpu/drm/i915/display/intel_display.c | 8 +- > drivers/gpu/drm/i915/gt/intel_gt.c | 6 +- > drivers/gpu/drm/i915/gt/intel_gt_irq.c | 5 +- > drivers/gpu/drm/i915/gt/intel_gt_pm.c | 49 +- > drivers/gpu/drm/i915/gt/intel_gt_pm.h | 1 + > drivers/gpu/drm/i915/gt/intel_gt_types.h | 2 + > drivers/gpu/drm/i915/gt/intel_llc.c | 2 +- > drivers/gpu/drm/i915/gt/intel_rps.c | 1872 +++++++++++++++ > drivers/gpu/drm/i915/gt/intel_rps.h | 37 + > drivers/gpu/drm/i915/gt/intel_rps_types.h | 93 + > drivers/gpu/drm/i915/gt/selftest_llc.c | 7 +- > .../gpu/drm/i915/gt/uc/intel_guc_submission.c | 4 +- > drivers/gpu/drm/i915/i915_debugfs.c | 73 +- > drivers/gpu/drm/i915/i915_drv.c | 2 + > drivers/gpu/drm/i915/i915_drv.h | 96 - > drivers/gpu/drm/i915/i915_gem.c | 3 +- > drivers/gpu/drm/i915/i915_irq.c | 359 +-- > drivers/gpu/drm/i915/i915_irq.h | 3 - > drivers/gpu/drm/i915/i915_pmu.c | 10 +- > drivers/gpu/drm/i915/i915_request.c | 7 +- > drivers/gpu/drm/i915/i915_sysfs.c | 74 +- > drivers/gpu/drm/i915/intel_pm.c | 2105 ++--------------- > drivers/gpu/drm/i915/intel_pm.h | 22 - > 24 files changed, 2359 insertions(+), 2484 deletions(-) > create mode 100644 drivers/gpu/drm/i915/gt/intel_rps.c > create mode 100644 drivers/gpu/drm/i915/gt/intel_rps.h > create mode 100644 drivers/gpu/drm/i915/gt/intel_rps_types.h > > diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile > index 2fd4bed188e5..133ca59e7c48 100644 > --- a/drivers/gpu/drm/i915/Makefile > +++ b/drivers/gpu/drm/i915/Makefile > @@ -90,11 +90,12 @@ gt-y += \ > gt/intel_hangcheck.o \ > gt/intel_llc.o \ > gt/intel_lrc.o \ > + gt/intel_mocs.o \ > gt/intel_rc6.o \ > gt/intel_renderstate.o \ > gt/intel_reset.o \ > gt/intel_ringbuffer.o \ > - gt/intel_mocs.o \ > + gt/intel_rps.o \ > gt/intel_sseu.o \ > gt/intel_timeline.o \ > gt/intel_workarounds.o > diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c > index 236fdf122e47..49c2c7f71394 100644 > --- a/drivers/gpu/drm/i915/display/intel_display.c > +++ b/drivers/gpu/drm/i915/display/intel_display.c > @@ -55,6 +55,8 @@ > #include "display/intel_tv.h" > #include "display/intel_vdsc.h" > > +#include "gt/intel_rps.h" > + > #include "i915_drv.h" > #include "i915_trace.h" > #include "intel_acpi.h" > @@ -14782,7 +14784,7 @@ static int do_rps_boost(struct wait_queue_entry *_wait, > * vblank without our intervention, so leave RPS alone. > */ > if (!i915_request_started(rq)) > - gen6_rps_boost(rq); > + intel_rps_boost(rq); > i915_request_put(rq); > > drm_crtc_vblank_put(wait->crtc); > @@ -14976,7 +14978,7 @@ intel_prepare_plane_fb(struct drm_plane *plane, > * maximum clocks following a vblank miss (see do_rps_boost()). > */ > if (!intel_state->rps_interactive) { > - intel_rps_mark_interactive(dev_priv, true); > + intel_rps_mark_interactive(&dev_priv->gt.rps, true); > intel_state->rps_interactive = true; > } > > @@ -15001,7 +15003,7 @@ intel_cleanup_plane_fb(struct drm_plane *plane, > struct drm_i915_private *dev_priv = to_i915(plane->dev); > > if (intel_state->rps_interactive) { > - intel_rps_mark_interactive(dev_priv, false); > + intel_rps_mark_interactive(&dev_priv->gt.rps, false); > intel_state->rps_interactive = false; > } > > diff --git a/drivers/gpu/drm/i915/gt/intel_gt.c b/drivers/gpu/drm/i915/gt/intel_gt.c > index 1c4b6c9642ad..7518852cb78a 100644 > --- a/drivers/gpu/drm/i915/gt/intel_gt.c > +++ b/drivers/gpu/drm/i915/gt/intel_gt.c > @@ -9,6 +9,7 @@ > #include "intel_gt_requests.h" > #include "intel_mocs.h" > #include "intel_rc6.h" > +#include "intel_rps.h" > #include "intel_uncore.h" > #include "intel_pm.h" > > @@ -321,8 +322,7 @@ void intel_gt_chipset_flush(struct intel_gt *gt) > > void intel_gt_driver_register(struct intel_gt *gt) > { > - if (IS_GEN(gt->i915, 5)) > - intel_gpu_ips_init(gt->i915); > + intel_rps_driver_register(>->rps); > } > > static int intel_gt_init_scratch(struct intel_gt *gt, unsigned int size) > @@ -385,7 +385,7 @@ void intel_gt_driver_remove(struct intel_gt *gt) > > void intel_gt_driver_unregister(struct intel_gt *gt) > { > - intel_gpu_ips_teardown(); > + intel_rps_driver_unregister(>->rps); > } > > void intel_gt_driver_release(struct intel_gt *gt) > diff --git a/drivers/gpu/drm/i915/gt/intel_gt_irq.c b/drivers/gpu/drm/i915/gt/intel_gt_irq.c > index 34a4fb624bf7..973ee7eded64 100644 > --- a/drivers/gpu/drm/i915/gt/intel_gt_irq.c > +++ b/drivers/gpu/drm/i915/gt/intel_gt_irq.c > @@ -11,6 +11,7 @@ > #include "intel_gt.h" > #include "intel_gt_irq.h" > #include "intel_uncore.h" > +#include "intel_rps.h" > > static void guc_irq_handler(struct intel_guc *guc, u16 iir) > { > @@ -77,7 +78,7 @@ gen11_other_irq_handler(struct intel_gt *gt, const u8 instance, > return guc_irq_handler(>->uc.guc, iir); > > if (instance == OTHER_GTPM_INSTANCE) > - return gen11_rps_irq_handler(gt, iir); > + return gen11_rps_irq_handler(>->rps, iir); > > WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n", > instance, iir); > @@ -336,7 +337,7 @@ void gen8_gt_irq_handler(struct intel_gt *gt, u32 master_ctl, u32 gt_iir[4]) > } > > if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) { > - gen6_rps_irq_handler(gt->i915, gt_iir[2]); > + gen6_rps_irq_handler(>->rps, gt_iir[2]); > guc_irq_handler(>->uc.guc, gt_iir[2] >> 16); > } > } > diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm.c b/drivers/gpu/drm/i915/gt/intel_gt_pm.c > index 06e73d56cfcf..32ca87265222 100644 > --- a/drivers/gpu/drm/i915/gt/intel_gt_pm.c > +++ b/drivers/gpu/drm/i915/gt/intel_gt_pm.c > @@ -12,8 +12,10 @@ > #include "intel_gt.h" > #include "intel_gt_pm.h" > #include "intel_gt_requests.h" > +#include "intel_llc.h" > #include "intel_pm.h" > #include "intel_rc6.h" > +#include "intel_rps.h" > #include "intel_wakeref.h" > > static int __gt_unpark(struct intel_wakeref *wf) > @@ -39,12 +41,8 @@ static int __gt_unpark(struct intel_wakeref *wf) > gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ); > GEM_BUG_ON(!gt->awake); > > - intel_enable_gt_powersave(i915); > - > - i915_update_gfx_val(i915); > - if (INTEL_GEN(i915) >= 6) > - gen6_rps_busy(i915); > - > + intel_rps_unpark(>->rps); > + intel_llc_enable(>->llc); > i915_pmu_gt_unparked(i915); > > intel_gt_queue_hangcheck(gt); > @@ -65,8 +63,8 @@ static int __gt_park(struct intel_wakeref *wf) > > i915_vma_parked(gt); > i915_pmu_gt_parked(i915); > - if (INTEL_GEN(i915) >= 6) > - gen6_rps_idle(i915); > + intel_llc_disable(>->llc); > + intel_rps_park(>->rps); > > /* Everything switched off, flush any residual interrupt just in case */ > intel_synchronize_irq(i915); > @@ -98,6 +96,7 @@ void intel_gt_pm_init(struct intel_gt *gt) > * user. > */ > intel_rc6_init(>->rc6); > + intel_rps_init(>->rps); > } > > static bool reset_engines(struct intel_gt *gt) > @@ -141,10 +140,39 @@ void intel_gt_sanitize(struct intel_gt *gt, bool force) > engine->reset.finish(engine); > } > > +void intel_gt_pm_enable(struct intel_gt *gt) > +{ > + struct intel_engine_cs *engine; > + enum intel_engine_id id; > + > + /* Powersaving is controlled by the host when inside a VM */ > + if (intel_vgpu_active(gt->i915)) > + return; > + > + if (is_mock_gt(gt)) > + return; > + > + intel_gt_pm_get(gt); > + intel_rps_enable(>->rps); > + intel_llc_enable(>->llc); > + > + for_each_engine(engine, gt->i915, id) { > + intel_engine_pm_get(engine); > + engine->serial++; /* force kernel context reload */ > + intel_engine_pm_put(engine); > + } > + > + intel_gt_pm_put(gt); > +} > + > void intel_gt_pm_disable(struct intel_gt *gt) > { > - if (!is_mock_gt(gt)) > - intel_sanitize_gt_powersave(gt->i915); > + if (is_mock_gt(gt)) > + return; > + > + intel_rc6_disable(>->rc6); > + intel_llc_disable(>->llc); > + intel_rps_disable(>->rps); > } > > void intel_gt_pm_fini(struct intel_gt *gt) > @@ -165,6 +193,7 @@ int intel_gt_resume(struct intel_gt *gt) > * allowing us to fixup the user contexts on their first pin. > */ > intel_gt_pm_get(gt); > + > intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL); > intel_rc6_sanitize(>->rc6); > > diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm.h b/drivers/gpu/drm/i915/gt/intel_gt_pm.h > index 0ed87da4bb68..df50f853748d 100644 > --- a/drivers/gpu/drm/i915/gt/intel_gt_pm.h > +++ b/drivers/gpu/drm/i915/gt/intel_gt_pm.h > @@ -40,6 +40,7 @@ static inline int intel_gt_pm_wait_for_idle(struct intel_gt *gt) > void intel_gt_pm_init_early(struct intel_gt *gt); > void intel_gt_pm_init(struct intel_gt *gt); > void intel_gt_pm_disable(struct intel_gt *gt); > +void intel_gt_pm_enable(struct intel_gt *gt); > void intel_gt_pm_fini(struct intel_gt *gt); > > void intel_gt_sanitize(struct intel_gt *gt, bool force); > diff --git a/drivers/gpu/drm/i915/gt/intel_gt_types.h b/drivers/gpu/drm/i915/gt/intel_gt_types.h > index 980973e66e7f..e4bd23eb8290 100644 > --- a/drivers/gpu/drm/i915/gt/intel_gt_types.h > +++ b/drivers/gpu/drm/i915/gt/intel_gt_types.h > @@ -20,6 +20,7 @@ > #include "intel_llc_types.h" > #include "intel_reset_types.h" > #include "intel_rc6_types.h" > +#include "intel_rps_types.h" > #include "intel_wakeref.h" > > struct drm_i915_private; > @@ -82,6 +83,7 @@ struct intel_gt { > > struct intel_llc llc; > struct intel_rc6 rc6; > + struct intel_rps rps; > > ktime_t last_init_time; > > diff --git a/drivers/gpu/drm/i915/gt/intel_llc.c b/drivers/gpu/drm/i915/gt/intel_llc.c > index 35093eb5f24e..ceb785b75c25 100644 > --- a/drivers/gpu/drm/i915/gt/intel_llc.c > +++ b/drivers/gpu/drm/i915/gt/intel_llc.c > @@ -48,7 +48,7 @@ static bool get_ia_constants(struct intel_llc *llc, > struct ia_constants *consts) > { > struct drm_i915_private *i915 = llc_to_gt(llc)->i915; > - struct intel_rps *rps = &i915->gt_pm.rps; > + struct intel_rps *rps = &llc_to_gt(llc)->rps; > > if (rps->max_freq <= rps->min_freq) > return false; > diff --git a/drivers/gpu/drm/i915/gt/intel_rps.c b/drivers/gpu/drm/i915/gt/intel_rps.c > new file mode 100644 > index 000000000000..30f56c786468 > --- /dev/null > +++ b/drivers/gpu/drm/i915/gt/intel_rps.c > @@ -0,0 +1,1872 @@ > +/* > + * SPDX-License-Identifier: MIT > + * > + * Copyright © 2019 Intel Corporation > + */ > + > +#include "i915_drv.h" > +#include "intel_gt.h" > +#include "intel_gt_irq.h" > +#include "intel_gt_pm_irq.h" > +#include "intel_rps.h" > +#include "intel_sideband.h" > +#include "../../../platform/x86/intel_ips.h" > + > +/* > + * Lock protecting IPS related data structures > + */ > +static DEFINE_SPINLOCK(mchdev_lock); > + > +static struct intel_gt *rps_to_gt(struct intel_rps *rps) > +{ > + return container_of(rps, struct intel_gt, rps); > +} > + > +static struct drm_i915_private *rps_to_i915(struct intel_rps *rps) > +{ > + return rps_to_gt(rps)->i915; > +} > + > +static struct intel_uncore *rps_to_uncore(struct intel_rps *rps) > +{ > + return rps_to_gt(rps)->uncore; > +} > + > +static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask) > +{ > + return mask & ~rps->pm_intrmsk_mbz; > +} > + > +static u32 rps_pm_mask(struct intel_rps *rps, u8 val) > +{ > + u32 mask = 0; > + > + /* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */ > + if (val > rps->min_freq_softlimit) > + mask |= (GEN6_PM_RP_UP_EI_EXPIRED | > + GEN6_PM_RP_DOWN_THRESHOLD | > + GEN6_PM_RP_DOWN_TIMEOUT); > + > + if (val < rps->max_freq_softlimit) > + mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD; > + > + mask &= rps->pm_events; > + > + return rps_pm_sanitize_mask(rps, ~mask); > +} > + > +static void rps_reset_ei(struct intel_rps *rps) > +{ > + memset(&rps->ei, 0, sizeof(rps->ei)); > +} > + > +static void rps_enable_interrupts(struct intel_rps *rps) > +{ > + struct intel_gt *gt = rps_to_gt(rps); > + > + rps_reset_ei(rps); > + > + if (IS_VALLEYVIEW(gt->i915)) > + /* WaGsvRC0ResidencyMethod:vlv */ > + rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED; > + else > + rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD | > + GEN6_PM_RP_DOWN_THRESHOLD | > + GEN6_PM_RP_DOWN_TIMEOUT); > + > + spin_lock_irq(>->irq_lock); > + gen6_gt_pm_enable_irq(gt, rps->pm_events); > + spin_unlock_irq(>->irq_lock); > + > + intel_uncore_write(gt->uncore, GEN6_PMINTRMSK, > + rps_pm_mask(rps, rps->cur_freq)); > +} > + > +static void gen6_rps_reset_interrupts(struct intel_rps *rps) > +{ > + gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS); > +} > + > +static void gen11_rps_reset_interrupts(struct intel_rps *rps) > +{ > + while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM)) > + ; > +} > + > +static void rps_reset_interrupts(struct intel_rps *rps) > +{ > + struct intel_gt *gt = rps_to_gt(rps); > + > + spin_lock_irq(>->irq_lock); > + if (INTEL_GEN(gt->i915) >= 11) > + gen11_rps_reset_interrupts(rps); > + else > + gen6_rps_reset_interrupts(rps); > + > + rps->pm_iir = 0; > + spin_unlock_irq(>->irq_lock); > +} > + > +static void rps_disable_interrupts(struct intel_rps *rps) > +{ > + struct intel_gt *gt = rps_to_gt(rps); > + > + rps->pm_events = 0; > + > + intel_uncore_write(gt->uncore, GEN6_PMINTRMSK, > + rps_pm_sanitize_mask(rps, ~0u)); > + > + spin_lock_irq(>->irq_lock); > + gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS); > + spin_unlock_irq(>->irq_lock); > + > + intel_synchronize_irq(gt->i915); > + > + /* > + * Now that we will not be generating any more work, flush any > + * outstanding tasks. As we are called on the RPS idle path, > + * we will reset the GPU to minimum frequencies, so the current > + * state of the worker can be discarded. > + */ > + cancel_work_sync(&rps->work); > + > + rps_reset_interrupts(rps); > +} > + > +static const struct cparams { > + u16 i; > + u16 t; > + u16 m; > + u16 c; > +} cparams[] = { > + { 1, 1333, 301, 28664 }, > + { 1, 1066, 294, 24460 }, > + { 1, 800, 294, 25192 }, > + { 0, 1333, 276, 27605 }, > + { 0, 1066, 276, 27605 }, > + { 0, 800, 231, 23784 }, > +}; > + > +static void gen5_rps_init(struct intel_rps *rps) > +{ > + struct drm_i915_private *i915 = rps_to_i915(rps); > + struct intel_uncore *uncore = rps_to_uncore(rps); > + u8 fmax, fmin, fstart; > + u32 rgvmodectl; > + int c_m, i; > + > + if (i915->fsb_freq <= 3200) > + c_m = 0; > + else if (i915->fsb_freq <= 4800) > + c_m = 1; > + else > + c_m = 2; > + > + for (i = 0; i < ARRAY_SIZE(cparams); i++) { > + if (cparams[i].i == c_m && cparams[i].t == i915->mem_freq) { > + rps->ips.m = cparams[i].m; > + rps->ips.c = cparams[i].c; > + break; > + } > + } > + > + rgvmodectl = intel_uncore_read(uncore, MEMMODECTL); > + > + /* Set up min, max, and cur for interrupt handling */ > + fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT; > + fmin = (rgvmodectl & MEMMODE_FMIN_MASK); > + fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> > + MEMMODE_FSTART_SHIFT; > + DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n", > + fmax, fmin, fstart); > + > + rps->min_freq = -fstart; > + rps->max_freq = -fmin; > + > + rps->idle_freq = rps->min_freq; > + rps->cur_freq = rps->idle_freq; > +} > + > +static unsigned long > +__ips_chipset_val(struct intel_ips *ips) > +{ > + struct intel_uncore *uncore = > + rps_to_uncore(container_of(ips, struct intel_rps, ips)); > + unsigned long now = jiffies_to_msecs(jiffies), dt; > + unsigned long result; > + u64 total, delta; > + > + lockdep_assert_held(&mchdev_lock); > + > + /* > + * Prevent division-by-zero if we are asking too fast. > + * Also, we don't get interesting results if we are polling > + * faster than once in 10ms, so just return the saved value > + * in such cases. > + */ > + dt = now - ips->last_time1; > + if (dt <= 10) > + return ips->chipset_power; > + > + /* FIXME: handle per-counter overflow */ > + total = intel_uncore_read(uncore, DMIEC); > + total += intel_uncore_read(uncore, DDREC); > + total += intel_uncore_read(uncore, CSIEC); > + > + delta = total - ips->last_count1; > + > + result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10); > + > + ips->last_count1 = total; > + ips->last_time1 = now; > + > + ips->chipset_power = result; > + > + return result; > +} > + > +static unsigned long ips_mch_val(struct intel_uncore *uncore) > +{ > + unsigned int m, x, b; > + u32 tsfs; > + > + tsfs = intel_uncore_read(uncore, TSFS); > + x = intel_uncore_read8(uncore, TR1); > + > + b = tsfs & TSFS_INTR_MASK; > + m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT; > + > + return m * x / 127 - b; > +} > + > +static int _pxvid_to_vd(u8 pxvid) > +{ > + if (pxvid == 0) > + return 0; > + > + if (pxvid >= 8 && pxvid < 31) > + pxvid = 31; > + > + return (pxvid + 2) * 125; > +} > + > +static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid) > +{ > + const int vd = _pxvid_to_vd(pxvid); > + > + if (INTEL_INFO(i915)->is_mobile) > + return max(vd - 1125, 0); > + > + return vd; > +} > + > +static void __gen5_ips_update(struct intel_ips *ips) > +{ > + struct intel_uncore *uncore = > + rps_to_uncore(container_of(ips, struct intel_rps, ips)); > + u64 now, delta, dt; > + u32 count; > + > + lockdep_assert_held(&mchdev_lock); > + > + now = ktime_get_raw_ns(); > + dt = now - ips->last_time2; > + do_div(dt, NSEC_PER_MSEC); > + > + /* Don't divide by 0 */ > + if (dt <= 10) > + return; > + > + count = intel_uncore_read(uncore, GFXEC); > + delta = count - ips->last_count2; > + > + ips->last_count2 = count; > + ips->last_time2 = now; > + > + /* More magic constants... */ > + ips->gfx_power = div_u64(delta * 1181, dt * 10); > +} > + > +static void gen5_rps_update(struct intel_rps *rps) > +{ > + spin_lock_irq(&mchdev_lock); > + __gen5_ips_update(&rps->ips); > + spin_unlock_irq(&mchdev_lock); > +} > + > +static bool gen5_rps_set(struct intel_rps *rps, u8 val) > +{ > + struct intel_uncore *uncore = rps_to_uncore(rps); > + u16 rgvswctl; > + > + lockdep_assert_held(&mchdev_lock); > + > + rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); > + if (rgvswctl & MEMCTL_CMD_STS) { > + DRM_DEBUG("gpu busy, RCS change rejected\n"); > + return false; /* still busy with another command */ > + } > + > + val = -val; > + > + rgvswctl = > + (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) | > + (val << MEMCTL_FREQ_SHIFT) | > + MEMCTL_SFCAVM; > + intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); > + intel_uncore_posting_read16(uncore, MEMSWCTL); > + > + rgvswctl |= MEMCTL_CMD_STS; > + intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); > + > + return true; > +} > + > +static unsigned long intel_pxfreq(u32 vidfreq) > +{ > + int div = (vidfreq & 0x3f0000) >> 16; > + int post = (vidfreq & 0x3000) >> 12; > + int pre = (vidfreq & 0x7); > + > + if (!pre) > + return 0; > + > + return div * 133333 / (pre << post); > +} > + > +static unsigned int init_emon(struct intel_uncore *uncore) > +{ > + u8 pxw[16]; > + int i; > + > + /* Disable to program */ > + intel_uncore_write(uncore, ECR, 0); > + intel_uncore_posting_read(uncore, ECR); > + > + /* Program energy weights for various events */ > + intel_uncore_write(uncore, SDEW, 0x15040d00); > + intel_uncore_write(uncore, CSIEW0, 0x007f0000); > + intel_uncore_write(uncore, CSIEW1, 0x1e220004); > + intel_uncore_write(uncore, CSIEW2, 0x04000004); > + > + for (i = 0; i < 5; i++) > + intel_uncore_write(uncore, PEW(i), 0); > + for (i = 0; i < 3; i++) > + intel_uncore_write(uncore, DEW(i), 0); > + > + /* Program P-state weights to account for frequency power adjustment */ > + for (i = 0; i < 16; i++) { > + u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i)); > + unsigned int freq = intel_pxfreq(pxvidfreq); > + unsigned int vid = > + (pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; > + unsigned int val; > + > + val = vid * vid * freq / 1000 * 255; > + val /= 127 * 127 * 900; > + > + pxw[i] = val; > + } > + /* Render standby states get 0 weight */ > + pxw[14] = 0; > + pxw[15] = 0; > + > + for (i = 0; i < 4; i++) { > + intel_uncore_write(uncore, PXW(i), > + pxw[i * 4 + 0] << 24 | > + pxw[i * 4 + 1] << 16 | > + pxw[i * 4 + 2] << 8 | > + pxw[i * 4 + 3] << 0); > + } > + > + /* Adjust magic regs to magic values (more experimental results) */ > + intel_uncore_write(uncore, OGW0, 0); > + intel_uncore_write(uncore, OGW1, 0); > + intel_uncore_write(uncore, EG0, 0x00007f00); > + intel_uncore_write(uncore, EG1, 0x0000000e); > + intel_uncore_write(uncore, EG2, 0x000e0000); > + intel_uncore_write(uncore, EG3, 0x68000300); > + intel_uncore_write(uncore, EG4, 0x42000000); > + intel_uncore_write(uncore, EG5, 0x00140031); > + intel_uncore_write(uncore, EG6, 0); > + intel_uncore_write(uncore, EG7, 0); > + > + for (i = 0; i < 8; i++) > + intel_uncore_write(uncore, PXWL(i), 0); > + > + /* Enable PMON + select events */ > + intel_uncore_write(uncore, ECR, 0x80000019); > + > + return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK; > +} > + > +static bool gen5_rps_enable(struct intel_rps *rps) > +{ > + struct intel_uncore *uncore = rps_to_uncore(rps); > + u8 fstart, vstart; > + u32 rgvmodectl; > + > + spin_lock_irq(&mchdev_lock); > + > + rgvmodectl = intel_uncore_read(uncore, MEMMODECTL); > + > + /* Enable temp reporting */ > + intel_uncore_write16(uncore, PMMISC, > + intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN); > + intel_uncore_write16(uncore, TSC1, > + intel_uncore_read16(uncore, TSC1) | TSE); > + > + /* 100ms RC evaluation intervals */ > + intel_uncore_write(uncore, RCUPEI, 100000); > + intel_uncore_write(uncore, RCDNEI, 100000); > + > + /* Set max/min thresholds to 90ms and 80ms respectively */ > + intel_uncore_write(uncore, RCBMAXAVG, 90000); > + intel_uncore_write(uncore, RCBMINAVG, 80000); > + > + intel_uncore_write(uncore, MEMIHYST, 1); > + > + /* Set up min, max, and cur for interrupt handling */ > + fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> > + MEMMODE_FSTART_SHIFT; > + > + vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) & > + PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; > + > + intel_uncore_write(uncore, > + MEMINTREN, > + MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN); > + > + intel_uncore_write(uncore, VIDSTART, vstart); > + intel_uncore_posting_read(uncore, VIDSTART); > + > + rgvmodectl |= MEMMODE_SWMODE_EN; > + intel_uncore_write(uncore, MEMMODECTL, rgvmodectl); > + > + if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) & > + MEMCTL_CMD_STS) == 0, 10)) > + DRM_ERROR("stuck trying to change perf mode\n"); > + mdelay(1); > + > + gen5_rps_set(rps, rps->cur_freq); > + > + rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC); > + rps->ips.last_count1 += intel_uncore_read(uncore, DDREC); > + rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC); > + rps->ips.last_time1 = jiffies_to_msecs(jiffies); > + > + rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC); > + rps->ips.last_time2 = ktime_get_raw_ns(); > + > + spin_unlock_irq(&mchdev_lock); > + > + rps->ips.corr = init_emon(uncore); > + > + return true; > +} > + > +static void gen5_rps_disable(struct intel_rps *rps) > +{ > + struct intel_uncore *uncore = rps_to_uncore(rps); > + u16 rgvswctl; > + > + spin_lock_irq(&mchdev_lock); > + > + rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); > + > + /* Ack interrupts, disable EFC interrupt */ > + intel_uncore_write(uncore, MEMINTREN, > + intel_uncore_read(uncore, MEMINTREN) & > + ~MEMINT_EVAL_CHG_EN); > + intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); > + intel_uncore_write(uncore, DEIER, > + intel_uncore_read(uncore, DEIER) & ~DE_PCU_EVENT); > + intel_uncore_write(uncore, DEIIR, DE_PCU_EVENT); > + intel_uncore_write(uncore, DEIMR, > + intel_uncore_read(uncore, DEIMR) | DE_PCU_EVENT); > + > + /* Go back to the starting frequency */ > + gen5_rps_set(rps, rps->idle_freq); > + mdelay(1); > + rgvswctl |= MEMCTL_CMD_STS; > + intel_uncore_write(uncore, MEMSWCTL, rgvswctl); > + mdelay(1); > + > + spin_unlock_irq(&mchdev_lock); > +} > + > +static u32 rps_limits(struct intel_rps *rps, u8 val) > +{ > + u32 limits; > + > + /* > + * Only set the down limit when we've reached the lowest level to avoid > + * getting more interrupts, otherwise leave this clear. This prevents a > + * race in the hw when coming out of rc6: There's a tiny window where > + * the hw runs at the minimal clock before selecting the desired > + * frequency, if the down threshold expires in that window we will not > + * receive a down interrupt. > + */ > + if (INTEL_GEN(rps_to_i915(rps)) >= 9) { > + limits = rps->max_freq_softlimit << 23; > + if (val <= rps->min_freq_softlimit) > + limits |= rps->min_freq_softlimit << 14; > + } else { > + limits = rps->max_freq_softlimit << 24; > + if (val <= rps->min_freq_softlimit) > + limits |= rps->min_freq_softlimit << 16; > + } > + > + return limits; > +} > + > +static void rps_set_power(struct intel_rps *rps, int new_power) > +{ > + struct intel_uncore *uncore = rps_to_uncore(rps); > + struct drm_i915_private *i915 = rps_to_i915(rps); > + u32 threshold_up = 0, threshold_down = 0; /* in % */ > + u32 ei_up = 0, ei_down = 0; > + > + lockdep_assert_held(&rps->power.mutex); > + > + if (new_power == rps->power.mode) > + return; > + > + /* Note the units here are not exactly 1us, but 1280ns. */ > + switch (new_power) { > + case LOW_POWER: > + /* Upclock if more than 95% busy over 16ms */ > + ei_up = 16000; > + threshold_up = 95; > + > + /* Downclock if less than 85% busy over 32ms */ > + ei_down = 32000; > + threshold_down = 85; > + break; > + > + case BETWEEN: > + /* Upclock if more than 90% busy over 13ms */ > + ei_up = 13000; > + threshold_up = 90; > + > + /* Downclock if less than 75% busy over 32ms */ > + ei_down = 32000; > + threshold_down = 75; > + break; > + > + case HIGH_POWER: > + /* Upclock if more than 85% busy over 10ms */ > + ei_up = 10000; > + threshold_up = 85; > + > + /* Downclock if less than 60% busy over 32ms */ > + ei_down = 32000; > + threshold_down = 60; > + break; > + } > + > + /* When byt can survive without system hang with dynamic > + * sw freq adjustments, this restriction can be lifted. > + */ > + if (IS_VALLEYVIEW(i915)) > + goto skip_hw_write; > + > + intel_uncore_write(uncore, GEN6_RP_UP_EI, > + GT_INTERVAL_FROM_US(i915, ei_up)); > + intel_uncore_write(uncore, GEN6_RP_UP_THRESHOLD, > + GT_INTERVAL_FROM_US(i915, > + ei_up * threshold_up / 100)); > + > + intel_uncore_write(uncore, GEN6_RP_DOWN_EI, > + GT_INTERVAL_FROM_US(i915, ei_down)); > + intel_uncore_write(uncore, GEN6_RP_DOWN_THRESHOLD, > + GT_INTERVAL_FROM_US(i915, > + ei_down * threshold_down / 100)); > + > + intel_uncore_write(uncore, GEN6_RP_CONTROL, > + (INTEL_GEN(i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) | > + GEN6_RP_MEDIA_HW_NORMAL_MODE | > + GEN6_RP_MEDIA_IS_GFX | > + GEN6_RP_ENABLE | > + GEN6_RP_UP_BUSY_AVG | > + GEN6_RP_DOWN_IDLE_AVG); > + > +skip_hw_write: > + rps->power.mode = new_power; > + rps->power.up_threshold = threshold_up; > + rps->power.down_threshold = threshold_down; > +} > + > +static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val) > +{ > + int new_power; > + > + new_power = rps->power.mode; > + switch (rps->power.mode) { > + case LOW_POWER: > + if (val > rps->efficient_freq + 1 && > + val > rps->cur_freq) > + new_power = BETWEEN; > + break; > + > + case BETWEEN: > + if (val <= rps->efficient_freq && > + val < rps->cur_freq) > + new_power = LOW_POWER; > + else if (val >= rps->rp0_freq && > + val > rps->cur_freq) > + new_power = HIGH_POWER; > + break; > + > + case HIGH_POWER: > + if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 && > + val < rps->cur_freq) > + new_power = BETWEEN; > + break; > + } > + /* Max/min bins are special */ > + if (val <= rps->min_freq_softlimit) > + new_power = LOW_POWER; > + if (val >= rps->max_freq_softlimit) > + new_power = HIGH_POWER; > + > + mutex_lock(&rps->power.mutex); > + if (rps->power.interactive) > + new_power = HIGH_POWER; > + rps_set_power(rps, new_power); > + mutex_unlock(&rps->power.mutex); > +} > + > +void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive) > +{ > + if (!rps->enabled) > + return; > + > + mutex_lock(&rps->power.mutex); > + if (interactive) { > + if (!rps->power.interactive++ && rps->active) > + rps_set_power(rps, HIGH_POWER); > + } else { > + GEM_BUG_ON(!rps->power.interactive); > + rps->power.interactive--; > + } > + mutex_unlock(&rps->power.mutex); > +} > + > +static int gen6_rps_set(struct intel_rps *rps, u8 val) > +{ > + struct intel_uncore *uncore = rps_to_uncore(rps); > + struct drm_i915_private *i915 = rps_to_i915(rps); > + u32 swreq; > + > + if (INTEL_GEN(i915) >= 9) > + swreq = GEN9_FREQUENCY(val); > + else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) > + swreq = HSW_FREQUENCY(val); > + else > + swreq = (GEN6_FREQUENCY(val) | > + GEN6_OFFSET(0) | > + GEN6_AGGRESSIVE_TURBO); > + intel_uncore_write(uncore, GEN6_RPNSWREQ, swreq); > + > + return 0; > +} > + > +static int vlv_rps_set(struct intel_rps *rps, u8 val) > +{ > + struct drm_i915_private *i915 = rps_to_i915(rps); > + int err; > + > + vlv_punit_get(i915); > + err = vlv_punit_write(i915, PUNIT_REG_GPU_FREQ_REQ, val); > + vlv_punit_put(i915); > + > + return err; > +} > + > +static int rps_set(struct intel_rps *rps, u8 val) > +{ > + struct drm_i915_private *i915 = rps_to_i915(rps); > + int err; > + > + if (INTEL_GEN(i915) < 6) > + return 0; > + > + if (val == rps->last_freq) > + return 0; > + > + if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) > + err = vlv_rps_set(rps, val); > + else > + err = gen6_rps_set(rps, val); > + if (err) > + return err; > + > + gen6_rps_set_thresholds(rps, val); > + rps->last_freq = val; > + > + return 0; > +} > + > +void intel_rps_unpark(struct intel_rps *rps) > +{ > + u8 freq; > + > + if (!rps->enabled) > + return; > + > + /* > + * Use the user's desired frequency as a guide, but for better > + * performance, jump directly to RPe as our starting frequency. > + */ > + mutex_lock(&rps->lock); > + rps->active = true; > + freq = max(rps->cur_freq, rps->efficient_freq), > + freq = clamp(freq, rps->min_freq_softlimit, rps->max_freq_softlimit); > + intel_rps_set(rps, freq); > + rps->last_adj = 0; > + mutex_unlock(&rps->lock); > + > + if (INTEL_GEN(rps_to_i915(rps)) >= 6) > + rps_enable_interrupts(rps); > + > + if (IS_GEN(rps_to_i915(rps), 5)) > + gen5_rps_update(rps); > +} > + > +void intel_rps_park(struct intel_rps *rps) > +{ > + struct drm_i915_private *i915 = rps_to_i915(rps); > + > + if (!rps->enabled) > + return; > + > + if (INTEL_GEN(i915) >= 6) > + rps_disable_interrupts(rps); > + > + rps->active = false; > + if (rps->last_freq <= rps->idle_freq) > + return; > + > + /* > + * The punit delays the write of the frequency and voltage until it > + * determines the GPU is awake. During normal usage we don't want to > + * waste power changing the frequency if the GPU is sleeping (rc6). > + * However, the GPU and driver is now idle and we do not want to delay > + * switching to minimum voltage (reducing power whilst idle) as we do > + * not expect to be woken in the near future and so must flush the > + * change by waking the device. > + * > + * We choose to take the media powerwell (either would do to trick the > + * punit into committing the voltage change) as that takes a lot less > + * power than the render powerwell. > + */ > + intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA); > + rps_set(rps, rps->idle_freq); > + intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA); > +} > + > +void intel_rps_boost(struct i915_request *rq) > +{ > + struct intel_rps *rps = &rq->engine->gt->rps; > + unsigned long flags; > + > + if (i915_request_signaled(rq) || !rps->active) > + return; > + > + /* Serializes with i915_request_retire() */ > + spin_lock_irqsave(&rq->lock, flags); > + if (!i915_request_has_waitboost(rq) && > + !dma_fence_is_signaled_locked(&rq->fence)) { > + rq->flags |= I915_REQUEST_WAITBOOST; > + > + if (!atomic_fetch_inc(&rps->num_waiters) && > + READ_ONCE(rps->cur_freq) < rps->boost_freq) > + schedule_work(&rps->work); > + > + atomic_inc(&rps->boosts); > + } > + spin_unlock_irqrestore(&rq->lock, flags); > +} > + > +int intel_rps_set(struct intel_rps *rps, u8 val) > +{ > + int err = 0; > + > + lockdep_assert_held(&rps->lock); > + GEM_BUG_ON(val > rps->max_freq); > + GEM_BUG_ON(val < rps->min_freq); > + > + if (rps->active) { > + err = rps_set(rps, val); > + > + /* > + * Make sure we continue to get interrupts > + * until we hit the minimum or maximum frequencies. > + */ > + if (INTEL_GEN(rps_to_i915(rps)) >= 6) { > + struct intel_uncore *uncore = rps_to_uncore(rps); > + > + intel_uncore_write(uncore, GEN6_RP_INTERRUPT_LIMITS, > + rps_limits(rps, val)); > + > + intel_uncore_write(uncore, GEN6_PMINTRMSK, > + rps_pm_mask(rps, val)); > + } > + } > + > + if (err == 0) > + rps->cur_freq = val; > + > + return err; > +} > + > +static void gen6_rps_init(struct intel_rps *rps) > +{ > + struct drm_i915_private *i915 = rps_to_i915(rps); > + struct intel_uncore *uncore = rps_to_uncore(rps); > + > + /* All of these values are in units of 50MHz */ > + > + /* static values from HW: RP0 > RP1 > RPn (min_freq) */ > + if (IS_GEN9_LP(i915)) { > + u32 rp_state_cap = intel_uncore_read(uncore, BXT_RP_STATE_CAP); > + > + rps->rp0_freq = (rp_state_cap >> 16) & 0xff; > + rps->rp1_freq = (rp_state_cap >> 8) & 0xff; > + rps->min_freq = (rp_state_cap >> 0) & 0xff; > + } else { > + u32 rp_state_cap = intel_uncore_read(uncore, GEN6_RP_STATE_CAP); > + > + rps->rp0_freq = (rp_state_cap >> 0) & 0xff; > + rps->rp1_freq = (rp_state_cap >> 8) & 0xff; > + rps->min_freq = (rp_state_cap >> 16) & 0xff; > + } > + > + /* hw_max = RP0 until we check for overclocking */ > + rps->max_freq = rps->rp0_freq; > + > + rps->efficient_freq = rps->rp1_freq; > + if (IS_HASWELL(i915) || IS_BROADWELL(i915) || > + IS_GEN9_BC(i915) || INTEL_GEN(i915) >= 10) { > + u32 ddcc_status = 0; > + > + if (sandybridge_pcode_read(i915, > + HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL, > + &ddcc_status, NULL) == 0) > + rps->efficient_freq = > + clamp_t(u8, > + (ddcc_status >> 8) & 0xff, > + rps->min_freq, > + rps->max_freq); > + } > + > + if (IS_GEN9_BC(i915) || INTEL_GEN(i915) >= 10) { > + /* Store the frequency values in 16.66 MHZ units, which is > + * the natural hardware unit for SKL > + */ > + rps->rp0_freq *= GEN9_FREQ_SCALER; > + rps->rp1_freq *= GEN9_FREQ_SCALER; > + rps->min_freq *= GEN9_FREQ_SCALER; > + rps->max_freq *= GEN9_FREQ_SCALER; > + rps->efficient_freq *= GEN9_FREQ_SCALER; > + } > +} > + > +static bool rps_reset(struct intel_rps *rps) > +{ > + /* force a reset */ > + rps->power.mode = -1; > + rps->last_freq = -1; > + > + if (rps_set(rps, rps->min_freq)) { > + DRM_ERROR("Failed to reset RPS to initial values\n"); > + return false; > + } > + > + rps->cur_freq = rps->min_freq; > + return true; > +} > + > +/* See the Gen9_GT_PM_Programming_Guide doc for the below */ > +static bool gen9_rps_enable(struct intel_rps *rps) > +{ > + struct drm_i915_private *i915 = rps_to_i915(rps); > + struct intel_uncore *uncore = rps_to_uncore(rps); > + > + /* Program defaults and thresholds for RPS */ > + if (IS_GEN(i915, 9)) > + intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ, > + GEN9_FREQUENCY(rps->rp1_freq)); > + > + /* 1 second timeout */ > + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, > + GT_INTERVAL_FROM_US(i915, 1000000)); > + > + intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa); > + > + return rps_reset(rps); > +} > + > +static bool gen8_rps_enable(struct intel_rps *rps) > +{ > + struct intel_uncore *uncore = rps_to_uncore(rps); > + > + intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ, > + HSW_FREQUENCY(rps->rp1_freq)); > + > + /* NB: Docs say 1s, and 1000000 - which aren't equivalent */ > + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, > + 100000000 / 128); /* 1 second timeout */ > + > + intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); > + > + return rps_reset(rps); > +} > + > +static bool gen6_rps_enable(struct intel_rps *rps) > +{ > + struct intel_uncore *uncore = rps_to_uncore(rps); > + > + /* Power down if completely idle for over 50ms */ > + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000); > + intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); > + > + return rps_reset(rps); > +} > + > +static int chv_rps_max_freq(struct intel_rps *rps) > +{ > + struct drm_i915_private *i915 = rps_to_i915(rps); > + u32 val; > + > + val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE); > + > + switch (RUNTIME_INFO(i915)->sseu.eu_total) { > + case 8: > + /* (2 * 4) config */ > + val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT; > + break; > + case 12: > + /* (2 * 6) config */ > + val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT; > + break; > + case 16: > + /* (2 * 8) config */ > + default: > + /* Setting (2 * 8) Min RP0 for any other combination */ > + val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT; > + break; > + } > + > + return val & FB_GFX_FREQ_FUSE_MASK; > +} > + > +static int chv_rps_rpe_freq(struct intel_rps *rps) > +{ > + struct drm_i915_private *i915 = rps_to_i915(rps); > + u32 val; > + > + val = vlv_punit_read(i915, PUNIT_GPU_DUTYCYCLE_REG); > + val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT; > + > + return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK; > +} > + > +static int chv_rps_guar_freq(struct intel_rps *rps) > +{ > + struct drm_i915_private *i915 = rps_to_i915(rps); > + u32 val; > + > + val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE); > + > + return val & FB_GFX_FREQ_FUSE_MASK; > +} > + > +static u32 chv_rps_min_freq(struct intel_rps *rps) > +{ > + struct drm_i915_private *i915 = rps_to_i915(rps); > + u32 val; > + > + val = vlv_punit_read(i915, FB_GFX_FMIN_AT_VMIN_FUSE); > + val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT; > + > + return val & FB_GFX_FREQ_FUSE_MASK; > +} > + > +static bool chv_rps_enable(struct intel_rps *rps) > +{ > + struct intel_uncore *uncore = rps_to_uncore(rps); > + struct drm_i915_private *i915 = rps_to_i915(rps); > + u32 val; > + > + /* 1: Program defaults and thresholds for RPS*/ > + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000); > + intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400); > + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000); > + intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000); > + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000); > + > + intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); > + > + /* 2: Enable RPS */ > + intel_uncore_write_fw(uncore, GEN6_RP_CONTROL, > + GEN6_RP_MEDIA_HW_NORMAL_MODE | > + GEN6_RP_MEDIA_IS_GFX | > + GEN6_RP_ENABLE | > + GEN6_RP_UP_BUSY_AVG | > + GEN6_RP_DOWN_IDLE_AVG); > + > + /* Setting Fixed Bias */ > + vlv_punit_get(i915); > + > + val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50; > + vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val); > + > + val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); > + > + vlv_punit_put(i915); > + > + /* RPS code assumes GPLL is used */ > + WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n"); > + > + DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE)); > + DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val); > + > + return rps_reset(rps); > +} > + > +static int vlv_rps_guar_freq(struct intel_rps *rps) > +{ > + struct drm_i915_private *i915 = rps_to_i915(rps); > + u32 val, rp1; > + > + val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE); > + > + rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK; > + rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT; > + > + return rp1; > +} > + > +static int vlv_rps_max_freq(struct intel_rps *rps) > +{ > + struct drm_i915_private *i915 = rps_to_i915(rps); > + u32 val, rp0; > + > + val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE); > + > + rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT; > + /* Clamp to max */ > + rp0 = min_t(u32, rp0, 0xea); > + > + return rp0; > +} > + > +static int vlv_rps_rpe_freq(struct intel_rps *rps) > +{ > + struct drm_i915_private *i915 = rps_to_i915(rps); > + u32 val, rpe; > + > + val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_LO); > + rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT; > + val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_HI); > + rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5; > + > + return rpe; > +} > + > +static int vlv_rps_min_freq(struct intel_rps *rps) > +{ > + struct drm_i915_private *i915 = rps_to_i915(rps); > + u32 val; > + > + val = vlv_punit_read(i915, PUNIT_REG_GPU_LFM) & 0xff; > + /* > + * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value > + * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on > + * a BYT-M B0 the above register contains 0xbf. Moreover when setting > + * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0 > + * to make sure it matches what Punit accepts. > + */ > + return max_t(u32, val, 0xc0); > +} > + > +static bool vlv_rps_enable(struct intel_rps *rps) > +{ > + struct intel_uncore *uncore = rps_to_uncore(rps); > + struct drm_i915_private *i915 = rps_to_i915(rps); > + u32 val; > + > + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000); > + intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400); > + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000); > + intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000); > + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000); > + > + intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); > + > + intel_uncore_write_fw(uncore, GEN6_RP_CONTROL, > + GEN6_RP_MEDIA_TURBO | > + GEN6_RP_MEDIA_HW_NORMAL_MODE | > + GEN6_RP_MEDIA_IS_GFX | > + GEN6_RP_ENABLE | > + GEN6_RP_UP_BUSY_AVG | > + GEN6_RP_DOWN_IDLE_CONT); > + > + vlv_punit_get(i915); > + > + /* Setting Fixed Bias */ > + val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875; > + vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val); > + > + val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); > + > + vlv_punit_put(i915); > + > + /* RPS code assumes GPLL is used */ > + WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n"); > + > + DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE)); > + DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val); > + > + return rps_reset(rps); > +} > + > +static unsigned long __ips_gfx_val(struct intel_ips *ips) > +{ > + struct intel_rps *rps = container_of(ips, typeof(*rps), ips); > + struct intel_uncore *uncore = rps_to_uncore(rps); > + unsigned long t, corr, state1, corr2, state2; > + u32 pxvid, ext_v; > + > + lockdep_assert_held(&mchdev_lock); > + > + pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq)); > + pxvid = (pxvid >> 24) & 0x7f; > + ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid); > + > + state1 = ext_v; > + > + /* Revel in the empirically derived constants */ > + > + /* Correction factor in 1/100000 units */ > + t = ips_mch_val(uncore); > + if (t > 80) > + corr = t * 2349 + 135940; > + else if (t >= 50) > + corr = t * 964 + 29317; > + else /* < 50 */ > + corr = t * 301 + 1004; > + > + corr = corr * 150142 * state1 / 10000 - 78642; > + corr /= 100000; > + corr2 = corr * ips->corr; > + > + state2 = corr2 * state1 / 10000; > + state2 /= 100; /* convert to mW */ > + > + __gen5_ips_update(ips); > + > + return ips->gfx_power + state2; > +} > + > +void intel_rps_enable(struct intel_rps *rps) > +{ > + struct drm_i915_private *i915 = rps_to_i915(rps); > + struct intel_uncore *uncore = rps_to_uncore(rps); > + > + intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); > + if (IS_CHERRYVIEW(i915)) > + rps->enabled = chv_rps_enable(rps); > + else if (IS_VALLEYVIEW(i915)) > + rps->enabled = vlv_rps_enable(rps); > + else if (INTEL_GEN(i915) >= 9) > + rps->enabled = gen9_rps_enable(rps); > + else if (INTEL_GEN(i915) >= 8) > + rps->enabled = gen8_rps_enable(rps); > + else if (INTEL_GEN(i915) >= 6) > + rps->enabled = gen6_rps_enable(rps); > + else if (IS_IRONLAKE_M(i915)) > + rps->enabled = gen5_rps_enable(rps); > + intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); > + if (!rps->enabled) > + return; > + > + WARN_ON(rps->max_freq < rps->min_freq); > + WARN_ON(rps->idle_freq > rps->max_freq); > + > + WARN_ON(rps->efficient_freq < rps->min_freq); > + WARN_ON(rps->efficient_freq > rps->max_freq); > +} > + > +static void gen6_rps_disable(struct intel_rps *rps) > +{ > + intel_uncore_write(rps_to_uncore(rps), GEN6_RP_CONTROL, 0); > +} > + > +void intel_rps_disable(struct intel_rps *rps) > +{ > + struct drm_i915_private *i915 = rps_to_i915(rps); > + > + rps->enabled = false; > + > + if (INTEL_GEN(i915) >= 6) > + gen6_rps_disable(rps); > + else if (IS_IRONLAKE_M(i915)) > + gen5_rps_disable(rps); > +} > + > +static int byt_gpu_freq(struct intel_rps *rps, int val) > +{ > + /* > + * N = val - 0xb7 > + * Slow = Fast = GPLL ref * N > + */ > + return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000); > +} > + > +static int byt_freq_opcode(struct intel_rps *rps, int val) > +{ > + return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7; > +} > + > +static int chv_gpu_freq(struct intel_rps *rps, int val) > +{ > + /* > + * N = val / 2 > + * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2 > + */ > + return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000); > +} > + > +static int chv_freq_opcode(struct intel_rps *rps, int val) > +{ > + /* CHV needs even values */ > + return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2; > +} > + > +int intel_gpu_freq(struct intel_rps *rps, int val) > +{ > + struct drm_i915_private *i915 = rps_to_i915(rps); > + > + if (INTEL_GEN(i915) >= 9) > + return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER, > + GEN9_FREQ_SCALER); > + else if (IS_CHERRYVIEW(i915)) > + return chv_gpu_freq(rps, val); > + else if (IS_VALLEYVIEW(i915)) > + return byt_gpu_freq(rps, val); > + else > + return val * GT_FREQUENCY_MULTIPLIER; > +} > + > +int intel_freq_opcode(struct intel_rps *rps, int val) > +{ > + struct drm_i915_private *i915 = rps_to_i915(rps); > + > + if (INTEL_GEN(i915) >= 9) > + return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER, > + GT_FREQUENCY_MULTIPLIER); > + else if (IS_CHERRYVIEW(i915)) > + return chv_freq_opcode(rps, val); > + else if (IS_VALLEYVIEW(i915)) > + return byt_freq_opcode(rps, val); > + else > + return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER); > +} > + > +static void vlv_init_gpll_ref_freq(struct intel_rps *rps) > +{ > + struct drm_i915_private *i915 = rps_to_i915(rps); > + > + rps->gpll_ref_freq = > + vlv_get_cck_clock(i915, "GPLL ref", > + CCK_GPLL_CLOCK_CONTROL, > + i915->czclk_freq); > + > + DRM_DEBUG_DRIVER("GPLL reference freq: %d kHz\n", rps->gpll_ref_freq); > +} > + > +static void vlv_rps_init(struct intel_rps *rps) > +{ > + struct drm_i915_private *i915 = rps_to_i915(rps); > + u32 val; > + > + vlv_iosf_sb_get(i915, > + BIT(VLV_IOSF_SB_PUNIT) | > + BIT(VLV_IOSF_SB_NC) | > + BIT(VLV_IOSF_SB_CCK)); > + > + vlv_init_gpll_ref_freq(rps); > + > + val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); > + switch ((val >> 6) & 3) { > + case 0: > + case 1: > + i915->mem_freq = 800; > + break; > + case 2: > + i915->mem_freq = 1066; > + break; > + case 3: > + i915->mem_freq = 1333; > + break; > + } > + DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", i915->mem_freq); > + > + rps->max_freq = vlv_rps_max_freq(rps); > + rps->rp0_freq = rps->max_freq; > + DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n", > + intel_gpu_freq(rps, rps->max_freq), > + rps->max_freq); > + > + rps->efficient_freq = vlv_rps_rpe_freq(rps); > + DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n", > + intel_gpu_freq(rps, rps->efficient_freq), > + rps->efficient_freq); > + > + rps->rp1_freq = vlv_rps_guar_freq(rps); > + DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n", > + intel_gpu_freq(rps, rps->rp1_freq), > + rps->rp1_freq); > + > + rps->min_freq = vlv_rps_min_freq(rps); > + DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n", > + intel_gpu_freq(rps, rps->min_freq), > + rps->min_freq); > + > + vlv_iosf_sb_put(i915, > + BIT(VLV_IOSF_SB_PUNIT) | > + BIT(VLV_IOSF_SB_NC) | > + BIT(VLV_IOSF_SB_CCK)); > +} > + > +static void chv_rps_init(struct intel_rps *rps) > +{ > + struct drm_i915_private *i915 = rps_to_i915(rps); > + u32 val; > + > + vlv_iosf_sb_get(i915, > + BIT(VLV_IOSF_SB_PUNIT) | > + BIT(VLV_IOSF_SB_NC) | > + BIT(VLV_IOSF_SB_CCK)); > + > + vlv_init_gpll_ref_freq(rps); > + > + val = vlv_cck_read(i915, CCK_FUSE_REG); > + > + switch ((val >> 2) & 0x7) { > + case 3: > + i915->mem_freq = 2000; > + break; > + default: > + i915->mem_freq = 1600; > + break; > + } > + DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", i915->mem_freq); > + > + rps->max_freq = chv_rps_max_freq(rps); > + rps->rp0_freq = rps->max_freq; > + DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n", > + intel_gpu_freq(rps, rps->max_freq), > + rps->max_freq); > + > + rps->efficient_freq = chv_rps_rpe_freq(rps); > + DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n", > + intel_gpu_freq(rps, rps->efficient_freq), > + rps->efficient_freq); > + > + rps->rp1_freq = chv_rps_guar_freq(rps); > + DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n", > + intel_gpu_freq(rps, rps->rp1_freq), > + rps->rp1_freq); > + > + rps->min_freq = chv_rps_min_freq(rps); > + DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n", > + intel_gpu_freq(rps, rps->min_freq), > + rps->min_freq); > + > + vlv_iosf_sb_put(i915, > + BIT(VLV_IOSF_SB_PUNIT) | > + BIT(VLV_IOSF_SB_NC) | > + BIT(VLV_IOSF_SB_CCK)); > + > + WARN_ONCE((rps->max_freq | rps->efficient_freq | rps->rp1_freq | > + rps->min_freq) & 1, > + "Odd GPU freq values\n"); > +} > + > +static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei) > +{ > + ei->ktime = ktime_get_raw(); > + ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT); > + ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT); > +} > + > +static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir) > +{ > + struct intel_uncore *uncore = rps_to_uncore(rps); > + const struct intel_rps_ei *prev = &rps->ei; > + struct intel_rps_ei now; > + u32 events = 0; > + > + if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0) > + return 0; > + > + vlv_c0_read(uncore, &now); > + > + if (prev->ktime) { > + u64 time, c0; > + u32 render, media; > + > + time = ktime_us_delta(now.ktime, prev->ktime); > + > + time *= rps_to_i915(rps)->czclk_freq; > + > + /* Workload can be split between render + media, > + * e.g. SwapBuffers being blitted in X after being rendered in > + * mesa. To account for this we need to combine both engines > + * into our activity counter. > + */ > + render = now.render_c0 - prev->render_c0; > + media = now.media_c0 - prev->media_c0; > + c0 = max(render, media); > + c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */ > + > + if (c0 > time * rps->power.up_threshold) > + events = GEN6_PM_RP_UP_THRESHOLD; > + else if (c0 < time * rps->power.down_threshold) > + events = GEN6_PM_RP_DOWN_THRESHOLD; > + } > + > + rps->ei = now; > + return events; > +} > + > +static void rps_work(struct work_struct *work) > +{ > + struct intel_rps *rps = container_of(work, typeof(*rps), work); > + struct intel_gt *gt = rps_to_gt(rps); > + bool client_boost = false; > + int new_freq, adj, min, max; > + u32 pm_iir = 0; > + > + spin_lock_irq(>->irq_lock); > + pm_iir = fetch_and_zero(&rps->pm_iir); > + client_boost = atomic_read(&rps->num_waiters); > + spin_unlock_irq(>->irq_lock); > + > + /* Make sure we didn't queue anything we're not going to process. */ > + if ((pm_iir & rps->pm_events) == 0 && !client_boost) > + goto out; > + > + mutex_lock(&rps->lock); > + > + pm_iir |= vlv_wa_c0_ei(rps, pm_iir); > + > + adj = rps->last_adj; > + new_freq = rps->cur_freq; > + min = rps->min_freq_softlimit; > + max = rps->max_freq_softlimit; > + if (client_boost) > + max = rps->max_freq; > + if (client_boost && new_freq < rps->boost_freq) { > + new_freq = rps->boost_freq; > + adj = 0; > + } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) { > + if (adj > 0) > + adj *= 2; > + else /* CHV needs even encode values */ > + adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1; > + > + if (new_freq >= rps->max_freq_softlimit) > + adj = 0; > + } else if (client_boost) { > + adj = 0; > + } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) { > + if (rps->cur_freq > rps->efficient_freq) > + new_freq = rps->efficient_freq; > + else if (rps->cur_freq > rps->min_freq_softlimit) > + new_freq = rps->min_freq_softlimit; > + adj = 0; > + } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) { > + if (adj < 0) > + adj *= 2; > + else /* CHV needs even encode values */ > + adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1; > + > + if (new_freq <= rps->min_freq_softlimit) > + adj = 0; > + } else { /* unknown event */ > + adj = 0; > + } > + > + rps->last_adj = adj; > + > + /* > + * Limit deboosting and boosting to keep ourselves at the extremes > + * when in the respective power modes (i.e. slowly decrease frequencies > + * while in the HIGH_POWER zone and slowly increase frequencies while > + * in the LOW_POWER zone). On idle, we will hit the timeout and drop > + * to the next level quickly, and conversely if busy we expect to > + * hit a waitboost and rapidly switch into max power. > + */ > + if ((adj < 0 && rps->power.mode == HIGH_POWER) || > + (adj > 0 && rps->power.mode == LOW_POWER)) > + rps->last_adj = 0; > + > + /* sysfs frequency interfaces may have snuck in while servicing the > + * interrupt > + */ > + new_freq += adj; > + new_freq = clamp_t(int, new_freq, min, max); > + > + if (intel_rps_set(rps, new_freq)) { > + DRM_DEBUG_DRIVER("Failed to set new GPU frequency\n"); > + rps->last_adj = 0; > + } > + > + mutex_unlock(&rps->lock); > + > +out: > + spin_lock_irq(>->irq_lock); > + gen6_gt_pm_unmask_irq(gt, rps->pm_events); > + spin_unlock_irq(>->irq_lock); > +} > + > +void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) > +{ > + struct intel_gt *gt = rps_to_gt(rps); > + const u32 events = rps->pm_events & pm_iir; > + > + lockdep_assert_held(>->irq_lock); > + > + if (unlikely(!events)) > + return; > + > + gen6_gt_pm_mask_irq(gt, events); > + > + rps->pm_iir |= events; > + schedule_work(&rps->work); > +} > + > +void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) > +{ > + struct drm_i915_private *i915 = rps_to_i915(rps); > + > + if (pm_iir & rps->pm_events) { > + struct intel_gt *gt = rps_to_gt(rps); > + > + spin_lock(>->irq_lock); > + gen6_gt_pm_mask_irq(gt, pm_iir & rps->pm_events); > + rps->pm_iir |= pm_iir & rps->pm_events; > + schedule_work(&rps->work); > + spin_unlock(>->irq_lock); > + } > + > + if (INTEL_GEN(i915) >= 8) > + return; > + > + if (pm_iir & PM_VEBOX_USER_INTERRUPT) > + intel_engine_breadcrumbs_irq(i915->engine[VECS0]); > + > + if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) > + DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir); > +} > + > +void gen5_rps_irq_handler(struct intel_rps *rps) > +{ > + struct intel_uncore *uncore = rps_to_uncore(rps); > + u32 busy_up, busy_down, max_avg, min_avg; > + u8 new_freq; > + > + spin_lock(&mchdev_lock); > + > + intel_uncore_write16(uncore, > + MEMINTRSTS, > + intel_uncore_read(uncore, MEMINTRSTS)); > + > + intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); > + busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG); > + busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG); > + max_avg = intel_uncore_read(uncore, RCBMAXAVG); > + min_avg = intel_uncore_read(uncore, RCBMINAVG); > + > + /* Handle RCS change request from hw */ > + new_freq = rps->cur_freq; > + if (busy_up > max_avg) > + new_freq++; > + else if (busy_down < min_avg) > + new_freq--; > + new_freq = clamp(new_freq, > + rps->min_freq_softlimit, > + rps->max_freq_softlimit); > + > + if (new_freq != rps->cur_freq && gen5_rps_set(rps, new_freq)) > + rps->cur_freq = new_freq; > + > + spin_unlock(&mchdev_lock); > +} > + > +void intel_rps_init(struct intel_rps *rps) > +{ > + struct drm_i915_private *i915 = rps_to_i915(rps); > + > + mutex_init(&rps->lock); > + mutex_init(&rps->power.mutex); > + > + INIT_WORK(&rps->work, rps_work); > + > + atomic_set(&rps->num_waiters, 0); > + > + if (IS_CHERRYVIEW(i915)) > + chv_rps_init(rps); > + else if (IS_VALLEYVIEW(i915)) > + vlv_rps_init(rps); > + else if (INTEL_GEN(i915) >= 6) > + gen6_rps_init(rps); > + else if (IS_IRONLAKE_M(i915)) > + gen5_rps_init(rps); > + > + /* Derive initial user preferences/limits from the hardware limits */ > + rps->max_freq_softlimit = rps->max_freq; > + rps->min_freq_softlimit = rps->min_freq; > + > + /* After setting max-softlimit, find the overclock max freq */ > + if (IS_GEN(i915, 6) || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) { > + u32 params = 0; > + > + sandybridge_pcode_read(i915, GEN6_READ_OC_PARAMS, > + ¶ms, NULL); > + if (params & BIT(31)) { /* OC supported */ > + DRM_DEBUG_DRIVER("Overclocking supported, max: %dMHz, overclock: %dMHz\n", > + (rps->max_freq & 0xff) * 50, > + (params & 0xff) * 50); > + rps->max_freq = params & 0xff; > + } > + } > + > + /* Finally allow us to boost to max by default */ > + rps->boost_freq = rps->max_freq; > + rps->idle_freq = rps->min_freq; > + rps->cur_freq = rps->idle_freq; > + > + rps->pm_intrmsk_mbz = 0; > + > + /* > + * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer > + * if GEN6_PM_UP_EI_EXPIRED is masked. > + * > + * TODO: verify if this can be reproduced on VLV,CHV. > + */ > + if (INTEL_GEN(i915) <= 7) > + rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED; > + > + if (INTEL_GEN(i915) >= 8) > + rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC; > +} > + > +u32 intel_get_cagf(struct intel_rps *rps, u32 rpstat) > +{ > + struct drm_i915_private *i915 = rps_to_i915(rps); > + u32 cagf; > + > + if (INTEL_GEN(i915) >= 9) > + cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT; > + else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) > + cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT; > + else > + cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT; > + > + return cagf; > +} > + > +/* External interface for intel_ips.ko */ > + > +static struct drm_i915_private __rcu *ips_mchdev; > + > +/** > + * Tells the intel_ips driver that the i915 driver is now loaded, if > + * IPS got loaded first. > + * > + * This awkward dance is so that neither module has to depend on the > + * other in order for IPS to do the appropriate communication of > + * GPU turbo limits to i915. > + */ > +static void > +ips_ping_for_i915_load(void) > +{ > + void (*link)(void); > + > + link = symbol_get(ips_link_to_i915_driver); > + if (link) { > + link(); > + symbol_put(ips_link_to_i915_driver); > + } > +} > + > +void intel_rps_driver_register(struct intel_rps *rps) > +{ > + struct intel_gt *gt = rps_to_gt(rps); > + > + /* > + * We only register the i915 ips part with intel-ips once everything is > + * set up, to avoid intel-ips sneaking in and reading bogus values. > + */ > + if (IS_GEN(gt->i915, 5)) { > + rcu_assign_pointer(ips_mchdev, gt->i915); > + ips_ping_for_i915_load(); > + } > +} > + > +void intel_rps_driver_unregister(struct intel_rps *rps) > +{ > + rcu_assign_pointer(ips_mchdev, NULL); > +} > + > +static struct drm_i915_private *mchdev_get(void) > +{ > + struct drm_i915_private *i915; > + > + rcu_read_lock(); > + i915 = rcu_dereference(ips_mchdev); > + if (!kref_get_unless_zero(&i915->drm.ref)) > + i915 = NULL; > + rcu_read_unlock(); > + > + return i915; > +} > + > +/** > + * i915_read_mch_val - return value for IPS use > + * > + * Calculate and return a value for the IPS driver to use when deciding whether > + * we have thermal and power headroom to increase CPU or GPU power budget. > + */ > +unsigned long i915_read_mch_val(void) > +{ > + struct drm_i915_private *i915; > + unsigned long chipset_val = 0; > + unsigned long graphics_val = 0; > + intel_wakeref_t wakeref; > + > + i915 = mchdev_get(); > + if (!i915) > + return 0; > + > + with_intel_runtime_pm(&i915->runtime_pm, wakeref) { > + struct intel_ips *ips = &i915->gt.rps.ips; > + > + spin_lock_irq(&mchdev_lock); > + chipset_val = __ips_chipset_val(ips); > + graphics_val = __ips_gfx_val(ips); > + spin_unlock_irq(&mchdev_lock); > + } > + > + drm_dev_put(&i915->drm); > + return chipset_val + graphics_val; > +} > +EXPORT_SYMBOL_GPL(i915_read_mch_val); > + > +/** > + * i915_gpu_raise - raise GPU frequency limit > + * > + * Raise the limit; IPS indicates we have thermal headroom. > + */ > +bool i915_gpu_raise(void) > +{ > + struct drm_i915_private *i915; > + struct intel_rps *rps; > + > + i915 = mchdev_get(); > + if (!i915) > + return false; > + > + rps = &i915->gt.rps; > + > + spin_lock_irq(&mchdev_lock); > + if (rps->max_freq_softlimit < rps->max_freq) > + rps->max_freq_softlimit++; > + spin_unlock_irq(&mchdev_lock); > + > + drm_dev_put(&i915->drm); > + return true; > +} > +EXPORT_SYMBOL_GPL(i915_gpu_raise); > + > +/** > + * i915_gpu_lower - lower GPU frequency limit > + * > + * IPS indicates we're close to a thermal limit, so throttle back the GPU > + * frequency maximum. > + */ > +bool i915_gpu_lower(void) > +{ > + struct drm_i915_private *i915; > + struct intel_rps *rps; > + > + i915 = mchdev_get(); > + if (!i915) > + return false; > + > + rps = &i915->gt.rps; > + > + spin_lock_irq(&mchdev_lock); > + if (rps->max_freq_softlimit > rps->min_freq) > + rps->max_freq_softlimit--; > + spin_unlock_irq(&mchdev_lock); > + > + drm_dev_put(&i915->drm); > + return true; > +} > +EXPORT_SYMBOL_GPL(i915_gpu_lower); > + > +/** > + * i915_gpu_busy - indicate GPU business to IPS > + * > + * Tell the IPS driver whether or not the GPU is busy. > + */ > +bool i915_gpu_busy(void) > +{ > + struct drm_i915_private *i915; > + bool ret; > + > + i915 = mchdev_get(); > + if (!i915) > + return false; > + > + ret = i915->gt.awake; > + > + drm_dev_put(&i915->drm); > + return ret; > +} > +EXPORT_SYMBOL_GPL(i915_gpu_busy); > + > +/** > + * i915_gpu_turbo_disable - disable graphics turbo > + * > + * Disable graphics turbo by resetting the max frequency and setting the > + * current frequency to the default. > + */ > +bool i915_gpu_turbo_disable(void) > +{ > + struct drm_i915_private *i915; > + struct intel_rps *rps; > + bool ret; > + > + i915 = mchdev_get(); > + if (!i915) > + return false; > + > + rps = &i915->gt.rps; > + > + spin_lock_irq(&mchdev_lock); > + rps->max_freq_softlimit = rps->min_freq; > + ret = gen5_rps_set(&i915->gt.rps, rps->min_freq); > + spin_unlock_irq(&mchdev_lock); > + > + drm_dev_put(&i915->drm); > + return ret; > +} > +EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable); > diff --git a/drivers/gpu/drm/i915/gt/intel_rps.h b/drivers/gpu/drm/i915/gt/intel_rps.h > new file mode 100644 > index 000000000000..997a4b4e0207 > --- /dev/null > +++ b/drivers/gpu/drm/i915/gt/intel_rps.h > @@ -0,0 +1,37 @@ > +/* > + * SPDX-License-Identifier: MIT > + * > + * Copyright © 2019 Intel Corporation > + */ > + > +#ifndef INTEL_RPS_H > +#define INTEL_RPS_H > + > +#include "intel_rps_types.h" > + > +struct i915_request; > + > +void intel_rps_init(struct intel_rps *rps); > + > +void intel_rps_driver_register(struct intel_rps *rps); > +void intel_rps_driver_unregister(struct intel_rps *rps); > + > +void intel_rps_enable(struct intel_rps *rps); > +void intel_rps_disable(struct intel_rps *rps); > + > +void intel_rps_park(struct intel_rps *rps); > +void intel_rps_unpark(struct intel_rps *rps); > +void intel_rps_boost(struct i915_request *rq); > + > +int intel_rps_set(struct intel_rps *rps, u8 val); > +void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive); > + > +int intel_gpu_freq(struct intel_rps *rps, int val); > +int intel_freq_opcode(struct intel_rps *rps, int val); > +u32 intel_get_cagf(struct intel_rps *rps, u32 rpstat1); > + > +void gen5_rps_irq_handler(struct intel_rps *rps); > +void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir); > +void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir); > + > +#endif /* INTEL_RPS_H */ > diff --git a/drivers/gpu/drm/i915/gt/intel_rps_types.h b/drivers/gpu/drm/i915/gt/intel_rps_types.h > new file mode 100644 > index 000000000000..40eb1fb651e7 > --- /dev/null > +++ b/drivers/gpu/drm/i915/gt/intel_rps_types.h > @@ -0,0 +1,93 @@ > +/* > + * SPDX-License-Identifier: MIT > + * > + * Copyright © 2019 Intel Corporation > + */ > + > +#ifndef INTEL_RPS_TYPES_H > +#define INTEL_RPS_TYPES_H > + > +#include <linux/atomic.h> > +#include <linux/ktime.h> > +#include <linux/mutex.h> > +#include <linux/types.h> > +#include <linux/workqueue.h> > + > +struct intel_ips { > + u64 last_count1; > + unsigned long last_time1; > + unsigned long chipset_power; > + u64 last_count2; > + u64 last_time2; > + unsigned long gfx_power; > + u8 corr; > + > + int c, m; > +}; > + > +struct intel_rps_ei { > + ktime_t ktime; > + u32 render_c0; > + u32 media_c0; > +}; > + > +struct intel_rps { > + struct mutex lock; /* protects enabling and the worker */ > + > + /* > + * work, interrupts_enabled and pm_iir are protected by > + * dev_priv->irq_lock > + */ > + struct work_struct work; > + bool enabled; > + bool active; > + u32 pm_iir; > + > + /* PM interrupt bits that should never be masked */ > + u32 pm_intrmsk_mbz; > + u32 pm_events; > + > + /* Frequencies are stored in potentially platform dependent multiples. > + * In other words, *_freq needs to be multiplied by X to be interesting. > + * Soft limits are those which are used for the dynamic reclocking done > + * by the driver (raise frequencies under heavy loads, and lower for > + * lighter loads). Hard limits are those imposed by the hardware. > + * > + * A distinction is made for overclocking, which is never enabled by > + * default, and is considered to be above the hard limit if it's > + * possible at all. > + */ > + u8 cur_freq; /* Current frequency (cached, may not == HW) */ > + u8 last_freq; /* Last swqreq frequency */ > + u8 min_freq_softlimit; /* Minimum frequency permitted by the driver */ > + u8 max_freq_softlimit; /* Max frequency permitted by the driver */ > + u8 max_freq; /* Maximum frequency, RP0 if not overclocking */ > + u8 min_freq; /* AKA RPn. Minimum frequency */ > + u8 boost_freq; /* Frequency to request when wait boosting */ > + u8 idle_freq; /* Frequency to request when we are idle */ > + u8 efficient_freq; /* AKA RPe. Pre-determined balanced frequency */ > + u8 rp1_freq; /* "less than" RP0 power/freqency */ > + u8 rp0_freq; /* Non-overclocked max frequency. */ > + u16 gpll_ref_freq; /* vlv/chv GPLL reference frequency */ > + > + int last_adj; > + > + struct { > + struct mutex mutex; > + > + enum { LOW_POWER, BETWEEN, HIGH_POWER } mode; > + unsigned int interactive; > + > + u8 up_threshold; /* Current %busy required to uplock */ > + u8 down_threshold; /* Current %busy required to downclock */ > + } power; > + > + atomic_t num_waiters; > + atomic_t boosts; > + > + /* manual wa residency calculations */ > + struct intel_rps_ei ei; > + struct intel_ips ips; > +}; > + > +#endif /* INTEL_RPS_TYPES_H */ > diff --git a/drivers/gpu/drm/i915/gt/selftest_llc.c b/drivers/gpu/drm/i915/gt/selftest_llc.c > index a7057785e420..fd3770e48ac7 100644 > --- a/drivers/gpu/drm/i915/gt/selftest_llc.c > +++ b/drivers/gpu/drm/i915/gt/selftest_llc.c > @@ -6,6 +6,7 @@ > > #include "intel_pm.h" /* intel_gpu_freq() */ > #include "selftest_llc.h" > +#include "intel_rps.h" > > static int gen6_verify_ring_freq(struct intel_llc *llc) > { > @@ -25,6 +26,8 @@ static int gen6_verify_ring_freq(struct intel_llc *llc) > for (gpu_freq = consts.min_gpu_freq; > gpu_freq <= consts.max_gpu_freq; > gpu_freq++) { > + struct intel_rps *rps = &llc_to_gt(llc)->rps; > + > unsigned int ia_freq, ring_freq, found; > u32 val; > > @@ -44,7 +47,7 @@ static int gen6_verify_ring_freq(struct intel_llc *llc) > if (found != ia_freq) { > pr_err("Min freq table(%d/[%d, %d]):%dMHz did not match expected CPU freq, found %d, expected %d\n", > gpu_freq, consts.min_gpu_freq, consts.max_gpu_freq, > - intel_gpu_freq(i915, gpu_freq * (INTEL_GEN(i915) >= 9 ? GEN9_FREQ_SCALER : 1)), > + intel_gpu_freq(rps, gpu_freq * (INTEL_GEN(i915) >= 9 ? GEN9_FREQ_SCALER : 1)), > found, ia_freq); > err = -EINVAL; > break; > @@ -54,7 +57,7 @@ static int gen6_verify_ring_freq(struct intel_llc *llc) > if (found != ring_freq) { > pr_err("Min freq table(%d/[%d, %d]):%dMHz did not match expected ring freq, found %d, expected %d\n", > gpu_freq, consts.min_gpu_freq, consts.max_gpu_freq, > - intel_gpu_freq(i915, gpu_freq * (INTEL_GEN(i915) >= 9 ? GEN9_FREQ_SCALER : 1)), > + intel_gpu_freq(rps, gpu_freq * (INTEL_GEN(i915) >= 9 ? GEN9_FREQ_SCALER : 1)), > found, ring_freq); > err = -EINVAL; > break; > diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c > index 009e54a3764f..97b858ce7682 100644 > --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c > +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c > @@ -1010,7 +1010,7 @@ void intel_guc_submission_fini(struct intel_guc *guc) > > static void guc_interrupts_capture(struct intel_gt *gt) > { > - struct intel_rps *rps = >->i915->gt_pm.rps; > + struct intel_rps *rps = >->rps; > struct intel_uncore *uncore = gt->uncore; > struct intel_engine_cs *engine; > enum intel_engine_id id; > @@ -1056,7 +1056,7 @@ static void guc_interrupts_capture(struct intel_gt *gt) > > static void guc_interrupts_release(struct intel_gt *gt) > { > - struct intel_rps *rps = >->i915->gt_pm.rps; > + struct intel_rps *rps = >->rps; > struct intel_uncore *uncore = gt->uncore; > struct intel_engine_cs *engine; > enum intel_engine_id id; > diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c > index 16211430eb78..22962373723b 100644 > --- a/drivers/gpu/drm/i915/i915_debugfs.c > +++ b/drivers/gpu/drm/i915/i915_debugfs.c > @@ -44,6 +44,7 @@ > #include "gt/intel_gt_requests.h" > #include "gt/intel_reset.h" > #include "gt/intel_rc6.h" > +#include "gt/intel_rps.h" > #include "gt/uc/intel_guc_submission.h" > > #include "i915_debugfs.h" > @@ -791,7 +792,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused) > { > struct drm_i915_private *dev_priv = node_to_i915(m->private); > struct intel_uncore *uncore = &dev_priv->uncore; > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > + struct intel_rps *rps = &dev_priv->gt.rps; > intel_wakeref_t wakeref; > int ret = 0; > > @@ -827,23 +828,23 @@ static int i915_frequency_info(struct seq_file *m, void *unused) > seq_printf(m, "DDR freq: %d MHz\n", dev_priv->mem_freq); > > seq_printf(m, "actual GPU freq: %d MHz\n", > - intel_gpu_freq(dev_priv, (freq_sts >> 8) & 0xff)); > + intel_gpu_freq(rps, (freq_sts >> 8) & 0xff)); > > seq_printf(m, "current GPU freq: %d MHz\n", > - intel_gpu_freq(dev_priv, rps->cur_freq)); > + intel_gpu_freq(rps, rps->cur_freq)); > > seq_printf(m, "max GPU freq: %d MHz\n", > - intel_gpu_freq(dev_priv, rps->max_freq)); > + intel_gpu_freq(rps, rps->max_freq)); > > seq_printf(m, "min GPU freq: %d MHz\n", > - intel_gpu_freq(dev_priv, rps->min_freq)); > + intel_gpu_freq(rps, rps->min_freq)); > > seq_printf(m, "idle GPU freq: %d MHz\n", > - intel_gpu_freq(dev_priv, rps->idle_freq)); > + intel_gpu_freq(rps, rps->idle_freq)); > > seq_printf(m, > "efficient (RPe) frequency: %d MHz\n", > - intel_gpu_freq(dev_priv, rps->efficient_freq)); > + intel_gpu_freq(rps, rps->efficient_freq)); > } else if (INTEL_GEN(dev_priv) >= 6) { > u32 rp_state_limits; > u32 gt_perf_status; > @@ -877,7 +878,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused) > else > reqf >>= 25; > } > - reqf = intel_gpu_freq(dev_priv, reqf); > + reqf = intel_gpu_freq(rps, reqf); > > rpmodectl = I915_READ(GEN6_RP_CONTROL); > rpinclimit = I915_READ(GEN6_RP_UP_THRESHOLD); > @@ -890,8 +891,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused) > rpdownei = I915_READ(GEN6_RP_CUR_DOWN_EI) & GEN6_CURIAVG_MASK; > rpcurdown = I915_READ(GEN6_RP_CUR_DOWN) & GEN6_CURBSYTAVG_MASK; > rpprevdown = I915_READ(GEN6_RP_PREV_DOWN) & GEN6_CURBSYTAVG_MASK; > - cagf = intel_gpu_freq(dev_priv, > - intel_get_cagf(dev_priv, rpstat)); > + cagf = intel_gpu_freq(rps, intel_get_cagf(rps, rpstat)); > > intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL); > > @@ -968,37 +968,37 @@ static int i915_frequency_info(struct seq_file *m, void *unused) > max_freq *= (IS_GEN9_BC(dev_priv) || > INTEL_GEN(dev_priv) >= 10 ? GEN9_FREQ_SCALER : 1); > seq_printf(m, "Lowest (RPN) frequency: %dMHz\n", > - intel_gpu_freq(dev_priv, max_freq)); > + intel_gpu_freq(rps, max_freq)); > > max_freq = (rp_state_cap & 0xff00) >> 8; > max_freq *= (IS_GEN9_BC(dev_priv) || > INTEL_GEN(dev_priv) >= 10 ? GEN9_FREQ_SCALER : 1); > seq_printf(m, "Nominal (RP1) frequency: %dMHz\n", > - intel_gpu_freq(dev_priv, max_freq)); > + intel_gpu_freq(rps, max_freq)); > > max_freq = (IS_GEN9_LP(dev_priv) ? rp_state_cap >> 16 : > rp_state_cap >> 0) & 0xff; > max_freq *= (IS_GEN9_BC(dev_priv) || > INTEL_GEN(dev_priv) >= 10 ? GEN9_FREQ_SCALER : 1); > seq_printf(m, "Max non-overclocked (RP0) frequency: %dMHz\n", > - intel_gpu_freq(dev_priv, max_freq)); > + intel_gpu_freq(rps, max_freq)); > seq_printf(m, "Max overclocked frequency: %dMHz\n", > - intel_gpu_freq(dev_priv, rps->max_freq)); > + intel_gpu_freq(rps, rps->max_freq)); > > seq_printf(m, "Current freq: %d MHz\n", > - intel_gpu_freq(dev_priv, rps->cur_freq)); > + intel_gpu_freq(rps, rps->cur_freq)); > seq_printf(m, "Actual freq: %d MHz\n", cagf); > seq_printf(m, "Idle freq: %d MHz\n", > - intel_gpu_freq(dev_priv, rps->idle_freq)); > + intel_gpu_freq(rps, rps->idle_freq)); > seq_printf(m, "Min freq: %d MHz\n", > - intel_gpu_freq(dev_priv, rps->min_freq)); > + intel_gpu_freq(rps, rps->min_freq)); > seq_printf(m, "Boost freq: %d MHz\n", > - intel_gpu_freq(dev_priv, rps->boost_freq)); > + intel_gpu_freq(rps, rps->boost_freq)); > seq_printf(m, "Max freq: %d MHz\n", > - intel_gpu_freq(dev_priv, rps->max_freq)); > + intel_gpu_freq(rps, rps->max_freq)); > seq_printf(m, > "efficient (RPe) frequency: %d MHz\n", > - intel_gpu_freq(dev_priv, rps->efficient_freq)); > + intel_gpu_freq(rps, rps->efficient_freq)); > } else { > seq_puts(m, "no P-state info available\n"); > } > @@ -1461,7 +1461,7 @@ static int i915_sr_status(struct seq_file *m, void *unused) > static int i915_ring_freq_table(struct seq_file *m, void *unused) > { > struct drm_i915_private *dev_priv = node_to_i915(m->private); > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > + struct intel_rps *rps = &dev_priv->gt.rps; > unsigned int max_gpu_freq, min_gpu_freq; > intel_wakeref_t wakeref; > int gpu_freq, ia_freq; > @@ -1486,10 +1486,11 @@ static int i915_ring_freq_table(struct seq_file *m, void *unused) > GEN6_PCODE_READ_MIN_FREQ_TABLE, > &ia_freq, NULL); > seq_printf(m, "%d\t\t%d\t\t\t\t%d\n", > - intel_gpu_freq(dev_priv, (gpu_freq * > - (IS_GEN9_BC(dev_priv) || > - INTEL_GEN(dev_priv) >= 10 ? > - GEN9_FREQ_SCALER : 1))), > + intel_gpu_freq(rps, > + (gpu_freq * > + (IS_GEN9_BC(dev_priv) || > + INTEL_GEN(dev_priv) >= 10 ? > + GEN9_FREQ_SCALER : 1))), > ((ia_freq >> 0) & 0xff) * 100, > ((ia_freq >> 8) & 0xff) * 100); > } > @@ -1717,7 +1718,7 @@ static const char *rps_power_to_str(unsigned int power) > static int i915_rps_boost_info(struct seq_file *m, void *data) > { > struct drm_i915_private *dev_priv = node_to_i915(m->private); > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > + struct intel_rps *rps = &dev_priv->gt.rps; > u32 act_freq = rps->cur_freq; > intel_wakeref_t wakeref; > > @@ -1729,7 +1730,7 @@ static int i915_rps_boost_info(struct seq_file *m, void *data) > vlv_punit_put(dev_priv); > act_freq = (act_freq >> 8) & 0xff; > } else { > - act_freq = intel_get_cagf(dev_priv, > + act_freq = intel_get_cagf(rps, > I915_READ(GEN6_RPSTAT1)); > } > } > @@ -1740,17 +1741,17 @@ static int i915_rps_boost_info(struct seq_file *m, void *data) > atomic_read(&rps->num_waiters)); > seq_printf(m, "Interactive? %d\n", READ_ONCE(rps->power.interactive)); > seq_printf(m, "Frequency requested %d, actual %d\n", > - intel_gpu_freq(dev_priv, rps->cur_freq), > - intel_gpu_freq(dev_priv, act_freq)); > + intel_gpu_freq(rps, rps->cur_freq), > + intel_gpu_freq(rps, act_freq)); > seq_printf(m, " min hard:%d, soft:%d; max soft:%d, hard:%d\n", > - intel_gpu_freq(dev_priv, rps->min_freq), > - intel_gpu_freq(dev_priv, rps->min_freq_softlimit), > - intel_gpu_freq(dev_priv, rps->max_freq_softlimit), > - intel_gpu_freq(dev_priv, rps->max_freq)); > + intel_gpu_freq(rps, rps->min_freq), > + intel_gpu_freq(rps, rps->min_freq_softlimit), > + intel_gpu_freq(rps, rps->max_freq_softlimit), > + intel_gpu_freq(rps, rps->max_freq)); > seq_printf(m, " idle:%d, efficient:%d, boost:%d\n", > - intel_gpu_freq(dev_priv, rps->idle_freq), > - intel_gpu_freq(dev_priv, rps->efficient_freq), > - intel_gpu_freq(dev_priv, rps->boost_freq)); > + intel_gpu_freq(rps, rps->idle_freq), > + intel_gpu_freq(rps, rps->efficient_freq), > + intel_gpu_freq(rps, rps->boost_freq)); > > seq_printf(m, "Wait boosts: %d\n", atomic_read(&rps->boosts)); > > diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c > index 5138d1eed306..78ac08db48a3 100644 > --- a/drivers/gpu/drm/i915/i915_drv.c > +++ b/drivers/gpu/drm/i915/i915_drv.c > @@ -1841,6 +1841,8 @@ static int i915_drm_resume(struct drm_device *dev) > > intel_dp_mst_resume(dev_priv); > > + intel_gt_pm_enable(&dev_priv->gt); > + > intel_display_resume(dev); > > drm_kms_helper_poll_enable(dev); > diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h > index 8882c0908c3b..eb5460290811 100644 > --- a/drivers/gpu/drm/i915/i915_drv.h > +++ b/drivers/gpu/drm/i915/i915_drv.h > @@ -543,94 +543,6 @@ struct i915_suspend_saved_registers { > > struct vlv_s0ix_state; > > -struct intel_rps_ei { > - ktime_t ktime; > - u32 render_c0; > - u32 media_c0; > -}; > - > -struct intel_rps { > - struct mutex lock; /* protects enabling and the worker */ > - > - /* > - * work, interrupts_enabled and pm_iir are protected by > - * dev_priv->irq_lock > - */ > - struct work_struct work; > - bool interrupts_enabled; > - u32 pm_iir; > - > - /* PM interrupt bits that should never be masked */ > - u32 pm_intrmsk_mbz; > - > - /* Frequencies are stored in potentially platform dependent multiples. > - * In other words, *_freq needs to be multiplied by X to be interesting. > - * Soft limits are those which are used for the dynamic reclocking done > - * by the driver (raise frequencies under heavy loads, and lower for > - * lighter loads). Hard limits are those imposed by the hardware. > - * > - * A distinction is made for overclocking, which is never enabled by > - * default, and is considered to be above the hard limit if it's > - * possible at all. > - */ > - u8 cur_freq; /* Current frequency (cached, may not == HW) */ > - u8 min_freq_softlimit; /* Minimum frequency permitted by the driver */ > - u8 max_freq_softlimit; /* Max frequency permitted by the driver */ > - u8 max_freq; /* Maximum frequency, RP0 if not overclocking */ > - u8 min_freq; /* AKA RPn. Minimum frequency */ > - u8 boost_freq; /* Frequency to request when wait boosting */ > - u8 idle_freq; /* Frequency to request when we are idle */ > - u8 efficient_freq; /* AKA RPe. Pre-determined balanced frequency */ > - u8 rp1_freq; /* "less than" RP0 power/freqency */ > - u8 rp0_freq; /* Non-overclocked max frequency. */ > - u16 gpll_ref_freq; /* vlv/chv GPLL reference frequency */ > - > - int last_adj; > - > - struct { > - struct mutex mutex; > - > - enum { LOW_POWER, BETWEEN, HIGH_POWER } mode; > - unsigned int interactive; > - > - u8 up_threshold; /* Current %busy required to uplock */ > - u8 down_threshold; /* Current %busy required to downclock */ > - } power; > - > - bool enabled; > - atomic_t num_waiters; > - atomic_t boosts; > - > - /* manual wa residency calculations */ > - struct intel_rps_ei ei; > -}; > - > -struct intel_gen6_power_mgmt { > - struct intel_rps rps; > -}; > - > -/* defined intel_pm.c */ > -extern spinlock_t mchdev_lock; > - > -struct intel_ilk_power_mgmt { > - u8 cur_delay; > - u8 min_delay; > - u8 max_delay; > - u8 fmax; > - u8 fstart; > - > - u64 last_count1; > - unsigned long last_time1; > - unsigned long chipset_power; > - u64 last_count2; > - u64 last_time2; > - unsigned long gfx_power; > - u8 corr; > - > - int c_m; > - int r_t; > -}; > - > #define MAX_L3_SLICES 2 > struct intel_l3_parity { > u32 *remap_info[MAX_L3_SLICES]; > @@ -1067,7 +979,6 @@ struct drm_i915_private { > u32 irq_mask; > u32 de_irq_mask[I915_MAX_PIPES]; > }; > - u32 pm_rps_events; > u32 pipestat_irq_mask[I915_MAX_PIPES]; > > struct i915_hotplug hotplug; > @@ -1202,13 +1113,6 @@ struct drm_i915_private { > */ > u32 edram_size_mb; > > - /* gen6+ GT PM state */ > - struct intel_gen6_power_mgmt gt_pm; > - > - /* ilk-only ips/rps state. Everything in here is protected by the global > - * mchdev_lock in intel_pm.c */ > - struct intel_ilk_power_mgmt ips; > - > struct i915_power_domains power_domains; > > struct i915_psr psr; > diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c > index b882988056bd..eab1709d1897 100644 > --- a/drivers/gpu/drm/i915/i915_gem.c > +++ b/drivers/gpu/drm/i915/i915_gem.c > @@ -52,6 +52,7 @@ > #include "gt/intel_mocs.h" > #include "gt/intel_reset.h" > #include "gt/intel_renderstate.h" > +#include "gt/intel_rps.h" > #include "gt/intel_workarounds.h" > > #include "i915_drv.h" > @@ -1272,8 +1273,6 @@ int i915_gem_init(struct drm_i915_private *dev_priv) > goto err_context; > } > > - intel_init_gt_powersave(dev_priv); > - > intel_uc_init(&dev_priv->gt.uc); > > ret = intel_gt_init_hw(&dev_priv->gt); > diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c > index 572a5c37cc61..598924b3c556 100644 > --- a/drivers/gpu/drm/i915/i915_irq.c > +++ b/drivers/gpu/drm/i915/i915_irq.c > @@ -45,6 +45,7 @@ > #include "gt/intel_gt.h" > #include "gt/intel_gt_irq.h" > #include "gt/intel_gt_pm_irq.h" > +#include "gt/intel_rps.h" > > #include "i915_drv.h" > #include "i915_irq.h" > @@ -327,87 +328,6 @@ static i915_reg_t gen6_pm_iir(struct drm_i915_private *dev_priv) > return INTEL_GEN(dev_priv) >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR; > } > > -void gen11_reset_rps_interrupts(struct drm_i915_private *dev_priv) > -{ > - struct intel_gt *gt = &dev_priv->gt; > - > - spin_lock_irq(>->irq_lock); > - > - while (gen11_gt_reset_one_iir(gt, 0, GEN11_GTPM)) > - ; > - > - dev_priv->gt_pm.rps.pm_iir = 0; > - > - spin_unlock_irq(>->irq_lock); > -} > - > -void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv) > -{ > - struct intel_gt *gt = &dev_priv->gt; > - > - spin_lock_irq(>->irq_lock); > - gen6_gt_pm_reset_iir(gt, GEN6_PM_RPS_EVENTS); > - dev_priv->gt_pm.rps.pm_iir = 0; > - spin_unlock_irq(>->irq_lock); > -} > - > -void gen6_enable_rps_interrupts(struct drm_i915_private *dev_priv) > -{ > - struct intel_gt *gt = &dev_priv->gt; > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > - > - if (READ_ONCE(rps->interrupts_enabled)) > - return; > - > - spin_lock_irq(>->irq_lock); > - WARN_ON_ONCE(rps->pm_iir); > - > - if (INTEL_GEN(dev_priv) >= 11) > - WARN_ON_ONCE(gen11_gt_reset_one_iir(gt, 0, GEN11_GTPM)); > - else > - WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events); > - > - rps->interrupts_enabled = true; > - gen6_gt_pm_enable_irq(gt, dev_priv->pm_rps_events); > - > - spin_unlock_irq(>->irq_lock); > -} > - > -u32 gen6_sanitize_rps_pm_mask(const struct drm_i915_private *i915, u32 mask) > -{ > - return mask & ~i915->gt_pm.rps.pm_intrmsk_mbz; > -} > - > -void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv) > -{ > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > - struct intel_gt *gt = &dev_priv->gt; > - > - if (!READ_ONCE(rps->interrupts_enabled)) > - return; > - > - spin_lock_irq(>->irq_lock); > - rps->interrupts_enabled = false; > - > - I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0u)); > - > - gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS); > - > - spin_unlock_irq(>->irq_lock); > - intel_synchronize_irq(dev_priv); > - > - /* Now that we will not be generating any more work, flush any > - * outstanding tasks. As we are called on the RPS idle path, > - * we will reset the GPU to minimum frequencies, so the current > - * state of the worker can be discarded. > - */ > - cancel_work_sync(&rps->work); > - if (INTEL_GEN(dev_priv) >= 11) > - gen11_reset_rps_interrupts(dev_priv); > - else > - gen6_reset_rps_interrupts(dev_priv); > -} > - > void gen9_reset_guc_interrupts(struct intel_guc *guc) > { > struct intel_gt *gt = guc_to_gt(guc); > @@ -1065,199 +985,6 @@ int intel_get_crtc_scanline(struct intel_crtc *crtc) > return position; > } > > -static void ironlake_rps_change_irq_handler(struct drm_i915_private *dev_priv) > -{ > - struct intel_uncore *uncore = &dev_priv->uncore; > - u32 busy_up, busy_down, max_avg, min_avg; > - u8 new_delay; > - > - spin_lock(&mchdev_lock); > - > - intel_uncore_write16(uncore, > - MEMINTRSTS, > - intel_uncore_read(uncore, MEMINTRSTS)); > - > - new_delay = dev_priv->ips.cur_delay; > - > - intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); > - busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG); > - busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG); > - max_avg = intel_uncore_read(uncore, RCBMAXAVG); > - min_avg = intel_uncore_read(uncore, RCBMINAVG); > - > - /* Handle RCS change request from hw */ > - if (busy_up > max_avg) { > - if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay) > - new_delay = dev_priv->ips.cur_delay - 1; > - if (new_delay < dev_priv->ips.max_delay) > - new_delay = dev_priv->ips.max_delay; > - } else if (busy_down < min_avg) { > - if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay) > - new_delay = dev_priv->ips.cur_delay + 1; > - if (new_delay > dev_priv->ips.min_delay) > - new_delay = dev_priv->ips.min_delay; > - } > - > - if (ironlake_set_drps(dev_priv, new_delay)) > - dev_priv->ips.cur_delay = new_delay; > - > - spin_unlock(&mchdev_lock); > - > - return; > -} > - > -static void vlv_c0_read(struct drm_i915_private *dev_priv, > - struct intel_rps_ei *ei) > -{ > - ei->ktime = ktime_get_raw(); > - ei->render_c0 = I915_READ(VLV_RENDER_C0_COUNT); > - ei->media_c0 = I915_READ(VLV_MEDIA_C0_COUNT); > -} > - > -void gen6_rps_reset_ei(struct drm_i915_private *dev_priv) > -{ > - memset(&dev_priv->gt_pm.rps.ei, 0, sizeof(dev_priv->gt_pm.rps.ei)); > -} > - > -static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir) > -{ > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > - const struct intel_rps_ei *prev = &rps->ei; > - struct intel_rps_ei now; > - u32 events = 0; > - > - if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0) > - return 0; > - > - vlv_c0_read(dev_priv, &now); > - > - if (prev->ktime) { > - u64 time, c0; > - u32 render, media; > - > - time = ktime_us_delta(now.ktime, prev->ktime); > - > - time *= dev_priv->czclk_freq; > - > - /* Workload can be split between render + media, > - * e.g. SwapBuffers being blitted in X after being rendered in > - * mesa. To account for this we need to combine both engines > - * into our activity counter. > - */ > - render = now.render_c0 - prev->render_c0; > - media = now.media_c0 - prev->media_c0; > - c0 = max(render, media); > - c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */ > - > - if (c0 > time * rps->power.up_threshold) > - events = GEN6_PM_RP_UP_THRESHOLD; > - else if (c0 < time * rps->power.down_threshold) > - events = GEN6_PM_RP_DOWN_THRESHOLD; > - } > - > - rps->ei = now; > - return events; > -} > - > -static void gen6_pm_rps_work(struct work_struct *work) > -{ > - struct drm_i915_private *dev_priv = > - container_of(work, struct drm_i915_private, gt_pm.rps.work); > - struct intel_gt *gt = &dev_priv->gt; > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > - bool client_boost = false; > - int new_delay, adj, min, max; > - u32 pm_iir = 0; > - > - spin_lock_irq(>->irq_lock); > - if (rps->interrupts_enabled) { > - pm_iir = fetch_and_zero(&rps->pm_iir); > - client_boost = atomic_read(&rps->num_waiters); > - } > - spin_unlock_irq(>->irq_lock); > - > - /* Make sure we didn't queue anything we're not going to process. */ > - WARN_ON(pm_iir & ~dev_priv->pm_rps_events); > - if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost) > - goto out; > - > - mutex_lock(&rps->lock); > - > - pm_iir |= vlv_wa_c0_ei(dev_priv, pm_iir); > - > - adj = rps->last_adj; > - new_delay = rps->cur_freq; > - min = rps->min_freq_softlimit; > - max = rps->max_freq_softlimit; > - if (client_boost) > - max = rps->max_freq; > - if (client_boost && new_delay < rps->boost_freq) { > - new_delay = rps->boost_freq; > - adj = 0; > - } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) { > - if (adj > 0) > - adj *= 2; > - else /* CHV needs even encode values */ > - adj = IS_CHERRYVIEW(dev_priv) ? 2 : 1; > - > - if (new_delay >= rps->max_freq_softlimit) > - adj = 0; > - } else if (client_boost) { > - adj = 0; > - } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) { > - if (rps->cur_freq > rps->efficient_freq) > - new_delay = rps->efficient_freq; > - else if (rps->cur_freq > rps->min_freq_softlimit) > - new_delay = rps->min_freq_softlimit; > - adj = 0; > - } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) { > - if (adj < 0) > - adj *= 2; > - else /* CHV needs even encode values */ > - adj = IS_CHERRYVIEW(dev_priv) ? -2 : -1; > - > - if (new_delay <= rps->min_freq_softlimit) > - adj = 0; > - } else { /* unknown event */ > - adj = 0; > - } > - > - rps->last_adj = adj; > - > - /* > - * Limit deboosting and boosting to keep ourselves at the extremes > - * when in the respective power modes (i.e. slowly decrease frequencies > - * while in the HIGH_POWER zone and slowly increase frequencies while > - * in the LOW_POWER zone). On idle, we will hit the timeout and drop > - * to the next level quickly, and conversely if busy we expect to > - * hit a waitboost and rapidly switch into max power. > - */ > - if ((adj < 0 && rps->power.mode == HIGH_POWER) || > - (adj > 0 && rps->power.mode == LOW_POWER)) > - rps->last_adj = 0; > - > - /* sysfs frequency interfaces may have snuck in while servicing the > - * interrupt > - */ > - new_delay += adj; > - new_delay = clamp_t(int, new_delay, min, max); > - > - if (intel_set_rps(dev_priv, new_delay)) { > - DRM_DEBUG_DRIVER("Failed to set new GPU frequency\n"); > - rps->last_adj = 0; > - } > - > - mutex_unlock(&rps->lock); > - > -out: > - /* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */ > - spin_lock_irq(>->irq_lock); > - if (rps->interrupts_enabled) > - gen6_gt_pm_unmask_irq(gt, dev_priv->pm_rps_events); > - spin_unlock_irq(>->irq_lock); > -} > - > - > /** > * ivybridge_parity_work - Workqueue called when a parity error interrupt > * occurred. > @@ -1631,54 +1358,6 @@ static void i9xx_pipe_crc_irq_handler(struct drm_i915_private *dev_priv, > res1, res2); > } > > -/* The RPS events need forcewake, so we add them to a work queue and mask their > - * IMR bits until the work is done. Other interrupts can be processed without > - * the work queue. */ > -void gen11_rps_irq_handler(struct intel_gt *gt, u32 pm_iir) > -{ > - struct drm_i915_private *i915 = gt->i915; > - struct intel_rps *rps = &i915->gt_pm.rps; > - const u32 events = i915->pm_rps_events & pm_iir; > - > - lockdep_assert_held(>->irq_lock); > - > - if (unlikely(!events)) > - return; > - > - gen6_gt_pm_mask_irq(gt, events); > - > - if (!rps->interrupts_enabled) > - return; > - > - rps->pm_iir |= events; > - schedule_work(&rps->work); > -} > - > -void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir) > -{ > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > - struct intel_gt *gt = &dev_priv->gt; > - > - if (pm_iir & dev_priv->pm_rps_events) { > - spin_lock(>->irq_lock); > - gen6_gt_pm_mask_irq(gt, pm_iir & dev_priv->pm_rps_events); > - if (rps->interrupts_enabled) { > - rps->pm_iir |= pm_iir & dev_priv->pm_rps_events; > - schedule_work(&rps->work); > - } > - spin_unlock(>->irq_lock); > - } > - > - if (INTEL_GEN(dev_priv) >= 8) > - return; > - > - if (pm_iir & PM_VEBOX_USER_INTERRUPT) > - intel_engine_breadcrumbs_irq(dev_priv->engine[VECS0]); > - > - if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) > - DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir); > -} > - > static void i9xx_pipestat_irq_reset(struct drm_i915_private *dev_priv) > { > enum pipe pipe; > @@ -1989,7 +1668,7 @@ static irqreturn_t valleyview_irq_handler(int irq, void *arg) > if (gt_iir) > gen6_gt_irq_handler(&dev_priv->gt, gt_iir); > if (pm_iir) > - gen6_rps_irq_handler(dev_priv, pm_iir); > + gen6_rps_irq_handler(&dev_priv->gt.rps, pm_iir); > > if (hotplug_status) > i9xx_hpd_irq_handler(dev_priv, hotplug_status); > @@ -2393,7 +2072,7 @@ static void ilk_display_irq_handler(struct drm_i915_private *dev_priv, > } > > if (IS_GEN(dev_priv, 5) && de_iir & DE_PCU_EVENT) > - ironlake_rps_change_irq_handler(dev_priv); > + gen5_rps_irq_handler(&dev_priv->gt.rps); > } > > static void ivb_display_irq_handler(struct drm_i915_private *dev_priv, > @@ -2498,7 +2177,7 @@ static irqreturn_t ironlake_irq_handler(int irq, void *arg) > if (pm_iir) { > I915_WRITE(GEN6_PMIIR, pm_iir); > ret = IRQ_HANDLED; > - gen6_rps_irq_handler(dev_priv, pm_iir); > + gen6_rps_irq_handler(&dev_priv->gt.rps, pm_iir); > } > } > > @@ -4270,13 +3949,10 @@ static irqreturn_t i965_irq_handler(int irq, void *arg) > void intel_irq_init(struct drm_i915_private *dev_priv) > { > struct drm_device *dev = &dev_priv->drm; > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > int i; > > intel_hpd_init_work(dev_priv); > > - INIT_WORK(&rps->work, gen6_pm_rps_work); > - > INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work); > for (i = 0; i < MAX_L3_SLICES; ++i) > dev_priv->l3_parity.remap_info[i] = NULL; > @@ -4285,33 +3961,6 @@ void intel_irq_init(struct drm_i915_private *dev_priv) > if (HAS_GT_UC(dev_priv) && INTEL_GEN(dev_priv) < 11) > dev_priv->gt.pm_guc_events = GUC_INTR_GUC2HOST << 16; > > - /* Let's track the enabled rps events */ > - if (IS_VALLEYVIEW(dev_priv)) > - /* WaGsvRC0ResidencyMethod:vlv */ > - dev_priv->pm_rps_events = GEN6_PM_RP_UP_EI_EXPIRED; > - else > - dev_priv->pm_rps_events = (GEN6_PM_RP_UP_THRESHOLD | > - GEN6_PM_RP_DOWN_THRESHOLD | > - GEN6_PM_RP_DOWN_TIMEOUT); > - > - /* We share the register with other engine */ > - if (INTEL_GEN(dev_priv) > 9) > - GEM_WARN_ON(dev_priv->pm_rps_events & 0xffff0000); > - > - rps->pm_intrmsk_mbz = 0; > - > - /* > - * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer > - * if GEN6_PM_UP_EI_EXPIRED is masked. > - * > - * TODO: verify if this can be reproduced on VLV,CHV. > - */ > - if (INTEL_GEN(dev_priv) <= 7) > - rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED; > - > - if (INTEL_GEN(dev_priv) >= 8) > - rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC; > - > dev->vblank_disable_immediate = true; > > /* Most platforms treat the display irq block as an always-on > diff --git a/drivers/gpu/drm/i915/i915_irq.h b/drivers/gpu/drm/i915/i915_irq.h > index 19a3bc019535..d0d91c6e00d7 100644 > --- a/drivers/gpu/drm/i915/i915_irq.h > +++ b/drivers/gpu/drm/i915/i915_irq.h > @@ -22,9 +22,6 @@ struct intel_gt; > struct intel_guc; > struct intel_uncore; > > -void gen11_rps_irq_handler(struct intel_gt *gt, u32 pm_iir); > -void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir); > - > void intel_irq_init(struct drm_i915_private *dev_priv); > void intel_irq_fini(struct drm_i915_private *dev_priv); > int intel_irq_install(struct drm_i915_private *dev_priv); > diff --git a/drivers/gpu/drm/i915/i915_pmu.c b/drivers/gpu/drm/i915/i915_pmu.c > index 85912917c062..266d66b1fb7b 100644 > --- a/drivers/gpu/drm/i915/i915_pmu.c > +++ b/drivers/gpu/drm/i915/i915_pmu.c > @@ -12,6 +12,7 @@ > #include "gt/intel_engine_user.h" > #include "gt/intel_gt_pm.h" > #include "gt/intel_rc6.h" > +#include "gt/intel_rps.h" > > #include "i915_drv.h" > #include "i915_pmu.h" > @@ -358,25 +359,26 @@ frequency_sample(struct intel_gt *gt, unsigned int period_ns) > struct drm_i915_private *i915 = gt->i915; > struct intel_uncore *uncore = gt->uncore; > struct i915_pmu *pmu = &i915->pmu; > + struct intel_rps *rps = >->rps; > > if (pmu->enable & config_enabled_mask(I915_PMU_ACTUAL_FREQUENCY)) { > u32 val; > > - val = i915->gt_pm.rps.cur_freq; > + val = rps->cur_freq; > if (intel_gt_pm_get_if_awake(gt)) { > val = intel_uncore_read_notrace(uncore, GEN6_RPSTAT1); > - val = intel_get_cagf(i915, val); > + val = intel_get_cagf(rps, val); > intel_gt_pm_put(gt); > } > > add_sample_mult(&pmu->sample[__I915_SAMPLE_FREQ_ACT], > - intel_gpu_freq(i915, val), > + intel_gpu_freq(rps, val), > period_ns / 1000); > } > > if (pmu->enable & config_enabled_mask(I915_PMU_REQUESTED_FREQUENCY)) { > add_sample_mult(&pmu->sample[__I915_SAMPLE_FREQ_REQ], > - intel_gpu_freq(i915, i915->gt_pm.rps.cur_freq), > + intel_gpu_freq(rps, rps->cur_freq), > period_ns / 1000); > } > } > diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c > index 4575f368455d..08ce2eeecf7e 100644 > --- a/drivers/gpu/drm/i915/i915_request.c > +++ b/drivers/gpu/drm/i915/i915_request.c > @@ -31,6 +31,7 @@ > > #include "gem/i915_gem_context.h" > #include "gt/intel_context.h" > +#include "gt/intel_rps.h" > > #include "i915_active.h" > #include "i915_drv.h" > @@ -257,8 +258,8 @@ bool i915_request_retire(struct i915_request *rq) > if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &rq->fence.flags)) > i915_request_cancel_breadcrumb(rq); > if (i915_request_has_waitboost(rq)) { > - GEM_BUG_ON(!atomic_read(&rq->i915->gt_pm.rps.num_waiters)); > - atomic_dec(&rq->i915->gt_pm.rps.num_waiters); > + GEM_BUG_ON(!atomic_read(&rq->engine->gt->rps.num_waiters)); > + atomic_dec(&rq->engine->gt->rps.num_waiters); > } > if (!test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) { > set_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags); > @@ -1466,7 +1467,7 @@ long i915_request_wait(struct i915_request *rq, > */ > if (flags & I915_WAIT_PRIORITY) { > if (!i915_request_started(rq) && INTEL_GEN(rq->i915) >= 6) > - gen6_rps_boost(rq); > + intel_rps_boost(rq); > i915_schedule_bump_priority(rq, I915_PRIORITY_WAIT); > } > > diff --git a/drivers/gpu/drm/i915/i915_sysfs.c b/drivers/gpu/drm/i915/i915_sysfs.c > index bf039b8ba593..65476909d1bf 100644 > --- a/drivers/gpu/drm/i915/i915_sysfs.c > +++ b/drivers/gpu/drm/i915/i915_sysfs.c > @@ -31,6 +31,7 @@ > #include <linux/sysfs.h> > > #include "gt/intel_rc6.h" > +#include "gt/intel_rps.h" > > #include "i915_drv.h" > #include "i915_sysfs.h" > @@ -259,6 +260,7 @@ static ssize_t gt_act_freq_mhz_show(struct device *kdev, > struct device_attribute *attr, char *buf) > { > struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); > + struct intel_rps *rps = &dev_priv->gt.rps; > intel_wakeref_t wakeref; > u32 freq; > > @@ -271,31 +273,31 @@ static ssize_t gt_act_freq_mhz_show(struct device *kdev, > > freq = (freq >> 8) & 0xff; > } else { > - freq = intel_get_cagf(dev_priv, I915_READ(GEN6_RPSTAT1)); > + freq = intel_get_cagf(rps, I915_READ(GEN6_RPSTAT1)); > } > > intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref); > > - return snprintf(buf, PAGE_SIZE, "%d\n", intel_gpu_freq(dev_priv, freq)); > + return snprintf(buf, PAGE_SIZE, "%d\n", intel_gpu_freq(rps, freq)); > } > > static ssize_t gt_cur_freq_mhz_show(struct device *kdev, > struct device_attribute *attr, char *buf) > { > struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); > + struct intel_rps *rps = &dev_priv->gt.rps; > > return snprintf(buf, PAGE_SIZE, "%d\n", > - intel_gpu_freq(dev_priv, > - dev_priv->gt_pm.rps.cur_freq)); > + intel_gpu_freq(rps, rps->cur_freq)); > } > > static ssize_t gt_boost_freq_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf) > { > struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); > + struct intel_rps *rps = &dev_priv->gt.rps; > > return snprintf(buf, PAGE_SIZE, "%d\n", > - intel_gpu_freq(dev_priv, > - dev_priv->gt_pm.rps.boost_freq)); > + intel_gpu_freq(rps, rps->boost_freq)); > } > > static ssize_t gt_boost_freq_mhz_store(struct device *kdev, > @@ -303,7 +305,7 @@ static ssize_t gt_boost_freq_mhz_store(struct device *kdev, > const char *buf, size_t count) > { > struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > + struct intel_rps *rps = &dev_priv->gt.rps; > bool boost = false; > ssize_t ret; > u32 val; > @@ -313,7 +315,7 @@ static ssize_t gt_boost_freq_mhz_store(struct device *kdev, > return ret; > > /* Validate against (static) hardware limits */ > - val = intel_freq_opcode(dev_priv, val); > + val = intel_freq_opcode(rps, val); > if (val < rps->min_freq || val > rps->max_freq) > return -EINVAL; > > @@ -333,19 +335,19 @@ static ssize_t vlv_rpe_freq_mhz_show(struct device *kdev, > struct device_attribute *attr, char *buf) > { > struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); > + struct intel_rps *rps = &dev_priv->gt.rps; > > return snprintf(buf, PAGE_SIZE, "%d\n", > - intel_gpu_freq(dev_priv, > - dev_priv->gt_pm.rps.efficient_freq)); > + intel_gpu_freq(rps, rps->efficient_freq)); > } > > static ssize_t gt_max_freq_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf) > { > struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); > + struct intel_rps *rps = &dev_priv->gt.rps; > > return snprintf(buf, PAGE_SIZE, "%d\n", > - intel_gpu_freq(dev_priv, > - dev_priv->gt_pm.rps.max_freq_softlimit)); > + intel_gpu_freq(rps, rps->max_freq_softlimit)); > } > > static ssize_t gt_max_freq_mhz_store(struct device *kdev, > @@ -353,19 +355,17 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev, > const char *buf, size_t count) > { > struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > - intel_wakeref_t wakeref; > - u32 val; > + struct intel_rps *rps = &dev_priv->gt.rps; > ssize_t ret; > + u32 val; > > ret = kstrtou32(buf, 0, &val); > if (ret) > return ret; > > - wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm); > mutex_lock(&rps->lock); > > - val = intel_freq_opcode(dev_priv, val); > + val = intel_freq_opcode(rps, val); > if (val < rps->min_freq || > val > rps->max_freq || > val < rps->min_freq_softlimit) { > @@ -375,7 +375,7 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev, > > if (val > rps->rp0_freq) > DRM_DEBUG("User requested overclocking to %d\n", > - intel_gpu_freq(dev_priv, val)); > + intel_gpu_freq(rps, val)); > > rps->max_freq_softlimit = val; > > @@ -383,14 +383,15 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev, > rps->min_freq_softlimit, > rps->max_freq_softlimit); > > - /* We still need *_set_rps to process the new max_delay and > + /* > + * We still need *_set_rps to process the new max_delay and > * update the interrupt limits and PMINTRMSK even though > - * frequency request may be unchanged. */ > - ret = intel_set_rps(dev_priv, val); > + * frequency request may be unchanged. > + */ > + intel_rps_set(rps, val); > > unlock: > mutex_unlock(&rps->lock); > - intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref); > > return ret ?: count; > } > @@ -398,10 +399,10 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev, > static ssize_t gt_min_freq_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf) > { > struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); > + struct intel_rps *rps = &dev_priv->gt.rps; > > return snprintf(buf, PAGE_SIZE, "%d\n", > - intel_gpu_freq(dev_priv, > - dev_priv->gt_pm.rps.min_freq_softlimit)); > + intel_gpu_freq(rps, rps->min_freq_softlimit)); > } > > static ssize_t gt_min_freq_mhz_store(struct device *kdev, > @@ -409,19 +410,17 @@ static ssize_t gt_min_freq_mhz_store(struct device *kdev, > const char *buf, size_t count) > { > struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > - intel_wakeref_t wakeref; > - u32 val; > + struct intel_rps *rps = &dev_priv->gt.rps; > ssize_t ret; > + u32 val; > > ret = kstrtou32(buf, 0, &val); > if (ret) > return ret; > > - wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm); > mutex_lock(&rps->lock); > > - val = intel_freq_opcode(dev_priv, val); > + val = intel_freq_opcode(rps, val); > if (val < rps->min_freq || > val > rps->max_freq || > val > rps->max_freq_softlimit) { > @@ -435,14 +434,15 @@ static ssize_t gt_min_freq_mhz_store(struct device *kdev, > rps->min_freq_softlimit, > rps->max_freq_softlimit); > > - /* We still need *_set_rps to process the new min_delay and > + /* > + * We still need *_set_rps to process the new min_delay and > * update the interrupt limits and PMINTRMSK even though > - * frequency request may be unchanged. */ > - ret = intel_set_rps(dev_priv, val); > + * frequency request may be unchanged. > + */ > + intel_rps_set(rps, val); > > unlock: > mutex_unlock(&rps->lock); > - intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref); > > return ret ?: count; > } > @@ -464,15 +464,15 @@ static DEVICE_ATTR(gt_RPn_freq_mhz, S_IRUGO, gt_rp_mhz_show, NULL); > static ssize_t gt_rp_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf) > { > struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > + struct intel_rps *rps = &dev_priv->gt.rps; > u32 val; > > if (attr == &dev_attr_gt_RP0_freq_mhz) > - val = intel_gpu_freq(dev_priv, rps->rp0_freq); > + val = intel_gpu_freq(rps, rps->rp0_freq); > else if (attr == &dev_attr_gt_RP1_freq_mhz) > - val = intel_gpu_freq(dev_priv, rps->rp1_freq); > + val = intel_gpu_freq(rps, rps->rp1_freq); > else if (attr == &dev_attr_gt_RPn_freq_mhz) > - val = intel_gpu_freq(dev_priv, rps->min_freq); > + val = intel_gpu_freq(rps, rps->min_freq); > else > BUG(); > > diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c > index 362234449087..6741507c74f3 100644 > --- a/drivers/gpu/drm/i915/intel_pm.c > +++ b/drivers/gpu/drm/i915/intel_pm.c > @@ -197,8 +197,6 @@ static void i915_ironlake_get_mem_freq(struct drm_i915_private *dev_priv) > break; > } > > - dev_priv->ips.r_t = dev_priv->mem_freq; > - > switch (csipll & 0x3ff) { > case 0x00c: > dev_priv->fsb_freq = 3200; > @@ -227,14 +225,6 @@ static void i915_ironlake_get_mem_freq(struct drm_i915_private *dev_priv) > dev_priv->fsb_freq = 0; > break; > } > - > - if (dev_priv->fsb_freq == 3200) { > - dev_priv->ips.c_m = 0; > - } else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) { > - dev_priv->ips.c_m = 1; > - } else { > - dev_priv->ips.c_m = 2; > - } > } > > static const struct cxsr_latency cxsr_latency_table[] = { > @@ -6339,1865 +6329,258 @@ void intel_init_ipc(struct drm_i915_private *dev_priv) > intel_enable_ipc(dev_priv); > } > > -/* > - * Lock protecting IPS related data structures > - */ > -DEFINE_SPINLOCK(mchdev_lock); > +static const struct cparams { > + u16 i; > + u16 t; > + u16 m; > + u16 c; > +} cparams[] = { > + { 1, 1333, 301, 28664 }, > + { 1, 1066, 294, 24460 }, > + { 1, 800, 294, 25192 }, > + { 0, 1333, 276, 27605 }, > + { 0, 1066, 276, 27605 }, > + { 0, 800, 231, 23784 }, > +}; > > -bool ironlake_set_drps(struct drm_i915_private *i915, u8 val) > +static void ibx_init_clock_gating(struct drm_i915_private *dev_priv) > { > - struct intel_uncore *uncore = &i915->uncore; > - u16 rgvswctl; > - > - lockdep_assert_held(&mchdev_lock); > - > - rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); > - if (rgvswctl & MEMCTL_CMD_STS) { > - DRM_DEBUG("gpu busy, RCS change rejected\n"); > - return false; /* still busy with another command */ > - } > - > - rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) | > - (val << MEMCTL_FREQ_SHIFT) | MEMCTL_SFCAVM; > - intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); > - intel_uncore_posting_read16(uncore, MEMSWCTL); > - > - rgvswctl |= MEMCTL_CMD_STS; > - intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); > - > - return true; > + /* > + * On Ibex Peak and Cougar Point, we need to disable clock > + * gating for the panel power sequencer or it will fail to > + * start up when no ports are active. > + */ > + I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE); > } > > -static void ironlake_enable_drps(struct drm_i915_private *dev_priv) > +static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv) > { > - struct intel_uncore *uncore = &dev_priv->uncore; > - u32 rgvmodectl; > - u8 fmax, fmin, fstart, vstart; > - > - spin_lock_irq(&mchdev_lock); > - > - rgvmodectl = intel_uncore_read(uncore, MEMMODECTL); > - > - /* Enable temp reporting */ > - intel_uncore_write16(uncore, PMMISC, I915_READ(PMMISC) | MCPPCE_EN); > - intel_uncore_write16(uncore, TSC1, I915_READ(TSC1) | TSE); > - > - /* 100ms RC evaluation intervals */ > - intel_uncore_write(uncore, RCUPEI, 100000); > - intel_uncore_write(uncore, RCDNEI, 100000); > - > - /* Set max/min thresholds to 90ms and 80ms respectively */ > - intel_uncore_write(uncore, RCBMAXAVG, 90000); > - intel_uncore_write(uncore, RCBMINAVG, 80000); > - > - intel_uncore_write(uncore, MEMIHYST, 1); > + enum pipe pipe; > > - /* Set up min, max, and cur for interrupt handling */ > - fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT; > - fmin = (rgvmodectl & MEMMODE_FMIN_MASK); > - fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> > - MEMMODE_FSTART_SHIFT; > + for_each_pipe(dev_priv, pipe) { > + I915_WRITE(DSPCNTR(pipe), > + I915_READ(DSPCNTR(pipe)) | > + DISPPLANE_TRICKLE_FEED_DISABLE); > > - vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) & > - PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; > + I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe))); > + POSTING_READ(DSPSURF(pipe)); > + } > +} > > - dev_priv->ips.fmax = fmax; /* IPS callback will increase this */ > - dev_priv->ips.fstart = fstart; > +static void ilk_init_clock_gating(struct drm_i915_private *dev_priv) > +{ > + u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE; > > - dev_priv->ips.max_delay = fstart; > - dev_priv->ips.min_delay = fmin; > - dev_priv->ips.cur_delay = fstart; > + /* > + * Required for FBC > + * WaFbcDisableDpfcClockGating:ilk > + */ > + dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE | > + ILK_DPFCUNIT_CLOCK_GATE_DISABLE | > + ILK_DPFDUNIT_CLOCK_GATE_ENABLE; > > - DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n", > - fmax, fmin, fstart); > + I915_WRITE(PCH_3DCGDIS0, > + MARIUNIT_CLOCK_GATE_DISABLE | > + SVSMUNIT_CLOCK_GATE_DISABLE); > + I915_WRITE(PCH_3DCGDIS1, > + VFMUNIT_CLOCK_GATE_DISABLE); > > - intel_uncore_write(uncore, > - MEMINTREN, > - MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN); > + /* > + * According to the spec the following bits should be set in > + * order to enable memory self-refresh > + * The bit 22/21 of 0x42004 > + * The bit 5 of 0x42020 > + * The bit 15 of 0x45000 > + */ > + I915_WRITE(ILK_DISPLAY_CHICKEN2, > + (I915_READ(ILK_DISPLAY_CHICKEN2) | > + ILK_DPARB_GATE | ILK_VSDPFD_FULL)); > + dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE; > + I915_WRITE(DISP_ARB_CTL, > + (I915_READ(DISP_ARB_CTL) | > + DISP_FBC_WM_DIS)); > > /* > - * Interrupts will be enabled in ironlake_irq_postinstall > + * Based on the document from hardware guys the following bits > + * should be set unconditionally in order to enable FBC. > + * The bit 22 of 0x42000 > + * The bit 22 of 0x42004 > + * The bit 7,8,9 of 0x42020. > */ > + if (IS_IRONLAKE_M(dev_priv)) { > + /* WaFbcAsynchFlipDisableFbcQueue:ilk */ > + I915_WRITE(ILK_DISPLAY_CHICKEN1, > + I915_READ(ILK_DISPLAY_CHICKEN1) | > + ILK_FBCQ_DIS); > + I915_WRITE(ILK_DISPLAY_CHICKEN2, > + I915_READ(ILK_DISPLAY_CHICKEN2) | > + ILK_DPARB_GATE); > + } > > - intel_uncore_write(uncore, VIDSTART, vstart); > - intel_uncore_posting_read(uncore, VIDSTART); > + I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate); > > - rgvmodectl |= MEMMODE_SWMODE_EN; > - intel_uncore_write(uncore, MEMMODECTL, rgvmodectl); > + I915_WRITE(ILK_DISPLAY_CHICKEN2, > + I915_READ(ILK_DISPLAY_CHICKEN2) | > + ILK_ELPIN_409_SELECT); > + I915_WRITE(_3D_CHICKEN2, > + _3D_CHICKEN2_WM_READ_PIPELINED << 16 | > + _3D_CHICKEN2_WM_READ_PIPELINED); > > - if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) & > - MEMCTL_CMD_STS) == 0, 10)) > - DRM_ERROR("stuck trying to change perf mode\n"); > - mdelay(1); > + /* WaDisableRenderCachePipelinedFlush:ilk */ > + I915_WRITE(CACHE_MODE_0, > + _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE)); > > - ironlake_set_drps(dev_priv, fstart); > + /* WaDisable_RenderCache_OperationalFlush:ilk */ > + I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); > > - dev_priv->ips.last_count1 = > - intel_uncore_read(uncore, DMIEC) + > - intel_uncore_read(uncore, DDREC) + > - intel_uncore_read(uncore, CSIEC); > - dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies); > - dev_priv->ips.last_count2 = intel_uncore_read(uncore, GFXEC); > - dev_priv->ips.last_time2 = ktime_get_raw_ns(); > + g4x_disable_trickle_feed(dev_priv); > > - spin_unlock_irq(&mchdev_lock); > + ibx_init_clock_gating(dev_priv); > } > > -static void ironlake_disable_drps(struct drm_i915_private *i915) > +static void cpt_init_clock_gating(struct drm_i915_private *dev_priv) > { > - struct intel_uncore *uncore = &i915->uncore; > - u16 rgvswctl; > - > - spin_lock_irq(&mchdev_lock); > - > - rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); > - > - /* Ack interrupts, disable EFC interrupt */ > - intel_uncore_write(uncore, > - MEMINTREN, > - intel_uncore_read(uncore, MEMINTREN) & > - ~MEMINT_EVAL_CHG_EN); > - intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); > - intel_uncore_write(uncore, > - DEIER, > - intel_uncore_read(uncore, DEIER) & ~DE_PCU_EVENT); > - intel_uncore_write(uncore, DEIIR, DE_PCU_EVENT); > - intel_uncore_write(uncore, > - DEIMR, > - intel_uncore_read(uncore, DEIMR) | DE_PCU_EVENT); > - > - /* Go back to the starting frequency */ > - ironlake_set_drps(i915, i915->ips.fstart); > - mdelay(1); > - rgvswctl |= MEMCTL_CMD_STS; > - intel_uncore_write(uncore, MEMSWCTL, rgvswctl); > - mdelay(1); > + enum pipe pipe; > + u32 val; > > - spin_unlock_irq(&mchdev_lock); > + /* > + * On Ibex Peak and Cougar Point, we need to disable clock > + * gating for the panel power sequencer or it will fail to > + * start up when no ports are active. > + */ > + I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE | > + PCH_DPLUNIT_CLOCK_GATE_DISABLE | > + PCH_CPUNIT_CLOCK_GATE_DISABLE); > + I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) | > + DPLS_EDP_PPS_FIX_DIS); > + /* The below fixes the weird display corruption, a few pixels shifted > + * downward, on (only) LVDS of some HP laptops with IVY. > + */ > + for_each_pipe(dev_priv, pipe) { > + val = I915_READ(TRANS_CHICKEN2(pipe)); > + val |= TRANS_CHICKEN2_TIMING_OVERRIDE; > + val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED; > + if (dev_priv->vbt.fdi_rx_polarity_inverted) > + val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED; > + val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK; > + val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER; > + val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH; > + I915_WRITE(TRANS_CHICKEN2(pipe), val); > + } > + /* WADP0ClockGatingDisable */ > + for_each_pipe(dev_priv, pipe) { > + I915_WRITE(TRANS_CHICKEN1(pipe), > + TRANS_CHICKEN1_DP0UNIT_GC_DISABLE); > + } > } > > -/* There's a funny hw issue where the hw returns all 0 when reading from > - * GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value > - * ourselves, instead of doing a rmw cycle (which might result in us clearing > - * all limits and the gpu stuck at whatever frequency it is at atm). > - */ > -static u32 intel_rps_limits(struct drm_i915_private *dev_priv, u8 val) > +static void gen6_check_mch_setup(struct drm_i915_private *dev_priv) > { > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > - u32 limits; > - > - /* Only set the down limit when we've reached the lowest level to avoid > - * getting more interrupts, otherwise leave this clear. This prevents a > - * race in the hw when coming out of rc6: There's a tiny window where > - * the hw runs at the minimal clock before selecting the desired > - * frequency, if the down threshold expires in that window we will not > - * receive a down interrupt. */ > - if (INTEL_GEN(dev_priv) >= 9) { > - limits = (rps->max_freq_softlimit) << 23; > - if (val <= rps->min_freq_softlimit) > - limits |= (rps->min_freq_softlimit) << 14; > - } else { > - limits = rps->max_freq_softlimit << 24; > - if (val <= rps->min_freq_softlimit) > - limits |= rps->min_freq_softlimit << 16; > - } > + u32 tmp; > > - return limits; > + tmp = I915_READ(MCH_SSKPD); > + if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL) > + DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n", > + tmp); > } > > -static void rps_set_power(struct drm_i915_private *dev_priv, int new_power) > +static void gen6_init_clock_gating(struct drm_i915_private *dev_priv) > { > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > - u32 threshold_up = 0, threshold_down = 0; /* in % */ > - u32 ei_up = 0, ei_down = 0; > - > - lockdep_assert_held(&rps->power.mutex); > + u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE; > > - if (new_power == rps->power.mode) > - return; > + I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate); > > - /* Note the units here are not exactly 1us, but 1280ns. */ > - switch (new_power) { > - case LOW_POWER: > - /* Upclock if more than 95% busy over 16ms */ > - ei_up = 16000; > - threshold_up = 95; > + I915_WRITE(ILK_DISPLAY_CHICKEN2, > + I915_READ(ILK_DISPLAY_CHICKEN2) | > + ILK_ELPIN_409_SELECT); > > - /* Downclock if less than 85% busy over 32ms */ > - ei_down = 32000; > - threshold_down = 85; > - break; > + /* WaDisableHiZPlanesWhenMSAAEnabled:snb */ > + I915_WRITE(_3D_CHICKEN, > + _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB)); > > - case BETWEEN: > - /* Upclock if more than 90% busy over 13ms */ > - ei_up = 13000; > - threshold_up = 90; > + /* WaDisable_RenderCache_OperationalFlush:snb */ > + I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); > > - /* Downclock if less than 75% busy over 32ms */ > - ei_down = 32000; > - threshold_down = 75; > - break; > + /* > + * BSpec recoomends 8x4 when MSAA is used, > + * however in practice 16x4 seems fastest. > + * > + * Note that PS/WM thread counts depend on the WIZ hashing > + * disable bit, which we don't touch here, but it's good > + * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). > + */ > + I915_WRITE(GEN6_GT_MODE, > + _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4)); > > - case HIGH_POWER: > - /* Upclock if more than 85% busy over 10ms */ > - ei_up = 10000; > - threshold_up = 85; > + I915_WRITE(CACHE_MODE_0, > + _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB)); > > - /* Downclock if less than 60% busy over 32ms */ > - ei_down = 32000; > - threshold_down = 60; > - break; > - } > + I915_WRITE(GEN6_UCGCTL1, > + I915_READ(GEN6_UCGCTL1) | > + GEN6_BLBUNIT_CLOCK_GATE_DISABLE | > + GEN6_CSUNIT_CLOCK_GATE_DISABLE); > > - /* When byt can survive without system hang with dynamic > - * sw freq adjustments, this restriction can be lifted. > + /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock > + * gating disable must be set. Failure to set it results in > + * flickering pixels due to Z write ordering failures after > + * some amount of runtime in the Mesa "fire" demo, and Unigine > + * Sanctuary and Tropics, and apparently anything else with > + * alpha test or pixel discard. > + * > + * According to the spec, bit 11 (RCCUNIT) must also be set, > + * but we didn't debug actual testcases to find it out. > + * > + * WaDisableRCCUnitClockGating:snb > + * WaDisableRCPBUnitClockGating:snb > */ > - if (IS_VALLEYVIEW(dev_priv)) > - goto skip_hw_write; > - > - I915_WRITE(GEN6_RP_UP_EI, > - GT_INTERVAL_FROM_US(dev_priv, ei_up)); > - I915_WRITE(GEN6_RP_UP_THRESHOLD, > - GT_INTERVAL_FROM_US(dev_priv, > - ei_up * threshold_up / 100)); > - > - I915_WRITE(GEN6_RP_DOWN_EI, > - GT_INTERVAL_FROM_US(dev_priv, ei_down)); > - I915_WRITE(GEN6_RP_DOWN_THRESHOLD, > - GT_INTERVAL_FROM_US(dev_priv, > - ei_down * threshold_down / 100)); > - > - I915_WRITE(GEN6_RP_CONTROL, > - (INTEL_GEN(dev_priv) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) | > - GEN6_RP_MEDIA_HW_NORMAL_MODE | > - GEN6_RP_MEDIA_IS_GFX | > - GEN6_RP_ENABLE | > - GEN6_RP_UP_BUSY_AVG | > - GEN6_RP_DOWN_IDLE_AVG); > - > -skip_hw_write: > - rps->power.mode = new_power; > - rps->power.up_threshold = threshold_up; > - rps->power.down_threshold = threshold_down; > -} > - > -static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val) > -{ > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > - int new_power; > - > - new_power = rps->power.mode; > - switch (rps->power.mode) { > - case LOW_POWER: > - if (val > rps->efficient_freq + 1 && > - val > rps->cur_freq) > - new_power = BETWEEN; > - break; > + I915_WRITE(GEN6_UCGCTL2, > + GEN6_RCPBUNIT_CLOCK_GATE_DISABLE | > + GEN6_RCCUNIT_CLOCK_GATE_DISABLE); > > - case BETWEEN: > - if (val <= rps->efficient_freq && > - val < rps->cur_freq) > - new_power = LOW_POWER; > - else if (val >= rps->rp0_freq && > - val > rps->cur_freq) > - new_power = HIGH_POWER; > - break; > + /* WaStripsFansDisableFastClipPerformanceFix:snb */ > + I915_WRITE(_3D_CHICKEN3, > + _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL)); > > - case HIGH_POWER: > - if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 && > - val < rps->cur_freq) > - new_power = BETWEEN; > - break; > - } > - /* Max/min bins are special */ > - if (val <= rps->min_freq_softlimit) > - new_power = LOW_POWER; > - if (val >= rps->max_freq_softlimit) > - new_power = HIGH_POWER; > + /* > + * Bspec says: > + * "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and > + * 3DSTATE_SF number of SF output attributes is more than 16." > + */ > + I915_WRITE(_3D_CHICKEN3, > + _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH)); > > - mutex_lock(&rps->power.mutex); > - if (rps->power.interactive) > - new_power = HIGH_POWER; > - rps_set_power(dev_priv, new_power); > - mutex_unlock(&rps->power.mutex); > -} > + /* > + * According to the spec the following bits should be > + * set in order to enable memory self-refresh and fbc: > + * The bit21 and bit22 of 0x42000 > + * The bit21 and bit22 of 0x42004 > + * The bit5 and bit7 of 0x42020 > + * The bit14 of 0x70180 > + * The bit14 of 0x71180 > + * > + * WaFbcAsynchFlipDisableFbcQueue:snb > + */ > + I915_WRITE(ILK_DISPLAY_CHICKEN1, > + I915_READ(ILK_DISPLAY_CHICKEN1) | > + ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS); > + I915_WRITE(ILK_DISPLAY_CHICKEN2, > + I915_READ(ILK_DISPLAY_CHICKEN2) | > + ILK_DPARB_GATE | ILK_VSDPFD_FULL); > + I915_WRITE(ILK_DSPCLK_GATE_D, > + I915_READ(ILK_DSPCLK_GATE_D) | > + ILK_DPARBUNIT_CLOCK_GATE_ENABLE | > + ILK_DPFDUNIT_CLOCK_GATE_ENABLE); > > -void intel_rps_mark_interactive(struct drm_i915_private *i915, bool interactive) > -{ > - struct intel_rps *rps = &i915->gt_pm.rps; > + g4x_disable_trickle_feed(dev_priv); > > - if (INTEL_GEN(i915) < 6) > - return; > + cpt_init_clock_gating(dev_priv); > > - mutex_lock(&rps->power.mutex); > - if (interactive) { > - if (!rps->power.interactive++ && READ_ONCE(i915->gt.awake)) > - rps_set_power(i915, HIGH_POWER); > - } else { > - GEM_BUG_ON(!rps->power.interactive); > - rps->power.interactive--; > - } > - mutex_unlock(&rps->power.mutex); > + gen6_check_mch_setup(dev_priv); > } > > -static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val) > +static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv) > { > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > - u32 mask = 0; > - > - /* We use UP_EI_EXPIRED interupts for both up/down in manual mode */ > - if (val > rps->min_freq_softlimit) > - mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT; > - if (val < rps->max_freq_softlimit) > - mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD; > - > - mask &= dev_priv->pm_rps_events; > - > - return gen6_sanitize_rps_pm_mask(dev_priv, ~mask); > -} > - > -/* gen6_set_rps is called to update the frequency request, but should also be > - * called when the range (min_delay and max_delay) is modified so that we can > - * update the GEN6_RP_INTERRUPT_LIMITS register accordingly. */ > -static int gen6_set_rps(struct drm_i915_private *dev_priv, u8 val) > -{ > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > - > - /* min/max delay may still have been modified so be sure to > - * write the limits value. > - */ > - if (val != rps->cur_freq) { > - gen6_set_rps_thresholds(dev_priv, val); > - > - if (INTEL_GEN(dev_priv) >= 9) > - I915_WRITE(GEN6_RPNSWREQ, > - GEN9_FREQUENCY(val)); > - else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) > - I915_WRITE(GEN6_RPNSWREQ, > - HSW_FREQUENCY(val)); > - else > - I915_WRITE(GEN6_RPNSWREQ, > - GEN6_FREQUENCY(val) | > - GEN6_OFFSET(0) | > - GEN6_AGGRESSIVE_TURBO); > - } > - > - /* Make sure we continue to get interrupts > - * until we hit the minimum or maximum frequencies. > - */ > - I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, intel_rps_limits(dev_priv, val)); > - I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val)); > - > - rps->cur_freq = val; > - trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val)); > - > - return 0; > -} > - > -static int valleyview_set_rps(struct drm_i915_private *dev_priv, u8 val) > -{ > - int err; > - > - if (WARN_ONCE(IS_CHERRYVIEW(dev_priv) && (val & 1), > - "Odd GPU freq value\n")) > - val &= ~1; > - > - I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val)); > - > - if (val != dev_priv->gt_pm.rps.cur_freq) { > - vlv_punit_get(dev_priv); > - err = vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val); > - vlv_punit_put(dev_priv); > - if (err) > - return err; > - > - gen6_set_rps_thresholds(dev_priv, val); > - } > - > - dev_priv->gt_pm.rps.cur_freq = val; > - trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val)); > - > - return 0; > -} > - > -/* vlv_set_rps_idle: Set the frequency to idle, if Gfx clocks are down > - * > - * * If Gfx is Idle, then > - * 1. Forcewake Media well. > - * 2. Request idle freq. > - * 3. Release Forcewake of Media well. > -*/ > -static void vlv_set_rps_idle(struct drm_i915_private *dev_priv) > -{ > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > - u32 val = rps->idle_freq; > - int err; > - > - if (rps->cur_freq <= val) > - return; > - > - /* The punit delays the write of the frequency and voltage until it > - * determines the GPU is awake. During normal usage we don't want to > - * waste power changing the frequency if the GPU is sleeping (rc6). > - * However, the GPU and driver is now idle and we do not want to delay > - * switching to minimum voltage (reducing power whilst idle) as we do > - * not expect to be woken in the near future and so must flush the > - * change by waking the device. > - * > - * We choose to take the media powerwell (either would do to trick the > - * punit into committing the voltage change) as that takes a lot less > - * power than the render powerwell. > - */ > - intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_MEDIA); > - err = valleyview_set_rps(dev_priv, val); > - intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_MEDIA); > - > - if (err) > - DRM_ERROR("Failed to set RPS for idle\n"); > -} > - > -void gen6_rps_busy(struct drm_i915_private *dev_priv) > -{ > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > - > - mutex_lock(&rps->lock); > - if (rps->enabled) { > - u8 freq; > - > - if (dev_priv->pm_rps_events & GEN6_PM_RP_UP_EI_EXPIRED) > - gen6_rps_reset_ei(dev_priv); > - I915_WRITE(GEN6_PMINTRMSK, > - gen6_rps_pm_mask(dev_priv, rps->cur_freq)); > - > - gen6_enable_rps_interrupts(dev_priv); > - > - /* Use the user's desired frequency as a guide, but for better > - * performance, jump directly to RPe as our starting frequency. > - */ > - freq = max(rps->cur_freq, > - rps->efficient_freq); > - > - if (intel_set_rps(dev_priv, > - clamp(freq, > - rps->min_freq_softlimit, > - rps->max_freq_softlimit))) > - DRM_DEBUG_DRIVER("Failed to set idle frequency\n"); > - } > - mutex_unlock(&rps->lock); > -} > - > -void gen6_rps_idle(struct drm_i915_private *dev_priv) > -{ > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > - > - /* Flush our bottom-half so that it does not race with us > - * setting the idle frequency and so that it is bounded by > - * our rpm wakeref. And then disable the interrupts to stop any > - * futher RPS reclocking whilst we are asleep. > - */ > - gen6_disable_rps_interrupts(dev_priv); > - > - mutex_lock(&rps->lock); > - if (rps->enabled) { > - if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) > - vlv_set_rps_idle(dev_priv); > - else > - gen6_set_rps(dev_priv, rps->idle_freq); > - rps->last_adj = 0; > - I915_WRITE(GEN6_PMINTRMSK, > - gen6_sanitize_rps_pm_mask(dev_priv, ~0)); > - } > - mutex_unlock(&rps->lock); > -} > - > -void gen6_rps_boost(struct i915_request *rq) > -{ > - struct intel_rps *rps = &rq->i915->gt_pm.rps; > - unsigned long flags; > - bool boost; > - > - /* This is intentionally racy! We peek at the state here, then > - * validate inside the RPS worker. > - */ > - if (!rps->enabled) > - return; > - > - if (i915_request_signaled(rq)) > - return; > - > - /* Serializes with i915_request_retire() */ > - boost = false; > - spin_lock_irqsave(&rq->lock, flags); > - if (!i915_request_has_waitboost(rq) && > - !dma_fence_is_signaled_locked(&rq->fence)) { > - boost = !atomic_fetch_inc(&rps->num_waiters); > - rq->flags |= I915_REQUEST_WAITBOOST; > - } > - spin_unlock_irqrestore(&rq->lock, flags); > - if (!boost) > - return; > - > - if (READ_ONCE(rps->cur_freq) < rps->boost_freq) > - schedule_work(&rps->work); > - > - atomic_inc(&rps->boosts); > -} > - > -int intel_set_rps(struct drm_i915_private *dev_priv, u8 val) > -{ > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > - int err; > - > - lockdep_assert_held(&rps->lock); > - GEM_BUG_ON(val > rps->max_freq); > - GEM_BUG_ON(val < rps->min_freq); > - > - if (!rps->enabled) { > - rps->cur_freq = val; > - return 0; > - } > - > - if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) > - err = valleyview_set_rps(dev_priv, val); > - else > - err = gen6_set_rps(dev_priv, val); > - > - return err; > -} > - > -static void gen9_disable_rps(struct drm_i915_private *dev_priv) > -{ > - I915_WRITE(GEN6_RP_CONTROL, 0); > -} > - > -static void gen6_disable_rps(struct drm_i915_private *dev_priv) > -{ > - I915_WRITE(GEN6_RPNSWREQ, 1 << 31); > - I915_WRITE(GEN6_RP_CONTROL, 0); > -} > - > -static void cherryview_disable_rps(struct drm_i915_private *dev_priv) > -{ > - I915_WRITE(GEN6_RP_CONTROL, 0); > -} > - > -static void valleyview_disable_rps(struct drm_i915_private *dev_priv) > -{ > - I915_WRITE(GEN6_RP_CONTROL, 0); > -} > - > -static void gen6_init_rps_frequencies(struct drm_i915_private *dev_priv) > -{ > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > - > - /* All of these values are in units of 50MHz */ > - > - /* static values from HW: RP0 > RP1 > RPn (min_freq) */ > - if (IS_GEN9_LP(dev_priv)) { > - u32 rp_state_cap = I915_READ(BXT_RP_STATE_CAP); > - rps->rp0_freq = (rp_state_cap >> 16) & 0xff; > - rps->rp1_freq = (rp_state_cap >> 8) & 0xff; > - rps->min_freq = (rp_state_cap >> 0) & 0xff; > - } else { > - u32 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP); > - rps->rp0_freq = (rp_state_cap >> 0) & 0xff; > - rps->rp1_freq = (rp_state_cap >> 8) & 0xff; > - rps->min_freq = (rp_state_cap >> 16) & 0xff; > - } > - /* hw_max = RP0 until we check for overclocking */ > - rps->max_freq = rps->rp0_freq; > - > - rps->efficient_freq = rps->rp1_freq; > - if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv) || > - IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) { > - u32 ddcc_status = 0; > - > - if (sandybridge_pcode_read(dev_priv, > - HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL, > - &ddcc_status, NULL) == 0) > - rps->efficient_freq = > - clamp_t(u8, > - ((ddcc_status >> 8) & 0xff), > - rps->min_freq, > - rps->max_freq); > - } > - > - if (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) { > - /* Store the frequency values in 16.66 MHZ units, which is > - * the natural hardware unit for SKL > - */ > - rps->rp0_freq *= GEN9_FREQ_SCALER; > - rps->rp1_freq *= GEN9_FREQ_SCALER; > - rps->min_freq *= GEN9_FREQ_SCALER; > - rps->max_freq *= GEN9_FREQ_SCALER; > - rps->efficient_freq *= GEN9_FREQ_SCALER; > - } > -} > - > -static void reset_rps(struct drm_i915_private *dev_priv, > - int (*set)(struct drm_i915_private *, u8)) > -{ > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > - u8 freq = rps->cur_freq; > - > - /* force a reset */ > - rps->power.mode = -1; > - rps->cur_freq = -1; > - > - if (set(dev_priv, freq)) > - DRM_ERROR("Failed to reset RPS to initial values\n"); > -} > - > -/* See the Gen9_GT_PM_Programming_Guide doc for the below */ > -static void gen9_enable_rps(struct drm_i915_private *dev_priv) > -{ > - intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL); > - > - /* Program defaults and thresholds for RPS */ > - if (IS_GEN(dev_priv, 9)) > - I915_WRITE(GEN6_RC_VIDEO_FREQ, > - GEN9_FREQUENCY(dev_priv->gt_pm.rps.rp1_freq)); > - > - /* 1 second timeout*/ > - I915_WRITE(GEN6_RP_DOWN_TIMEOUT, > - GT_INTERVAL_FROM_US(dev_priv, 1000000)); > - > - I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 0xa); > - > - /* Leaning on the below call to gen6_set_rps to program/setup the > - * Up/Down EI & threshold registers, as well as the RP_CONTROL, > - * RP_INTERRUPT_LIMITS & RPNSWREQ registers */ > - reset_rps(dev_priv, gen6_set_rps); > - > - intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL); > -} > - > -static void gen8_enable_rps(struct drm_i915_private *dev_priv) > -{ > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > - > - intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL); > - > - /* 1 Program defaults and thresholds for RPS*/ > - I915_WRITE(GEN6_RPNSWREQ, > - HSW_FREQUENCY(rps->rp1_freq)); > - I915_WRITE(GEN6_RC_VIDEO_FREQ, > - HSW_FREQUENCY(rps->rp1_freq)); > - /* NB: Docs say 1s, and 1000000 - which aren't equivalent */ > - I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128); /* 1 second timeout */ > - > - /* Docs recommend 900MHz, and 300 MHz respectively */ > - I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, > - rps->max_freq_softlimit << 24 | > - rps->min_freq_softlimit << 16); > - > - I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */ > - I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/ > - I915_WRITE(GEN6_RP_UP_EI, 66000); /* 84.48ms, XXX: random? */ > - I915_WRITE(GEN6_RP_DOWN_EI, 350000); /* 448ms, XXX: random? */ > - > - I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10); > - > - /* 2: Enable RPS */ > - I915_WRITE(GEN6_RP_CONTROL, > - GEN6_RP_MEDIA_TURBO | > - GEN6_RP_MEDIA_HW_NORMAL_MODE | > - GEN6_RP_MEDIA_IS_GFX | > - GEN6_RP_ENABLE | > - GEN6_RP_UP_BUSY_AVG | > - GEN6_RP_DOWN_IDLE_AVG); > - > - reset_rps(dev_priv, gen6_set_rps); > - > - intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL); > -} > - > -static void gen6_enable_rps(struct drm_i915_private *dev_priv) > -{ > - /* Here begins a magic sequence of register writes to enable > - * auto-downclocking. > - * > - * Perhaps there might be some value in exposing these to > - * userspace... > - */ > - intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL); > - > - /* Power down if completely idle for over 50ms */ > - I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000); > - I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10); > - > - reset_rps(dev_priv, gen6_set_rps); > - > - intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL); > -} > - > -static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv) > -{ > - u32 val, rp0; > - > - val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE); > - > - switch (RUNTIME_INFO(dev_priv)->sseu.eu_total) { > - case 8: > - /* (2 * 4) config */ > - rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT); > - break; > - case 12: > - /* (2 * 6) config */ > - rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT); > - break; > - case 16: > - /* (2 * 8) config */ > - default: > - /* Setting (2 * 8) Min RP0 for any other combination */ > - rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT); > - break; > - } > - > - rp0 = (rp0 & FB_GFX_FREQ_FUSE_MASK); > - > - return rp0; > -} > - > -static int cherryview_rps_rpe_freq(struct drm_i915_private *dev_priv) > -{ > - u32 val, rpe; > - > - val = vlv_punit_read(dev_priv, PUNIT_GPU_DUTYCYCLE_REG); > - rpe = (val >> PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT) & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK; > - > - return rpe; > -} > - > -static int cherryview_rps_guar_freq(struct drm_i915_private *dev_priv) > -{ > - u32 val, rp1; > - > - val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE); > - rp1 = (val & FB_GFX_FREQ_FUSE_MASK); > - > - return rp1; > -} > - > -static u32 cherryview_rps_min_freq(struct drm_i915_private *dev_priv) > -{ > - u32 val, rpn; > - > - val = vlv_punit_read(dev_priv, FB_GFX_FMIN_AT_VMIN_FUSE); > - rpn = ((val >> FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT) & > - FB_GFX_FREQ_FUSE_MASK); > - > - return rpn; > -} > - > -static int valleyview_rps_guar_freq(struct drm_i915_private *dev_priv) > -{ > - u32 val, rp1; > - > - val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE); > - > - rp1 = (val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK) >> FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT; > - > - return rp1; > -} > - > -static int valleyview_rps_max_freq(struct drm_i915_private *dev_priv) > -{ > - u32 val, rp0; > - > - val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE); > - > - rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT; > - /* Clamp to max */ > - rp0 = min_t(u32, rp0, 0xea); > - > - return rp0; > -} > - > -static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv) > -{ > - u32 val, rpe; > - > - val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO); > - rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT; > - val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI); > - rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5; > - > - return rpe; > -} > - > -static int valleyview_rps_min_freq(struct drm_i915_private *dev_priv) > -{ > - u32 val; > - > - val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff; > - /* > - * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value > - * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on > - * a BYT-M B0 the above register contains 0xbf. Moreover when setting > - * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0 > - * to make sure it matches what Punit accepts. > - */ > - return max_t(u32, val, 0xc0); > -} > - > -static void vlv_init_gpll_ref_freq(struct drm_i915_private *dev_priv) > -{ > - dev_priv->gt_pm.rps.gpll_ref_freq = > - vlv_get_cck_clock(dev_priv, "GPLL ref", > - CCK_GPLL_CLOCK_CONTROL, > - dev_priv->czclk_freq); > - > - DRM_DEBUG_DRIVER("GPLL reference freq: %d kHz\n", > - dev_priv->gt_pm.rps.gpll_ref_freq); > -} > - > -static void valleyview_init_gt_powersave(struct drm_i915_private *dev_priv) > -{ > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > - u32 val; > - > - vlv_iosf_sb_get(dev_priv, > - BIT(VLV_IOSF_SB_PUNIT) | > - BIT(VLV_IOSF_SB_NC) | > - BIT(VLV_IOSF_SB_CCK)); > - > - vlv_init_gpll_ref_freq(dev_priv); > - > - val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); > - switch ((val >> 6) & 3) { > - case 0: > - case 1: > - dev_priv->mem_freq = 800; > - break; > - case 2: > - dev_priv->mem_freq = 1066; > - break; > - case 3: > - dev_priv->mem_freq = 1333; > - break; > - } > - DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq); > - > - rps->max_freq = valleyview_rps_max_freq(dev_priv); > - rps->rp0_freq = rps->max_freq; > - DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n", > - intel_gpu_freq(dev_priv, rps->max_freq), > - rps->max_freq); > - > - rps->efficient_freq = valleyview_rps_rpe_freq(dev_priv); > - DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n", > - intel_gpu_freq(dev_priv, rps->efficient_freq), > - rps->efficient_freq); > - > - rps->rp1_freq = valleyview_rps_guar_freq(dev_priv); > - DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n", > - intel_gpu_freq(dev_priv, rps->rp1_freq), > - rps->rp1_freq); > - > - rps->min_freq = valleyview_rps_min_freq(dev_priv); > - DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n", > - intel_gpu_freq(dev_priv, rps->min_freq), > - rps->min_freq); > - > - vlv_iosf_sb_put(dev_priv, > - BIT(VLV_IOSF_SB_PUNIT) | > - BIT(VLV_IOSF_SB_NC) | > - BIT(VLV_IOSF_SB_CCK)); > -} > - > -static void cherryview_init_gt_powersave(struct drm_i915_private *dev_priv) > -{ > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > - u32 val; > - > - vlv_iosf_sb_get(dev_priv, > - BIT(VLV_IOSF_SB_PUNIT) | > - BIT(VLV_IOSF_SB_NC) | > - BIT(VLV_IOSF_SB_CCK)); > - > - vlv_init_gpll_ref_freq(dev_priv); > - > - val = vlv_cck_read(dev_priv, CCK_FUSE_REG); > - > - switch ((val >> 2) & 0x7) { > - case 3: > - dev_priv->mem_freq = 2000; > - break; > - default: > - dev_priv->mem_freq = 1600; > - break; > - } > - DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq); > - > - rps->max_freq = cherryview_rps_max_freq(dev_priv); > - rps->rp0_freq = rps->max_freq; > - DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n", > - intel_gpu_freq(dev_priv, rps->max_freq), > - rps->max_freq); > - > - rps->efficient_freq = cherryview_rps_rpe_freq(dev_priv); > - DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n", > - intel_gpu_freq(dev_priv, rps->efficient_freq), > - rps->efficient_freq); > - > - rps->rp1_freq = cherryview_rps_guar_freq(dev_priv); > - DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n", > - intel_gpu_freq(dev_priv, rps->rp1_freq), > - rps->rp1_freq); > - > - rps->min_freq = cherryview_rps_min_freq(dev_priv); > - DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n", > - intel_gpu_freq(dev_priv, rps->min_freq), > - rps->min_freq); > - > - vlv_iosf_sb_put(dev_priv, > - BIT(VLV_IOSF_SB_PUNIT) | > - BIT(VLV_IOSF_SB_NC) | > - BIT(VLV_IOSF_SB_CCK)); > - > - WARN_ONCE((rps->max_freq | rps->efficient_freq | rps->rp1_freq | > - rps->min_freq) & 1, > - "Odd GPU freq values\n"); > -} > - > -static void cherryview_enable_rps(struct drm_i915_private *dev_priv) > -{ > - u32 val; > - > - intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL); > - > - /* 1: Program defaults and thresholds for RPS*/ > - I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000); > - I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400); > - I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000); > - I915_WRITE(GEN6_RP_UP_EI, 66000); > - I915_WRITE(GEN6_RP_DOWN_EI, 350000); > - > - I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10); > - > - /* 2: Enable RPS */ > - I915_WRITE(GEN6_RP_CONTROL, > - GEN6_RP_MEDIA_HW_NORMAL_MODE | > - GEN6_RP_MEDIA_IS_GFX | > - GEN6_RP_ENABLE | > - GEN6_RP_UP_BUSY_AVG | > - GEN6_RP_DOWN_IDLE_AVG); > - > - /* Setting Fixed Bias */ > - vlv_punit_get(dev_priv); > - > - val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50; > - vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val); > - > - val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); > - > - vlv_punit_put(dev_priv); > - > - /* RPS code assumes GPLL is used */ > - WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n"); > - > - DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE)); > - DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val); > - > - reset_rps(dev_priv, valleyview_set_rps); > - > - intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL); > -} > - > -static void valleyview_enable_rps(struct drm_i915_private *dev_priv) > -{ > - u32 val; > - > - intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL); > - > - I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000); > - I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400); > - I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000); > - I915_WRITE(GEN6_RP_UP_EI, 66000); > - I915_WRITE(GEN6_RP_DOWN_EI, 350000); > - > - I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10); > - > - I915_WRITE(GEN6_RP_CONTROL, > - GEN6_RP_MEDIA_TURBO | > - GEN6_RP_MEDIA_HW_NORMAL_MODE | > - GEN6_RP_MEDIA_IS_GFX | > - GEN6_RP_ENABLE | > - GEN6_RP_UP_BUSY_AVG | > - GEN6_RP_DOWN_IDLE_CONT); > - > - vlv_punit_get(dev_priv); > - > - /* Setting Fixed Bias */ > - val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875; > - vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val); > - > - val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); > - > - vlv_punit_put(dev_priv); > - > - /* RPS code assumes GPLL is used */ > - WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n"); > - > - DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE)); > - DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val); > - > - reset_rps(dev_priv, valleyview_set_rps); > - > - intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL); > -} > - > -static unsigned long intel_pxfreq(u32 vidfreq) > -{ > - unsigned long freq; > - int div = (vidfreq & 0x3f0000) >> 16; > - int post = (vidfreq & 0x3000) >> 12; > - int pre = (vidfreq & 0x7); > - > - if (!pre) > - return 0; > - > - freq = ((div * 133333) / ((1<<post) * pre)); > - > - return freq; > -} > - > -static const struct cparams { > - u16 i; > - u16 t; > - u16 m; > - u16 c; > -} cparams[] = { > - { 1, 1333, 301, 28664 }, > - { 1, 1066, 294, 24460 }, > - { 1, 800, 294, 25192 }, > - { 0, 1333, 276, 27605 }, > - { 0, 1066, 276, 27605 }, > - { 0, 800, 231, 23784 }, > -}; > - > -static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv) > -{ > - u64 total_count, diff, ret; > - u32 count1, count2, count3, m = 0, c = 0; > - unsigned long now = jiffies_to_msecs(jiffies), diff1; > - int i; > - > - lockdep_assert_held(&mchdev_lock); > - > - diff1 = now - dev_priv->ips.last_time1; > - > - /* Prevent division-by-zero if we are asking too fast. > - * Also, we don't get interesting results if we are polling > - * faster than once in 10ms, so just return the saved value > - * in such cases. > - */ > - if (diff1 <= 10) > - return dev_priv->ips.chipset_power; > - > - count1 = I915_READ(DMIEC); > - count2 = I915_READ(DDREC); > - count3 = I915_READ(CSIEC); > - > - total_count = count1 + count2 + count3; > - > - /* FIXME: handle per-counter overflow */ > - if (total_count < dev_priv->ips.last_count1) { > - diff = ~0UL - dev_priv->ips.last_count1; > - diff += total_count; > - } else { > - diff = total_count - dev_priv->ips.last_count1; > - } > - > - for (i = 0; i < ARRAY_SIZE(cparams); i++) { > - if (cparams[i].i == dev_priv->ips.c_m && > - cparams[i].t == dev_priv->ips.r_t) { > - m = cparams[i].m; > - c = cparams[i].c; > - break; > - } > - } > - > - diff = div_u64(diff, diff1); > - ret = ((m * diff) + c); > - ret = div_u64(ret, 10); > - > - dev_priv->ips.last_count1 = total_count; > - dev_priv->ips.last_time1 = now; > - > - dev_priv->ips.chipset_power = ret; > - > - return ret; > -} > - > -unsigned long i915_chipset_val(struct drm_i915_private *dev_priv) > -{ > - intel_wakeref_t wakeref; > - unsigned long val = 0; > - > - if (!IS_GEN(dev_priv, 5)) > - return 0; > - > - with_intel_runtime_pm(&dev_priv->runtime_pm, wakeref) { > - spin_lock_irq(&mchdev_lock); > - val = __i915_chipset_val(dev_priv); > - spin_unlock_irq(&mchdev_lock); > - } > - > - return val; > -} > - > -unsigned long i915_mch_val(struct drm_i915_private *i915) > -{ > - unsigned long m, x, b; > - u32 tsfs; > - > - tsfs = intel_uncore_read(&i915->uncore, TSFS); > - > - m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT); > - x = intel_uncore_read8(&i915->uncore, TR1); > - > - b = tsfs & TSFS_INTR_MASK; > - > - return ((m * x) / 127) - b; > -} > - > -static int _pxvid_to_vd(u8 pxvid) > -{ > - if (pxvid == 0) > - return 0; > - > - if (pxvid >= 8 && pxvid < 31) > - pxvid = 31; > - > - return (pxvid + 2) * 125; > -} > - > -static u32 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid) > -{ > - const int vd = _pxvid_to_vd(pxvid); > - const int vm = vd - 1125; > - > - if (INTEL_INFO(dev_priv)->is_mobile) > - return vm > 0 ? vm : 0; > - > - return vd; > -} > - > -static void __i915_update_gfx_val(struct drm_i915_private *dev_priv) > -{ > - u64 now, diff, diffms; > - u32 count; > - > - lockdep_assert_held(&mchdev_lock); > - > - now = ktime_get_raw_ns(); > - diffms = now - dev_priv->ips.last_time2; > - do_div(diffms, NSEC_PER_MSEC); > - > - /* Don't divide by 0 */ > - if (!diffms) > - return; > - > - count = I915_READ(GFXEC); > - > - if (count < dev_priv->ips.last_count2) { > - diff = ~0UL - dev_priv->ips.last_count2; > - diff += count; > - } else { > - diff = count - dev_priv->ips.last_count2; > - } > - > - dev_priv->ips.last_count2 = count; > - dev_priv->ips.last_time2 = now; > - > - /* More magic constants... */ > - diff = diff * 1181; > - diff = div_u64(diff, diffms * 10); > - dev_priv->ips.gfx_power = diff; > -} > - > -void i915_update_gfx_val(struct drm_i915_private *dev_priv) > -{ > - intel_wakeref_t wakeref; > - > - if (!IS_GEN(dev_priv, 5)) > - return; > - > - with_intel_runtime_pm(&dev_priv->runtime_pm, wakeref) { > - spin_lock_irq(&mchdev_lock); > - __i915_update_gfx_val(dev_priv); > - spin_unlock_irq(&mchdev_lock); > - } > -} > - > -static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv) > -{ > - unsigned long t, corr, state1, corr2, state2; > - u32 pxvid, ext_v; > - > - lockdep_assert_held(&mchdev_lock); > - > - pxvid = I915_READ(PXVFREQ(dev_priv->gt_pm.rps.cur_freq)); > - pxvid = (pxvid >> 24) & 0x7f; > - ext_v = pvid_to_extvid(dev_priv, pxvid); > - > - state1 = ext_v; > - > - t = i915_mch_val(dev_priv); > - > - /* Revel in the empirically derived constants */ > - > - /* Correction factor in 1/100000 units */ > - if (t > 80) > - corr = ((t * 2349) + 135940); > - else if (t >= 50) > - corr = ((t * 964) + 29317); > - else /* < 50 */ > - corr = ((t * 301) + 1004); > - > - corr = corr * ((150142 * state1) / 10000 - 78642); > - corr /= 100000; > - corr2 = (corr * dev_priv->ips.corr); > - > - state2 = (corr2 * state1) / 10000; > - state2 /= 100; /* convert to mW */ > - > - __i915_update_gfx_val(dev_priv); > - > - return dev_priv->ips.gfx_power + state2; > -} > - > -unsigned long i915_gfx_val(struct drm_i915_private *dev_priv) > -{ > - intel_wakeref_t wakeref; > - unsigned long val = 0; > - > - if (!IS_GEN(dev_priv, 5)) > - return 0; > - > - with_intel_runtime_pm(&dev_priv->runtime_pm, wakeref) { > - spin_lock_irq(&mchdev_lock); > - val = __i915_gfx_val(dev_priv); > - spin_unlock_irq(&mchdev_lock); > - } > - > - return val; > -} > - > -static struct drm_i915_private __rcu *i915_mch_dev; > - > -static struct drm_i915_private *mchdev_get(void) > -{ > - struct drm_i915_private *i915; > - > - rcu_read_lock(); > - i915 = rcu_dereference(i915_mch_dev); > - if (!kref_get_unless_zero(&i915->drm.ref)) > - i915 = NULL; > - rcu_read_unlock(); > - > - return i915; > -} > - > -/** > - * i915_read_mch_val - return value for IPS use > - * > - * Calculate and return a value for the IPS driver to use when deciding whether > - * we have thermal and power headroom to increase CPU or GPU power budget. > - */ > -unsigned long i915_read_mch_val(void) > -{ > - struct drm_i915_private *i915; > - unsigned long chipset_val = 0; > - unsigned long graphics_val = 0; > - intel_wakeref_t wakeref; > - > - i915 = mchdev_get(); > - if (!i915) > - return 0; > - > - with_intel_runtime_pm(&i915->runtime_pm, wakeref) { > - spin_lock_irq(&mchdev_lock); > - chipset_val = __i915_chipset_val(i915); > - graphics_val = __i915_gfx_val(i915); > - spin_unlock_irq(&mchdev_lock); > - } > - > - drm_dev_put(&i915->drm); > - return chipset_val + graphics_val; > -} > -EXPORT_SYMBOL_GPL(i915_read_mch_val); > - > -/** > - * i915_gpu_raise - raise GPU frequency limit > - * > - * Raise the limit; IPS indicates we have thermal headroom. > - */ > -bool i915_gpu_raise(void) > -{ > - struct drm_i915_private *i915; > - > - i915 = mchdev_get(); > - if (!i915) > - return false; > - > - spin_lock_irq(&mchdev_lock); > - if (i915->ips.max_delay > i915->ips.fmax) > - i915->ips.max_delay--; > - spin_unlock_irq(&mchdev_lock); > - > - drm_dev_put(&i915->drm); > - return true; > -} > -EXPORT_SYMBOL_GPL(i915_gpu_raise); > - > -/** > - * i915_gpu_lower - lower GPU frequency limit > - * > - * IPS indicates we're close to a thermal limit, so throttle back the GPU > - * frequency maximum. > - */ > -bool i915_gpu_lower(void) > -{ > - struct drm_i915_private *i915; > - > - i915 = mchdev_get(); > - if (!i915) > - return false; > - > - spin_lock_irq(&mchdev_lock); > - if (i915->ips.max_delay < i915->ips.min_delay) > - i915->ips.max_delay++; > - spin_unlock_irq(&mchdev_lock); > - > - drm_dev_put(&i915->drm); > - return true; > -} > -EXPORT_SYMBOL_GPL(i915_gpu_lower); > - > -/** > - * i915_gpu_busy - indicate GPU business to IPS > - * > - * Tell the IPS driver whether or not the GPU is busy. > - */ > -bool i915_gpu_busy(void) > -{ > - struct drm_i915_private *i915; > - bool ret; > - > - i915 = mchdev_get(); > - if (!i915) > - return false; > - > - ret = i915->gt.awake; > - > - drm_dev_put(&i915->drm); > - return ret; > -} > -EXPORT_SYMBOL_GPL(i915_gpu_busy); > - > -/** > - * i915_gpu_turbo_disable - disable graphics turbo > - * > - * Disable graphics turbo by resetting the max frequency and setting the > - * current frequency to the default. > - */ > -bool i915_gpu_turbo_disable(void) > -{ > - struct drm_i915_private *i915; > - bool ret; > - > - i915 = mchdev_get(); > - if (!i915) > - return false; > - > - spin_lock_irq(&mchdev_lock); > - i915->ips.max_delay = i915->ips.fstart; > - ret = ironlake_set_drps(i915, i915->ips.fstart); > - spin_unlock_irq(&mchdev_lock); > - > - drm_dev_put(&i915->drm); > - return ret; > -} > -EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable); > - > -/** > - * Tells the intel_ips driver that the i915 driver is now loaded, if > - * IPS got loaded first. > - * > - * This awkward dance is so that neither module has to depend on the > - * other in order for IPS to do the appropriate communication of > - * GPU turbo limits to i915. > - */ > -static void > -ips_ping_for_i915_load(void) > -{ > - void (*link)(void); > - > - link = symbol_get(ips_link_to_i915_driver); > - if (link) { > - link(); > - symbol_put(ips_link_to_i915_driver); > - } > -} > - > -void intel_gpu_ips_init(struct drm_i915_private *dev_priv) > -{ > - /* We only register the i915 ips part with intel-ips once everything is > - * set up, to avoid intel-ips sneaking in and reading bogus values. */ > - rcu_assign_pointer(i915_mch_dev, dev_priv); > - > - ips_ping_for_i915_load(); > -} > - > -void intel_gpu_ips_teardown(void) > -{ > - rcu_assign_pointer(i915_mch_dev, NULL); > -} > - > -static void intel_init_emon(struct drm_i915_private *dev_priv) > -{ > - u32 lcfuse; > - u8 pxw[16]; > - int i; > - > - /* Disable to program */ > - I915_WRITE(ECR, 0); > - POSTING_READ(ECR); > - > - /* Program energy weights for various events */ > - I915_WRITE(SDEW, 0x15040d00); > - I915_WRITE(CSIEW0, 0x007f0000); > - I915_WRITE(CSIEW1, 0x1e220004); > - I915_WRITE(CSIEW2, 0x04000004); > - > - for (i = 0; i < 5; i++) > - I915_WRITE(PEW(i), 0); > - for (i = 0; i < 3; i++) > - I915_WRITE(DEW(i), 0); > - > - /* Program P-state weights to account for frequency power adjustment */ > - for (i = 0; i < 16; i++) { > - u32 pxvidfreq = I915_READ(PXVFREQ(i)); > - unsigned long freq = intel_pxfreq(pxvidfreq); > - unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >> > - PXVFREQ_PX_SHIFT; > - unsigned long val; > - > - val = vid * vid; > - val *= (freq / 1000); > - val *= 255; > - val /= (127*127*900); > - if (val > 0xff) > - DRM_ERROR("bad pxval: %ld\n", val); > - pxw[i] = val; > - } > - /* Render standby states get 0 weight */ > - pxw[14] = 0; > - pxw[15] = 0; > - > - for (i = 0; i < 4; i++) { > - u32 val = (pxw[i*4] << 24) | (pxw[(i*4)+1] << 16) | > - (pxw[(i*4)+2] << 8) | (pxw[(i*4)+3]); > - I915_WRITE(PXW(i), val); > - } > - > - /* Adjust magic regs to magic values (more experimental results) */ > - I915_WRITE(OGW0, 0); > - I915_WRITE(OGW1, 0); > - I915_WRITE(EG0, 0x00007f00); > - I915_WRITE(EG1, 0x0000000e); > - I915_WRITE(EG2, 0x000e0000); > - I915_WRITE(EG3, 0x68000300); > - I915_WRITE(EG4, 0x42000000); > - I915_WRITE(EG5, 0x00140031); > - I915_WRITE(EG6, 0); > - I915_WRITE(EG7, 0); > - > - for (i = 0; i < 8; i++) > - I915_WRITE(PXWL(i), 0); > - > - /* Enable PMON + select events */ > - I915_WRITE(ECR, 0x80000019); > - > - lcfuse = I915_READ(LCFUSE02); > - > - dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK); > -} > - > -void intel_init_gt_powersave(struct drm_i915_private *dev_priv) > -{ > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > - > - /* Powersaving is controlled by the host when inside a VM */ > - if (intel_vgpu_active(dev_priv)) > - mkwrite_device_info(dev_priv)->has_rps = false; > - > - /* Initialize RPS limits (for userspace) */ > - if (IS_CHERRYVIEW(dev_priv)) > - cherryview_init_gt_powersave(dev_priv); > - else if (IS_VALLEYVIEW(dev_priv)) > - valleyview_init_gt_powersave(dev_priv); > - else if (INTEL_GEN(dev_priv) >= 6) > - gen6_init_rps_frequencies(dev_priv); > - > - /* Derive initial user preferences/limits from the hardware limits */ > - rps->max_freq_softlimit = rps->max_freq; > - rps->min_freq_softlimit = rps->min_freq; > - > - /* After setting max-softlimit, find the overclock max freq */ > - if (IS_GEN(dev_priv, 6) || > - IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv)) { > - u32 params = 0; > - > - sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, > - ¶ms, NULL); > - if (params & BIT(31)) { /* OC supported */ > - DRM_DEBUG_DRIVER("Overclocking supported, max: %dMHz, overclock: %dMHz\n", > - (rps->max_freq & 0xff) * 50, > - (params & 0xff) * 50); > - rps->max_freq = params & 0xff; > - } > - } > - > - /* Finally allow us to boost to max by default */ > - rps->boost_freq = rps->max_freq; > - rps->idle_freq = rps->min_freq; > - rps->cur_freq = rps->idle_freq; > -} > - > -void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv) > -{ > - dev_priv->gt_pm.rps.enabled = true; /* force RPS disabling */ > - intel_disable_gt_powersave(dev_priv); > - > - if (INTEL_GEN(dev_priv) >= 11) > - gen11_reset_rps_interrupts(dev_priv); > - else if (INTEL_GEN(dev_priv) >= 6) > - gen6_reset_rps_interrupts(dev_priv); > -} > - > -static void intel_disable_rps(struct drm_i915_private *dev_priv) > -{ > - lockdep_assert_held(&dev_priv->gt_pm.rps.lock); > - > - if (!dev_priv->gt_pm.rps.enabled) > - return; > - > - if (INTEL_GEN(dev_priv) >= 9) > - gen9_disable_rps(dev_priv); > - else if (IS_CHERRYVIEW(dev_priv)) > - cherryview_disable_rps(dev_priv); > - else if (IS_VALLEYVIEW(dev_priv)) > - valleyview_disable_rps(dev_priv); > - else if (INTEL_GEN(dev_priv) >= 6) > - gen6_disable_rps(dev_priv); > - else if (IS_IRONLAKE_M(dev_priv)) > - ironlake_disable_drps(dev_priv); > - > - dev_priv->gt_pm.rps.enabled = false; > -} > - > -void intel_disable_gt_powersave(struct drm_i915_private *dev_priv) > -{ > - mutex_lock(&dev_priv->gt_pm.rps.lock); > - > - intel_disable_rps(dev_priv); > - if (HAS_LLC(dev_priv)) > - intel_llc_disable(&dev_priv->gt.llc); > - > - mutex_unlock(&dev_priv->gt_pm.rps.lock); > -} > - > -static void intel_enable_rps(struct drm_i915_private *dev_priv) > -{ > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > - > - lockdep_assert_held(&rps->lock); > - > - if (rps->enabled) > - return; > - > - if (IS_CHERRYVIEW(dev_priv)) { > - cherryview_enable_rps(dev_priv); > - } else if (IS_VALLEYVIEW(dev_priv)) { > - valleyview_enable_rps(dev_priv); > - } else if (INTEL_GEN(dev_priv) >= 9) { > - gen9_enable_rps(dev_priv); > - } else if (IS_BROADWELL(dev_priv)) { > - gen8_enable_rps(dev_priv); > - } else if (INTEL_GEN(dev_priv) >= 6) { > - gen6_enable_rps(dev_priv); > - } else if (IS_IRONLAKE_M(dev_priv)) { > - ironlake_enable_drps(dev_priv); > - intel_init_emon(dev_priv); > - } > - > - WARN_ON(rps->max_freq < rps->min_freq); > - WARN_ON(rps->idle_freq > rps->max_freq); > - > - WARN_ON(rps->efficient_freq < rps->min_freq); > - WARN_ON(rps->efficient_freq > rps->max_freq); > - > - rps->enabled = true; > -} > - > -void intel_enable_gt_powersave(struct drm_i915_private *dev_priv) > -{ > - /* Powersaving is controlled by the host when inside a VM */ > - if (intel_vgpu_active(dev_priv)) > - return; > - > - mutex_lock(&dev_priv->gt_pm.rps.lock); > - > - if (HAS_RPS(dev_priv)) > - intel_enable_rps(dev_priv); > - > - intel_llc_enable(&dev_priv->gt.llc); > - > - mutex_unlock(&dev_priv->gt_pm.rps.lock); > -} > - > -static void ibx_init_clock_gating(struct drm_i915_private *dev_priv) > -{ > - /* > - * On Ibex Peak and Cougar Point, we need to disable clock > - * gating for the panel power sequencer or it will fail to > - * start up when no ports are active. > - */ > - I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE); > -} > - > -static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv) > -{ > - enum pipe pipe; > - > - for_each_pipe(dev_priv, pipe) { > - I915_WRITE(DSPCNTR(pipe), > - I915_READ(DSPCNTR(pipe)) | > - DISPPLANE_TRICKLE_FEED_DISABLE); > - > - I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe))); > - POSTING_READ(DSPSURF(pipe)); > - } > -} > - > -static void ilk_init_clock_gating(struct drm_i915_private *dev_priv) > -{ > - u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE; > - > - /* > - * Required for FBC > - * WaFbcDisableDpfcClockGating:ilk > - */ > - dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE | > - ILK_DPFCUNIT_CLOCK_GATE_DISABLE | > - ILK_DPFDUNIT_CLOCK_GATE_ENABLE; > - > - I915_WRITE(PCH_3DCGDIS0, > - MARIUNIT_CLOCK_GATE_DISABLE | > - SVSMUNIT_CLOCK_GATE_DISABLE); > - I915_WRITE(PCH_3DCGDIS1, > - VFMUNIT_CLOCK_GATE_DISABLE); > - > - /* > - * According to the spec the following bits should be set in > - * order to enable memory self-refresh > - * The bit 22/21 of 0x42004 > - * The bit 5 of 0x42020 > - * The bit 15 of 0x45000 > - */ > - I915_WRITE(ILK_DISPLAY_CHICKEN2, > - (I915_READ(ILK_DISPLAY_CHICKEN2) | > - ILK_DPARB_GATE | ILK_VSDPFD_FULL)); > - dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE; > - I915_WRITE(DISP_ARB_CTL, > - (I915_READ(DISP_ARB_CTL) | > - DISP_FBC_WM_DIS)); > - > - /* > - * Based on the document from hardware guys the following bits > - * should be set unconditionally in order to enable FBC. > - * The bit 22 of 0x42000 > - * The bit 22 of 0x42004 > - * The bit 7,8,9 of 0x42020. > - */ > - if (IS_IRONLAKE_M(dev_priv)) { > - /* WaFbcAsynchFlipDisableFbcQueue:ilk */ > - I915_WRITE(ILK_DISPLAY_CHICKEN1, > - I915_READ(ILK_DISPLAY_CHICKEN1) | > - ILK_FBCQ_DIS); > - I915_WRITE(ILK_DISPLAY_CHICKEN2, > - I915_READ(ILK_DISPLAY_CHICKEN2) | > - ILK_DPARB_GATE); > - } > - > - I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate); > - > - I915_WRITE(ILK_DISPLAY_CHICKEN2, > - I915_READ(ILK_DISPLAY_CHICKEN2) | > - ILK_ELPIN_409_SELECT); > - I915_WRITE(_3D_CHICKEN2, > - _3D_CHICKEN2_WM_READ_PIPELINED << 16 | > - _3D_CHICKEN2_WM_READ_PIPELINED); > - > - /* WaDisableRenderCachePipelinedFlush:ilk */ > - I915_WRITE(CACHE_MODE_0, > - _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE)); > - > - /* WaDisable_RenderCache_OperationalFlush:ilk */ > - I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); > - > - g4x_disable_trickle_feed(dev_priv); > - > - ibx_init_clock_gating(dev_priv); > -} > - > -static void cpt_init_clock_gating(struct drm_i915_private *dev_priv) > -{ > - enum pipe pipe; > - u32 val; > - > - /* > - * On Ibex Peak and Cougar Point, we need to disable clock > - * gating for the panel power sequencer or it will fail to > - * start up when no ports are active. > - */ > - I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE | > - PCH_DPLUNIT_CLOCK_GATE_DISABLE | > - PCH_CPUNIT_CLOCK_GATE_DISABLE); > - I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) | > - DPLS_EDP_PPS_FIX_DIS); > - /* The below fixes the weird display corruption, a few pixels shifted > - * downward, on (only) LVDS of some HP laptops with IVY. > - */ > - for_each_pipe(dev_priv, pipe) { > - val = I915_READ(TRANS_CHICKEN2(pipe)); > - val |= TRANS_CHICKEN2_TIMING_OVERRIDE; > - val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED; > - if (dev_priv->vbt.fdi_rx_polarity_inverted) > - val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED; > - val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK; > - val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER; > - val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH; > - I915_WRITE(TRANS_CHICKEN2(pipe), val); > - } > - /* WADP0ClockGatingDisable */ > - for_each_pipe(dev_priv, pipe) { > - I915_WRITE(TRANS_CHICKEN1(pipe), > - TRANS_CHICKEN1_DP0UNIT_GC_DISABLE); > - } > -} > - > -static void gen6_check_mch_setup(struct drm_i915_private *dev_priv) > -{ > - u32 tmp; > - > - tmp = I915_READ(MCH_SSKPD); > - if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL) > - DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n", > - tmp); > -} > - > -static void gen6_init_clock_gating(struct drm_i915_private *dev_priv) > -{ > - u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE; > - > - I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate); > - > - I915_WRITE(ILK_DISPLAY_CHICKEN2, > - I915_READ(ILK_DISPLAY_CHICKEN2) | > - ILK_ELPIN_409_SELECT); > - > - /* WaDisableHiZPlanesWhenMSAAEnabled:snb */ > - I915_WRITE(_3D_CHICKEN, > - _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB)); > - > - /* WaDisable_RenderCache_OperationalFlush:snb */ > - I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); > - > - /* > - * BSpec recoomends 8x4 when MSAA is used, > - * however in practice 16x4 seems fastest. > - * > - * Note that PS/WM thread counts depend on the WIZ hashing > - * disable bit, which we don't touch here, but it's good > - * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). > - */ > - I915_WRITE(GEN6_GT_MODE, > - _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4)); > - > - I915_WRITE(CACHE_MODE_0, > - _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB)); > - > - I915_WRITE(GEN6_UCGCTL1, > - I915_READ(GEN6_UCGCTL1) | > - GEN6_BLBUNIT_CLOCK_GATE_DISABLE | > - GEN6_CSUNIT_CLOCK_GATE_DISABLE); > - > - /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock > - * gating disable must be set. Failure to set it results in > - * flickering pixels due to Z write ordering failures after > - * some amount of runtime in the Mesa "fire" demo, and Unigine > - * Sanctuary and Tropics, and apparently anything else with > - * alpha test or pixel discard. > - * > - * According to the spec, bit 11 (RCCUNIT) must also be set, > - * but we didn't debug actual testcases to find it out. > - * > - * WaDisableRCCUnitClockGating:snb > - * WaDisableRCPBUnitClockGating:snb > - */ > - I915_WRITE(GEN6_UCGCTL2, > - GEN6_RCPBUNIT_CLOCK_GATE_DISABLE | > - GEN6_RCCUNIT_CLOCK_GATE_DISABLE); > - > - /* WaStripsFansDisableFastClipPerformanceFix:snb */ > - I915_WRITE(_3D_CHICKEN3, > - _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL)); > - > - /* > - * Bspec says: > - * "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and > - * 3DSTATE_SF number of SF output attributes is more than 16." > - */ > - I915_WRITE(_3D_CHICKEN3, > - _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH)); > - > - /* > - * According to the spec the following bits should be > - * set in order to enable memory self-refresh and fbc: > - * The bit21 and bit22 of 0x42000 > - * The bit21 and bit22 of 0x42004 > - * The bit5 and bit7 of 0x42020 > - * The bit14 of 0x70180 > - * The bit14 of 0x71180 > - * > - * WaFbcAsynchFlipDisableFbcQueue:snb > - */ > - I915_WRITE(ILK_DISPLAY_CHICKEN1, > - I915_READ(ILK_DISPLAY_CHICKEN1) | > - ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS); > - I915_WRITE(ILK_DISPLAY_CHICKEN2, > - I915_READ(ILK_DISPLAY_CHICKEN2) | > - ILK_DPARB_GATE | ILK_VSDPFD_FULL); > - I915_WRITE(ILK_DSPCLK_GATE_D, > - I915_READ(ILK_DSPCLK_GATE_D) | > - ILK_DPARBUNIT_CLOCK_GATE_ENABLE | > - ILK_DPFDUNIT_CLOCK_GATE_ENABLE); > - > - g4x_disable_trickle_feed(dev_priv); > - > - cpt_init_clock_gating(dev_priv); > - > - gen6_check_mch_setup(dev_priv); > -} > - > -static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv) > -{ > - u32 reg = I915_READ(GEN7_FF_THREAD_MODE); > + u32 reg = I915_READ(GEN7_FF_THREAD_MODE); > > /* > * WaVSThreadDispatchOverride:ivb,vlv > @@ -8942,90 +7325,8 @@ void intel_init_pm(struct drm_i915_private *dev_priv) > } > } > > -static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val) > -{ > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > - > - /* > - * N = val - 0xb7 > - * Slow = Fast = GPLL ref * N > - */ > - return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000); > -} > - > -static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val) > -{ > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > - > - return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7; > -} > - > -static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val) > -{ > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > - > - /* > - * N = val / 2 > - * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2 > - */ > - return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000); > -} > - > -static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val) > -{ > - struct intel_rps *rps = &dev_priv->gt_pm.rps; > - > - /* CHV needs even values */ > - return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2; > -} > - > -int intel_gpu_freq(struct drm_i915_private *dev_priv, int val) > -{ > - if (INTEL_GEN(dev_priv) >= 9) > - return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER, > - GEN9_FREQ_SCALER); > - else if (IS_CHERRYVIEW(dev_priv)) > - return chv_gpu_freq(dev_priv, val); > - else if (IS_VALLEYVIEW(dev_priv)) > - return byt_gpu_freq(dev_priv, val); > - else > - return val * GT_FREQUENCY_MULTIPLIER; > -} > - > -int intel_freq_opcode(struct drm_i915_private *dev_priv, int val) > -{ > - if (INTEL_GEN(dev_priv) >= 9) > - return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER, > - GT_FREQUENCY_MULTIPLIER); > - else if (IS_CHERRYVIEW(dev_priv)) > - return chv_freq_opcode(dev_priv, val); > - else if (IS_VALLEYVIEW(dev_priv)) > - return byt_freq_opcode(dev_priv, val); > - else > - return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER); > -} > - > void intel_pm_setup(struct drm_i915_private *dev_priv) > { > - mutex_init(&dev_priv->gt_pm.rps.lock); > - mutex_init(&dev_priv->gt_pm.rps.power.mutex); > - > - atomic_set(&dev_priv->gt_pm.rps.num_waiters, 0); > - > dev_priv->runtime_pm.suspended = false; > atomic_set(&dev_priv->runtime_pm.wakeref_count, 0); > } > - > -u32 intel_get_cagf(struct drm_i915_private *dev_priv, u32 rpstat) > -{ > - u32 cagf; > - > - if (INTEL_GEN(dev_priv) >= 9) > - cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT; > - else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) > - cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT; > - else > - cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT; > - > - return cagf; > -} > diff --git a/drivers/gpu/drm/i915/intel_pm.h b/drivers/gpu/drm/i915/intel_pm.h > index 93d192d0610a..b56e6285d1c3 100644 > --- a/drivers/gpu/drm/i915/intel_pm.h > +++ b/drivers/gpu/drm/i915/intel_pm.h > @@ -29,15 +29,6 @@ void intel_update_watermarks(struct intel_crtc *crtc); > void intel_init_pm(struct drm_i915_private *dev_priv); > void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv); > void intel_pm_setup(struct drm_i915_private *dev_priv); > -void intel_gpu_ips_init(struct drm_i915_private *dev_priv); > -void intel_gpu_ips_teardown(void); > -void intel_init_gt_powersave(struct drm_i915_private *dev_priv); > -void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv); > -void intel_enable_gt_powersave(struct drm_i915_private *dev_priv); > -void intel_disable_gt_powersave(struct drm_i915_private *dev_priv); > -void gen6_rps_busy(struct drm_i915_private *dev_priv); > -void gen6_rps_idle(struct drm_i915_private *dev_priv); > -void gen6_rps_boost(struct i915_request *rq); > void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv); > void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv); > void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv); > @@ -69,19 +60,6 @@ int skl_check_pipe_max_pixel_rate(struct intel_crtc *intel_crtc, > void intel_init_ipc(struct drm_i915_private *dev_priv); > void intel_enable_ipc(struct drm_i915_private *dev_priv); > > -int intel_gpu_freq(struct drm_i915_private *dev_priv, int val); > -int intel_freq_opcode(struct drm_i915_private *dev_priv, int val); > - > -u32 intel_get_cagf(struct drm_i915_private *dev_priv, u32 rpstat1); > - > -unsigned long i915_chipset_val(struct drm_i915_private *dev_priv); > -unsigned long i915_mch_val(struct drm_i915_private *dev_priv); > -unsigned long i915_gfx_val(struct drm_i915_private *dev_priv); > -void i915_update_gfx_val(struct drm_i915_private *dev_priv); > - > -bool ironlake_set_drps(struct drm_i915_private *dev_priv, u8 val); > -int intel_set_rps(struct drm_i915_private *dev_priv, u8 val); > -void intel_rps_mark_interactive(struct drm_i915_private *i915, bool interactive); > bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable); > > #endif /* __INTEL_PM_H__ */ > -- > 2.23.0
Quoting Andi Shyti (2019-10-23 10:50:00) > @@ -14976,7 +14978,7 @@ intel_prepare_plane_fb(struct drm_plane *plane, > * maximum clocks following a vblank miss (see do_rps_boost()). > */ > if (!intel_state->rps_interactive) { > - intel_rps_mark_interactive(dev_priv, true); > + intel_rps_mark_interactive(&dev_priv->gt.rps, true); I wonder if we can do &plane->vma->vm->gt->rps Mwahaha. > intel_state->rps_interactive = true; > } > > @@ -15001,7 +15003,7 @@ intel_cleanup_plane_fb(struct drm_plane *plane, > struct drm_i915_private *dev_priv = to_i915(plane->dev); > > if (intel_state->rps_interactive) { > - intel_rps_mark_interactive(dev_priv, false); > + intel_rps_mark_interactive(&dev_priv->gt.rps, false); > intel_state->rps_interactive = false; > }
> > * maximum clocks following a vblank miss (see do_rps_boost()). > > */ > > if (!intel_state->rps_interactive) { > > - intel_rps_mark_interactive(dev_priv, true); > > + intel_rps_mark_interactive(&dev_priv->gt.rps, true); > > I wonder if we can do &plane->vma->vm->gt->rps agree, looks ugly, I could fix it by extracting rps. Shall I do it now in a v3 or after the patch gets merged? As you can gues I'd prefer doing after the patch is merged :) Andi
Quoting Andi Shyti (2019-10-23 13:32:16) > > > * maximum clocks following a vblank miss (see do_rps_boost()). > > > */ > > > if (!intel_state->rps_interactive) { > > > - intel_rps_mark_interactive(dev_priv, true); > > > + intel_rps_mark_interactive(&dev_priv->gt.rps, true); > > > > I wonder if we can do &plane->vma->vm->gt->rps > > agree, looks ugly, I could fix it by extracting rps. Shall I do > it now in a v3 or after the patch gets merged? As you can gues > I'd prefer doing after the patch is merged :) Mostly thinking aloud. I plan on soak testing this first; something did not look quite right around pm_enable/pm_disable vs gt_resume/gt_suspend (i.e. did not match my current expectations of where to push the gt init next). -Chris
> > > > * maximum clocks following a vblank miss (see do_rps_boost()). > > > > */ > > > > if (!intel_state->rps_interactive) { > > > > - intel_rps_mark_interactive(dev_priv, true); > > > > + intel_rps_mark_interactive(&dev_priv->gt.rps, true); > > > > > > I wonder if we can do &plane->vma->vm->gt->rps > > > > agree, looks ugly, I could fix it by extracting rps. Shall I do > > it now in a v3 or after the patch gets merged? As you can gues > > I'd prefer doing after the patch is merged :) > > Mostly thinking aloud. I plan on soak testing this first; something did > not look quite right around pm_enable/pm_disable vs gt_resume/gt_suspend > (i.e. did not match my current expectations of where to push the gt init > next). yes, it's confusing and to me they look inverted in meaining :). Do we want to have a unique resume/suspend rather than both? For this refactoring it was just easier to keep it this way. Andi
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile index 2fd4bed188e5..133ca59e7c48 100644 --- a/drivers/gpu/drm/i915/Makefile +++ b/drivers/gpu/drm/i915/Makefile @@ -90,11 +90,12 @@ gt-y += \ gt/intel_hangcheck.o \ gt/intel_llc.o \ gt/intel_lrc.o \ + gt/intel_mocs.o \ gt/intel_rc6.o \ gt/intel_renderstate.o \ gt/intel_reset.o \ gt/intel_ringbuffer.o \ - gt/intel_mocs.o \ + gt/intel_rps.o \ gt/intel_sseu.o \ gt/intel_timeline.o \ gt/intel_workarounds.o diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c index 236fdf122e47..49c2c7f71394 100644 --- a/drivers/gpu/drm/i915/display/intel_display.c +++ b/drivers/gpu/drm/i915/display/intel_display.c @@ -55,6 +55,8 @@ #include "display/intel_tv.h" #include "display/intel_vdsc.h" +#include "gt/intel_rps.h" + #include "i915_drv.h" #include "i915_trace.h" #include "intel_acpi.h" @@ -14782,7 +14784,7 @@ static int do_rps_boost(struct wait_queue_entry *_wait, * vblank without our intervention, so leave RPS alone. */ if (!i915_request_started(rq)) - gen6_rps_boost(rq); + intel_rps_boost(rq); i915_request_put(rq); drm_crtc_vblank_put(wait->crtc); @@ -14976,7 +14978,7 @@ intel_prepare_plane_fb(struct drm_plane *plane, * maximum clocks following a vblank miss (see do_rps_boost()). */ if (!intel_state->rps_interactive) { - intel_rps_mark_interactive(dev_priv, true); + intel_rps_mark_interactive(&dev_priv->gt.rps, true); intel_state->rps_interactive = true; } @@ -15001,7 +15003,7 @@ intel_cleanup_plane_fb(struct drm_plane *plane, struct drm_i915_private *dev_priv = to_i915(plane->dev); if (intel_state->rps_interactive) { - intel_rps_mark_interactive(dev_priv, false); + intel_rps_mark_interactive(&dev_priv->gt.rps, false); intel_state->rps_interactive = false; } diff --git a/drivers/gpu/drm/i915/gt/intel_gt.c b/drivers/gpu/drm/i915/gt/intel_gt.c index 1c4b6c9642ad..7518852cb78a 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt.c +++ b/drivers/gpu/drm/i915/gt/intel_gt.c @@ -9,6 +9,7 @@ #include "intel_gt_requests.h" #include "intel_mocs.h" #include "intel_rc6.h" +#include "intel_rps.h" #include "intel_uncore.h" #include "intel_pm.h" @@ -321,8 +322,7 @@ void intel_gt_chipset_flush(struct intel_gt *gt) void intel_gt_driver_register(struct intel_gt *gt) { - if (IS_GEN(gt->i915, 5)) - intel_gpu_ips_init(gt->i915); + intel_rps_driver_register(>->rps); } static int intel_gt_init_scratch(struct intel_gt *gt, unsigned int size) @@ -385,7 +385,7 @@ void intel_gt_driver_remove(struct intel_gt *gt) void intel_gt_driver_unregister(struct intel_gt *gt) { - intel_gpu_ips_teardown(); + intel_rps_driver_unregister(>->rps); } void intel_gt_driver_release(struct intel_gt *gt) diff --git a/drivers/gpu/drm/i915/gt/intel_gt_irq.c b/drivers/gpu/drm/i915/gt/intel_gt_irq.c index 34a4fb624bf7..973ee7eded64 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_irq.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_irq.c @@ -11,6 +11,7 @@ #include "intel_gt.h" #include "intel_gt_irq.h" #include "intel_uncore.h" +#include "intel_rps.h" static void guc_irq_handler(struct intel_guc *guc, u16 iir) { @@ -77,7 +78,7 @@ gen11_other_irq_handler(struct intel_gt *gt, const u8 instance, return guc_irq_handler(>->uc.guc, iir); if (instance == OTHER_GTPM_INSTANCE) - return gen11_rps_irq_handler(gt, iir); + return gen11_rps_irq_handler(>->rps, iir); WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n", instance, iir); @@ -336,7 +337,7 @@ void gen8_gt_irq_handler(struct intel_gt *gt, u32 master_ctl, u32 gt_iir[4]) } if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) { - gen6_rps_irq_handler(gt->i915, gt_iir[2]); + gen6_rps_irq_handler(>->rps, gt_iir[2]); guc_irq_handler(>->uc.guc, gt_iir[2] >> 16); } } diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm.c b/drivers/gpu/drm/i915/gt/intel_gt_pm.c index 06e73d56cfcf..32ca87265222 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_pm.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_pm.c @@ -12,8 +12,10 @@ #include "intel_gt.h" #include "intel_gt_pm.h" #include "intel_gt_requests.h" +#include "intel_llc.h" #include "intel_pm.h" #include "intel_rc6.h" +#include "intel_rps.h" #include "intel_wakeref.h" static int __gt_unpark(struct intel_wakeref *wf) @@ -39,12 +41,8 @@ static int __gt_unpark(struct intel_wakeref *wf) gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ); GEM_BUG_ON(!gt->awake); - intel_enable_gt_powersave(i915); - - i915_update_gfx_val(i915); - if (INTEL_GEN(i915) >= 6) - gen6_rps_busy(i915); - + intel_rps_unpark(>->rps); + intel_llc_enable(>->llc); i915_pmu_gt_unparked(i915); intel_gt_queue_hangcheck(gt); @@ -65,8 +63,8 @@ static int __gt_park(struct intel_wakeref *wf) i915_vma_parked(gt); i915_pmu_gt_parked(i915); - if (INTEL_GEN(i915) >= 6) - gen6_rps_idle(i915); + intel_llc_disable(>->llc); + intel_rps_park(>->rps); /* Everything switched off, flush any residual interrupt just in case */ intel_synchronize_irq(i915); @@ -98,6 +96,7 @@ void intel_gt_pm_init(struct intel_gt *gt) * user. */ intel_rc6_init(>->rc6); + intel_rps_init(>->rps); } static bool reset_engines(struct intel_gt *gt) @@ -141,10 +140,39 @@ void intel_gt_sanitize(struct intel_gt *gt, bool force) engine->reset.finish(engine); } +void intel_gt_pm_enable(struct intel_gt *gt) +{ + struct intel_engine_cs *engine; + enum intel_engine_id id; + + /* Powersaving is controlled by the host when inside a VM */ + if (intel_vgpu_active(gt->i915)) + return; + + if (is_mock_gt(gt)) + return; + + intel_gt_pm_get(gt); + intel_rps_enable(>->rps); + intel_llc_enable(>->llc); + + for_each_engine(engine, gt->i915, id) { + intel_engine_pm_get(engine); + engine->serial++; /* force kernel context reload */ + intel_engine_pm_put(engine); + } + + intel_gt_pm_put(gt); +} + void intel_gt_pm_disable(struct intel_gt *gt) { - if (!is_mock_gt(gt)) - intel_sanitize_gt_powersave(gt->i915); + if (is_mock_gt(gt)) + return; + + intel_rc6_disable(>->rc6); + intel_llc_disable(>->llc); + intel_rps_disable(>->rps); } void intel_gt_pm_fini(struct intel_gt *gt) @@ -165,6 +193,7 @@ int intel_gt_resume(struct intel_gt *gt) * allowing us to fixup the user contexts on their first pin. */ intel_gt_pm_get(gt); + intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL); intel_rc6_sanitize(>->rc6); diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm.h b/drivers/gpu/drm/i915/gt/intel_gt_pm.h index 0ed87da4bb68..df50f853748d 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_pm.h +++ b/drivers/gpu/drm/i915/gt/intel_gt_pm.h @@ -40,6 +40,7 @@ static inline int intel_gt_pm_wait_for_idle(struct intel_gt *gt) void intel_gt_pm_init_early(struct intel_gt *gt); void intel_gt_pm_init(struct intel_gt *gt); void intel_gt_pm_disable(struct intel_gt *gt); +void intel_gt_pm_enable(struct intel_gt *gt); void intel_gt_pm_fini(struct intel_gt *gt); void intel_gt_sanitize(struct intel_gt *gt, bool force); diff --git a/drivers/gpu/drm/i915/gt/intel_gt_types.h b/drivers/gpu/drm/i915/gt/intel_gt_types.h index 980973e66e7f..e4bd23eb8290 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_types.h +++ b/drivers/gpu/drm/i915/gt/intel_gt_types.h @@ -20,6 +20,7 @@ #include "intel_llc_types.h" #include "intel_reset_types.h" #include "intel_rc6_types.h" +#include "intel_rps_types.h" #include "intel_wakeref.h" struct drm_i915_private; @@ -82,6 +83,7 @@ struct intel_gt { struct intel_llc llc; struct intel_rc6 rc6; + struct intel_rps rps; ktime_t last_init_time; diff --git a/drivers/gpu/drm/i915/gt/intel_llc.c b/drivers/gpu/drm/i915/gt/intel_llc.c index 35093eb5f24e..ceb785b75c25 100644 --- a/drivers/gpu/drm/i915/gt/intel_llc.c +++ b/drivers/gpu/drm/i915/gt/intel_llc.c @@ -48,7 +48,7 @@ static bool get_ia_constants(struct intel_llc *llc, struct ia_constants *consts) { struct drm_i915_private *i915 = llc_to_gt(llc)->i915; - struct intel_rps *rps = &i915->gt_pm.rps; + struct intel_rps *rps = &llc_to_gt(llc)->rps; if (rps->max_freq <= rps->min_freq) return false; diff --git a/drivers/gpu/drm/i915/gt/intel_rps.c b/drivers/gpu/drm/i915/gt/intel_rps.c new file mode 100644 index 000000000000..30f56c786468 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/intel_rps.c @@ -0,0 +1,1872 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright © 2019 Intel Corporation + */ + +#include "i915_drv.h" +#include "intel_gt.h" +#include "intel_gt_irq.h" +#include "intel_gt_pm_irq.h" +#include "intel_rps.h" +#include "intel_sideband.h" +#include "../../../platform/x86/intel_ips.h" + +/* + * Lock protecting IPS related data structures + */ +static DEFINE_SPINLOCK(mchdev_lock); + +static struct intel_gt *rps_to_gt(struct intel_rps *rps) +{ + return container_of(rps, struct intel_gt, rps); +} + +static struct drm_i915_private *rps_to_i915(struct intel_rps *rps) +{ + return rps_to_gt(rps)->i915; +} + +static struct intel_uncore *rps_to_uncore(struct intel_rps *rps) +{ + return rps_to_gt(rps)->uncore; +} + +static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask) +{ + return mask & ~rps->pm_intrmsk_mbz; +} + +static u32 rps_pm_mask(struct intel_rps *rps, u8 val) +{ + u32 mask = 0; + + /* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */ + if (val > rps->min_freq_softlimit) + mask |= (GEN6_PM_RP_UP_EI_EXPIRED | + GEN6_PM_RP_DOWN_THRESHOLD | + GEN6_PM_RP_DOWN_TIMEOUT); + + if (val < rps->max_freq_softlimit) + mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD; + + mask &= rps->pm_events; + + return rps_pm_sanitize_mask(rps, ~mask); +} + +static void rps_reset_ei(struct intel_rps *rps) +{ + memset(&rps->ei, 0, sizeof(rps->ei)); +} + +static void rps_enable_interrupts(struct intel_rps *rps) +{ + struct intel_gt *gt = rps_to_gt(rps); + + rps_reset_ei(rps); + + if (IS_VALLEYVIEW(gt->i915)) + /* WaGsvRC0ResidencyMethod:vlv */ + rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED; + else + rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD | + GEN6_PM_RP_DOWN_THRESHOLD | + GEN6_PM_RP_DOWN_TIMEOUT); + + spin_lock_irq(>->irq_lock); + gen6_gt_pm_enable_irq(gt, rps->pm_events); + spin_unlock_irq(>->irq_lock); + + intel_uncore_write(gt->uncore, GEN6_PMINTRMSK, + rps_pm_mask(rps, rps->cur_freq)); +} + +static void gen6_rps_reset_interrupts(struct intel_rps *rps) +{ + gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS); +} + +static void gen11_rps_reset_interrupts(struct intel_rps *rps) +{ + while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM)) + ; +} + +static void rps_reset_interrupts(struct intel_rps *rps) +{ + struct intel_gt *gt = rps_to_gt(rps); + + spin_lock_irq(>->irq_lock); + if (INTEL_GEN(gt->i915) >= 11) + gen11_rps_reset_interrupts(rps); + else + gen6_rps_reset_interrupts(rps); + + rps->pm_iir = 0; + spin_unlock_irq(>->irq_lock); +} + +static void rps_disable_interrupts(struct intel_rps *rps) +{ + struct intel_gt *gt = rps_to_gt(rps); + + rps->pm_events = 0; + + intel_uncore_write(gt->uncore, GEN6_PMINTRMSK, + rps_pm_sanitize_mask(rps, ~0u)); + + spin_lock_irq(>->irq_lock); + gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS); + spin_unlock_irq(>->irq_lock); + + intel_synchronize_irq(gt->i915); + + /* + * Now that we will not be generating any more work, flush any + * outstanding tasks. As we are called on the RPS idle path, + * we will reset the GPU to minimum frequencies, so the current + * state of the worker can be discarded. + */ + cancel_work_sync(&rps->work); + + rps_reset_interrupts(rps); +} + +static const struct cparams { + u16 i; + u16 t; + u16 m; + u16 c; +} cparams[] = { + { 1, 1333, 301, 28664 }, + { 1, 1066, 294, 24460 }, + { 1, 800, 294, 25192 }, + { 0, 1333, 276, 27605 }, + { 0, 1066, 276, 27605 }, + { 0, 800, 231, 23784 }, +}; + +static void gen5_rps_init(struct intel_rps *rps) +{ + struct drm_i915_private *i915 = rps_to_i915(rps); + struct intel_uncore *uncore = rps_to_uncore(rps); + u8 fmax, fmin, fstart; + u32 rgvmodectl; + int c_m, i; + + if (i915->fsb_freq <= 3200) + c_m = 0; + else if (i915->fsb_freq <= 4800) + c_m = 1; + else + c_m = 2; + + for (i = 0; i < ARRAY_SIZE(cparams); i++) { + if (cparams[i].i == c_m && cparams[i].t == i915->mem_freq) { + rps->ips.m = cparams[i].m; + rps->ips.c = cparams[i].c; + break; + } + } + + rgvmodectl = intel_uncore_read(uncore, MEMMODECTL); + + /* Set up min, max, and cur for interrupt handling */ + fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT; + fmin = (rgvmodectl & MEMMODE_FMIN_MASK); + fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> + MEMMODE_FSTART_SHIFT; + DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n", + fmax, fmin, fstart); + + rps->min_freq = -fstart; + rps->max_freq = -fmin; + + rps->idle_freq = rps->min_freq; + rps->cur_freq = rps->idle_freq; +} + +static unsigned long +__ips_chipset_val(struct intel_ips *ips) +{ + struct intel_uncore *uncore = + rps_to_uncore(container_of(ips, struct intel_rps, ips)); + unsigned long now = jiffies_to_msecs(jiffies), dt; + unsigned long result; + u64 total, delta; + + lockdep_assert_held(&mchdev_lock); + + /* + * Prevent division-by-zero if we are asking too fast. + * Also, we don't get interesting results if we are polling + * faster than once in 10ms, so just return the saved value + * in such cases. + */ + dt = now - ips->last_time1; + if (dt <= 10) + return ips->chipset_power; + + /* FIXME: handle per-counter overflow */ + total = intel_uncore_read(uncore, DMIEC); + total += intel_uncore_read(uncore, DDREC); + total += intel_uncore_read(uncore, CSIEC); + + delta = total - ips->last_count1; + + result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10); + + ips->last_count1 = total; + ips->last_time1 = now; + + ips->chipset_power = result; + + return result; +} + +static unsigned long ips_mch_val(struct intel_uncore *uncore) +{ + unsigned int m, x, b; + u32 tsfs; + + tsfs = intel_uncore_read(uncore, TSFS); + x = intel_uncore_read8(uncore, TR1); + + b = tsfs & TSFS_INTR_MASK; + m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT; + + return m * x / 127 - b; +} + +static int _pxvid_to_vd(u8 pxvid) +{ + if (pxvid == 0) + return 0; + + if (pxvid >= 8 && pxvid < 31) + pxvid = 31; + + return (pxvid + 2) * 125; +} + +static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid) +{ + const int vd = _pxvid_to_vd(pxvid); + + if (INTEL_INFO(i915)->is_mobile) + return max(vd - 1125, 0); + + return vd; +} + +static void __gen5_ips_update(struct intel_ips *ips) +{ + struct intel_uncore *uncore = + rps_to_uncore(container_of(ips, struct intel_rps, ips)); + u64 now, delta, dt; + u32 count; + + lockdep_assert_held(&mchdev_lock); + + now = ktime_get_raw_ns(); + dt = now - ips->last_time2; + do_div(dt, NSEC_PER_MSEC); + + /* Don't divide by 0 */ + if (dt <= 10) + return; + + count = intel_uncore_read(uncore, GFXEC); + delta = count - ips->last_count2; + + ips->last_count2 = count; + ips->last_time2 = now; + + /* More magic constants... */ + ips->gfx_power = div_u64(delta * 1181, dt * 10); +} + +static void gen5_rps_update(struct intel_rps *rps) +{ + spin_lock_irq(&mchdev_lock); + __gen5_ips_update(&rps->ips); + spin_unlock_irq(&mchdev_lock); +} + +static bool gen5_rps_set(struct intel_rps *rps, u8 val) +{ + struct intel_uncore *uncore = rps_to_uncore(rps); + u16 rgvswctl; + + lockdep_assert_held(&mchdev_lock); + + rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); + if (rgvswctl & MEMCTL_CMD_STS) { + DRM_DEBUG("gpu busy, RCS change rejected\n"); + return false; /* still busy with another command */ + } + + val = -val; + + rgvswctl = + (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) | + (val << MEMCTL_FREQ_SHIFT) | + MEMCTL_SFCAVM; + intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); + intel_uncore_posting_read16(uncore, MEMSWCTL); + + rgvswctl |= MEMCTL_CMD_STS; + intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); + + return true; +} + +static unsigned long intel_pxfreq(u32 vidfreq) +{ + int div = (vidfreq & 0x3f0000) >> 16; + int post = (vidfreq & 0x3000) >> 12; + int pre = (vidfreq & 0x7); + + if (!pre) + return 0; + + return div * 133333 / (pre << post); +} + +static unsigned int init_emon(struct intel_uncore *uncore) +{ + u8 pxw[16]; + int i; + + /* Disable to program */ + intel_uncore_write(uncore, ECR, 0); + intel_uncore_posting_read(uncore, ECR); + + /* Program energy weights for various events */ + intel_uncore_write(uncore, SDEW, 0x15040d00); + intel_uncore_write(uncore, CSIEW0, 0x007f0000); + intel_uncore_write(uncore, CSIEW1, 0x1e220004); + intel_uncore_write(uncore, CSIEW2, 0x04000004); + + for (i = 0; i < 5; i++) + intel_uncore_write(uncore, PEW(i), 0); + for (i = 0; i < 3; i++) + intel_uncore_write(uncore, DEW(i), 0); + + /* Program P-state weights to account for frequency power adjustment */ + for (i = 0; i < 16; i++) { + u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i)); + unsigned int freq = intel_pxfreq(pxvidfreq); + unsigned int vid = + (pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; + unsigned int val; + + val = vid * vid * freq / 1000 * 255; + val /= 127 * 127 * 900; + + pxw[i] = val; + } + /* Render standby states get 0 weight */ + pxw[14] = 0; + pxw[15] = 0; + + for (i = 0; i < 4; i++) { + intel_uncore_write(uncore, PXW(i), + pxw[i * 4 + 0] << 24 | + pxw[i * 4 + 1] << 16 | + pxw[i * 4 + 2] << 8 | + pxw[i * 4 + 3] << 0); + } + + /* Adjust magic regs to magic values (more experimental results) */ + intel_uncore_write(uncore, OGW0, 0); + intel_uncore_write(uncore, OGW1, 0); + intel_uncore_write(uncore, EG0, 0x00007f00); + intel_uncore_write(uncore, EG1, 0x0000000e); + intel_uncore_write(uncore, EG2, 0x000e0000); + intel_uncore_write(uncore, EG3, 0x68000300); + intel_uncore_write(uncore, EG4, 0x42000000); + intel_uncore_write(uncore, EG5, 0x00140031); + intel_uncore_write(uncore, EG6, 0); + intel_uncore_write(uncore, EG7, 0); + + for (i = 0; i < 8; i++) + intel_uncore_write(uncore, PXWL(i), 0); + + /* Enable PMON + select events */ + intel_uncore_write(uncore, ECR, 0x80000019); + + return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK; +} + +static bool gen5_rps_enable(struct intel_rps *rps) +{ + struct intel_uncore *uncore = rps_to_uncore(rps); + u8 fstart, vstart; + u32 rgvmodectl; + + spin_lock_irq(&mchdev_lock); + + rgvmodectl = intel_uncore_read(uncore, MEMMODECTL); + + /* Enable temp reporting */ + intel_uncore_write16(uncore, PMMISC, + intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN); + intel_uncore_write16(uncore, TSC1, + intel_uncore_read16(uncore, TSC1) | TSE); + + /* 100ms RC evaluation intervals */ + intel_uncore_write(uncore, RCUPEI, 100000); + intel_uncore_write(uncore, RCDNEI, 100000); + + /* Set max/min thresholds to 90ms and 80ms respectively */ + intel_uncore_write(uncore, RCBMAXAVG, 90000); + intel_uncore_write(uncore, RCBMINAVG, 80000); + + intel_uncore_write(uncore, MEMIHYST, 1); + + /* Set up min, max, and cur for interrupt handling */ + fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> + MEMMODE_FSTART_SHIFT; + + vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) & + PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; + + intel_uncore_write(uncore, + MEMINTREN, + MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN); + + intel_uncore_write(uncore, VIDSTART, vstart); + intel_uncore_posting_read(uncore, VIDSTART); + + rgvmodectl |= MEMMODE_SWMODE_EN; + intel_uncore_write(uncore, MEMMODECTL, rgvmodectl); + + if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) & + MEMCTL_CMD_STS) == 0, 10)) + DRM_ERROR("stuck trying to change perf mode\n"); + mdelay(1); + + gen5_rps_set(rps, rps->cur_freq); + + rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC); + rps->ips.last_count1 += intel_uncore_read(uncore, DDREC); + rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC); + rps->ips.last_time1 = jiffies_to_msecs(jiffies); + + rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC); + rps->ips.last_time2 = ktime_get_raw_ns(); + + spin_unlock_irq(&mchdev_lock); + + rps->ips.corr = init_emon(uncore); + + return true; +} + +static void gen5_rps_disable(struct intel_rps *rps) +{ + struct intel_uncore *uncore = rps_to_uncore(rps); + u16 rgvswctl; + + spin_lock_irq(&mchdev_lock); + + rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); + + /* Ack interrupts, disable EFC interrupt */ + intel_uncore_write(uncore, MEMINTREN, + intel_uncore_read(uncore, MEMINTREN) & + ~MEMINT_EVAL_CHG_EN); + intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); + intel_uncore_write(uncore, DEIER, + intel_uncore_read(uncore, DEIER) & ~DE_PCU_EVENT); + intel_uncore_write(uncore, DEIIR, DE_PCU_EVENT); + intel_uncore_write(uncore, DEIMR, + intel_uncore_read(uncore, DEIMR) | DE_PCU_EVENT); + + /* Go back to the starting frequency */ + gen5_rps_set(rps, rps->idle_freq); + mdelay(1); + rgvswctl |= MEMCTL_CMD_STS; + intel_uncore_write(uncore, MEMSWCTL, rgvswctl); + mdelay(1); + + spin_unlock_irq(&mchdev_lock); +} + +static u32 rps_limits(struct intel_rps *rps, u8 val) +{ + u32 limits; + + /* + * Only set the down limit when we've reached the lowest level to avoid + * getting more interrupts, otherwise leave this clear. This prevents a + * race in the hw when coming out of rc6: There's a tiny window where + * the hw runs at the minimal clock before selecting the desired + * frequency, if the down threshold expires in that window we will not + * receive a down interrupt. + */ + if (INTEL_GEN(rps_to_i915(rps)) >= 9) { + limits = rps->max_freq_softlimit << 23; + if (val <= rps->min_freq_softlimit) + limits |= rps->min_freq_softlimit << 14; + } else { + limits = rps->max_freq_softlimit << 24; + if (val <= rps->min_freq_softlimit) + limits |= rps->min_freq_softlimit << 16; + } + + return limits; +} + +static void rps_set_power(struct intel_rps *rps, int new_power) +{ + struct intel_uncore *uncore = rps_to_uncore(rps); + struct drm_i915_private *i915 = rps_to_i915(rps); + u32 threshold_up = 0, threshold_down = 0; /* in % */ + u32 ei_up = 0, ei_down = 0; + + lockdep_assert_held(&rps->power.mutex); + + if (new_power == rps->power.mode) + return; + + /* Note the units here are not exactly 1us, but 1280ns. */ + switch (new_power) { + case LOW_POWER: + /* Upclock if more than 95% busy over 16ms */ + ei_up = 16000; + threshold_up = 95; + + /* Downclock if less than 85% busy over 32ms */ + ei_down = 32000; + threshold_down = 85; + break; + + case BETWEEN: + /* Upclock if more than 90% busy over 13ms */ + ei_up = 13000; + threshold_up = 90; + + /* Downclock if less than 75% busy over 32ms */ + ei_down = 32000; + threshold_down = 75; + break; + + case HIGH_POWER: + /* Upclock if more than 85% busy over 10ms */ + ei_up = 10000; + threshold_up = 85; + + /* Downclock if less than 60% busy over 32ms */ + ei_down = 32000; + threshold_down = 60; + break; + } + + /* When byt can survive without system hang with dynamic + * sw freq adjustments, this restriction can be lifted. + */ + if (IS_VALLEYVIEW(i915)) + goto skip_hw_write; + + intel_uncore_write(uncore, GEN6_RP_UP_EI, + GT_INTERVAL_FROM_US(i915, ei_up)); + intel_uncore_write(uncore, GEN6_RP_UP_THRESHOLD, + GT_INTERVAL_FROM_US(i915, + ei_up * threshold_up / 100)); + + intel_uncore_write(uncore, GEN6_RP_DOWN_EI, + GT_INTERVAL_FROM_US(i915, ei_down)); + intel_uncore_write(uncore, GEN6_RP_DOWN_THRESHOLD, + GT_INTERVAL_FROM_US(i915, + ei_down * threshold_down / 100)); + + intel_uncore_write(uncore, GEN6_RP_CONTROL, + (INTEL_GEN(i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) | + GEN6_RP_MEDIA_HW_NORMAL_MODE | + GEN6_RP_MEDIA_IS_GFX | + GEN6_RP_ENABLE | + GEN6_RP_UP_BUSY_AVG | + GEN6_RP_DOWN_IDLE_AVG); + +skip_hw_write: + rps->power.mode = new_power; + rps->power.up_threshold = threshold_up; + rps->power.down_threshold = threshold_down; +} + +static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val) +{ + int new_power; + + new_power = rps->power.mode; + switch (rps->power.mode) { + case LOW_POWER: + if (val > rps->efficient_freq + 1 && + val > rps->cur_freq) + new_power = BETWEEN; + break; + + case BETWEEN: + if (val <= rps->efficient_freq && + val < rps->cur_freq) + new_power = LOW_POWER; + else if (val >= rps->rp0_freq && + val > rps->cur_freq) + new_power = HIGH_POWER; + break; + + case HIGH_POWER: + if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 && + val < rps->cur_freq) + new_power = BETWEEN; + break; + } + /* Max/min bins are special */ + if (val <= rps->min_freq_softlimit) + new_power = LOW_POWER; + if (val >= rps->max_freq_softlimit) + new_power = HIGH_POWER; + + mutex_lock(&rps->power.mutex); + if (rps->power.interactive) + new_power = HIGH_POWER; + rps_set_power(rps, new_power); + mutex_unlock(&rps->power.mutex); +} + +void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive) +{ + if (!rps->enabled) + return; + + mutex_lock(&rps->power.mutex); + if (interactive) { + if (!rps->power.interactive++ && rps->active) + rps_set_power(rps, HIGH_POWER); + } else { + GEM_BUG_ON(!rps->power.interactive); + rps->power.interactive--; + } + mutex_unlock(&rps->power.mutex); +} + +static int gen6_rps_set(struct intel_rps *rps, u8 val) +{ + struct intel_uncore *uncore = rps_to_uncore(rps); + struct drm_i915_private *i915 = rps_to_i915(rps); + u32 swreq; + + if (INTEL_GEN(i915) >= 9) + swreq = GEN9_FREQUENCY(val); + else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) + swreq = HSW_FREQUENCY(val); + else + swreq = (GEN6_FREQUENCY(val) | + GEN6_OFFSET(0) | + GEN6_AGGRESSIVE_TURBO); + intel_uncore_write(uncore, GEN6_RPNSWREQ, swreq); + + return 0; +} + +static int vlv_rps_set(struct intel_rps *rps, u8 val) +{ + struct drm_i915_private *i915 = rps_to_i915(rps); + int err; + + vlv_punit_get(i915); + err = vlv_punit_write(i915, PUNIT_REG_GPU_FREQ_REQ, val); + vlv_punit_put(i915); + + return err; +} + +static int rps_set(struct intel_rps *rps, u8 val) +{ + struct drm_i915_private *i915 = rps_to_i915(rps); + int err; + + if (INTEL_GEN(i915) < 6) + return 0; + + if (val == rps->last_freq) + return 0; + + if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) + err = vlv_rps_set(rps, val); + else + err = gen6_rps_set(rps, val); + if (err) + return err; + + gen6_rps_set_thresholds(rps, val); + rps->last_freq = val; + + return 0; +} + +void intel_rps_unpark(struct intel_rps *rps) +{ + u8 freq; + + if (!rps->enabled) + return; + + /* + * Use the user's desired frequency as a guide, but for better + * performance, jump directly to RPe as our starting frequency. + */ + mutex_lock(&rps->lock); + rps->active = true; + freq = max(rps->cur_freq, rps->efficient_freq), + freq = clamp(freq, rps->min_freq_softlimit, rps->max_freq_softlimit); + intel_rps_set(rps, freq); + rps->last_adj = 0; + mutex_unlock(&rps->lock); + + if (INTEL_GEN(rps_to_i915(rps)) >= 6) + rps_enable_interrupts(rps); + + if (IS_GEN(rps_to_i915(rps), 5)) + gen5_rps_update(rps); +} + +void intel_rps_park(struct intel_rps *rps) +{ + struct drm_i915_private *i915 = rps_to_i915(rps); + + if (!rps->enabled) + return; + + if (INTEL_GEN(i915) >= 6) + rps_disable_interrupts(rps); + + rps->active = false; + if (rps->last_freq <= rps->idle_freq) + return; + + /* + * The punit delays the write of the frequency and voltage until it + * determines the GPU is awake. During normal usage we don't want to + * waste power changing the frequency if the GPU is sleeping (rc6). + * However, the GPU and driver is now idle and we do not want to delay + * switching to minimum voltage (reducing power whilst idle) as we do + * not expect to be woken in the near future and so must flush the + * change by waking the device. + * + * We choose to take the media powerwell (either would do to trick the + * punit into committing the voltage change) as that takes a lot less + * power than the render powerwell. + */ + intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA); + rps_set(rps, rps->idle_freq); + intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA); +} + +void intel_rps_boost(struct i915_request *rq) +{ + struct intel_rps *rps = &rq->engine->gt->rps; + unsigned long flags; + + if (i915_request_signaled(rq) || !rps->active) + return; + + /* Serializes with i915_request_retire() */ + spin_lock_irqsave(&rq->lock, flags); + if (!i915_request_has_waitboost(rq) && + !dma_fence_is_signaled_locked(&rq->fence)) { + rq->flags |= I915_REQUEST_WAITBOOST; + + if (!atomic_fetch_inc(&rps->num_waiters) && + READ_ONCE(rps->cur_freq) < rps->boost_freq) + schedule_work(&rps->work); + + atomic_inc(&rps->boosts); + } + spin_unlock_irqrestore(&rq->lock, flags); +} + +int intel_rps_set(struct intel_rps *rps, u8 val) +{ + int err = 0; + + lockdep_assert_held(&rps->lock); + GEM_BUG_ON(val > rps->max_freq); + GEM_BUG_ON(val < rps->min_freq); + + if (rps->active) { + err = rps_set(rps, val); + + /* + * Make sure we continue to get interrupts + * until we hit the minimum or maximum frequencies. + */ + if (INTEL_GEN(rps_to_i915(rps)) >= 6) { + struct intel_uncore *uncore = rps_to_uncore(rps); + + intel_uncore_write(uncore, GEN6_RP_INTERRUPT_LIMITS, + rps_limits(rps, val)); + + intel_uncore_write(uncore, GEN6_PMINTRMSK, + rps_pm_mask(rps, val)); + } + } + + if (err == 0) + rps->cur_freq = val; + + return err; +} + +static void gen6_rps_init(struct intel_rps *rps) +{ + struct drm_i915_private *i915 = rps_to_i915(rps); + struct intel_uncore *uncore = rps_to_uncore(rps); + + /* All of these values are in units of 50MHz */ + + /* static values from HW: RP0 > RP1 > RPn (min_freq) */ + if (IS_GEN9_LP(i915)) { + u32 rp_state_cap = intel_uncore_read(uncore, BXT_RP_STATE_CAP); + + rps->rp0_freq = (rp_state_cap >> 16) & 0xff; + rps->rp1_freq = (rp_state_cap >> 8) & 0xff; + rps->min_freq = (rp_state_cap >> 0) & 0xff; + } else { + u32 rp_state_cap = intel_uncore_read(uncore, GEN6_RP_STATE_CAP); + + rps->rp0_freq = (rp_state_cap >> 0) & 0xff; + rps->rp1_freq = (rp_state_cap >> 8) & 0xff; + rps->min_freq = (rp_state_cap >> 16) & 0xff; + } + + /* hw_max = RP0 until we check for overclocking */ + rps->max_freq = rps->rp0_freq; + + rps->efficient_freq = rps->rp1_freq; + if (IS_HASWELL(i915) || IS_BROADWELL(i915) || + IS_GEN9_BC(i915) || INTEL_GEN(i915) >= 10) { + u32 ddcc_status = 0; + + if (sandybridge_pcode_read(i915, + HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL, + &ddcc_status, NULL) == 0) + rps->efficient_freq = + clamp_t(u8, + (ddcc_status >> 8) & 0xff, + rps->min_freq, + rps->max_freq); + } + + if (IS_GEN9_BC(i915) || INTEL_GEN(i915) >= 10) { + /* Store the frequency values in 16.66 MHZ units, which is + * the natural hardware unit for SKL + */ + rps->rp0_freq *= GEN9_FREQ_SCALER; + rps->rp1_freq *= GEN9_FREQ_SCALER; + rps->min_freq *= GEN9_FREQ_SCALER; + rps->max_freq *= GEN9_FREQ_SCALER; + rps->efficient_freq *= GEN9_FREQ_SCALER; + } +} + +static bool rps_reset(struct intel_rps *rps) +{ + /* force a reset */ + rps->power.mode = -1; + rps->last_freq = -1; + + if (rps_set(rps, rps->min_freq)) { + DRM_ERROR("Failed to reset RPS to initial values\n"); + return false; + } + + rps->cur_freq = rps->min_freq; + return true; +} + +/* See the Gen9_GT_PM_Programming_Guide doc for the below */ +static bool gen9_rps_enable(struct intel_rps *rps) +{ + struct drm_i915_private *i915 = rps_to_i915(rps); + struct intel_uncore *uncore = rps_to_uncore(rps); + + /* Program defaults and thresholds for RPS */ + if (IS_GEN(i915, 9)) + intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ, + GEN9_FREQUENCY(rps->rp1_freq)); + + /* 1 second timeout */ + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, + GT_INTERVAL_FROM_US(i915, 1000000)); + + intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa); + + return rps_reset(rps); +} + +static bool gen8_rps_enable(struct intel_rps *rps) +{ + struct intel_uncore *uncore = rps_to_uncore(rps); + + intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ, + HSW_FREQUENCY(rps->rp1_freq)); + + /* NB: Docs say 1s, and 1000000 - which aren't equivalent */ + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, + 100000000 / 128); /* 1 second timeout */ + + intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); + + return rps_reset(rps); +} + +static bool gen6_rps_enable(struct intel_rps *rps) +{ + struct intel_uncore *uncore = rps_to_uncore(rps); + + /* Power down if completely idle for over 50ms */ + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000); + intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); + + return rps_reset(rps); +} + +static int chv_rps_max_freq(struct intel_rps *rps) +{ + struct drm_i915_private *i915 = rps_to_i915(rps); + u32 val; + + val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE); + + switch (RUNTIME_INFO(i915)->sseu.eu_total) { + case 8: + /* (2 * 4) config */ + val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT; + break; + case 12: + /* (2 * 6) config */ + val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT; + break; + case 16: + /* (2 * 8) config */ + default: + /* Setting (2 * 8) Min RP0 for any other combination */ + val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT; + break; + } + + return val & FB_GFX_FREQ_FUSE_MASK; +} + +static int chv_rps_rpe_freq(struct intel_rps *rps) +{ + struct drm_i915_private *i915 = rps_to_i915(rps); + u32 val; + + val = vlv_punit_read(i915, PUNIT_GPU_DUTYCYCLE_REG); + val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT; + + return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK; +} + +static int chv_rps_guar_freq(struct intel_rps *rps) +{ + struct drm_i915_private *i915 = rps_to_i915(rps); + u32 val; + + val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE); + + return val & FB_GFX_FREQ_FUSE_MASK; +} + +static u32 chv_rps_min_freq(struct intel_rps *rps) +{ + struct drm_i915_private *i915 = rps_to_i915(rps); + u32 val; + + val = vlv_punit_read(i915, FB_GFX_FMIN_AT_VMIN_FUSE); + val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT; + + return val & FB_GFX_FREQ_FUSE_MASK; +} + +static bool chv_rps_enable(struct intel_rps *rps) +{ + struct intel_uncore *uncore = rps_to_uncore(rps); + struct drm_i915_private *i915 = rps_to_i915(rps); + u32 val; + + /* 1: Program defaults and thresholds for RPS*/ + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000); + intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400); + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000); + intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000); + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000); + + intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); + + /* 2: Enable RPS */ + intel_uncore_write_fw(uncore, GEN6_RP_CONTROL, + GEN6_RP_MEDIA_HW_NORMAL_MODE | + GEN6_RP_MEDIA_IS_GFX | + GEN6_RP_ENABLE | + GEN6_RP_UP_BUSY_AVG | + GEN6_RP_DOWN_IDLE_AVG); + + /* Setting Fixed Bias */ + vlv_punit_get(i915); + + val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50; + vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val); + + val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); + + vlv_punit_put(i915); + + /* RPS code assumes GPLL is used */ + WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n"); + + DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE)); + DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val); + + return rps_reset(rps); +} + +static int vlv_rps_guar_freq(struct intel_rps *rps) +{ + struct drm_i915_private *i915 = rps_to_i915(rps); + u32 val, rp1; + + val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE); + + rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK; + rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT; + + return rp1; +} + +static int vlv_rps_max_freq(struct intel_rps *rps) +{ + struct drm_i915_private *i915 = rps_to_i915(rps); + u32 val, rp0; + + val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE); + + rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT; + /* Clamp to max */ + rp0 = min_t(u32, rp0, 0xea); + + return rp0; +} + +static int vlv_rps_rpe_freq(struct intel_rps *rps) +{ + struct drm_i915_private *i915 = rps_to_i915(rps); + u32 val, rpe; + + val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_LO); + rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT; + val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_HI); + rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5; + + return rpe; +} + +static int vlv_rps_min_freq(struct intel_rps *rps) +{ + struct drm_i915_private *i915 = rps_to_i915(rps); + u32 val; + + val = vlv_punit_read(i915, PUNIT_REG_GPU_LFM) & 0xff; + /* + * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value + * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on + * a BYT-M B0 the above register contains 0xbf. Moreover when setting + * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0 + * to make sure it matches what Punit accepts. + */ + return max_t(u32, val, 0xc0); +} + +static bool vlv_rps_enable(struct intel_rps *rps) +{ + struct intel_uncore *uncore = rps_to_uncore(rps); + struct drm_i915_private *i915 = rps_to_i915(rps); + u32 val; + + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000); + intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400); + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000); + intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000); + intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000); + + intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); + + intel_uncore_write_fw(uncore, GEN6_RP_CONTROL, + GEN6_RP_MEDIA_TURBO | + GEN6_RP_MEDIA_HW_NORMAL_MODE | + GEN6_RP_MEDIA_IS_GFX | + GEN6_RP_ENABLE | + GEN6_RP_UP_BUSY_AVG | + GEN6_RP_DOWN_IDLE_CONT); + + vlv_punit_get(i915); + + /* Setting Fixed Bias */ + val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875; + vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val); + + val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); + + vlv_punit_put(i915); + + /* RPS code assumes GPLL is used */ + WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n"); + + DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE)); + DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val); + + return rps_reset(rps); +} + +static unsigned long __ips_gfx_val(struct intel_ips *ips) +{ + struct intel_rps *rps = container_of(ips, typeof(*rps), ips); + struct intel_uncore *uncore = rps_to_uncore(rps); + unsigned long t, corr, state1, corr2, state2; + u32 pxvid, ext_v; + + lockdep_assert_held(&mchdev_lock); + + pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq)); + pxvid = (pxvid >> 24) & 0x7f; + ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid); + + state1 = ext_v; + + /* Revel in the empirically derived constants */ + + /* Correction factor in 1/100000 units */ + t = ips_mch_val(uncore); + if (t > 80) + corr = t * 2349 + 135940; + else if (t >= 50) + corr = t * 964 + 29317; + else /* < 50 */ + corr = t * 301 + 1004; + + corr = corr * 150142 * state1 / 10000 - 78642; + corr /= 100000; + corr2 = corr * ips->corr; + + state2 = corr2 * state1 / 10000; + state2 /= 100; /* convert to mW */ + + __gen5_ips_update(ips); + + return ips->gfx_power + state2; +} + +void intel_rps_enable(struct intel_rps *rps) +{ + struct drm_i915_private *i915 = rps_to_i915(rps); + struct intel_uncore *uncore = rps_to_uncore(rps); + + intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); + if (IS_CHERRYVIEW(i915)) + rps->enabled = chv_rps_enable(rps); + else if (IS_VALLEYVIEW(i915)) + rps->enabled = vlv_rps_enable(rps); + else if (INTEL_GEN(i915) >= 9) + rps->enabled = gen9_rps_enable(rps); + else if (INTEL_GEN(i915) >= 8) + rps->enabled = gen8_rps_enable(rps); + else if (INTEL_GEN(i915) >= 6) + rps->enabled = gen6_rps_enable(rps); + else if (IS_IRONLAKE_M(i915)) + rps->enabled = gen5_rps_enable(rps); + intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); + if (!rps->enabled) + return; + + WARN_ON(rps->max_freq < rps->min_freq); + WARN_ON(rps->idle_freq > rps->max_freq); + + WARN_ON(rps->efficient_freq < rps->min_freq); + WARN_ON(rps->efficient_freq > rps->max_freq); +} + +static void gen6_rps_disable(struct intel_rps *rps) +{ + intel_uncore_write(rps_to_uncore(rps), GEN6_RP_CONTROL, 0); +} + +void intel_rps_disable(struct intel_rps *rps) +{ + struct drm_i915_private *i915 = rps_to_i915(rps); + + rps->enabled = false; + + if (INTEL_GEN(i915) >= 6) + gen6_rps_disable(rps); + else if (IS_IRONLAKE_M(i915)) + gen5_rps_disable(rps); +} + +static int byt_gpu_freq(struct intel_rps *rps, int val) +{ + /* + * N = val - 0xb7 + * Slow = Fast = GPLL ref * N + */ + return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000); +} + +static int byt_freq_opcode(struct intel_rps *rps, int val) +{ + return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7; +} + +static int chv_gpu_freq(struct intel_rps *rps, int val) +{ + /* + * N = val / 2 + * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2 + */ + return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000); +} + +static int chv_freq_opcode(struct intel_rps *rps, int val) +{ + /* CHV needs even values */ + return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2; +} + +int intel_gpu_freq(struct intel_rps *rps, int val) +{ + struct drm_i915_private *i915 = rps_to_i915(rps); + + if (INTEL_GEN(i915) >= 9) + return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER, + GEN9_FREQ_SCALER); + else if (IS_CHERRYVIEW(i915)) + return chv_gpu_freq(rps, val); + else if (IS_VALLEYVIEW(i915)) + return byt_gpu_freq(rps, val); + else + return val * GT_FREQUENCY_MULTIPLIER; +} + +int intel_freq_opcode(struct intel_rps *rps, int val) +{ + struct drm_i915_private *i915 = rps_to_i915(rps); + + if (INTEL_GEN(i915) >= 9) + return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER, + GT_FREQUENCY_MULTIPLIER); + else if (IS_CHERRYVIEW(i915)) + return chv_freq_opcode(rps, val); + else if (IS_VALLEYVIEW(i915)) + return byt_freq_opcode(rps, val); + else + return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER); +} + +static void vlv_init_gpll_ref_freq(struct intel_rps *rps) +{ + struct drm_i915_private *i915 = rps_to_i915(rps); + + rps->gpll_ref_freq = + vlv_get_cck_clock(i915, "GPLL ref", + CCK_GPLL_CLOCK_CONTROL, + i915->czclk_freq); + + DRM_DEBUG_DRIVER("GPLL reference freq: %d kHz\n", rps->gpll_ref_freq); +} + +static void vlv_rps_init(struct intel_rps *rps) +{ + struct drm_i915_private *i915 = rps_to_i915(rps); + u32 val; + + vlv_iosf_sb_get(i915, + BIT(VLV_IOSF_SB_PUNIT) | + BIT(VLV_IOSF_SB_NC) | + BIT(VLV_IOSF_SB_CCK)); + + vlv_init_gpll_ref_freq(rps); + + val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); + switch ((val >> 6) & 3) { + case 0: + case 1: + i915->mem_freq = 800; + break; + case 2: + i915->mem_freq = 1066; + break; + case 3: + i915->mem_freq = 1333; + break; + } + DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", i915->mem_freq); + + rps->max_freq = vlv_rps_max_freq(rps); + rps->rp0_freq = rps->max_freq; + DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n", + intel_gpu_freq(rps, rps->max_freq), + rps->max_freq); + + rps->efficient_freq = vlv_rps_rpe_freq(rps); + DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n", + intel_gpu_freq(rps, rps->efficient_freq), + rps->efficient_freq); + + rps->rp1_freq = vlv_rps_guar_freq(rps); + DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n", + intel_gpu_freq(rps, rps->rp1_freq), + rps->rp1_freq); + + rps->min_freq = vlv_rps_min_freq(rps); + DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n", + intel_gpu_freq(rps, rps->min_freq), + rps->min_freq); + + vlv_iosf_sb_put(i915, + BIT(VLV_IOSF_SB_PUNIT) | + BIT(VLV_IOSF_SB_NC) | + BIT(VLV_IOSF_SB_CCK)); +} + +static void chv_rps_init(struct intel_rps *rps) +{ + struct drm_i915_private *i915 = rps_to_i915(rps); + u32 val; + + vlv_iosf_sb_get(i915, + BIT(VLV_IOSF_SB_PUNIT) | + BIT(VLV_IOSF_SB_NC) | + BIT(VLV_IOSF_SB_CCK)); + + vlv_init_gpll_ref_freq(rps); + + val = vlv_cck_read(i915, CCK_FUSE_REG); + + switch ((val >> 2) & 0x7) { + case 3: + i915->mem_freq = 2000; + break; + default: + i915->mem_freq = 1600; + break; + } + DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", i915->mem_freq); + + rps->max_freq = chv_rps_max_freq(rps); + rps->rp0_freq = rps->max_freq; + DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n", + intel_gpu_freq(rps, rps->max_freq), + rps->max_freq); + + rps->efficient_freq = chv_rps_rpe_freq(rps); + DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n", + intel_gpu_freq(rps, rps->efficient_freq), + rps->efficient_freq); + + rps->rp1_freq = chv_rps_guar_freq(rps); + DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n", + intel_gpu_freq(rps, rps->rp1_freq), + rps->rp1_freq); + + rps->min_freq = chv_rps_min_freq(rps); + DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n", + intel_gpu_freq(rps, rps->min_freq), + rps->min_freq); + + vlv_iosf_sb_put(i915, + BIT(VLV_IOSF_SB_PUNIT) | + BIT(VLV_IOSF_SB_NC) | + BIT(VLV_IOSF_SB_CCK)); + + WARN_ONCE((rps->max_freq | rps->efficient_freq | rps->rp1_freq | + rps->min_freq) & 1, + "Odd GPU freq values\n"); +} + +static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei) +{ + ei->ktime = ktime_get_raw(); + ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT); + ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT); +} + +static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir) +{ + struct intel_uncore *uncore = rps_to_uncore(rps); + const struct intel_rps_ei *prev = &rps->ei; + struct intel_rps_ei now; + u32 events = 0; + + if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0) + return 0; + + vlv_c0_read(uncore, &now); + + if (prev->ktime) { + u64 time, c0; + u32 render, media; + + time = ktime_us_delta(now.ktime, prev->ktime); + + time *= rps_to_i915(rps)->czclk_freq; + + /* Workload can be split between render + media, + * e.g. SwapBuffers being blitted in X after being rendered in + * mesa. To account for this we need to combine both engines + * into our activity counter. + */ + render = now.render_c0 - prev->render_c0; + media = now.media_c0 - prev->media_c0; + c0 = max(render, media); + c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */ + + if (c0 > time * rps->power.up_threshold) + events = GEN6_PM_RP_UP_THRESHOLD; + else if (c0 < time * rps->power.down_threshold) + events = GEN6_PM_RP_DOWN_THRESHOLD; + } + + rps->ei = now; + return events; +} + +static void rps_work(struct work_struct *work) +{ + struct intel_rps *rps = container_of(work, typeof(*rps), work); + struct intel_gt *gt = rps_to_gt(rps); + bool client_boost = false; + int new_freq, adj, min, max; + u32 pm_iir = 0; + + spin_lock_irq(>->irq_lock); + pm_iir = fetch_and_zero(&rps->pm_iir); + client_boost = atomic_read(&rps->num_waiters); + spin_unlock_irq(>->irq_lock); + + /* Make sure we didn't queue anything we're not going to process. */ + if ((pm_iir & rps->pm_events) == 0 && !client_boost) + goto out; + + mutex_lock(&rps->lock); + + pm_iir |= vlv_wa_c0_ei(rps, pm_iir); + + adj = rps->last_adj; + new_freq = rps->cur_freq; + min = rps->min_freq_softlimit; + max = rps->max_freq_softlimit; + if (client_boost) + max = rps->max_freq; + if (client_boost && new_freq < rps->boost_freq) { + new_freq = rps->boost_freq; + adj = 0; + } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) { + if (adj > 0) + adj *= 2; + else /* CHV needs even encode values */ + adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1; + + if (new_freq >= rps->max_freq_softlimit) + adj = 0; + } else if (client_boost) { + adj = 0; + } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) { + if (rps->cur_freq > rps->efficient_freq) + new_freq = rps->efficient_freq; + else if (rps->cur_freq > rps->min_freq_softlimit) + new_freq = rps->min_freq_softlimit; + adj = 0; + } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) { + if (adj < 0) + adj *= 2; + else /* CHV needs even encode values */ + adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1; + + if (new_freq <= rps->min_freq_softlimit) + adj = 0; + } else { /* unknown event */ + adj = 0; + } + + rps->last_adj = adj; + + /* + * Limit deboosting and boosting to keep ourselves at the extremes + * when in the respective power modes (i.e. slowly decrease frequencies + * while in the HIGH_POWER zone and slowly increase frequencies while + * in the LOW_POWER zone). On idle, we will hit the timeout and drop + * to the next level quickly, and conversely if busy we expect to + * hit a waitboost and rapidly switch into max power. + */ + if ((adj < 0 && rps->power.mode == HIGH_POWER) || + (adj > 0 && rps->power.mode == LOW_POWER)) + rps->last_adj = 0; + + /* sysfs frequency interfaces may have snuck in while servicing the + * interrupt + */ + new_freq += adj; + new_freq = clamp_t(int, new_freq, min, max); + + if (intel_rps_set(rps, new_freq)) { + DRM_DEBUG_DRIVER("Failed to set new GPU frequency\n"); + rps->last_adj = 0; + } + + mutex_unlock(&rps->lock); + +out: + spin_lock_irq(>->irq_lock); + gen6_gt_pm_unmask_irq(gt, rps->pm_events); + spin_unlock_irq(>->irq_lock); +} + +void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) +{ + struct intel_gt *gt = rps_to_gt(rps); + const u32 events = rps->pm_events & pm_iir; + + lockdep_assert_held(>->irq_lock); + + if (unlikely(!events)) + return; + + gen6_gt_pm_mask_irq(gt, events); + + rps->pm_iir |= events; + schedule_work(&rps->work); +} + +void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) +{ + struct drm_i915_private *i915 = rps_to_i915(rps); + + if (pm_iir & rps->pm_events) { + struct intel_gt *gt = rps_to_gt(rps); + + spin_lock(>->irq_lock); + gen6_gt_pm_mask_irq(gt, pm_iir & rps->pm_events); + rps->pm_iir |= pm_iir & rps->pm_events; + schedule_work(&rps->work); + spin_unlock(>->irq_lock); + } + + if (INTEL_GEN(i915) >= 8) + return; + + if (pm_iir & PM_VEBOX_USER_INTERRUPT) + intel_engine_breadcrumbs_irq(i915->engine[VECS0]); + + if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) + DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir); +} + +void gen5_rps_irq_handler(struct intel_rps *rps) +{ + struct intel_uncore *uncore = rps_to_uncore(rps); + u32 busy_up, busy_down, max_avg, min_avg; + u8 new_freq; + + spin_lock(&mchdev_lock); + + intel_uncore_write16(uncore, + MEMINTRSTS, + intel_uncore_read(uncore, MEMINTRSTS)); + + intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); + busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG); + busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG); + max_avg = intel_uncore_read(uncore, RCBMAXAVG); + min_avg = intel_uncore_read(uncore, RCBMINAVG); + + /* Handle RCS change request from hw */ + new_freq = rps->cur_freq; + if (busy_up > max_avg) + new_freq++; + else if (busy_down < min_avg) + new_freq--; + new_freq = clamp(new_freq, + rps->min_freq_softlimit, + rps->max_freq_softlimit); + + if (new_freq != rps->cur_freq && gen5_rps_set(rps, new_freq)) + rps->cur_freq = new_freq; + + spin_unlock(&mchdev_lock); +} + +void intel_rps_init(struct intel_rps *rps) +{ + struct drm_i915_private *i915 = rps_to_i915(rps); + + mutex_init(&rps->lock); + mutex_init(&rps->power.mutex); + + INIT_WORK(&rps->work, rps_work); + + atomic_set(&rps->num_waiters, 0); + + if (IS_CHERRYVIEW(i915)) + chv_rps_init(rps); + else if (IS_VALLEYVIEW(i915)) + vlv_rps_init(rps); + else if (INTEL_GEN(i915) >= 6) + gen6_rps_init(rps); + else if (IS_IRONLAKE_M(i915)) + gen5_rps_init(rps); + + /* Derive initial user preferences/limits from the hardware limits */ + rps->max_freq_softlimit = rps->max_freq; + rps->min_freq_softlimit = rps->min_freq; + + /* After setting max-softlimit, find the overclock max freq */ + if (IS_GEN(i915, 6) || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) { + u32 params = 0; + + sandybridge_pcode_read(i915, GEN6_READ_OC_PARAMS, + ¶ms, NULL); + if (params & BIT(31)) { /* OC supported */ + DRM_DEBUG_DRIVER("Overclocking supported, max: %dMHz, overclock: %dMHz\n", + (rps->max_freq & 0xff) * 50, + (params & 0xff) * 50); + rps->max_freq = params & 0xff; + } + } + + /* Finally allow us to boost to max by default */ + rps->boost_freq = rps->max_freq; + rps->idle_freq = rps->min_freq; + rps->cur_freq = rps->idle_freq; + + rps->pm_intrmsk_mbz = 0; + + /* + * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer + * if GEN6_PM_UP_EI_EXPIRED is masked. + * + * TODO: verify if this can be reproduced on VLV,CHV. + */ + if (INTEL_GEN(i915) <= 7) + rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED; + + if (INTEL_GEN(i915) >= 8) + rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC; +} + +u32 intel_get_cagf(struct intel_rps *rps, u32 rpstat) +{ + struct drm_i915_private *i915 = rps_to_i915(rps); + u32 cagf; + + if (INTEL_GEN(i915) >= 9) + cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT; + else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) + cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT; + else + cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT; + + return cagf; +} + +/* External interface for intel_ips.ko */ + +static struct drm_i915_private __rcu *ips_mchdev; + +/** + * Tells the intel_ips driver that the i915 driver is now loaded, if + * IPS got loaded first. + * + * This awkward dance is so that neither module has to depend on the + * other in order for IPS to do the appropriate communication of + * GPU turbo limits to i915. + */ +static void +ips_ping_for_i915_load(void) +{ + void (*link)(void); + + link = symbol_get(ips_link_to_i915_driver); + if (link) { + link(); + symbol_put(ips_link_to_i915_driver); + } +} + +void intel_rps_driver_register(struct intel_rps *rps) +{ + struct intel_gt *gt = rps_to_gt(rps); + + /* + * We only register the i915 ips part with intel-ips once everything is + * set up, to avoid intel-ips sneaking in and reading bogus values. + */ + if (IS_GEN(gt->i915, 5)) { + rcu_assign_pointer(ips_mchdev, gt->i915); + ips_ping_for_i915_load(); + } +} + +void intel_rps_driver_unregister(struct intel_rps *rps) +{ + rcu_assign_pointer(ips_mchdev, NULL); +} + +static struct drm_i915_private *mchdev_get(void) +{ + struct drm_i915_private *i915; + + rcu_read_lock(); + i915 = rcu_dereference(ips_mchdev); + if (!kref_get_unless_zero(&i915->drm.ref)) + i915 = NULL; + rcu_read_unlock(); + + return i915; +} + +/** + * i915_read_mch_val - return value for IPS use + * + * Calculate and return a value for the IPS driver to use when deciding whether + * we have thermal and power headroom to increase CPU or GPU power budget. + */ +unsigned long i915_read_mch_val(void) +{ + struct drm_i915_private *i915; + unsigned long chipset_val = 0; + unsigned long graphics_val = 0; + intel_wakeref_t wakeref; + + i915 = mchdev_get(); + if (!i915) + return 0; + + with_intel_runtime_pm(&i915->runtime_pm, wakeref) { + struct intel_ips *ips = &i915->gt.rps.ips; + + spin_lock_irq(&mchdev_lock); + chipset_val = __ips_chipset_val(ips); + graphics_val = __ips_gfx_val(ips); + spin_unlock_irq(&mchdev_lock); + } + + drm_dev_put(&i915->drm); + return chipset_val + graphics_val; +} +EXPORT_SYMBOL_GPL(i915_read_mch_val); + +/** + * i915_gpu_raise - raise GPU frequency limit + * + * Raise the limit; IPS indicates we have thermal headroom. + */ +bool i915_gpu_raise(void) +{ + struct drm_i915_private *i915; + struct intel_rps *rps; + + i915 = mchdev_get(); + if (!i915) + return false; + + rps = &i915->gt.rps; + + spin_lock_irq(&mchdev_lock); + if (rps->max_freq_softlimit < rps->max_freq) + rps->max_freq_softlimit++; + spin_unlock_irq(&mchdev_lock); + + drm_dev_put(&i915->drm); + return true; +} +EXPORT_SYMBOL_GPL(i915_gpu_raise); + +/** + * i915_gpu_lower - lower GPU frequency limit + * + * IPS indicates we're close to a thermal limit, so throttle back the GPU + * frequency maximum. + */ +bool i915_gpu_lower(void) +{ + struct drm_i915_private *i915; + struct intel_rps *rps; + + i915 = mchdev_get(); + if (!i915) + return false; + + rps = &i915->gt.rps; + + spin_lock_irq(&mchdev_lock); + if (rps->max_freq_softlimit > rps->min_freq) + rps->max_freq_softlimit--; + spin_unlock_irq(&mchdev_lock); + + drm_dev_put(&i915->drm); + return true; +} +EXPORT_SYMBOL_GPL(i915_gpu_lower); + +/** + * i915_gpu_busy - indicate GPU business to IPS + * + * Tell the IPS driver whether or not the GPU is busy. + */ +bool i915_gpu_busy(void) +{ + struct drm_i915_private *i915; + bool ret; + + i915 = mchdev_get(); + if (!i915) + return false; + + ret = i915->gt.awake; + + drm_dev_put(&i915->drm); + return ret; +} +EXPORT_SYMBOL_GPL(i915_gpu_busy); + +/** + * i915_gpu_turbo_disable - disable graphics turbo + * + * Disable graphics turbo by resetting the max frequency and setting the + * current frequency to the default. + */ +bool i915_gpu_turbo_disable(void) +{ + struct drm_i915_private *i915; + struct intel_rps *rps; + bool ret; + + i915 = mchdev_get(); + if (!i915) + return false; + + rps = &i915->gt.rps; + + spin_lock_irq(&mchdev_lock); + rps->max_freq_softlimit = rps->min_freq; + ret = gen5_rps_set(&i915->gt.rps, rps->min_freq); + spin_unlock_irq(&mchdev_lock); + + drm_dev_put(&i915->drm); + return ret; +} +EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable); diff --git a/drivers/gpu/drm/i915/gt/intel_rps.h b/drivers/gpu/drm/i915/gt/intel_rps.h new file mode 100644 index 000000000000..997a4b4e0207 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/intel_rps.h @@ -0,0 +1,37 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright © 2019 Intel Corporation + */ + +#ifndef INTEL_RPS_H +#define INTEL_RPS_H + +#include "intel_rps_types.h" + +struct i915_request; + +void intel_rps_init(struct intel_rps *rps); + +void intel_rps_driver_register(struct intel_rps *rps); +void intel_rps_driver_unregister(struct intel_rps *rps); + +void intel_rps_enable(struct intel_rps *rps); +void intel_rps_disable(struct intel_rps *rps); + +void intel_rps_park(struct intel_rps *rps); +void intel_rps_unpark(struct intel_rps *rps); +void intel_rps_boost(struct i915_request *rq); + +int intel_rps_set(struct intel_rps *rps, u8 val); +void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive); + +int intel_gpu_freq(struct intel_rps *rps, int val); +int intel_freq_opcode(struct intel_rps *rps, int val); +u32 intel_get_cagf(struct intel_rps *rps, u32 rpstat1); + +void gen5_rps_irq_handler(struct intel_rps *rps); +void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir); +void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir); + +#endif /* INTEL_RPS_H */ diff --git a/drivers/gpu/drm/i915/gt/intel_rps_types.h b/drivers/gpu/drm/i915/gt/intel_rps_types.h new file mode 100644 index 000000000000..40eb1fb651e7 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/intel_rps_types.h @@ -0,0 +1,93 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright © 2019 Intel Corporation + */ + +#ifndef INTEL_RPS_TYPES_H +#define INTEL_RPS_TYPES_H + +#include <linux/atomic.h> +#include <linux/ktime.h> +#include <linux/mutex.h> +#include <linux/types.h> +#include <linux/workqueue.h> + +struct intel_ips { + u64 last_count1; + unsigned long last_time1; + unsigned long chipset_power; + u64 last_count2; + u64 last_time2; + unsigned long gfx_power; + u8 corr; + + int c, m; +}; + +struct intel_rps_ei { + ktime_t ktime; + u32 render_c0; + u32 media_c0; +}; + +struct intel_rps { + struct mutex lock; /* protects enabling and the worker */ + + /* + * work, interrupts_enabled and pm_iir are protected by + * dev_priv->irq_lock + */ + struct work_struct work; + bool enabled; + bool active; + u32 pm_iir; + + /* PM interrupt bits that should never be masked */ + u32 pm_intrmsk_mbz; + u32 pm_events; + + /* Frequencies are stored in potentially platform dependent multiples. + * In other words, *_freq needs to be multiplied by X to be interesting. + * Soft limits are those which are used for the dynamic reclocking done + * by the driver (raise frequencies under heavy loads, and lower for + * lighter loads). Hard limits are those imposed by the hardware. + * + * A distinction is made for overclocking, which is never enabled by + * default, and is considered to be above the hard limit if it's + * possible at all. + */ + u8 cur_freq; /* Current frequency (cached, may not == HW) */ + u8 last_freq; /* Last swqreq frequency */ + u8 min_freq_softlimit; /* Minimum frequency permitted by the driver */ + u8 max_freq_softlimit; /* Max frequency permitted by the driver */ + u8 max_freq; /* Maximum frequency, RP0 if not overclocking */ + u8 min_freq; /* AKA RPn. Minimum frequency */ + u8 boost_freq; /* Frequency to request when wait boosting */ + u8 idle_freq; /* Frequency to request when we are idle */ + u8 efficient_freq; /* AKA RPe. Pre-determined balanced frequency */ + u8 rp1_freq; /* "less than" RP0 power/freqency */ + u8 rp0_freq; /* Non-overclocked max frequency. */ + u16 gpll_ref_freq; /* vlv/chv GPLL reference frequency */ + + int last_adj; + + struct { + struct mutex mutex; + + enum { LOW_POWER, BETWEEN, HIGH_POWER } mode; + unsigned int interactive; + + u8 up_threshold; /* Current %busy required to uplock */ + u8 down_threshold; /* Current %busy required to downclock */ + } power; + + atomic_t num_waiters; + atomic_t boosts; + + /* manual wa residency calculations */ + struct intel_rps_ei ei; + struct intel_ips ips; +}; + +#endif /* INTEL_RPS_TYPES_H */ diff --git a/drivers/gpu/drm/i915/gt/selftest_llc.c b/drivers/gpu/drm/i915/gt/selftest_llc.c index a7057785e420..fd3770e48ac7 100644 --- a/drivers/gpu/drm/i915/gt/selftest_llc.c +++ b/drivers/gpu/drm/i915/gt/selftest_llc.c @@ -6,6 +6,7 @@ #include "intel_pm.h" /* intel_gpu_freq() */ #include "selftest_llc.h" +#include "intel_rps.h" static int gen6_verify_ring_freq(struct intel_llc *llc) { @@ -25,6 +26,8 @@ static int gen6_verify_ring_freq(struct intel_llc *llc) for (gpu_freq = consts.min_gpu_freq; gpu_freq <= consts.max_gpu_freq; gpu_freq++) { + struct intel_rps *rps = &llc_to_gt(llc)->rps; + unsigned int ia_freq, ring_freq, found; u32 val; @@ -44,7 +47,7 @@ static int gen6_verify_ring_freq(struct intel_llc *llc) if (found != ia_freq) { pr_err("Min freq table(%d/[%d, %d]):%dMHz did not match expected CPU freq, found %d, expected %d\n", gpu_freq, consts.min_gpu_freq, consts.max_gpu_freq, - intel_gpu_freq(i915, gpu_freq * (INTEL_GEN(i915) >= 9 ? GEN9_FREQ_SCALER : 1)), + intel_gpu_freq(rps, gpu_freq * (INTEL_GEN(i915) >= 9 ? GEN9_FREQ_SCALER : 1)), found, ia_freq); err = -EINVAL; break; @@ -54,7 +57,7 @@ static int gen6_verify_ring_freq(struct intel_llc *llc) if (found != ring_freq) { pr_err("Min freq table(%d/[%d, %d]):%dMHz did not match expected ring freq, found %d, expected %d\n", gpu_freq, consts.min_gpu_freq, consts.max_gpu_freq, - intel_gpu_freq(i915, gpu_freq * (INTEL_GEN(i915) >= 9 ? GEN9_FREQ_SCALER : 1)), + intel_gpu_freq(rps, gpu_freq * (INTEL_GEN(i915) >= 9 ? GEN9_FREQ_SCALER : 1)), found, ring_freq); err = -EINVAL; break; diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c index 009e54a3764f..97b858ce7682 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c @@ -1010,7 +1010,7 @@ void intel_guc_submission_fini(struct intel_guc *guc) static void guc_interrupts_capture(struct intel_gt *gt) { - struct intel_rps *rps = >->i915->gt_pm.rps; + struct intel_rps *rps = >->rps; struct intel_uncore *uncore = gt->uncore; struct intel_engine_cs *engine; enum intel_engine_id id; @@ -1056,7 +1056,7 @@ static void guc_interrupts_capture(struct intel_gt *gt) static void guc_interrupts_release(struct intel_gt *gt) { - struct intel_rps *rps = >->i915->gt_pm.rps; + struct intel_rps *rps = >->rps; struct intel_uncore *uncore = gt->uncore; struct intel_engine_cs *engine; enum intel_engine_id id; diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index 16211430eb78..22962373723b 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -44,6 +44,7 @@ #include "gt/intel_gt_requests.h" #include "gt/intel_reset.h" #include "gt/intel_rc6.h" +#include "gt/intel_rps.h" #include "gt/uc/intel_guc_submission.h" #include "i915_debugfs.h" @@ -791,7 +792,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused) { struct drm_i915_private *dev_priv = node_to_i915(m->private); struct intel_uncore *uncore = &dev_priv->uncore; - struct intel_rps *rps = &dev_priv->gt_pm.rps; + struct intel_rps *rps = &dev_priv->gt.rps; intel_wakeref_t wakeref; int ret = 0; @@ -827,23 +828,23 @@ static int i915_frequency_info(struct seq_file *m, void *unused) seq_printf(m, "DDR freq: %d MHz\n", dev_priv->mem_freq); seq_printf(m, "actual GPU freq: %d MHz\n", - intel_gpu_freq(dev_priv, (freq_sts >> 8) & 0xff)); + intel_gpu_freq(rps, (freq_sts >> 8) & 0xff)); seq_printf(m, "current GPU freq: %d MHz\n", - intel_gpu_freq(dev_priv, rps->cur_freq)); + intel_gpu_freq(rps, rps->cur_freq)); seq_printf(m, "max GPU freq: %d MHz\n", - intel_gpu_freq(dev_priv, rps->max_freq)); + intel_gpu_freq(rps, rps->max_freq)); seq_printf(m, "min GPU freq: %d MHz\n", - intel_gpu_freq(dev_priv, rps->min_freq)); + intel_gpu_freq(rps, rps->min_freq)); seq_printf(m, "idle GPU freq: %d MHz\n", - intel_gpu_freq(dev_priv, rps->idle_freq)); + intel_gpu_freq(rps, rps->idle_freq)); seq_printf(m, "efficient (RPe) frequency: %d MHz\n", - intel_gpu_freq(dev_priv, rps->efficient_freq)); + intel_gpu_freq(rps, rps->efficient_freq)); } else if (INTEL_GEN(dev_priv) >= 6) { u32 rp_state_limits; u32 gt_perf_status; @@ -877,7 +878,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused) else reqf >>= 25; } - reqf = intel_gpu_freq(dev_priv, reqf); + reqf = intel_gpu_freq(rps, reqf); rpmodectl = I915_READ(GEN6_RP_CONTROL); rpinclimit = I915_READ(GEN6_RP_UP_THRESHOLD); @@ -890,8 +891,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused) rpdownei = I915_READ(GEN6_RP_CUR_DOWN_EI) & GEN6_CURIAVG_MASK; rpcurdown = I915_READ(GEN6_RP_CUR_DOWN) & GEN6_CURBSYTAVG_MASK; rpprevdown = I915_READ(GEN6_RP_PREV_DOWN) & GEN6_CURBSYTAVG_MASK; - cagf = intel_gpu_freq(dev_priv, - intel_get_cagf(dev_priv, rpstat)); + cagf = intel_gpu_freq(rps, intel_get_cagf(rps, rpstat)); intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL); @@ -968,37 +968,37 @@ static int i915_frequency_info(struct seq_file *m, void *unused) max_freq *= (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10 ? GEN9_FREQ_SCALER : 1); seq_printf(m, "Lowest (RPN) frequency: %dMHz\n", - intel_gpu_freq(dev_priv, max_freq)); + intel_gpu_freq(rps, max_freq)); max_freq = (rp_state_cap & 0xff00) >> 8; max_freq *= (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10 ? GEN9_FREQ_SCALER : 1); seq_printf(m, "Nominal (RP1) frequency: %dMHz\n", - intel_gpu_freq(dev_priv, max_freq)); + intel_gpu_freq(rps, max_freq)); max_freq = (IS_GEN9_LP(dev_priv) ? rp_state_cap >> 16 : rp_state_cap >> 0) & 0xff; max_freq *= (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10 ? GEN9_FREQ_SCALER : 1); seq_printf(m, "Max non-overclocked (RP0) frequency: %dMHz\n", - intel_gpu_freq(dev_priv, max_freq)); + intel_gpu_freq(rps, max_freq)); seq_printf(m, "Max overclocked frequency: %dMHz\n", - intel_gpu_freq(dev_priv, rps->max_freq)); + intel_gpu_freq(rps, rps->max_freq)); seq_printf(m, "Current freq: %d MHz\n", - intel_gpu_freq(dev_priv, rps->cur_freq)); + intel_gpu_freq(rps, rps->cur_freq)); seq_printf(m, "Actual freq: %d MHz\n", cagf); seq_printf(m, "Idle freq: %d MHz\n", - intel_gpu_freq(dev_priv, rps->idle_freq)); + intel_gpu_freq(rps, rps->idle_freq)); seq_printf(m, "Min freq: %d MHz\n", - intel_gpu_freq(dev_priv, rps->min_freq)); + intel_gpu_freq(rps, rps->min_freq)); seq_printf(m, "Boost freq: %d MHz\n", - intel_gpu_freq(dev_priv, rps->boost_freq)); + intel_gpu_freq(rps, rps->boost_freq)); seq_printf(m, "Max freq: %d MHz\n", - intel_gpu_freq(dev_priv, rps->max_freq)); + intel_gpu_freq(rps, rps->max_freq)); seq_printf(m, "efficient (RPe) frequency: %d MHz\n", - intel_gpu_freq(dev_priv, rps->efficient_freq)); + intel_gpu_freq(rps, rps->efficient_freq)); } else { seq_puts(m, "no P-state info available\n"); } @@ -1461,7 +1461,7 @@ static int i915_sr_status(struct seq_file *m, void *unused) static int i915_ring_freq_table(struct seq_file *m, void *unused) { struct drm_i915_private *dev_priv = node_to_i915(m->private); - struct intel_rps *rps = &dev_priv->gt_pm.rps; + struct intel_rps *rps = &dev_priv->gt.rps; unsigned int max_gpu_freq, min_gpu_freq; intel_wakeref_t wakeref; int gpu_freq, ia_freq; @@ -1486,10 +1486,11 @@ static int i915_ring_freq_table(struct seq_file *m, void *unused) GEN6_PCODE_READ_MIN_FREQ_TABLE, &ia_freq, NULL); seq_printf(m, "%d\t\t%d\t\t\t\t%d\n", - intel_gpu_freq(dev_priv, (gpu_freq * - (IS_GEN9_BC(dev_priv) || - INTEL_GEN(dev_priv) >= 10 ? - GEN9_FREQ_SCALER : 1))), + intel_gpu_freq(rps, + (gpu_freq * + (IS_GEN9_BC(dev_priv) || + INTEL_GEN(dev_priv) >= 10 ? + GEN9_FREQ_SCALER : 1))), ((ia_freq >> 0) & 0xff) * 100, ((ia_freq >> 8) & 0xff) * 100); } @@ -1717,7 +1718,7 @@ static const char *rps_power_to_str(unsigned int power) static int i915_rps_boost_info(struct seq_file *m, void *data) { struct drm_i915_private *dev_priv = node_to_i915(m->private); - struct intel_rps *rps = &dev_priv->gt_pm.rps; + struct intel_rps *rps = &dev_priv->gt.rps; u32 act_freq = rps->cur_freq; intel_wakeref_t wakeref; @@ -1729,7 +1730,7 @@ static int i915_rps_boost_info(struct seq_file *m, void *data) vlv_punit_put(dev_priv); act_freq = (act_freq >> 8) & 0xff; } else { - act_freq = intel_get_cagf(dev_priv, + act_freq = intel_get_cagf(rps, I915_READ(GEN6_RPSTAT1)); } } @@ -1740,17 +1741,17 @@ static int i915_rps_boost_info(struct seq_file *m, void *data) atomic_read(&rps->num_waiters)); seq_printf(m, "Interactive? %d\n", READ_ONCE(rps->power.interactive)); seq_printf(m, "Frequency requested %d, actual %d\n", - intel_gpu_freq(dev_priv, rps->cur_freq), - intel_gpu_freq(dev_priv, act_freq)); + intel_gpu_freq(rps, rps->cur_freq), + intel_gpu_freq(rps, act_freq)); seq_printf(m, " min hard:%d, soft:%d; max soft:%d, hard:%d\n", - intel_gpu_freq(dev_priv, rps->min_freq), - intel_gpu_freq(dev_priv, rps->min_freq_softlimit), - intel_gpu_freq(dev_priv, rps->max_freq_softlimit), - intel_gpu_freq(dev_priv, rps->max_freq)); + intel_gpu_freq(rps, rps->min_freq), + intel_gpu_freq(rps, rps->min_freq_softlimit), + intel_gpu_freq(rps, rps->max_freq_softlimit), + intel_gpu_freq(rps, rps->max_freq)); seq_printf(m, " idle:%d, efficient:%d, boost:%d\n", - intel_gpu_freq(dev_priv, rps->idle_freq), - intel_gpu_freq(dev_priv, rps->efficient_freq), - intel_gpu_freq(dev_priv, rps->boost_freq)); + intel_gpu_freq(rps, rps->idle_freq), + intel_gpu_freq(rps, rps->efficient_freq), + intel_gpu_freq(rps, rps->boost_freq)); seq_printf(m, "Wait boosts: %d\n", atomic_read(&rps->boosts)); diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index 5138d1eed306..78ac08db48a3 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -1841,6 +1841,8 @@ static int i915_drm_resume(struct drm_device *dev) intel_dp_mst_resume(dev_priv); + intel_gt_pm_enable(&dev_priv->gt); + intel_display_resume(dev); drm_kms_helper_poll_enable(dev); diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 8882c0908c3b..eb5460290811 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -543,94 +543,6 @@ struct i915_suspend_saved_registers { struct vlv_s0ix_state; -struct intel_rps_ei { - ktime_t ktime; - u32 render_c0; - u32 media_c0; -}; - -struct intel_rps { - struct mutex lock; /* protects enabling and the worker */ - - /* - * work, interrupts_enabled and pm_iir are protected by - * dev_priv->irq_lock - */ - struct work_struct work; - bool interrupts_enabled; - u32 pm_iir; - - /* PM interrupt bits that should never be masked */ - u32 pm_intrmsk_mbz; - - /* Frequencies are stored in potentially platform dependent multiples. - * In other words, *_freq needs to be multiplied by X to be interesting. - * Soft limits are those which are used for the dynamic reclocking done - * by the driver (raise frequencies under heavy loads, and lower for - * lighter loads). Hard limits are those imposed by the hardware. - * - * A distinction is made for overclocking, which is never enabled by - * default, and is considered to be above the hard limit if it's - * possible at all. - */ - u8 cur_freq; /* Current frequency (cached, may not == HW) */ - u8 min_freq_softlimit; /* Minimum frequency permitted by the driver */ - u8 max_freq_softlimit; /* Max frequency permitted by the driver */ - u8 max_freq; /* Maximum frequency, RP0 if not overclocking */ - u8 min_freq; /* AKA RPn. Minimum frequency */ - u8 boost_freq; /* Frequency to request when wait boosting */ - u8 idle_freq; /* Frequency to request when we are idle */ - u8 efficient_freq; /* AKA RPe. Pre-determined balanced frequency */ - u8 rp1_freq; /* "less than" RP0 power/freqency */ - u8 rp0_freq; /* Non-overclocked max frequency. */ - u16 gpll_ref_freq; /* vlv/chv GPLL reference frequency */ - - int last_adj; - - struct { - struct mutex mutex; - - enum { LOW_POWER, BETWEEN, HIGH_POWER } mode; - unsigned int interactive; - - u8 up_threshold; /* Current %busy required to uplock */ - u8 down_threshold; /* Current %busy required to downclock */ - } power; - - bool enabled; - atomic_t num_waiters; - atomic_t boosts; - - /* manual wa residency calculations */ - struct intel_rps_ei ei; -}; - -struct intel_gen6_power_mgmt { - struct intel_rps rps; -}; - -/* defined intel_pm.c */ -extern spinlock_t mchdev_lock; - -struct intel_ilk_power_mgmt { - u8 cur_delay; - u8 min_delay; - u8 max_delay; - u8 fmax; - u8 fstart; - - u64 last_count1; - unsigned long last_time1; - unsigned long chipset_power; - u64 last_count2; - u64 last_time2; - unsigned long gfx_power; - u8 corr; - - int c_m; - int r_t; -}; - #define MAX_L3_SLICES 2 struct intel_l3_parity { u32 *remap_info[MAX_L3_SLICES]; @@ -1067,7 +979,6 @@ struct drm_i915_private { u32 irq_mask; u32 de_irq_mask[I915_MAX_PIPES]; }; - u32 pm_rps_events; u32 pipestat_irq_mask[I915_MAX_PIPES]; struct i915_hotplug hotplug; @@ -1202,13 +1113,6 @@ struct drm_i915_private { */ u32 edram_size_mb; - /* gen6+ GT PM state */ - struct intel_gen6_power_mgmt gt_pm; - - /* ilk-only ips/rps state. Everything in here is protected by the global - * mchdev_lock in intel_pm.c */ - struct intel_ilk_power_mgmt ips; - struct i915_power_domains power_domains; struct i915_psr psr; diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index b882988056bd..eab1709d1897 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -52,6 +52,7 @@ #include "gt/intel_mocs.h" #include "gt/intel_reset.h" #include "gt/intel_renderstate.h" +#include "gt/intel_rps.h" #include "gt/intel_workarounds.h" #include "i915_drv.h" @@ -1272,8 +1273,6 @@ int i915_gem_init(struct drm_i915_private *dev_priv) goto err_context; } - intel_init_gt_powersave(dev_priv); - intel_uc_init(&dev_priv->gt.uc); ret = intel_gt_init_hw(&dev_priv->gt); diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index 572a5c37cc61..598924b3c556 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -45,6 +45,7 @@ #include "gt/intel_gt.h" #include "gt/intel_gt_irq.h" #include "gt/intel_gt_pm_irq.h" +#include "gt/intel_rps.h" #include "i915_drv.h" #include "i915_irq.h" @@ -327,87 +328,6 @@ static i915_reg_t gen6_pm_iir(struct drm_i915_private *dev_priv) return INTEL_GEN(dev_priv) >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR; } -void gen11_reset_rps_interrupts(struct drm_i915_private *dev_priv) -{ - struct intel_gt *gt = &dev_priv->gt; - - spin_lock_irq(>->irq_lock); - - while (gen11_gt_reset_one_iir(gt, 0, GEN11_GTPM)) - ; - - dev_priv->gt_pm.rps.pm_iir = 0; - - spin_unlock_irq(>->irq_lock); -} - -void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv) -{ - struct intel_gt *gt = &dev_priv->gt; - - spin_lock_irq(>->irq_lock); - gen6_gt_pm_reset_iir(gt, GEN6_PM_RPS_EVENTS); - dev_priv->gt_pm.rps.pm_iir = 0; - spin_unlock_irq(>->irq_lock); -} - -void gen6_enable_rps_interrupts(struct drm_i915_private *dev_priv) -{ - struct intel_gt *gt = &dev_priv->gt; - struct intel_rps *rps = &dev_priv->gt_pm.rps; - - if (READ_ONCE(rps->interrupts_enabled)) - return; - - spin_lock_irq(>->irq_lock); - WARN_ON_ONCE(rps->pm_iir); - - if (INTEL_GEN(dev_priv) >= 11) - WARN_ON_ONCE(gen11_gt_reset_one_iir(gt, 0, GEN11_GTPM)); - else - WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events); - - rps->interrupts_enabled = true; - gen6_gt_pm_enable_irq(gt, dev_priv->pm_rps_events); - - spin_unlock_irq(>->irq_lock); -} - -u32 gen6_sanitize_rps_pm_mask(const struct drm_i915_private *i915, u32 mask) -{ - return mask & ~i915->gt_pm.rps.pm_intrmsk_mbz; -} - -void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv) -{ - struct intel_rps *rps = &dev_priv->gt_pm.rps; - struct intel_gt *gt = &dev_priv->gt; - - if (!READ_ONCE(rps->interrupts_enabled)) - return; - - spin_lock_irq(>->irq_lock); - rps->interrupts_enabled = false; - - I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0u)); - - gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS); - - spin_unlock_irq(>->irq_lock); - intel_synchronize_irq(dev_priv); - - /* Now that we will not be generating any more work, flush any - * outstanding tasks. As we are called on the RPS idle path, - * we will reset the GPU to minimum frequencies, so the current - * state of the worker can be discarded. - */ - cancel_work_sync(&rps->work); - if (INTEL_GEN(dev_priv) >= 11) - gen11_reset_rps_interrupts(dev_priv); - else - gen6_reset_rps_interrupts(dev_priv); -} - void gen9_reset_guc_interrupts(struct intel_guc *guc) { struct intel_gt *gt = guc_to_gt(guc); @@ -1065,199 +985,6 @@ int intel_get_crtc_scanline(struct intel_crtc *crtc) return position; } -static void ironlake_rps_change_irq_handler(struct drm_i915_private *dev_priv) -{ - struct intel_uncore *uncore = &dev_priv->uncore; - u32 busy_up, busy_down, max_avg, min_avg; - u8 new_delay; - - spin_lock(&mchdev_lock); - - intel_uncore_write16(uncore, - MEMINTRSTS, - intel_uncore_read(uncore, MEMINTRSTS)); - - new_delay = dev_priv->ips.cur_delay; - - intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); - busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG); - busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG); - max_avg = intel_uncore_read(uncore, RCBMAXAVG); - min_avg = intel_uncore_read(uncore, RCBMINAVG); - - /* Handle RCS change request from hw */ - if (busy_up > max_avg) { - if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay) - new_delay = dev_priv->ips.cur_delay - 1; - if (new_delay < dev_priv->ips.max_delay) - new_delay = dev_priv->ips.max_delay; - } else if (busy_down < min_avg) { - if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay) - new_delay = dev_priv->ips.cur_delay + 1; - if (new_delay > dev_priv->ips.min_delay) - new_delay = dev_priv->ips.min_delay; - } - - if (ironlake_set_drps(dev_priv, new_delay)) - dev_priv->ips.cur_delay = new_delay; - - spin_unlock(&mchdev_lock); - - return; -} - -static void vlv_c0_read(struct drm_i915_private *dev_priv, - struct intel_rps_ei *ei) -{ - ei->ktime = ktime_get_raw(); - ei->render_c0 = I915_READ(VLV_RENDER_C0_COUNT); - ei->media_c0 = I915_READ(VLV_MEDIA_C0_COUNT); -} - -void gen6_rps_reset_ei(struct drm_i915_private *dev_priv) -{ - memset(&dev_priv->gt_pm.rps.ei, 0, sizeof(dev_priv->gt_pm.rps.ei)); -} - -static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir) -{ - struct intel_rps *rps = &dev_priv->gt_pm.rps; - const struct intel_rps_ei *prev = &rps->ei; - struct intel_rps_ei now; - u32 events = 0; - - if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0) - return 0; - - vlv_c0_read(dev_priv, &now); - - if (prev->ktime) { - u64 time, c0; - u32 render, media; - - time = ktime_us_delta(now.ktime, prev->ktime); - - time *= dev_priv->czclk_freq; - - /* Workload can be split between render + media, - * e.g. SwapBuffers being blitted in X after being rendered in - * mesa. To account for this we need to combine both engines - * into our activity counter. - */ - render = now.render_c0 - prev->render_c0; - media = now.media_c0 - prev->media_c0; - c0 = max(render, media); - c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */ - - if (c0 > time * rps->power.up_threshold) - events = GEN6_PM_RP_UP_THRESHOLD; - else if (c0 < time * rps->power.down_threshold) - events = GEN6_PM_RP_DOWN_THRESHOLD; - } - - rps->ei = now; - return events; -} - -static void gen6_pm_rps_work(struct work_struct *work) -{ - struct drm_i915_private *dev_priv = - container_of(work, struct drm_i915_private, gt_pm.rps.work); - struct intel_gt *gt = &dev_priv->gt; - struct intel_rps *rps = &dev_priv->gt_pm.rps; - bool client_boost = false; - int new_delay, adj, min, max; - u32 pm_iir = 0; - - spin_lock_irq(>->irq_lock); - if (rps->interrupts_enabled) { - pm_iir = fetch_and_zero(&rps->pm_iir); - client_boost = atomic_read(&rps->num_waiters); - } - spin_unlock_irq(>->irq_lock); - - /* Make sure we didn't queue anything we're not going to process. */ - WARN_ON(pm_iir & ~dev_priv->pm_rps_events); - if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost) - goto out; - - mutex_lock(&rps->lock); - - pm_iir |= vlv_wa_c0_ei(dev_priv, pm_iir); - - adj = rps->last_adj; - new_delay = rps->cur_freq; - min = rps->min_freq_softlimit; - max = rps->max_freq_softlimit; - if (client_boost) - max = rps->max_freq; - if (client_boost && new_delay < rps->boost_freq) { - new_delay = rps->boost_freq; - adj = 0; - } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) { - if (adj > 0) - adj *= 2; - else /* CHV needs even encode values */ - adj = IS_CHERRYVIEW(dev_priv) ? 2 : 1; - - if (new_delay >= rps->max_freq_softlimit) - adj = 0; - } else if (client_boost) { - adj = 0; - } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) { - if (rps->cur_freq > rps->efficient_freq) - new_delay = rps->efficient_freq; - else if (rps->cur_freq > rps->min_freq_softlimit) - new_delay = rps->min_freq_softlimit; - adj = 0; - } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) { - if (adj < 0) - adj *= 2; - else /* CHV needs even encode values */ - adj = IS_CHERRYVIEW(dev_priv) ? -2 : -1; - - if (new_delay <= rps->min_freq_softlimit) - adj = 0; - } else { /* unknown event */ - adj = 0; - } - - rps->last_adj = adj; - - /* - * Limit deboosting and boosting to keep ourselves at the extremes - * when in the respective power modes (i.e. slowly decrease frequencies - * while in the HIGH_POWER zone and slowly increase frequencies while - * in the LOW_POWER zone). On idle, we will hit the timeout and drop - * to the next level quickly, and conversely if busy we expect to - * hit a waitboost and rapidly switch into max power. - */ - if ((adj < 0 && rps->power.mode == HIGH_POWER) || - (adj > 0 && rps->power.mode == LOW_POWER)) - rps->last_adj = 0; - - /* sysfs frequency interfaces may have snuck in while servicing the - * interrupt - */ - new_delay += adj; - new_delay = clamp_t(int, new_delay, min, max); - - if (intel_set_rps(dev_priv, new_delay)) { - DRM_DEBUG_DRIVER("Failed to set new GPU frequency\n"); - rps->last_adj = 0; - } - - mutex_unlock(&rps->lock); - -out: - /* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */ - spin_lock_irq(>->irq_lock); - if (rps->interrupts_enabled) - gen6_gt_pm_unmask_irq(gt, dev_priv->pm_rps_events); - spin_unlock_irq(>->irq_lock); -} - - /** * ivybridge_parity_work - Workqueue called when a parity error interrupt * occurred. @@ -1631,54 +1358,6 @@ static void i9xx_pipe_crc_irq_handler(struct drm_i915_private *dev_priv, res1, res2); } -/* The RPS events need forcewake, so we add them to a work queue and mask their - * IMR bits until the work is done. Other interrupts can be processed without - * the work queue. */ -void gen11_rps_irq_handler(struct intel_gt *gt, u32 pm_iir) -{ - struct drm_i915_private *i915 = gt->i915; - struct intel_rps *rps = &i915->gt_pm.rps; - const u32 events = i915->pm_rps_events & pm_iir; - - lockdep_assert_held(>->irq_lock); - - if (unlikely(!events)) - return; - - gen6_gt_pm_mask_irq(gt, events); - - if (!rps->interrupts_enabled) - return; - - rps->pm_iir |= events; - schedule_work(&rps->work); -} - -void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir) -{ - struct intel_rps *rps = &dev_priv->gt_pm.rps; - struct intel_gt *gt = &dev_priv->gt; - - if (pm_iir & dev_priv->pm_rps_events) { - spin_lock(>->irq_lock); - gen6_gt_pm_mask_irq(gt, pm_iir & dev_priv->pm_rps_events); - if (rps->interrupts_enabled) { - rps->pm_iir |= pm_iir & dev_priv->pm_rps_events; - schedule_work(&rps->work); - } - spin_unlock(>->irq_lock); - } - - if (INTEL_GEN(dev_priv) >= 8) - return; - - if (pm_iir & PM_VEBOX_USER_INTERRUPT) - intel_engine_breadcrumbs_irq(dev_priv->engine[VECS0]); - - if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) - DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir); -} - static void i9xx_pipestat_irq_reset(struct drm_i915_private *dev_priv) { enum pipe pipe; @@ -1989,7 +1668,7 @@ static irqreturn_t valleyview_irq_handler(int irq, void *arg) if (gt_iir) gen6_gt_irq_handler(&dev_priv->gt, gt_iir); if (pm_iir) - gen6_rps_irq_handler(dev_priv, pm_iir); + gen6_rps_irq_handler(&dev_priv->gt.rps, pm_iir); if (hotplug_status) i9xx_hpd_irq_handler(dev_priv, hotplug_status); @@ -2393,7 +2072,7 @@ static void ilk_display_irq_handler(struct drm_i915_private *dev_priv, } if (IS_GEN(dev_priv, 5) && de_iir & DE_PCU_EVENT) - ironlake_rps_change_irq_handler(dev_priv); + gen5_rps_irq_handler(&dev_priv->gt.rps); } static void ivb_display_irq_handler(struct drm_i915_private *dev_priv, @@ -2498,7 +2177,7 @@ static irqreturn_t ironlake_irq_handler(int irq, void *arg) if (pm_iir) { I915_WRITE(GEN6_PMIIR, pm_iir); ret = IRQ_HANDLED; - gen6_rps_irq_handler(dev_priv, pm_iir); + gen6_rps_irq_handler(&dev_priv->gt.rps, pm_iir); } } @@ -4270,13 +3949,10 @@ static irqreturn_t i965_irq_handler(int irq, void *arg) void intel_irq_init(struct drm_i915_private *dev_priv) { struct drm_device *dev = &dev_priv->drm; - struct intel_rps *rps = &dev_priv->gt_pm.rps; int i; intel_hpd_init_work(dev_priv); - INIT_WORK(&rps->work, gen6_pm_rps_work); - INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work); for (i = 0; i < MAX_L3_SLICES; ++i) dev_priv->l3_parity.remap_info[i] = NULL; @@ -4285,33 +3961,6 @@ void intel_irq_init(struct drm_i915_private *dev_priv) if (HAS_GT_UC(dev_priv) && INTEL_GEN(dev_priv) < 11) dev_priv->gt.pm_guc_events = GUC_INTR_GUC2HOST << 16; - /* Let's track the enabled rps events */ - if (IS_VALLEYVIEW(dev_priv)) - /* WaGsvRC0ResidencyMethod:vlv */ - dev_priv->pm_rps_events = GEN6_PM_RP_UP_EI_EXPIRED; - else - dev_priv->pm_rps_events = (GEN6_PM_RP_UP_THRESHOLD | - GEN6_PM_RP_DOWN_THRESHOLD | - GEN6_PM_RP_DOWN_TIMEOUT); - - /* We share the register with other engine */ - if (INTEL_GEN(dev_priv) > 9) - GEM_WARN_ON(dev_priv->pm_rps_events & 0xffff0000); - - rps->pm_intrmsk_mbz = 0; - - /* - * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer - * if GEN6_PM_UP_EI_EXPIRED is masked. - * - * TODO: verify if this can be reproduced on VLV,CHV. - */ - if (INTEL_GEN(dev_priv) <= 7) - rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED; - - if (INTEL_GEN(dev_priv) >= 8) - rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC; - dev->vblank_disable_immediate = true; /* Most platforms treat the display irq block as an always-on diff --git a/drivers/gpu/drm/i915/i915_irq.h b/drivers/gpu/drm/i915/i915_irq.h index 19a3bc019535..d0d91c6e00d7 100644 --- a/drivers/gpu/drm/i915/i915_irq.h +++ b/drivers/gpu/drm/i915/i915_irq.h @@ -22,9 +22,6 @@ struct intel_gt; struct intel_guc; struct intel_uncore; -void gen11_rps_irq_handler(struct intel_gt *gt, u32 pm_iir); -void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir); - void intel_irq_init(struct drm_i915_private *dev_priv); void intel_irq_fini(struct drm_i915_private *dev_priv); int intel_irq_install(struct drm_i915_private *dev_priv); diff --git a/drivers/gpu/drm/i915/i915_pmu.c b/drivers/gpu/drm/i915/i915_pmu.c index 85912917c062..266d66b1fb7b 100644 --- a/drivers/gpu/drm/i915/i915_pmu.c +++ b/drivers/gpu/drm/i915/i915_pmu.c @@ -12,6 +12,7 @@ #include "gt/intel_engine_user.h" #include "gt/intel_gt_pm.h" #include "gt/intel_rc6.h" +#include "gt/intel_rps.h" #include "i915_drv.h" #include "i915_pmu.h" @@ -358,25 +359,26 @@ frequency_sample(struct intel_gt *gt, unsigned int period_ns) struct drm_i915_private *i915 = gt->i915; struct intel_uncore *uncore = gt->uncore; struct i915_pmu *pmu = &i915->pmu; + struct intel_rps *rps = >->rps; if (pmu->enable & config_enabled_mask(I915_PMU_ACTUAL_FREQUENCY)) { u32 val; - val = i915->gt_pm.rps.cur_freq; + val = rps->cur_freq; if (intel_gt_pm_get_if_awake(gt)) { val = intel_uncore_read_notrace(uncore, GEN6_RPSTAT1); - val = intel_get_cagf(i915, val); + val = intel_get_cagf(rps, val); intel_gt_pm_put(gt); } add_sample_mult(&pmu->sample[__I915_SAMPLE_FREQ_ACT], - intel_gpu_freq(i915, val), + intel_gpu_freq(rps, val), period_ns / 1000); } if (pmu->enable & config_enabled_mask(I915_PMU_REQUESTED_FREQUENCY)) { add_sample_mult(&pmu->sample[__I915_SAMPLE_FREQ_REQ], - intel_gpu_freq(i915, i915->gt_pm.rps.cur_freq), + intel_gpu_freq(rps, rps->cur_freq), period_ns / 1000); } } diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c index 4575f368455d..08ce2eeecf7e 100644 --- a/drivers/gpu/drm/i915/i915_request.c +++ b/drivers/gpu/drm/i915/i915_request.c @@ -31,6 +31,7 @@ #include "gem/i915_gem_context.h" #include "gt/intel_context.h" +#include "gt/intel_rps.h" #include "i915_active.h" #include "i915_drv.h" @@ -257,8 +258,8 @@ bool i915_request_retire(struct i915_request *rq) if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &rq->fence.flags)) i915_request_cancel_breadcrumb(rq); if (i915_request_has_waitboost(rq)) { - GEM_BUG_ON(!atomic_read(&rq->i915->gt_pm.rps.num_waiters)); - atomic_dec(&rq->i915->gt_pm.rps.num_waiters); + GEM_BUG_ON(!atomic_read(&rq->engine->gt->rps.num_waiters)); + atomic_dec(&rq->engine->gt->rps.num_waiters); } if (!test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) { set_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags); @@ -1466,7 +1467,7 @@ long i915_request_wait(struct i915_request *rq, */ if (flags & I915_WAIT_PRIORITY) { if (!i915_request_started(rq) && INTEL_GEN(rq->i915) >= 6) - gen6_rps_boost(rq); + intel_rps_boost(rq); i915_schedule_bump_priority(rq, I915_PRIORITY_WAIT); } diff --git a/drivers/gpu/drm/i915/i915_sysfs.c b/drivers/gpu/drm/i915/i915_sysfs.c index bf039b8ba593..65476909d1bf 100644 --- a/drivers/gpu/drm/i915/i915_sysfs.c +++ b/drivers/gpu/drm/i915/i915_sysfs.c @@ -31,6 +31,7 @@ #include <linux/sysfs.h> #include "gt/intel_rc6.h" +#include "gt/intel_rps.h" #include "i915_drv.h" #include "i915_sysfs.h" @@ -259,6 +260,7 @@ static ssize_t gt_act_freq_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf) { struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); + struct intel_rps *rps = &dev_priv->gt.rps; intel_wakeref_t wakeref; u32 freq; @@ -271,31 +273,31 @@ static ssize_t gt_act_freq_mhz_show(struct device *kdev, freq = (freq >> 8) & 0xff; } else { - freq = intel_get_cagf(dev_priv, I915_READ(GEN6_RPSTAT1)); + freq = intel_get_cagf(rps, I915_READ(GEN6_RPSTAT1)); } intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref); - return snprintf(buf, PAGE_SIZE, "%d\n", intel_gpu_freq(dev_priv, freq)); + return snprintf(buf, PAGE_SIZE, "%d\n", intel_gpu_freq(rps, freq)); } static ssize_t gt_cur_freq_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf) { struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); + struct intel_rps *rps = &dev_priv->gt.rps; return snprintf(buf, PAGE_SIZE, "%d\n", - intel_gpu_freq(dev_priv, - dev_priv->gt_pm.rps.cur_freq)); + intel_gpu_freq(rps, rps->cur_freq)); } static ssize_t gt_boost_freq_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf) { struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); + struct intel_rps *rps = &dev_priv->gt.rps; return snprintf(buf, PAGE_SIZE, "%d\n", - intel_gpu_freq(dev_priv, - dev_priv->gt_pm.rps.boost_freq)); + intel_gpu_freq(rps, rps->boost_freq)); } static ssize_t gt_boost_freq_mhz_store(struct device *kdev, @@ -303,7 +305,7 @@ static ssize_t gt_boost_freq_mhz_store(struct device *kdev, const char *buf, size_t count) { struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); - struct intel_rps *rps = &dev_priv->gt_pm.rps; + struct intel_rps *rps = &dev_priv->gt.rps; bool boost = false; ssize_t ret; u32 val; @@ -313,7 +315,7 @@ static ssize_t gt_boost_freq_mhz_store(struct device *kdev, return ret; /* Validate against (static) hardware limits */ - val = intel_freq_opcode(dev_priv, val); + val = intel_freq_opcode(rps, val); if (val < rps->min_freq || val > rps->max_freq) return -EINVAL; @@ -333,19 +335,19 @@ static ssize_t vlv_rpe_freq_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf) { struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); + struct intel_rps *rps = &dev_priv->gt.rps; return snprintf(buf, PAGE_SIZE, "%d\n", - intel_gpu_freq(dev_priv, - dev_priv->gt_pm.rps.efficient_freq)); + intel_gpu_freq(rps, rps->efficient_freq)); } static ssize_t gt_max_freq_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf) { struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); + struct intel_rps *rps = &dev_priv->gt.rps; return snprintf(buf, PAGE_SIZE, "%d\n", - intel_gpu_freq(dev_priv, - dev_priv->gt_pm.rps.max_freq_softlimit)); + intel_gpu_freq(rps, rps->max_freq_softlimit)); } static ssize_t gt_max_freq_mhz_store(struct device *kdev, @@ -353,19 +355,17 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev, const char *buf, size_t count) { struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); - struct intel_rps *rps = &dev_priv->gt_pm.rps; - intel_wakeref_t wakeref; - u32 val; + struct intel_rps *rps = &dev_priv->gt.rps; ssize_t ret; + u32 val; ret = kstrtou32(buf, 0, &val); if (ret) return ret; - wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm); mutex_lock(&rps->lock); - val = intel_freq_opcode(dev_priv, val); + val = intel_freq_opcode(rps, val); if (val < rps->min_freq || val > rps->max_freq || val < rps->min_freq_softlimit) { @@ -375,7 +375,7 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev, if (val > rps->rp0_freq) DRM_DEBUG("User requested overclocking to %d\n", - intel_gpu_freq(dev_priv, val)); + intel_gpu_freq(rps, val)); rps->max_freq_softlimit = val; @@ -383,14 +383,15 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev, rps->min_freq_softlimit, rps->max_freq_softlimit); - /* We still need *_set_rps to process the new max_delay and + /* + * We still need *_set_rps to process the new max_delay and * update the interrupt limits and PMINTRMSK even though - * frequency request may be unchanged. */ - ret = intel_set_rps(dev_priv, val); + * frequency request may be unchanged. + */ + intel_rps_set(rps, val); unlock: mutex_unlock(&rps->lock); - intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref); return ret ?: count; } @@ -398,10 +399,10 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev, static ssize_t gt_min_freq_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf) { struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); + struct intel_rps *rps = &dev_priv->gt.rps; return snprintf(buf, PAGE_SIZE, "%d\n", - intel_gpu_freq(dev_priv, - dev_priv->gt_pm.rps.min_freq_softlimit)); + intel_gpu_freq(rps, rps->min_freq_softlimit)); } static ssize_t gt_min_freq_mhz_store(struct device *kdev, @@ -409,19 +410,17 @@ static ssize_t gt_min_freq_mhz_store(struct device *kdev, const char *buf, size_t count) { struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); - struct intel_rps *rps = &dev_priv->gt_pm.rps; - intel_wakeref_t wakeref; - u32 val; + struct intel_rps *rps = &dev_priv->gt.rps; ssize_t ret; + u32 val; ret = kstrtou32(buf, 0, &val); if (ret) return ret; - wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm); mutex_lock(&rps->lock); - val = intel_freq_opcode(dev_priv, val); + val = intel_freq_opcode(rps, val); if (val < rps->min_freq || val > rps->max_freq || val > rps->max_freq_softlimit) { @@ -435,14 +434,15 @@ static ssize_t gt_min_freq_mhz_store(struct device *kdev, rps->min_freq_softlimit, rps->max_freq_softlimit); - /* We still need *_set_rps to process the new min_delay and + /* + * We still need *_set_rps to process the new min_delay and * update the interrupt limits and PMINTRMSK even though - * frequency request may be unchanged. */ - ret = intel_set_rps(dev_priv, val); + * frequency request may be unchanged. + */ + intel_rps_set(rps, val); unlock: mutex_unlock(&rps->lock); - intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref); return ret ?: count; } @@ -464,15 +464,15 @@ static DEVICE_ATTR(gt_RPn_freq_mhz, S_IRUGO, gt_rp_mhz_show, NULL); static ssize_t gt_rp_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf) { struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); - struct intel_rps *rps = &dev_priv->gt_pm.rps; + struct intel_rps *rps = &dev_priv->gt.rps; u32 val; if (attr == &dev_attr_gt_RP0_freq_mhz) - val = intel_gpu_freq(dev_priv, rps->rp0_freq); + val = intel_gpu_freq(rps, rps->rp0_freq); else if (attr == &dev_attr_gt_RP1_freq_mhz) - val = intel_gpu_freq(dev_priv, rps->rp1_freq); + val = intel_gpu_freq(rps, rps->rp1_freq); else if (attr == &dev_attr_gt_RPn_freq_mhz) - val = intel_gpu_freq(dev_priv, rps->min_freq); + val = intel_gpu_freq(rps, rps->min_freq); else BUG(); diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index 362234449087..6741507c74f3 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -197,8 +197,6 @@ static void i915_ironlake_get_mem_freq(struct drm_i915_private *dev_priv) break; } - dev_priv->ips.r_t = dev_priv->mem_freq; - switch (csipll & 0x3ff) { case 0x00c: dev_priv->fsb_freq = 3200; @@ -227,14 +225,6 @@ static void i915_ironlake_get_mem_freq(struct drm_i915_private *dev_priv) dev_priv->fsb_freq = 0; break; } - - if (dev_priv->fsb_freq == 3200) { - dev_priv->ips.c_m = 0; - } else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) { - dev_priv->ips.c_m = 1; - } else { - dev_priv->ips.c_m = 2; - } } static const struct cxsr_latency cxsr_latency_table[] = { @@ -6339,1865 +6329,258 @@ void intel_init_ipc(struct drm_i915_private *dev_priv) intel_enable_ipc(dev_priv); } -/* - * Lock protecting IPS related data structures - */ -DEFINE_SPINLOCK(mchdev_lock); +static const struct cparams { + u16 i; + u16 t; + u16 m; + u16 c; +} cparams[] = { + { 1, 1333, 301, 28664 }, + { 1, 1066, 294, 24460 }, + { 1, 800, 294, 25192 }, + { 0, 1333, 276, 27605 }, + { 0, 1066, 276, 27605 }, + { 0, 800, 231, 23784 }, +}; -bool ironlake_set_drps(struct drm_i915_private *i915, u8 val) +static void ibx_init_clock_gating(struct drm_i915_private *dev_priv) { - struct intel_uncore *uncore = &i915->uncore; - u16 rgvswctl; - - lockdep_assert_held(&mchdev_lock); - - rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); - if (rgvswctl & MEMCTL_CMD_STS) { - DRM_DEBUG("gpu busy, RCS change rejected\n"); - return false; /* still busy with another command */ - } - - rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) | - (val << MEMCTL_FREQ_SHIFT) | MEMCTL_SFCAVM; - intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); - intel_uncore_posting_read16(uncore, MEMSWCTL); - - rgvswctl |= MEMCTL_CMD_STS; - intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); - - return true; + /* + * On Ibex Peak and Cougar Point, we need to disable clock + * gating for the panel power sequencer or it will fail to + * start up when no ports are active. + */ + I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE); } -static void ironlake_enable_drps(struct drm_i915_private *dev_priv) +static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv) { - struct intel_uncore *uncore = &dev_priv->uncore; - u32 rgvmodectl; - u8 fmax, fmin, fstart, vstart; - - spin_lock_irq(&mchdev_lock); - - rgvmodectl = intel_uncore_read(uncore, MEMMODECTL); - - /* Enable temp reporting */ - intel_uncore_write16(uncore, PMMISC, I915_READ(PMMISC) | MCPPCE_EN); - intel_uncore_write16(uncore, TSC1, I915_READ(TSC1) | TSE); - - /* 100ms RC evaluation intervals */ - intel_uncore_write(uncore, RCUPEI, 100000); - intel_uncore_write(uncore, RCDNEI, 100000); - - /* Set max/min thresholds to 90ms and 80ms respectively */ - intel_uncore_write(uncore, RCBMAXAVG, 90000); - intel_uncore_write(uncore, RCBMINAVG, 80000); - - intel_uncore_write(uncore, MEMIHYST, 1); + enum pipe pipe; - /* Set up min, max, and cur for interrupt handling */ - fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT; - fmin = (rgvmodectl & MEMMODE_FMIN_MASK); - fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> - MEMMODE_FSTART_SHIFT; + for_each_pipe(dev_priv, pipe) { + I915_WRITE(DSPCNTR(pipe), + I915_READ(DSPCNTR(pipe)) | + DISPPLANE_TRICKLE_FEED_DISABLE); - vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) & - PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; + I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe))); + POSTING_READ(DSPSURF(pipe)); + } +} - dev_priv->ips.fmax = fmax; /* IPS callback will increase this */ - dev_priv->ips.fstart = fstart; +static void ilk_init_clock_gating(struct drm_i915_private *dev_priv) +{ + u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE; - dev_priv->ips.max_delay = fstart; - dev_priv->ips.min_delay = fmin; - dev_priv->ips.cur_delay = fstart; + /* + * Required for FBC + * WaFbcDisableDpfcClockGating:ilk + */ + dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE | + ILK_DPFCUNIT_CLOCK_GATE_DISABLE | + ILK_DPFDUNIT_CLOCK_GATE_ENABLE; - DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n", - fmax, fmin, fstart); + I915_WRITE(PCH_3DCGDIS0, + MARIUNIT_CLOCK_GATE_DISABLE | + SVSMUNIT_CLOCK_GATE_DISABLE); + I915_WRITE(PCH_3DCGDIS1, + VFMUNIT_CLOCK_GATE_DISABLE); - intel_uncore_write(uncore, - MEMINTREN, - MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN); + /* + * According to the spec the following bits should be set in + * order to enable memory self-refresh + * The bit 22/21 of 0x42004 + * The bit 5 of 0x42020 + * The bit 15 of 0x45000 + */ + I915_WRITE(ILK_DISPLAY_CHICKEN2, + (I915_READ(ILK_DISPLAY_CHICKEN2) | + ILK_DPARB_GATE | ILK_VSDPFD_FULL)); + dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE; + I915_WRITE(DISP_ARB_CTL, + (I915_READ(DISP_ARB_CTL) | + DISP_FBC_WM_DIS)); /* - * Interrupts will be enabled in ironlake_irq_postinstall + * Based on the document from hardware guys the following bits + * should be set unconditionally in order to enable FBC. + * The bit 22 of 0x42000 + * The bit 22 of 0x42004 + * The bit 7,8,9 of 0x42020. */ + if (IS_IRONLAKE_M(dev_priv)) { + /* WaFbcAsynchFlipDisableFbcQueue:ilk */ + I915_WRITE(ILK_DISPLAY_CHICKEN1, + I915_READ(ILK_DISPLAY_CHICKEN1) | + ILK_FBCQ_DIS); + I915_WRITE(ILK_DISPLAY_CHICKEN2, + I915_READ(ILK_DISPLAY_CHICKEN2) | + ILK_DPARB_GATE); + } - intel_uncore_write(uncore, VIDSTART, vstart); - intel_uncore_posting_read(uncore, VIDSTART); + I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate); - rgvmodectl |= MEMMODE_SWMODE_EN; - intel_uncore_write(uncore, MEMMODECTL, rgvmodectl); + I915_WRITE(ILK_DISPLAY_CHICKEN2, + I915_READ(ILK_DISPLAY_CHICKEN2) | + ILK_ELPIN_409_SELECT); + I915_WRITE(_3D_CHICKEN2, + _3D_CHICKEN2_WM_READ_PIPELINED << 16 | + _3D_CHICKEN2_WM_READ_PIPELINED); - if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) & - MEMCTL_CMD_STS) == 0, 10)) - DRM_ERROR("stuck trying to change perf mode\n"); - mdelay(1); + /* WaDisableRenderCachePipelinedFlush:ilk */ + I915_WRITE(CACHE_MODE_0, + _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE)); - ironlake_set_drps(dev_priv, fstart); + /* WaDisable_RenderCache_OperationalFlush:ilk */ + I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); - dev_priv->ips.last_count1 = - intel_uncore_read(uncore, DMIEC) + - intel_uncore_read(uncore, DDREC) + - intel_uncore_read(uncore, CSIEC); - dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies); - dev_priv->ips.last_count2 = intel_uncore_read(uncore, GFXEC); - dev_priv->ips.last_time2 = ktime_get_raw_ns(); + g4x_disable_trickle_feed(dev_priv); - spin_unlock_irq(&mchdev_lock); + ibx_init_clock_gating(dev_priv); } -static void ironlake_disable_drps(struct drm_i915_private *i915) +static void cpt_init_clock_gating(struct drm_i915_private *dev_priv) { - struct intel_uncore *uncore = &i915->uncore; - u16 rgvswctl; - - spin_lock_irq(&mchdev_lock); - - rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); - - /* Ack interrupts, disable EFC interrupt */ - intel_uncore_write(uncore, - MEMINTREN, - intel_uncore_read(uncore, MEMINTREN) & - ~MEMINT_EVAL_CHG_EN); - intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); - intel_uncore_write(uncore, - DEIER, - intel_uncore_read(uncore, DEIER) & ~DE_PCU_EVENT); - intel_uncore_write(uncore, DEIIR, DE_PCU_EVENT); - intel_uncore_write(uncore, - DEIMR, - intel_uncore_read(uncore, DEIMR) | DE_PCU_EVENT); - - /* Go back to the starting frequency */ - ironlake_set_drps(i915, i915->ips.fstart); - mdelay(1); - rgvswctl |= MEMCTL_CMD_STS; - intel_uncore_write(uncore, MEMSWCTL, rgvswctl); - mdelay(1); + enum pipe pipe; + u32 val; - spin_unlock_irq(&mchdev_lock); + /* + * On Ibex Peak and Cougar Point, we need to disable clock + * gating for the panel power sequencer or it will fail to + * start up when no ports are active. + */ + I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE | + PCH_DPLUNIT_CLOCK_GATE_DISABLE | + PCH_CPUNIT_CLOCK_GATE_DISABLE); + I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) | + DPLS_EDP_PPS_FIX_DIS); + /* The below fixes the weird display corruption, a few pixels shifted + * downward, on (only) LVDS of some HP laptops with IVY. + */ + for_each_pipe(dev_priv, pipe) { + val = I915_READ(TRANS_CHICKEN2(pipe)); + val |= TRANS_CHICKEN2_TIMING_OVERRIDE; + val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED; + if (dev_priv->vbt.fdi_rx_polarity_inverted) + val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED; + val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK; + val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER; + val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH; + I915_WRITE(TRANS_CHICKEN2(pipe), val); + } + /* WADP0ClockGatingDisable */ + for_each_pipe(dev_priv, pipe) { + I915_WRITE(TRANS_CHICKEN1(pipe), + TRANS_CHICKEN1_DP0UNIT_GC_DISABLE); + } } -/* There's a funny hw issue where the hw returns all 0 when reading from - * GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value - * ourselves, instead of doing a rmw cycle (which might result in us clearing - * all limits and the gpu stuck at whatever frequency it is at atm). - */ -static u32 intel_rps_limits(struct drm_i915_private *dev_priv, u8 val) +static void gen6_check_mch_setup(struct drm_i915_private *dev_priv) { - struct intel_rps *rps = &dev_priv->gt_pm.rps; - u32 limits; - - /* Only set the down limit when we've reached the lowest level to avoid - * getting more interrupts, otherwise leave this clear. This prevents a - * race in the hw when coming out of rc6: There's a tiny window where - * the hw runs at the minimal clock before selecting the desired - * frequency, if the down threshold expires in that window we will not - * receive a down interrupt. */ - if (INTEL_GEN(dev_priv) >= 9) { - limits = (rps->max_freq_softlimit) << 23; - if (val <= rps->min_freq_softlimit) - limits |= (rps->min_freq_softlimit) << 14; - } else { - limits = rps->max_freq_softlimit << 24; - if (val <= rps->min_freq_softlimit) - limits |= rps->min_freq_softlimit << 16; - } + u32 tmp; - return limits; + tmp = I915_READ(MCH_SSKPD); + if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL) + DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n", + tmp); } -static void rps_set_power(struct drm_i915_private *dev_priv, int new_power) +static void gen6_init_clock_gating(struct drm_i915_private *dev_priv) { - struct intel_rps *rps = &dev_priv->gt_pm.rps; - u32 threshold_up = 0, threshold_down = 0; /* in % */ - u32 ei_up = 0, ei_down = 0; - - lockdep_assert_held(&rps->power.mutex); + u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE; - if (new_power == rps->power.mode) - return; + I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate); - /* Note the units here are not exactly 1us, but 1280ns. */ - switch (new_power) { - case LOW_POWER: - /* Upclock if more than 95% busy over 16ms */ - ei_up = 16000; - threshold_up = 95; + I915_WRITE(ILK_DISPLAY_CHICKEN2, + I915_READ(ILK_DISPLAY_CHICKEN2) | + ILK_ELPIN_409_SELECT); - /* Downclock if less than 85% busy over 32ms */ - ei_down = 32000; - threshold_down = 85; - break; + /* WaDisableHiZPlanesWhenMSAAEnabled:snb */ + I915_WRITE(_3D_CHICKEN, + _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB)); - case BETWEEN: - /* Upclock if more than 90% busy over 13ms */ - ei_up = 13000; - threshold_up = 90; + /* WaDisable_RenderCache_OperationalFlush:snb */ + I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); - /* Downclock if less than 75% busy over 32ms */ - ei_down = 32000; - threshold_down = 75; - break; + /* + * BSpec recoomends 8x4 when MSAA is used, + * however in practice 16x4 seems fastest. + * + * Note that PS/WM thread counts depend on the WIZ hashing + * disable bit, which we don't touch here, but it's good + * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). + */ + I915_WRITE(GEN6_GT_MODE, + _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4)); - case HIGH_POWER: - /* Upclock if more than 85% busy over 10ms */ - ei_up = 10000; - threshold_up = 85; + I915_WRITE(CACHE_MODE_0, + _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB)); - /* Downclock if less than 60% busy over 32ms */ - ei_down = 32000; - threshold_down = 60; - break; - } + I915_WRITE(GEN6_UCGCTL1, + I915_READ(GEN6_UCGCTL1) | + GEN6_BLBUNIT_CLOCK_GATE_DISABLE | + GEN6_CSUNIT_CLOCK_GATE_DISABLE); - /* When byt can survive without system hang with dynamic - * sw freq adjustments, this restriction can be lifted. + /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock + * gating disable must be set. Failure to set it results in + * flickering pixels due to Z write ordering failures after + * some amount of runtime in the Mesa "fire" demo, and Unigine + * Sanctuary and Tropics, and apparently anything else with + * alpha test or pixel discard. + * + * According to the spec, bit 11 (RCCUNIT) must also be set, + * but we didn't debug actual testcases to find it out. + * + * WaDisableRCCUnitClockGating:snb + * WaDisableRCPBUnitClockGating:snb */ - if (IS_VALLEYVIEW(dev_priv)) - goto skip_hw_write; - - I915_WRITE(GEN6_RP_UP_EI, - GT_INTERVAL_FROM_US(dev_priv, ei_up)); - I915_WRITE(GEN6_RP_UP_THRESHOLD, - GT_INTERVAL_FROM_US(dev_priv, - ei_up * threshold_up / 100)); - - I915_WRITE(GEN6_RP_DOWN_EI, - GT_INTERVAL_FROM_US(dev_priv, ei_down)); - I915_WRITE(GEN6_RP_DOWN_THRESHOLD, - GT_INTERVAL_FROM_US(dev_priv, - ei_down * threshold_down / 100)); - - I915_WRITE(GEN6_RP_CONTROL, - (INTEL_GEN(dev_priv) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) | - GEN6_RP_MEDIA_HW_NORMAL_MODE | - GEN6_RP_MEDIA_IS_GFX | - GEN6_RP_ENABLE | - GEN6_RP_UP_BUSY_AVG | - GEN6_RP_DOWN_IDLE_AVG); - -skip_hw_write: - rps->power.mode = new_power; - rps->power.up_threshold = threshold_up; - rps->power.down_threshold = threshold_down; -} - -static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val) -{ - struct intel_rps *rps = &dev_priv->gt_pm.rps; - int new_power; - - new_power = rps->power.mode; - switch (rps->power.mode) { - case LOW_POWER: - if (val > rps->efficient_freq + 1 && - val > rps->cur_freq) - new_power = BETWEEN; - break; + I915_WRITE(GEN6_UCGCTL2, + GEN6_RCPBUNIT_CLOCK_GATE_DISABLE | + GEN6_RCCUNIT_CLOCK_GATE_DISABLE); - case BETWEEN: - if (val <= rps->efficient_freq && - val < rps->cur_freq) - new_power = LOW_POWER; - else if (val >= rps->rp0_freq && - val > rps->cur_freq) - new_power = HIGH_POWER; - break; + /* WaStripsFansDisableFastClipPerformanceFix:snb */ + I915_WRITE(_3D_CHICKEN3, + _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL)); - case HIGH_POWER: - if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 && - val < rps->cur_freq) - new_power = BETWEEN; - break; - } - /* Max/min bins are special */ - if (val <= rps->min_freq_softlimit) - new_power = LOW_POWER; - if (val >= rps->max_freq_softlimit) - new_power = HIGH_POWER; + /* + * Bspec says: + * "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and + * 3DSTATE_SF number of SF output attributes is more than 16." + */ + I915_WRITE(_3D_CHICKEN3, + _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH)); - mutex_lock(&rps->power.mutex); - if (rps->power.interactive) - new_power = HIGH_POWER; - rps_set_power(dev_priv, new_power); - mutex_unlock(&rps->power.mutex); -} + /* + * According to the spec the following bits should be + * set in order to enable memory self-refresh and fbc: + * The bit21 and bit22 of 0x42000 + * The bit21 and bit22 of 0x42004 + * The bit5 and bit7 of 0x42020 + * The bit14 of 0x70180 + * The bit14 of 0x71180 + * + * WaFbcAsynchFlipDisableFbcQueue:snb + */ + I915_WRITE(ILK_DISPLAY_CHICKEN1, + I915_READ(ILK_DISPLAY_CHICKEN1) | + ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS); + I915_WRITE(ILK_DISPLAY_CHICKEN2, + I915_READ(ILK_DISPLAY_CHICKEN2) | + ILK_DPARB_GATE | ILK_VSDPFD_FULL); + I915_WRITE(ILK_DSPCLK_GATE_D, + I915_READ(ILK_DSPCLK_GATE_D) | + ILK_DPARBUNIT_CLOCK_GATE_ENABLE | + ILK_DPFDUNIT_CLOCK_GATE_ENABLE); -void intel_rps_mark_interactive(struct drm_i915_private *i915, bool interactive) -{ - struct intel_rps *rps = &i915->gt_pm.rps; + g4x_disable_trickle_feed(dev_priv); - if (INTEL_GEN(i915) < 6) - return; + cpt_init_clock_gating(dev_priv); - mutex_lock(&rps->power.mutex); - if (interactive) { - if (!rps->power.interactive++ && READ_ONCE(i915->gt.awake)) - rps_set_power(i915, HIGH_POWER); - } else { - GEM_BUG_ON(!rps->power.interactive); - rps->power.interactive--; - } - mutex_unlock(&rps->power.mutex); + gen6_check_mch_setup(dev_priv); } -static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val) +static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv) { - struct intel_rps *rps = &dev_priv->gt_pm.rps; - u32 mask = 0; - - /* We use UP_EI_EXPIRED interupts for both up/down in manual mode */ - if (val > rps->min_freq_softlimit) - mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT; - if (val < rps->max_freq_softlimit) - mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD; - - mask &= dev_priv->pm_rps_events; - - return gen6_sanitize_rps_pm_mask(dev_priv, ~mask); -} - -/* gen6_set_rps is called to update the frequency request, but should also be - * called when the range (min_delay and max_delay) is modified so that we can - * update the GEN6_RP_INTERRUPT_LIMITS register accordingly. */ -static int gen6_set_rps(struct drm_i915_private *dev_priv, u8 val) -{ - struct intel_rps *rps = &dev_priv->gt_pm.rps; - - /* min/max delay may still have been modified so be sure to - * write the limits value. - */ - if (val != rps->cur_freq) { - gen6_set_rps_thresholds(dev_priv, val); - - if (INTEL_GEN(dev_priv) >= 9) - I915_WRITE(GEN6_RPNSWREQ, - GEN9_FREQUENCY(val)); - else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) - I915_WRITE(GEN6_RPNSWREQ, - HSW_FREQUENCY(val)); - else - I915_WRITE(GEN6_RPNSWREQ, - GEN6_FREQUENCY(val) | - GEN6_OFFSET(0) | - GEN6_AGGRESSIVE_TURBO); - } - - /* Make sure we continue to get interrupts - * until we hit the minimum or maximum frequencies. - */ - I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, intel_rps_limits(dev_priv, val)); - I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val)); - - rps->cur_freq = val; - trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val)); - - return 0; -} - -static int valleyview_set_rps(struct drm_i915_private *dev_priv, u8 val) -{ - int err; - - if (WARN_ONCE(IS_CHERRYVIEW(dev_priv) && (val & 1), - "Odd GPU freq value\n")) - val &= ~1; - - I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val)); - - if (val != dev_priv->gt_pm.rps.cur_freq) { - vlv_punit_get(dev_priv); - err = vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val); - vlv_punit_put(dev_priv); - if (err) - return err; - - gen6_set_rps_thresholds(dev_priv, val); - } - - dev_priv->gt_pm.rps.cur_freq = val; - trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val)); - - return 0; -} - -/* vlv_set_rps_idle: Set the frequency to idle, if Gfx clocks are down - * - * * If Gfx is Idle, then - * 1. Forcewake Media well. - * 2. Request idle freq. - * 3. Release Forcewake of Media well. -*/ -static void vlv_set_rps_idle(struct drm_i915_private *dev_priv) -{ - struct intel_rps *rps = &dev_priv->gt_pm.rps; - u32 val = rps->idle_freq; - int err; - - if (rps->cur_freq <= val) - return; - - /* The punit delays the write of the frequency and voltage until it - * determines the GPU is awake. During normal usage we don't want to - * waste power changing the frequency if the GPU is sleeping (rc6). - * However, the GPU and driver is now idle and we do not want to delay - * switching to minimum voltage (reducing power whilst idle) as we do - * not expect to be woken in the near future and so must flush the - * change by waking the device. - * - * We choose to take the media powerwell (either would do to trick the - * punit into committing the voltage change) as that takes a lot less - * power than the render powerwell. - */ - intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_MEDIA); - err = valleyview_set_rps(dev_priv, val); - intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_MEDIA); - - if (err) - DRM_ERROR("Failed to set RPS for idle\n"); -} - -void gen6_rps_busy(struct drm_i915_private *dev_priv) -{ - struct intel_rps *rps = &dev_priv->gt_pm.rps; - - mutex_lock(&rps->lock); - if (rps->enabled) { - u8 freq; - - if (dev_priv->pm_rps_events & GEN6_PM_RP_UP_EI_EXPIRED) - gen6_rps_reset_ei(dev_priv); - I915_WRITE(GEN6_PMINTRMSK, - gen6_rps_pm_mask(dev_priv, rps->cur_freq)); - - gen6_enable_rps_interrupts(dev_priv); - - /* Use the user's desired frequency as a guide, but for better - * performance, jump directly to RPe as our starting frequency. - */ - freq = max(rps->cur_freq, - rps->efficient_freq); - - if (intel_set_rps(dev_priv, - clamp(freq, - rps->min_freq_softlimit, - rps->max_freq_softlimit))) - DRM_DEBUG_DRIVER("Failed to set idle frequency\n"); - } - mutex_unlock(&rps->lock); -} - -void gen6_rps_idle(struct drm_i915_private *dev_priv) -{ - struct intel_rps *rps = &dev_priv->gt_pm.rps; - - /* Flush our bottom-half so that it does not race with us - * setting the idle frequency and so that it is bounded by - * our rpm wakeref. And then disable the interrupts to stop any - * futher RPS reclocking whilst we are asleep. - */ - gen6_disable_rps_interrupts(dev_priv); - - mutex_lock(&rps->lock); - if (rps->enabled) { - if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) - vlv_set_rps_idle(dev_priv); - else - gen6_set_rps(dev_priv, rps->idle_freq); - rps->last_adj = 0; - I915_WRITE(GEN6_PMINTRMSK, - gen6_sanitize_rps_pm_mask(dev_priv, ~0)); - } - mutex_unlock(&rps->lock); -} - -void gen6_rps_boost(struct i915_request *rq) -{ - struct intel_rps *rps = &rq->i915->gt_pm.rps; - unsigned long flags; - bool boost; - - /* This is intentionally racy! We peek at the state here, then - * validate inside the RPS worker. - */ - if (!rps->enabled) - return; - - if (i915_request_signaled(rq)) - return; - - /* Serializes with i915_request_retire() */ - boost = false; - spin_lock_irqsave(&rq->lock, flags); - if (!i915_request_has_waitboost(rq) && - !dma_fence_is_signaled_locked(&rq->fence)) { - boost = !atomic_fetch_inc(&rps->num_waiters); - rq->flags |= I915_REQUEST_WAITBOOST; - } - spin_unlock_irqrestore(&rq->lock, flags); - if (!boost) - return; - - if (READ_ONCE(rps->cur_freq) < rps->boost_freq) - schedule_work(&rps->work); - - atomic_inc(&rps->boosts); -} - -int intel_set_rps(struct drm_i915_private *dev_priv, u8 val) -{ - struct intel_rps *rps = &dev_priv->gt_pm.rps; - int err; - - lockdep_assert_held(&rps->lock); - GEM_BUG_ON(val > rps->max_freq); - GEM_BUG_ON(val < rps->min_freq); - - if (!rps->enabled) { - rps->cur_freq = val; - return 0; - } - - if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) - err = valleyview_set_rps(dev_priv, val); - else - err = gen6_set_rps(dev_priv, val); - - return err; -} - -static void gen9_disable_rps(struct drm_i915_private *dev_priv) -{ - I915_WRITE(GEN6_RP_CONTROL, 0); -} - -static void gen6_disable_rps(struct drm_i915_private *dev_priv) -{ - I915_WRITE(GEN6_RPNSWREQ, 1 << 31); - I915_WRITE(GEN6_RP_CONTROL, 0); -} - -static void cherryview_disable_rps(struct drm_i915_private *dev_priv) -{ - I915_WRITE(GEN6_RP_CONTROL, 0); -} - -static void valleyview_disable_rps(struct drm_i915_private *dev_priv) -{ - I915_WRITE(GEN6_RP_CONTROL, 0); -} - -static void gen6_init_rps_frequencies(struct drm_i915_private *dev_priv) -{ - struct intel_rps *rps = &dev_priv->gt_pm.rps; - - /* All of these values are in units of 50MHz */ - - /* static values from HW: RP0 > RP1 > RPn (min_freq) */ - if (IS_GEN9_LP(dev_priv)) { - u32 rp_state_cap = I915_READ(BXT_RP_STATE_CAP); - rps->rp0_freq = (rp_state_cap >> 16) & 0xff; - rps->rp1_freq = (rp_state_cap >> 8) & 0xff; - rps->min_freq = (rp_state_cap >> 0) & 0xff; - } else { - u32 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP); - rps->rp0_freq = (rp_state_cap >> 0) & 0xff; - rps->rp1_freq = (rp_state_cap >> 8) & 0xff; - rps->min_freq = (rp_state_cap >> 16) & 0xff; - } - /* hw_max = RP0 until we check for overclocking */ - rps->max_freq = rps->rp0_freq; - - rps->efficient_freq = rps->rp1_freq; - if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv) || - IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) { - u32 ddcc_status = 0; - - if (sandybridge_pcode_read(dev_priv, - HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL, - &ddcc_status, NULL) == 0) - rps->efficient_freq = - clamp_t(u8, - ((ddcc_status >> 8) & 0xff), - rps->min_freq, - rps->max_freq); - } - - if (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) { - /* Store the frequency values in 16.66 MHZ units, which is - * the natural hardware unit for SKL - */ - rps->rp0_freq *= GEN9_FREQ_SCALER; - rps->rp1_freq *= GEN9_FREQ_SCALER; - rps->min_freq *= GEN9_FREQ_SCALER; - rps->max_freq *= GEN9_FREQ_SCALER; - rps->efficient_freq *= GEN9_FREQ_SCALER; - } -} - -static void reset_rps(struct drm_i915_private *dev_priv, - int (*set)(struct drm_i915_private *, u8)) -{ - struct intel_rps *rps = &dev_priv->gt_pm.rps; - u8 freq = rps->cur_freq; - - /* force a reset */ - rps->power.mode = -1; - rps->cur_freq = -1; - - if (set(dev_priv, freq)) - DRM_ERROR("Failed to reset RPS to initial values\n"); -} - -/* See the Gen9_GT_PM_Programming_Guide doc for the below */ -static void gen9_enable_rps(struct drm_i915_private *dev_priv) -{ - intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL); - - /* Program defaults and thresholds for RPS */ - if (IS_GEN(dev_priv, 9)) - I915_WRITE(GEN6_RC_VIDEO_FREQ, - GEN9_FREQUENCY(dev_priv->gt_pm.rps.rp1_freq)); - - /* 1 second timeout*/ - I915_WRITE(GEN6_RP_DOWN_TIMEOUT, - GT_INTERVAL_FROM_US(dev_priv, 1000000)); - - I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 0xa); - - /* Leaning on the below call to gen6_set_rps to program/setup the - * Up/Down EI & threshold registers, as well as the RP_CONTROL, - * RP_INTERRUPT_LIMITS & RPNSWREQ registers */ - reset_rps(dev_priv, gen6_set_rps); - - intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL); -} - -static void gen8_enable_rps(struct drm_i915_private *dev_priv) -{ - struct intel_rps *rps = &dev_priv->gt_pm.rps; - - intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL); - - /* 1 Program defaults and thresholds for RPS*/ - I915_WRITE(GEN6_RPNSWREQ, - HSW_FREQUENCY(rps->rp1_freq)); - I915_WRITE(GEN6_RC_VIDEO_FREQ, - HSW_FREQUENCY(rps->rp1_freq)); - /* NB: Docs say 1s, and 1000000 - which aren't equivalent */ - I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128); /* 1 second timeout */ - - /* Docs recommend 900MHz, and 300 MHz respectively */ - I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, - rps->max_freq_softlimit << 24 | - rps->min_freq_softlimit << 16); - - I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */ - I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/ - I915_WRITE(GEN6_RP_UP_EI, 66000); /* 84.48ms, XXX: random? */ - I915_WRITE(GEN6_RP_DOWN_EI, 350000); /* 448ms, XXX: random? */ - - I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10); - - /* 2: Enable RPS */ - I915_WRITE(GEN6_RP_CONTROL, - GEN6_RP_MEDIA_TURBO | - GEN6_RP_MEDIA_HW_NORMAL_MODE | - GEN6_RP_MEDIA_IS_GFX | - GEN6_RP_ENABLE | - GEN6_RP_UP_BUSY_AVG | - GEN6_RP_DOWN_IDLE_AVG); - - reset_rps(dev_priv, gen6_set_rps); - - intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL); -} - -static void gen6_enable_rps(struct drm_i915_private *dev_priv) -{ - /* Here begins a magic sequence of register writes to enable - * auto-downclocking. - * - * Perhaps there might be some value in exposing these to - * userspace... - */ - intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL); - - /* Power down if completely idle for over 50ms */ - I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000); - I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10); - - reset_rps(dev_priv, gen6_set_rps); - - intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL); -} - -static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv) -{ - u32 val, rp0; - - val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE); - - switch (RUNTIME_INFO(dev_priv)->sseu.eu_total) { - case 8: - /* (2 * 4) config */ - rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT); - break; - case 12: - /* (2 * 6) config */ - rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT); - break; - case 16: - /* (2 * 8) config */ - default: - /* Setting (2 * 8) Min RP0 for any other combination */ - rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT); - break; - } - - rp0 = (rp0 & FB_GFX_FREQ_FUSE_MASK); - - return rp0; -} - -static int cherryview_rps_rpe_freq(struct drm_i915_private *dev_priv) -{ - u32 val, rpe; - - val = vlv_punit_read(dev_priv, PUNIT_GPU_DUTYCYCLE_REG); - rpe = (val >> PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT) & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK; - - return rpe; -} - -static int cherryview_rps_guar_freq(struct drm_i915_private *dev_priv) -{ - u32 val, rp1; - - val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE); - rp1 = (val & FB_GFX_FREQ_FUSE_MASK); - - return rp1; -} - -static u32 cherryview_rps_min_freq(struct drm_i915_private *dev_priv) -{ - u32 val, rpn; - - val = vlv_punit_read(dev_priv, FB_GFX_FMIN_AT_VMIN_FUSE); - rpn = ((val >> FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT) & - FB_GFX_FREQ_FUSE_MASK); - - return rpn; -} - -static int valleyview_rps_guar_freq(struct drm_i915_private *dev_priv) -{ - u32 val, rp1; - - val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE); - - rp1 = (val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK) >> FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT; - - return rp1; -} - -static int valleyview_rps_max_freq(struct drm_i915_private *dev_priv) -{ - u32 val, rp0; - - val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE); - - rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT; - /* Clamp to max */ - rp0 = min_t(u32, rp0, 0xea); - - return rp0; -} - -static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv) -{ - u32 val, rpe; - - val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO); - rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT; - val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI); - rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5; - - return rpe; -} - -static int valleyview_rps_min_freq(struct drm_i915_private *dev_priv) -{ - u32 val; - - val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff; - /* - * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value - * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on - * a BYT-M B0 the above register contains 0xbf. Moreover when setting - * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0 - * to make sure it matches what Punit accepts. - */ - return max_t(u32, val, 0xc0); -} - -static void vlv_init_gpll_ref_freq(struct drm_i915_private *dev_priv) -{ - dev_priv->gt_pm.rps.gpll_ref_freq = - vlv_get_cck_clock(dev_priv, "GPLL ref", - CCK_GPLL_CLOCK_CONTROL, - dev_priv->czclk_freq); - - DRM_DEBUG_DRIVER("GPLL reference freq: %d kHz\n", - dev_priv->gt_pm.rps.gpll_ref_freq); -} - -static void valleyview_init_gt_powersave(struct drm_i915_private *dev_priv) -{ - struct intel_rps *rps = &dev_priv->gt_pm.rps; - u32 val; - - vlv_iosf_sb_get(dev_priv, - BIT(VLV_IOSF_SB_PUNIT) | - BIT(VLV_IOSF_SB_NC) | - BIT(VLV_IOSF_SB_CCK)); - - vlv_init_gpll_ref_freq(dev_priv); - - val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); - switch ((val >> 6) & 3) { - case 0: - case 1: - dev_priv->mem_freq = 800; - break; - case 2: - dev_priv->mem_freq = 1066; - break; - case 3: - dev_priv->mem_freq = 1333; - break; - } - DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq); - - rps->max_freq = valleyview_rps_max_freq(dev_priv); - rps->rp0_freq = rps->max_freq; - DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n", - intel_gpu_freq(dev_priv, rps->max_freq), - rps->max_freq); - - rps->efficient_freq = valleyview_rps_rpe_freq(dev_priv); - DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n", - intel_gpu_freq(dev_priv, rps->efficient_freq), - rps->efficient_freq); - - rps->rp1_freq = valleyview_rps_guar_freq(dev_priv); - DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n", - intel_gpu_freq(dev_priv, rps->rp1_freq), - rps->rp1_freq); - - rps->min_freq = valleyview_rps_min_freq(dev_priv); - DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n", - intel_gpu_freq(dev_priv, rps->min_freq), - rps->min_freq); - - vlv_iosf_sb_put(dev_priv, - BIT(VLV_IOSF_SB_PUNIT) | - BIT(VLV_IOSF_SB_NC) | - BIT(VLV_IOSF_SB_CCK)); -} - -static void cherryview_init_gt_powersave(struct drm_i915_private *dev_priv) -{ - struct intel_rps *rps = &dev_priv->gt_pm.rps; - u32 val; - - vlv_iosf_sb_get(dev_priv, - BIT(VLV_IOSF_SB_PUNIT) | - BIT(VLV_IOSF_SB_NC) | - BIT(VLV_IOSF_SB_CCK)); - - vlv_init_gpll_ref_freq(dev_priv); - - val = vlv_cck_read(dev_priv, CCK_FUSE_REG); - - switch ((val >> 2) & 0x7) { - case 3: - dev_priv->mem_freq = 2000; - break; - default: - dev_priv->mem_freq = 1600; - break; - } - DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq); - - rps->max_freq = cherryview_rps_max_freq(dev_priv); - rps->rp0_freq = rps->max_freq; - DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n", - intel_gpu_freq(dev_priv, rps->max_freq), - rps->max_freq); - - rps->efficient_freq = cherryview_rps_rpe_freq(dev_priv); - DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n", - intel_gpu_freq(dev_priv, rps->efficient_freq), - rps->efficient_freq); - - rps->rp1_freq = cherryview_rps_guar_freq(dev_priv); - DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n", - intel_gpu_freq(dev_priv, rps->rp1_freq), - rps->rp1_freq); - - rps->min_freq = cherryview_rps_min_freq(dev_priv); - DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n", - intel_gpu_freq(dev_priv, rps->min_freq), - rps->min_freq); - - vlv_iosf_sb_put(dev_priv, - BIT(VLV_IOSF_SB_PUNIT) | - BIT(VLV_IOSF_SB_NC) | - BIT(VLV_IOSF_SB_CCK)); - - WARN_ONCE((rps->max_freq | rps->efficient_freq | rps->rp1_freq | - rps->min_freq) & 1, - "Odd GPU freq values\n"); -} - -static void cherryview_enable_rps(struct drm_i915_private *dev_priv) -{ - u32 val; - - intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL); - - /* 1: Program defaults and thresholds for RPS*/ - I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000); - I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400); - I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000); - I915_WRITE(GEN6_RP_UP_EI, 66000); - I915_WRITE(GEN6_RP_DOWN_EI, 350000); - - I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10); - - /* 2: Enable RPS */ - I915_WRITE(GEN6_RP_CONTROL, - GEN6_RP_MEDIA_HW_NORMAL_MODE | - GEN6_RP_MEDIA_IS_GFX | - GEN6_RP_ENABLE | - GEN6_RP_UP_BUSY_AVG | - GEN6_RP_DOWN_IDLE_AVG); - - /* Setting Fixed Bias */ - vlv_punit_get(dev_priv); - - val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50; - vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val); - - val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); - - vlv_punit_put(dev_priv); - - /* RPS code assumes GPLL is used */ - WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n"); - - DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE)); - DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val); - - reset_rps(dev_priv, valleyview_set_rps); - - intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL); -} - -static void valleyview_enable_rps(struct drm_i915_private *dev_priv) -{ - u32 val; - - intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL); - - I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000); - I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400); - I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000); - I915_WRITE(GEN6_RP_UP_EI, 66000); - I915_WRITE(GEN6_RP_DOWN_EI, 350000); - - I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10); - - I915_WRITE(GEN6_RP_CONTROL, - GEN6_RP_MEDIA_TURBO | - GEN6_RP_MEDIA_HW_NORMAL_MODE | - GEN6_RP_MEDIA_IS_GFX | - GEN6_RP_ENABLE | - GEN6_RP_UP_BUSY_AVG | - GEN6_RP_DOWN_IDLE_CONT); - - vlv_punit_get(dev_priv); - - /* Setting Fixed Bias */ - val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875; - vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val); - - val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); - - vlv_punit_put(dev_priv); - - /* RPS code assumes GPLL is used */ - WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n"); - - DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE)); - DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val); - - reset_rps(dev_priv, valleyview_set_rps); - - intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL); -} - -static unsigned long intel_pxfreq(u32 vidfreq) -{ - unsigned long freq; - int div = (vidfreq & 0x3f0000) >> 16; - int post = (vidfreq & 0x3000) >> 12; - int pre = (vidfreq & 0x7); - - if (!pre) - return 0; - - freq = ((div * 133333) / ((1<<post) * pre)); - - return freq; -} - -static const struct cparams { - u16 i; - u16 t; - u16 m; - u16 c; -} cparams[] = { - { 1, 1333, 301, 28664 }, - { 1, 1066, 294, 24460 }, - { 1, 800, 294, 25192 }, - { 0, 1333, 276, 27605 }, - { 0, 1066, 276, 27605 }, - { 0, 800, 231, 23784 }, -}; - -static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv) -{ - u64 total_count, diff, ret; - u32 count1, count2, count3, m = 0, c = 0; - unsigned long now = jiffies_to_msecs(jiffies), diff1; - int i; - - lockdep_assert_held(&mchdev_lock); - - diff1 = now - dev_priv->ips.last_time1; - - /* Prevent division-by-zero if we are asking too fast. - * Also, we don't get interesting results if we are polling - * faster than once in 10ms, so just return the saved value - * in such cases. - */ - if (diff1 <= 10) - return dev_priv->ips.chipset_power; - - count1 = I915_READ(DMIEC); - count2 = I915_READ(DDREC); - count3 = I915_READ(CSIEC); - - total_count = count1 + count2 + count3; - - /* FIXME: handle per-counter overflow */ - if (total_count < dev_priv->ips.last_count1) { - diff = ~0UL - dev_priv->ips.last_count1; - diff += total_count; - } else { - diff = total_count - dev_priv->ips.last_count1; - } - - for (i = 0; i < ARRAY_SIZE(cparams); i++) { - if (cparams[i].i == dev_priv->ips.c_m && - cparams[i].t == dev_priv->ips.r_t) { - m = cparams[i].m; - c = cparams[i].c; - break; - } - } - - diff = div_u64(diff, diff1); - ret = ((m * diff) + c); - ret = div_u64(ret, 10); - - dev_priv->ips.last_count1 = total_count; - dev_priv->ips.last_time1 = now; - - dev_priv->ips.chipset_power = ret; - - return ret; -} - -unsigned long i915_chipset_val(struct drm_i915_private *dev_priv) -{ - intel_wakeref_t wakeref; - unsigned long val = 0; - - if (!IS_GEN(dev_priv, 5)) - return 0; - - with_intel_runtime_pm(&dev_priv->runtime_pm, wakeref) { - spin_lock_irq(&mchdev_lock); - val = __i915_chipset_val(dev_priv); - spin_unlock_irq(&mchdev_lock); - } - - return val; -} - -unsigned long i915_mch_val(struct drm_i915_private *i915) -{ - unsigned long m, x, b; - u32 tsfs; - - tsfs = intel_uncore_read(&i915->uncore, TSFS); - - m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT); - x = intel_uncore_read8(&i915->uncore, TR1); - - b = tsfs & TSFS_INTR_MASK; - - return ((m * x) / 127) - b; -} - -static int _pxvid_to_vd(u8 pxvid) -{ - if (pxvid == 0) - return 0; - - if (pxvid >= 8 && pxvid < 31) - pxvid = 31; - - return (pxvid + 2) * 125; -} - -static u32 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid) -{ - const int vd = _pxvid_to_vd(pxvid); - const int vm = vd - 1125; - - if (INTEL_INFO(dev_priv)->is_mobile) - return vm > 0 ? vm : 0; - - return vd; -} - -static void __i915_update_gfx_val(struct drm_i915_private *dev_priv) -{ - u64 now, diff, diffms; - u32 count; - - lockdep_assert_held(&mchdev_lock); - - now = ktime_get_raw_ns(); - diffms = now - dev_priv->ips.last_time2; - do_div(diffms, NSEC_PER_MSEC); - - /* Don't divide by 0 */ - if (!diffms) - return; - - count = I915_READ(GFXEC); - - if (count < dev_priv->ips.last_count2) { - diff = ~0UL - dev_priv->ips.last_count2; - diff += count; - } else { - diff = count - dev_priv->ips.last_count2; - } - - dev_priv->ips.last_count2 = count; - dev_priv->ips.last_time2 = now; - - /* More magic constants... */ - diff = diff * 1181; - diff = div_u64(diff, diffms * 10); - dev_priv->ips.gfx_power = diff; -} - -void i915_update_gfx_val(struct drm_i915_private *dev_priv) -{ - intel_wakeref_t wakeref; - - if (!IS_GEN(dev_priv, 5)) - return; - - with_intel_runtime_pm(&dev_priv->runtime_pm, wakeref) { - spin_lock_irq(&mchdev_lock); - __i915_update_gfx_val(dev_priv); - spin_unlock_irq(&mchdev_lock); - } -} - -static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv) -{ - unsigned long t, corr, state1, corr2, state2; - u32 pxvid, ext_v; - - lockdep_assert_held(&mchdev_lock); - - pxvid = I915_READ(PXVFREQ(dev_priv->gt_pm.rps.cur_freq)); - pxvid = (pxvid >> 24) & 0x7f; - ext_v = pvid_to_extvid(dev_priv, pxvid); - - state1 = ext_v; - - t = i915_mch_val(dev_priv); - - /* Revel in the empirically derived constants */ - - /* Correction factor in 1/100000 units */ - if (t > 80) - corr = ((t * 2349) + 135940); - else if (t >= 50) - corr = ((t * 964) + 29317); - else /* < 50 */ - corr = ((t * 301) + 1004); - - corr = corr * ((150142 * state1) / 10000 - 78642); - corr /= 100000; - corr2 = (corr * dev_priv->ips.corr); - - state2 = (corr2 * state1) / 10000; - state2 /= 100; /* convert to mW */ - - __i915_update_gfx_val(dev_priv); - - return dev_priv->ips.gfx_power + state2; -} - -unsigned long i915_gfx_val(struct drm_i915_private *dev_priv) -{ - intel_wakeref_t wakeref; - unsigned long val = 0; - - if (!IS_GEN(dev_priv, 5)) - return 0; - - with_intel_runtime_pm(&dev_priv->runtime_pm, wakeref) { - spin_lock_irq(&mchdev_lock); - val = __i915_gfx_val(dev_priv); - spin_unlock_irq(&mchdev_lock); - } - - return val; -} - -static struct drm_i915_private __rcu *i915_mch_dev; - -static struct drm_i915_private *mchdev_get(void) -{ - struct drm_i915_private *i915; - - rcu_read_lock(); - i915 = rcu_dereference(i915_mch_dev); - if (!kref_get_unless_zero(&i915->drm.ref)) - i915 = NULL; - rcu_read_unlock(); - - return i915; -} - -/** - * i915_read_mch_val - return value for IPS use - * - * Calculate and return a value for the IPS driver to use when deciding whether - * we have thermal and power headroom to increase CPU or GPU power budget. - */ -unsigned long i915_read_mch_val(void) -{ - struct drm_i915_private *i915; - unsigned long chipset_val = 0; - unsigned long graphics_val = 0; - intel_wakeref_t wakeref; - - i915 = mchdev_get(); - if (!i915) - return 0; - - with_intel_runtime_pm(&i915->runtime_pm, wakeref) { - spin_lock_irq(&mchdev_lock); - chipset_val = __i915_chipset_val(i915); - graphics_val = __i915_gfx_val(i915); - spin_unlock_irq(&mchdev_lock); - } - - drm_dev_put(&i915->drm); - return chipset_val + graphics_val; -} -EXPORT_SYMBOL_GPL(i915_read_mch_val); - -/** - * i915_gpu_raise - raise GPU frequency limit - * - * Raise the limit; IPS indicates we have thermal headroom. - */ -bool i915_gpu_raise(void) -{ - struct drm_i915_private *i915; - - i915 = mchdev_get(); - if (!i915) - return false; - - spin_lock_irq(&mchdev_lock); - if (i915->ips.max_delay > i915->ips.fmax) - i915->ips.max_delay--; - spin_unlock_irq(&mchdev_lock); - - drm_dev_put(&i915->drm); - return true; -} -EXPORT_SYMBOL_GPL(i915_gpu_raise); - -/** - * i915_gpu_lower - lower GPU frequency limit - * - * IPS indicates we're close to a thermal limit, so throttle back the GPU - * frequency maximum. - */ -bool i915_gpu_lower(void) -{ - struct drm_i915_private *i915; - - i915 = mchdev_get(); - if (!i915) - return false; - - spin_lock_irq(&mchdev_lock); - if (i915->ips.max_delay < i915->ips.min_delay) - i915->ips.max_delay++; - spin_unlock_irq(&mchdev_lock); - - drm_dev_put(&i915->drm); - return true; -} -EXPORT_SYMBOL_GPL(i915_gpu_lower); - -/** - * i915_gpu_busy - indicate GPU business to IPS - * - * Tell the IPS driver whether or not the GPU is busy. - */ -bool i915_gpu_busy(void) -{ - struct drm_i915_private *i915; - bool ret; - - i915 = mchdev_get(); - if (!i915) - return false; - - ret = i915->gt.awake; - - drm_dev_put(&i915->drm); - return ret; -} -EXPORT_SYMBOL_GPL(i915_gpu_busy); - -/** - * i915_gpu_turbo_disable - disable graphics turbo - * - * Disable graphics turbo by resetting the max frequency and setting the - * current frequency to the default. - */ -bool i915_gpu_turbo_disable(void) -{ - struct drm_i915_private *i915; - bool ret; - - i915 = mchdev_get(); - if (!i915) - return false; - - spin_lock_irq(&mchdev_lock); - i915->ips.max_delay = i915->ips.fstart; - ret = ironlake_set_drps(i915, i915->ips.fstart); - spin_unlock_irq(&mchdev_lock); - - drm_dev_put(&i915->drm); - return ret; -} -EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable); - -/** - * Tells the intel_ips driver that the i915 driver is now loaded, if - * IPS got loaded first. - * - * This awkward dance is so that neither module has to depend on the - * other in order for IPS to do the appropriate communication of - * GPU turbo limits to i915. - */ -static void -ips_ping_for_i915_load(void) -{ - void (*link)(void); - - link = symbol_get(ips_link_to_i915_driver); - if (link) { - link(); - symbol_put(ips_link_to_i915_driver); - } -} - -void intel_gpu_ips_init(struct drm_i915_private *dev_priv) -{ - /* We only register the i915 ips part with intel-ips once everything is - * set up, to avoid intel-ips sneaking in and reading bogus values. */ - rcu_assign_pointer(i915_mch_dev, dev_priv); - - ips_ping_for_i915_load(); -} - -void intel_gpu_ips_teardown(void) -{ - rcu_assign_pointer(i915_mch_dev, NULL); -} - -static void intel_init_emon(struct drm_i915_private *dev_priv) -{ - u32 lcfuse; - u8 pxw[16]; - int i; - - /* Disable to program */ - I915_WRITE(ECR, 0); - POSTING_READ(ECR); - - /* Program energy weights for various events */ - I915_WRITE(SDEW, 0x15040d00); - I915_WRITE(CSIEW0, 0x007f0000); - I915_WRITE(CSIEW1, 0x1e220004); - I915_WRITE(CSIEW2, 0x04000004); - - for (i = 0; i < 5; i++) - I915_WRITE(PEW(i), 0); - for (i = 0; i < 3; i++) - I915_WRITE(DEW(i), 0); - - /* Program P-state weights to account for frequency power adjustment */ - for (i = 0; i < 16; i++) { - u32 pxvidfreq = I915_READ(PXVFREQ(i)); - unsigned long freq = intel_pxfreq(pxvidfreq); - unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >> - PXVFREQ_PX_SHIFT; - unsigned long val; - - val = vid * vid; - val *= (freq / 1000); - val *= 255; - val /= (127*127*900); - if (val > 0xff) - DRM_ERROR("bad pxval: %ld\n", val); - pxw[i] = val; - } - /* Render standby states get 0 weight */ - pxw[14] = 0; - pxw[15] = 0; - - for (i = 0; i < 4; i++) { - u32 val = (pxw[i*4] << 24) | (pxw[(i*4)+1] << 16) | - (pxw[(i*4)+2] << 8) | (pxw[(i*4)+3]); - I915_WRITE(PXW(i), val); - } - - /* Adjust magic regs to magic values (more experimental results) */ - I915_WRITE(OGW0, 0); - I915_WRITE(OGW1, 0); - I915_WRITE(EG0, 0x00007f00); - I915_WRITE(EG1, 0x0000000e); - I915_WRITE(EG2, 0x000e0000); - I915_WRITE(EG3, 0x68000300); - I915_WRITE(EG4, 0x42000000); - I915_WRITE(EG5, 0x00140031); - I915_WRITE(EG6, 0); - I915_WRITE(EG7, 0); - - for (i = 0; i < 8; i++) - I915_WRITE(PXWL(i), 0); - - /* Enable PMON + select events */ - I915_WRITE(ECR, 0x80000019); - - lcfuse = I915_READ(LCFUSE02); - - dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK); -} - -void intel_init_gt_powersave(struct drm_i915_private *dev_priv) -{ - struct intel_rps *rps = &dev_priv->gt_pm.rps; - - /* Powersaving is controlled by the host when inside a VM */ - if (intel_vgpu_active(dev_priv)) - mkwrite_device_info(dev_priv)->has_rps = false; - - /* Initialize RPS limits (for userspace) */ - if (IS_CHERRYVIEW(dev_priv)) - cherryview_init_gt_powersave(dev_priv); - else if (IS_VALLEYVIEW(dev_priv)) - valleyview_init_gt_powersave(dev_priv); - else if (INTEL_GEN(dev_priv) >= 6) - gen6_init_rps_frequencies(dev_priv); - - /* Derive initial user preferences/limits from the hardware limits */ - rps->max_freq_softlimit = rps->max_freq; - rps->min_freq_softlimit = rps->min_freq; - - /* After setting max-softlimit, find the overclock max freq */ - if (IS_GEN(dev_priv, 6) || - IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv)) { - u32 params = 0; - - sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, - ¶ms, NULL); - if (params & BIT(31)) { /* OC supported */ - DRM_DEBUG_DRIVER("Overclocking supported, max: %dMHz, overclock: %dMHz\n", - (rps->max_freq & 0xff) * 50, - (params & 0xff) * 50); - rps->max_freq = params & 0xff; - } - } - - /* Finally allow us to boost to max by default */ - rps->boost_freq = rps->max_freq; - rps->idle_freq = rps->min_freq; - rps->cur_freq = rps->idle_freq; -} - -void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv) -{ - dev_priv->gt_pm.rps.enabled = true; /* force RPS disabling */ - intel_disable_gt_powersave(dev_priv); - - if (INTEL_GEN(dev_priv) >= 11) - gen11_reset_rps_interrupts(dev_priv); - else if (INTEL_GEN(dev_priv) >= 6) - gen6_reset_rps_interrupts(dev_priv); -} - -static void intel_disable_rps(struct drm_i915_private *dev_priv) -{ - lockdep_assert_held(&dev_priv->gt_pm.rps.lock); - - if (!dev_priv->gt_pm.rps.enabled) - return; - - if (INTEL_GEN(dev_priv) >= 9) - gen9_disable_rps(dev_priv); - else if (IS_CHERRYVIEW(dev_priv)) - cherryview_disable_rps(dev_priv); - else if (IS_VALLEYVIEW(dev_priv)) - valleyview_disable_rps(dev_priv); - else if (INTEL_GEN(dev_priv) >= 6) - gen6_disable_rps(dev_priv); - else if (IS_IRONLAKE_M(dev_priv)) - ironlake_disable_drps(dev_priv); - - dev_priv->gt_pm.rps.enabled = false; -} - -void intel_disable_gt_powersave(struct drm_i915_private *dev_priv) -{ - mutex_lock(&dev_priv->gt_pm.rps.lock); - - intel_disable_rps(dev_priv); - if (HAS_LLC(dev_priv)) - intel_llc_disable(&dev_priv->gt.llc); - - mutex_unlock(&dev_priv->gt_pm.rps.lock); -} - -static void intel_enable_rps(struct drm_i915_private *dev_priv) -{ - struct intel_rps *rps = &dev_priv->gt_pm.rps; - - lockdep_assert_held(&rps->lock); - - if (rps->enabled) - return; - - if (IS_CHERRYVIEW(dev_priv)) { - cherryview_enable_rps(dev_priv); - } else if (IS_VALLEYVIEW(dev_priv)) { - valleyview_enable_rps(dev_priv); - } else if (INTEL_GEN(dev_priv) >= 9) { - gen9_enable_rps(dev_priv); - } else if (IS_BROADWELL(dev_priv)) { - gen8_enable_rps(dev_priv); - } else if (INTEL_GEN(dev_priv) >= 6) { - gen6_enable_rps(dev_priv); - } else if (IS_IRONLAKE_M(dev_priv)) { - ironlake_enable_drps(dev_priv); - intel_init_emon(dev_priv); - } - - WARN_ON(rps->max_freq < rps->min_freq); - WARN_ON(rps->idle_freq > rps->max_freq); - - WARN_ON(rps->efficient_freq < rps->min_freq); - WARN_ON(rps->efficient_freq > rps->max_freq); - - rps->enabled = true; -} - -void intel_enable_gt_powersave(struct drm_i915_private *dev_priv) -{ - /* Powersaving is controlled by the host when inside a VM */ - if (intel_vgpu_active(dev_priv)) - return; - - mutex_lock(&dev_priv->gt_pm.rps.lock); - - if (HAS_RPS(dev_priv)) - intel_enable_rps(dev_priv); - - intel_llc_enable(&dev_priv->gt.llc); - - mutex_unlock(&dev_priv->gt_pm.rps.lock); -} - -static void ibx_init_clock_gating(struct drm_i915_private *dev_priv) -{ - /* - * On Ibex Peak and Cougar Point, we need to disable clock - * gating for the panel power sequencer or it will fail to - * start up when no ports are active. - */ - I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE); -} - -static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv) -{ - enum pipe pipe; - - for_each_pipe(dev_priv, pipe) { - I915_WRITE(DSPCNTR(pipe), - I915_READ(DSPCNTR(pipe)) | - DISPPLANE_TRICKLE_FEED_DISABLE); - - I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe))); - POSTING_READ(DSPSURF(pipe)); - } -} - -static void ilk_init_clock_gating(struct drm_i915_private *dev_priv) -{ - u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE; - - /* - * Required for FBC - * WaFbcDisableDpfcClockGating:ilk - */ - dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE | - ILK_DPFCUNIT_CLOCK_GATE_DISABLE | - ILK_DPFDUNIT_CLOCK_GATE_ENABLE; - - I915_WRITE(PCH_3DCGDIS0, - MARIUNIT_CLOCK_GATE_DISABLE | - SVSMUNIT_CLOCK_GATE_DISABLE); - I915_WRITE(PCH_3DCGDIS1, - VFMUNIT_CLOCK_GATE_DISABLE); - - /* - * According to the spec the following bits should be set in - * order to enable memory self-refresh - * The bit 22/21 of 0x42004 - * The bit 5 of 0x42020 - * The bit 15 of 0x45000 - */ - I915_WRITE(ILK_DISPLAY_CHICKEN2, - (I915_READ(ILK_DISPLAY_CHICKEN2) | - ILK_DPARB_GATE | ILK_VSDPFD_FULL)); - dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE; - I915_WRITE(DISP_ARB_CTL, - (I915_READ(DISP_ARB_CTL) | - DISP_FBC_WM_DIS)); - - /* - * Based on the document from hardware guys the following bits - * should be set unconditionally in order to enable FBC. - * The bit 22 of 0x42000 - * The bit 22 of 0x42004 - * The bit 7,8,9 of 0x42020. - */ - if (IS_IRONLAKE_M(dev_priv)) { - /* WaFbcAsynchFlipDisableFbcQueue:ilk */ - I915_WRITE(ILK_DISPLAY_CHICKEN1, - I915_READ(ILK_DISPLAY_CHICKEN1) | - ILK_FBCQ_DIS); - I915_WRITE(ILK_DISPLAY_CHICKEN2, - I915_READ(ILK_DISPLAY_CHICKEN2) | - ILK_DPARB_GATE); - } - - I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate); - - I915_WRITE(ILK_DISPLAY_CHICKEN2, - I915_READ(ILK_DISPLAY_CHICKEN2) | - ILK_ELPIN_409_SELECT); - I915_WRITE(_3D_CHICKEN2, - _3D_CHICKEN2_WM_READ_PIPELINED << 16 | - _3D_CHICKEN2_WM_READ_PIPELINED); - - /* WaDisableRenderCachePipelinedFlush:ilk */ - I915_WRITE(CACHE_MODE_0, - _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE)); - - /* WaDisable_RenderCache_OperationalFlush:ilk */ - I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); - - g4x_disable_trickle_feed(dev_priv); - - ibx_init_clock_gating(dev_priv); -} - -static void cpt_init_clock_gating(struct drm_i915_private *dev_priv) -{ - enum pipe pipe; - u32 val; - - /* - * On Ibex Peak and Cougar Point, we need to disable clock - * gating for the panel power sequencer or it will fail to - * start up when no ports are active. - */ - I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE | - PCH_DPLUNIT_CLOCK_GATE_DISABLE | - PCH_CPUNIT_CLOCK_GATE_DISABLE); - I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) | - DPLS_EDP_PPS_FIX_DIS); - /* The below fixes the weird display corruption, a few pixels shifted - * downward, on (only) LVDS of some HP laptops with IVY. - */ - for_each_pipe(dev_priv, pipe) { - val = I915_READ(TRANS_CHICKEN2(pipe)); - val |= TRANS_CHICKEN2_TIMING_OVERRIDE; - val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED; - if (dev_priv->vbt.fdi_rx_polarity_inverted) - val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED; - val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK; - val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER; - val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH; - I915_WRITE(TRANS_CHICKEN2(pipe), val); - } - /* WADP0ClockGatingDisable */ - for_each_pipe(dev_priv, pipe) { - I915_WRITE(TRANS_CHICKEN1(pipe), - TRANS_CHICKEN1_DP0UNIT_GC_DISABLE); - } -} - -static void gen6_check_mch_setup(struct drm_i915_private *dev_priv) -{ - u32 tmp; - - tmp = I915_READ(MCH_SSKPD); - if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL) - DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n", - tmp); -} - -static void gen6_init_clock_gating(struct drm_i915_private *dev_priv) -{ - u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE; - - I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate); - - I915_WRITE(ILK_DISPLAY_CHICKEN2, - I915_READ(ILK_DISPLAY_CHICKEN2) | - ILK_ELPIN_409_SELECT); - - /* WaDisableHiZPlanesWhenMSAAEnabled:snb */ - I915_WRITE(_3D_CHICKEN, - _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB)); - - /* WaDisable_RenderCache_OperationalFlush:snb */ - I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); - - /* - * BSpec recoomends 8x4 when MSAA is used, - * however in practice 16x4 seems fastest. - * - * Note that PS/WM thread counts depend on the WIZ hashing - * disable bit, which we don't touch here, but it's good - * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). - */ - I915_WRITE(GEN6_GT_MODE, - _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4)); - - I915_WRITE(CACHE_MODE_0, - _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB)); - - I915_WRITE(GEN6_UCGCTL1, - I915_READ(GEN6_UCGCTL1) | - GEN6_BLBUNIT_CLOCK_GATE_DISABLE | - GEN6_CSUNIT_CLOCK_GATE_DISABLE); - - /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock - * gating disable must be set. Failure to set it results in - * flickering pixels due to Z write ordering failures after - * some amount of runtime in the Mesa "fire" demo, and Unigine - * Sanctuary and Tropics, and apparently anything else with - * alpha test or pixel discard. - * - * According to the spec, bit 11 (RCCUNIT) must also be set, - * but we didn't debug actual testcases to find it out. - * - * WaDisableRCCUnitClockGating:snb - * WaDisableRCPBUnitClockGating:snb - */ - I915_WRITE(GEN6_UCGCTL2, - GEN6_RCPBUNIT_CLOCK_GATE_DISABLE | - GEN6_RCCUNIT_CLOCK_GATE_DISABLE); - - /* WaStripsFansDisableFastClipPerformanceFix:snb */ - I915_WRITE(_3D_CHICKEN3, - _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL)); - - /* - * Bspec says: - * "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and - * 3DSTATE_SF number of SF output attributes is more than 16." - */ - I915_WRITE(_3D_CHICKEN3, - _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH)); - - /* - * According to the spec the following bits should be - * set in order to enable memory self-refresh and fbc: - * The bit21 and bit22 of 0x42000 - * The bit21 and bit22 of 0x42004 - * The bit5 and bit7 of 0x42020 - * The bit14 of 0x70180 - * The bit14 of 0x71180 - * - * WaFbcAsynchFlipDisableFbcQueue:snb - */ - I915_WRITE(ILK_DISPLAY_CHICKEN1, - I915_READ(ILK_DISPLAY_CHICKEN1) | - ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS); - I915_WRITE(ILK_DISPLAY_CHICKEN2, - I915_READ(ILK_DISPLAY_CHICKEN2) | - ILK_DPARB_GATE | ILK_VSDPFD_FULL); - I915_WRITE(ILK_DSPCLK_GATE_D, - I915_READ(ILK_DSPCLK_GATE_D) | - ILK_DPARBUNIT_CLOCK_GATE_ENABLE | - ILK_DPFDUNIT_CLOCK_GATE_ENABLE); - - g4x_disable_trickle_feed(dev_priv); - - cpt_init_clock_gating(dev_priv); - - gen6_check_mch_setup(dev_priv); -} - -static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv) -{ - u32 reg = I915_READ(GEN7_FF_THREAD_MODE); + u32 reg = I915_READ(GEN7_FF_THREAD_MODE); /* * WaVSThreadDispatchOverride:ivb,vlv @@ -8942,90 +7325,8 @@ void intel_init_pm(struct drm_i915_private *dev_priv) } } -static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val) -{ - struct intel_rps *rps = &dev_priv->gt_pm.rps; - - /* - * N = val - 0xb7 - * Slow = Fast = GPLL ref * N - */ - return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000); -} - -static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val) -{ - struct intel_rps *rps = &dev_priv->gt_pm.rps; - - return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7; -} - -static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val) -{ - struct intel_rps *rps = &dev_priv->gt_pm.rps; - - /* - * N = val / 2 - * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2 - */ - return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000); -} - -static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val) -{ - struct intel_rps *rps = &dev_priv->gt_pm.rps; - - /* CHV needs even values */ - return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2; -} - -int intel_gpu_freq(struct drm_i915_private *dev_priv, int val) -{ - if (INTEL_GEN(dev_priv) >= 9) - return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER, - GEN9_FREQ_SCALER); - else if (IS_CHERRYVIEW(dev_priv)) - return chv_gpu_freq(dev_priv, val); - else if (IS_VALLEYVIEW(dev_priv)) - return byt_gpu_freq(dev_priv, val); - else - return val * GT_FREQUENCY_MULTIPLIER; -} - -int intel_freq_opcode(struct drm_i915_private *dev_priv, int val) -{ - if (INTEL_GEN(dev_priv) >= 9) - return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER, - GT_FREQUENCY_MULTIPLIER); - else if (IS_CHERRYVIEW(dev_priv)) - return chv_freq_opcode(dev_priv, val); - else if (IS_VALLEYVIEW(dev_priv)) - return byt_freq_opcode(dev_priv, val); - else - return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER); -} - void intel_pm_setup(struct drm_i915_private *dev_priv) { - mutex_init(&dev_priv->gt_pm.rps.lock); - mutex_init(&dev_priv->gt_pm.rps.power.mutex); - - atomic_set(&dev_priv->gt_pm.rps.num_waiters, 0); - dev_priv->runtime_pm.suspended = false; atomic_set(&dev_priv->runtime_pm.wakeref_count, 0); } - -u32 intel_get_cagf(struct drm_i915_private *dev_priv, u32 rpstat) -{ - u32 cagf; - - if (INTEL_GEN(dev_priv) >= 9) - cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT; - else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) - cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT; - else - cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT; - - return cagf; -} diff --git a/drivers/gpu/drm/i915/intel_pm.h b/drivers/gpu/drm/i915/intel_pm.h index 93d192d0610a..b56e6285d1c3 100644 --- a/drivers/gpu/drm/i915/intel_pm.h +++ b/drivers/gpu/drm/i915/intel_pm.h @@ -29,15 +29,6 @@ void intel_update_watermarks(struct intel_crtc *crtc); void intel_init_pm(struct drm_i915_private *dev_priv); void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv); void intel_pm_setup(struct drm_i915_private *dev_priv); -void intel_gpu_ips_init(struct drm_i915_private *dev_priv); -void intel_gpu_ips_teardown(void); -void intel_init_gt_powersave(struct drm_i915_private *dev_priv); -void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv); -void intel_enable_gt_powersave(struct drm_i915_private *dev_priv); -void intel_disable_gt_powersave(struct drm_i915_private *dev_priv); -void gen6_rps_busy(struct drm_i915_private *dev_priv); -void gen6_rps_idle(struct drm_i915_private *dev_priv); -void gen6_rps_boost(struct i915_request *rq); void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv); void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv); void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv); @@ -69,19 +60,6 @@ int skl_check_pipe_max_pixel_rate(struct intel_crtc *intel_crtc, void intel_init_ipc(struct drm_i915_private *dev_priv); void intel_enable_ipc(struct drm_i915_private *dev_priv); -int intel_gpu_freq(struct drm_i915_private *dev_priv, int val); -int intel_freq_opcode(struct drm_i915_private *dev_priv, int val); - -u32 intel_get_cagf(struct drm_i915_private *dev_priv, u32 rpstat1); - -unsigned long i915_chipset_val(struct drm_i915_private *dev_priv); -unsigned long i915_mch_val(struct drm_i915_private *dev_priv); -unsigned long i915_gfx_val(struct drm_i915_private *dev_priv); -void i915_update_gfx_val(struct drm_i915_private *dev_priv); - -bool ironlake_set_drps(struct drm_i915_private *dev_priv, u8 val); -int intel_set_rps(struct drm_i915_private *dev_priv, u8 val); -void intel_rps_mark_interactive(struct drm_i915_private *i915, bool interactive); bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable); #endif /* __INTEL_PM_H__ */
i915_irq.c is large. One reason for this is that has a large chunk of the GT render power management stashed away in it. Extract that logic out of i915_irq.c and intel_pm.c and put it under one roof. Based on a patch by Chris Wilson. Signed-off-by: Andi Shyti <andi.shyti@intel.com> Cc: Chris Wilson <chris@chris-wilson.co.uk> --- drivers/gpu/drm/i915/Makefile | 3 +- drivers/gpu/drm/i915/display/intel_display.c | 8 +- drivers/gpu/drm/i915/gt/intel_gt.c | 6 +- drivers/gpu/drm/i915/gt/intel_gt_irq.c | 5 +- drivers/gpu/drm/i915/gt/intel_gt_pm.c | 49 +- drivers/gpu/drm/i915/gt/intel_gt_pm.h | 1 + drivers/gpu/drm/i915/gt/intel_gt_types.h | 2 + drivers/gpu/drm/i915/gt/intel_llc.c | 2 +- drivers/gpu/drm/i915/gt/intel_rps.c | 1872 +++++++++++++++ drivers/gpu/drm/i915/gt/intel_rps.h | 37 + drivers/gpu/drm/i915/gt/intel_rps_types.h | 93 + drivers/gpu/drm/i915/gt/selftest_llc.c | 7 +- .../gpu/drm/i915/gt/uc/intel_guc_submission.c | 4 +- drivers/gpu/drm/i915/i915_debugfs.c | 73 +- drivers/gpu/drm/i915/i915_drv.c | 2 + drivers/gpu/drm/i915/i915_drv.h | 96 - drivers/gpu/drm/i915/i915_gem.c | 3 +- drivers/gpu/drm/i915/i915_irq.c | 359 +-- drivers/gpu/drm/i915/i915_irq.h | 3 - drivers/gpu/drm/i915/i915_pmu.c | 10 +- drivers/gpu/drm/i915/i915_request.c | 7 +- drivers/gpu/drm/i915/i915_sysfs.c | 74 +- drivers/gpu/drm/i915/intel_pm.c | 2105 ++--------------- drivers/gpu/drm/i915/intel_pm.h | 22 - 24 files changed, 2359 insertions(+), 2484 deletions(-) create mode 100644 drivers/gpu/drm/i915/gt/intel_rps.c create mode 100644 drivers/gpu/drm/i915/gt/intel_rps.h create mode 100644 drivers/gpu/drm/i915/gt/intel_rps_types.h