@@ -54,7 +54,8 @@ i915-y += i915_drv.o \
intel_runtime_pm.o \
intel_sideband.o \
intel_uncore.o \
- intel_wakeref.o
+ intel_wakeref.o \
+ vlv_suspend.o
# core library code
i915-y += \
@@ -80,71 +80,10 @@
#include "intel_csr.h"
#include "intel_memory_region.h"
#include "intel_pm.h"
+#include "vlv_suspend.h"
static struct drm_driver driver;
-struct vlv_s0ix_state {
- /* GAM */
- u32 wr_watermark;
- u32 gfx_prio_ctrl;
- u32 arb_mode;
- u32 gfx_pend_tlb0;
- u32 gfx_pend_tlb1;
- u32 lra_limits[GEN7_LRA_LIMITS_REG_NUM];
- u32 media_max_req_count;
- u32 gfx_max_req_count;
- u32 render_hwsp;
- u32 ecochk;
- u32 bsd_hwsp;
- u32 blt_hwsp;
- u32 tlb_rd_addr;
-
- /* MBC */
- u32 g3dctl;
- u32 gsckgctl;
- u32 mbctl;
-
- /* GCP */
- u32 ucgctl1;
- u32 ucgctl3;
- u32 rcgctl1;
- u32 rcgctl2;
- u32 rstctl;
- u32 misccpctl;
-
- /* GPM */
- u32 gfxpause;
- u32 rpdeuhwtc;
- u32 rpdeuc;
- u32 ecobus;
- u32 pwrdwnupctl;
- u32 rp_down_timeout;
- u32 rp_deucsw;
- u32 rcubmabdtmr;
- u32 rcedata;
- u32 spare2gh;
-
- /* Display 1 CZ domain */
- u32 gt_imr;
- u32 gt_ier;
- u32 pm_imr;
- u32 pm_ier;
- u32 gt_scratch[GEN7_GT_SCRATCH_REG_NUM];
-
- /* GT SA CZ domain */
- u32 tilectl;
- u32 gt_fifoctl;
- u32 gtlc_wake_ctrl;
- u32 gtlc_survive;
- u32 pmwgicz;
-
- /* Display 2 CZ domain */
- u32 gu_ctl0;
- u32 gu_ctl1;
- u32 pcbr;
- u32 clock_gate_dis2;
-};
-
static int i915_get_bridge_dev(struct drm_i915_private *dev_priv)
{
int domain = pci_domain_nr(dev_priv->drm.pdev->bus);
@@ -446,29 +385,6 @@ static void intel_detect_preproduction_hw(struct drm_i915_private *dev_priv)
}
}
-static int vlv_alloc_s0ix_state(struct drm_i915_private *i915)
-{
- if (!IS_VALLEYVIEW(i915))
- return 0;
-
- /* we write all the values in the struct, so no need to zero it out */
- i915->vlv_s0ix_state = kmalloc(sizeof(*i915->vlv_s0ix_state),
- GFP_KERNEL);
- if (!i915->vlv_s0ix_state)
- return -ENOMEM;
-
- return 0;
-}
-
-static void vlv_free_s0ix_state(struct drm_i915_private *i915)
-{
- if (!i915->vlv_s0ix_state)
- return;
-
- kfree(i915->vlv_s0ix_state);
- i915->vlv_s0ix_state = NULL;
-}
-
static void sanitize_gpu(struct drm_i915_private *i915)
{
if (!INTEL_INFO(i915)->gpu_reset_clobbers_display)
@@ -517,7 +433,7 @@ static int i915_driver_early_probe(struct drm_i915_private *dev_priv)
if (ret < 0)
return ret;
- ret = vlv_alloc_s0ix_state(dev_priv);
+ ret = vlv_suspend_init(dev_priv);
if (ret < 0)
goto err_workqueues;
@@ -548,7 +464,7 @@ static int i915_driver_early_probe(struct drm_i915_private *dev_priv)
err_gem:
i915_gem_cleanup_early(dev_priv);
intel_gt_driver_late_release(&dev_priv->gt);
- vlv_free_s0ix_state(dev_priv);
+ vlv_suspend_cleanup(dev_priv);
err_workqueues:
i915_workqueues_cleanup(dev_priv);
return ret;
@@ -565,7 +481,7 @@ static void i915_driver_late_release(struct drm_i915_private *dev_priv)
intel_power_domains_cleanup(dev_priv);
i915_gem_cleanup_early(dev_priv);
intel_gt_driver_late_release(&dev_priv->gt);
- vlv_free_s0ix_state(dev_priv);
+ vlv_suspend_cleanup(dev_priv);
i915_workqueues_cleanup(dev_priv);
pm_qos_remove_request(&dev_priv->sb_qos);
@@ -1673,10 +1589,6 @@ static void intel_suspend_encoders(struct drm_i915_private *dev_priv)
drm_modeset_unlock_all(dev);
}
-static int vlv_resume_prepare(struct drm_i915_private *dev_priv,
- bool rpm_resume);
-static int vlv_suspend_complete(struct drm_i915_private *dev_priv);
-
static bool suspend_to_idle(struct drm_i915_private *dev_priv)
{
#if IS_ENABLED(CONFIG_ACPI_SLEEP)
@@ -1763,7 +1675,7 @@ static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation)
struct drm_i915_private *dev_priv = to_i915(dev);
struct pci_dev *pdev = dev_priv->drm.pdev;
struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
- int ret = 0;
+ int ret;
disable_rpm_wakeref_asserts(rpm);
@@ -1776,9 +1688,7 @@ static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation)
intel_display_power_suspend_late(dev_priv);
- if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
- ret = vlv_suspend_complete(dev_priv);
-
+ ret = vlv_suspend_complete(dev_priv);
if (ret) {
drm_err(&dev_priv->drm, "Suspend complete failed: %d\n", ret);
intel_power_domains_resume(dev_priv);
@@ -1953,12 +1863,10 @@ static int i915_drm_resume_early(struct drm_device *dev)
disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
- if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
- ret = vlv_resume_prepare(dev_priv, false);
+ ret = vlv_resume_prepare(dev_priv, false);
if (ret)
drm_err(&dev_priv->drm,
- "Resume prepare failed: %d, continuing anyway\n",
- ret);
+ "Resume prepare failed: %d, continuing anyway\n", ret);
intel_uncore_resume_early(&dev_priv->uncore);
@@ -2125,390 +2033,11 @@ static int i915_pm_restore(struct device *kdev)
return i915_pm_resume(kdev);
}
-/*
- * Save all Gunit registers that may be lost after a D3 and a subsequent
- * S0i[R123] transition. The list of registers needing a save/restore is
- * defined in the VLV2_S0IXRegs document. This documents marks all Gunit
- * registers in the following way:
- * - Driver: saved/restored by the driver
- * - Punit : saved/restored by the Punit firmware
- * - No, w/o marking: no need to save/restore, since the register is R/O or
- * used internally by the HW in a way that doesn't depend
- * keeping the content across a suspend/resume.
- * - Debug : used for debugging
- *
- * We save/restore all registers marked with 'Driver', with the following
- * exceptions:
- * - Registers out of use, including also registers marked with 'Debug'.
- * These have no effect on the driver's operation, so we don't save/restore
- * them to reduce the overhead.
- * - Registers that are fully setup by an initialization function called from
- * the resume path. For example many clock gating and RPS/RC6 registers.
- * - Registers that provide the right functionality with their reset defaults.
- *
- * TODO: Except for registers that based on the above 3 criteria can be safely
- * ignored, we save/restore all others, practically treating the HW context as
- * a black-box for the driver. Further investigation is needed to reduce the
- * saved/restored registers even further, by following the same 3 criteria.
- */
-static void vlv_save_gunit_s0ix_state(struct drm_i915_private *dev_priv)
-{
- struct vlv_s0ix_state *s = dev_priv->vlv_s0ix_state;
- int i;
-
- if (!s)
- return;
-
- /* GAM 0x4000-0x4770 */
- s->wr_watermark = I915_READ(GEN7_WR_WATERMARK);
- s->gfx_prio_ctrl = I915_READ(GEN7_GFX_PRIO_CTRL);
- s->arb_mode = I915_READ(ARB_MODE);
- s->gfx_pend_tlb0 = I915_READ(GEN7_GFX_PEND_TLB0);
- s->gfx_pend_tlb1 = I915_READ(GEN7_GFX_PEND_TLB1);
-
- for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
- s->lra_limits[i] = I915_READ(GEN7_LRA_LIMITS(i));
-
- s->media_max_req_count = I915_READ(GEN7_MEDIA_MAX_REQ_COUNT);
- s->gfx_max_req_count = I915_READ(GEN7_GFX_MAX_REQ_COUNT);
-
- s->render_hwsp = I915_READ(RENDER_HWS_PGA_GEN7);
- s->ecochk = I915_READ(GAM_ECOCHK);
- s->bsd_hwsp = I915_READ(BSD_HWS_PGA_GEN7);
- s->blt_hwsp = I915_READ(BLT_HWS_PGA_GEN7);
-
- s->tlb_rd_addr = I915_READ(GEN7_TLB_RD_ADDR);
-
- /* MBC 0x9024-0x91D0, 0x8500 */
- s->g3dctl = I915_READ(VLV_G3DCTL);
- s->gsckgctl = I915_READ(VLV_GSCKGCTL);
- s->mbctl = I915_READ(GEN6_MBCTL);
-
- /* GCP 0x9400-0x9424, 0x8100-0x810C */
- s->ucgctl1 = I915_READ(GEN6_UCGCTL1);
- s->ucgctl3 = I915_READ(GEN6_UCGCTL3);
- s->rcgctl1 = I915_READ(GEN6_RCGCTL1);
- s->rcgctl2 = I915_READ(GEN6_RCGCTL2);
- s->rstctl = I915_READ(GEN6_RSTCTL);
- s->misccpctl = I915_READ(GEN7_MISCCPCTL);
-
- /* GPM 0xA000-0xAA84, 0x8000-0x80FC */
- s->gfxpause = I915_READ(GEN6_GFXPAUSE);
- s->rpdeuhwtc = I915_READ(GEN6_RPDEUHWTC);
- s->rpdeuc = I915_READ(GEN6_RPDEUC);
- s->ecobus = I915_READ(ECOBUS);
- s->pwrdwnupctl = I915_READ(VLV_PWRDWNUPCTL);
- s->rp_down_timeout = I915_READ(GEN6_RP_DOWN_TIMEOUT);
- s->rp_deucsw = I915_READ(GEN6_RPDEUCSW);
- s->rcubmabdtmr = I915_READ(GEN6_RCUBMABDTMR);
- s->rcedata = I915_READ(VLV_RCEDATA);
- s->spare2gh = I915_READ(VLV_SPAREG2H);
-
- /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */
- s->gt_imr = I915_READ(GTIMR);
- s->gt_ier = I915_READ(GTIER);
- s->pm_imr = I915_READ(GEN6_PMIMR);
- s->pm_ier = I915_READ(GEN6_PMIER);
-
- for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
- s->gt_scratch[i] = I915_READ(GEN7_GT_SCRATCH(i));
-
- /* GT SA CZ domain, 0x100000-0x138124 */
- s->tilectl = I915_READ(TILECTL);
- s->gt_fifoctl = I915_READ(GTFIFOCTL);
- s->gtlc_wake_ctrl = I915_READ(VLV_GTLC_WAKE_CTRL);
- s->gtlc_survive = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
- s->pmwgicz = I915_READ(VLV_PMWGICZ);
-
- /* Gunit-Display CZ domain, 0x182028-0x1821CF */
- s->gu_ctl0 = I915_READ(VLV_GU_CTL0);
- s->gu_ctl1 = I915_READ(VLV_GU_CTL1);
- s->pcbr = I915_READ(VLV_PCBR);
- s->clock_gate_dis2 = I915_READ(VLV_GUNIT_CLOCK_GATE2);
-
- /*
- * Not saving any of:
- * DFT, 0x9800-0x9EC0
- * SARB, 0xB000-0xB1FC
- * GAC, 0x5208-0x524C, 0x14000-0x14C000
- * PCI CFG
- */
-}
-
-static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *dev_priv)
-{
- struct vlv_s0ix_state *s = dev_priv->vlv_s0ix_state;
- u32 val;
- int i;
-
- if (!s)
- return;
-
- /* GAM 0x4000-0x4770 */
- I915_WRITE(GEN7_WR_WATERMARK, s->wr_watermark);
- I915_WRITE(GEN7_GFX_PRIO_CTRL, s->gfx_prio_ctrl);
- I915_WRITE(ARB_MODE, s->arb_mode | (0xffff << 16));
- I915_WRITE(GEN7_GFX_PEND_TLB0, s->gfx_pend_tlb0);
- I915_WRITE(GEN7_GFX_PEND_TLB1, s->gfx_pend_tlb1);
-
- for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
- I915_WRITE(GEN7_LRA_LIMITS(i), s->lra_limits[i]);
-
- I915_WRITE(GEN7_MEDIA_MAX_REQ_COUNT, s->media_max_req_count);
- I915_WRITE(GEN7_GFX_MAX_REQ_COUNT, s->gfx_max_req_count);
-
- I915_WRITE(RENDER_HWS_PGA_GEN7, s->render_hwsp);
- I915_WRITE(GAM_ECOCHK, s->ecochk);
- I915_WRITE(BSD_HWS_PGA_GEN7, s->bsd_hwsp);
- I915_WRITE(BLT_HWS_PGA_GEN7, s->blt_hwsp);
-
- I915_WRITE(GEN7_TLB_RD_ADDR, s->tlb_rd_addr);
-
- /* MBC 0x9024-0x91D0, 0x8500 */
- I915_WRITE(VLV_G3DCTL, s->g3dctl);
- I915_WRITE(VLV_GSCKGCTL, s->gsckgctl);
- I915_WRITE(GEN6_MBCTL, s->mbctl);
-
- /* GCP 0x9400-0x9424, 0x8100-0x810C */
- I915_WRITE(GEN6_UCGCTL1, s->ucgctl1);
- I915_WRITE(GEN6_UCGCTL3, s->ucgctl3);
- I915_WRITE(GEN6_RCGCTL1, s->rcgctl1);
- I915_WRITE(GEN6_RCGCTL2, s->rcgctl2);
- I915_WRITE(GEN6_RSTCTL, s->rstctl);
- I915_WRITE(GEN7_MISCCPCTL, s->misccpctl);
-
- /* GPM 0xA000-0xAA84, 0x8000-0x80FC */
- I915_WRITE(GEN6_GFXPAUSE, s->gfxpause);
- I915_WRITE(GEN6_RPDEUHWTC, s->rpdeuhwtc);
- I915_WRITE(GEN6_RPDEUC, s->rpdeuc);
- I915_WRITE(ECOBUS, s->ecobus);
- I915_WRITE(VLV_PWRDWNUPCTL, s->pwrdwnupctl);
- I915_WRITE(GEN6_RP_DOWN_TIMEOUT,s->rp_down_timeout);
- I915_WRITE(GEN6_RPDEUCSW, s->rp_deucsw);
- I915_WRITE(GEN6_RCUBMABDTMR, s->rcubmabdtmr);
- I915_WRITE(VLV_RCEDATA, s->rcedata);
- I915_WRITE(VLV_SPAREG2H, s->spare2gh);
-
- /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */
- I915_WRITE(GTIMR, s->gt_imr);
- I915_WRITE(GTIER, s->gt_ier);
- I915_WRITE(GEN6_PMIMR, s->pm_imr);
- I915_WRITE(GEN6_PMIER, s->pm_ier);
-
- for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
- I915_WRITE(GEN7_GT_SCRATCH(i), s->gt_scratch[i]);
-
- /* GT SA CZ domain, 0x100000-0x138124 */
- I915_WRITE(TILECTL, s->tilectl);
- I915_WRITE(GTFIFOCTL, s->gt_fifoctl);
- /*
- * Preserve the GT allow wake and GFX force clock bit, they are not
- * be restored, as they are used to control the s0ix suspend/resume
- * sequence by the caller.
- */
- val = I915_READ(VLV_GTLC_WAKE_CTRL);
- val &= VLV_GTLC_ALLOWWAKEREQ;
- val |= s->gtlc_wake_ctrl & ~VLV_GTLC_ALLOWWAKEREQ;
- I915_WRITE(VLV_GTLC_WAKE_CTRL, val);
-
- val = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
- val &= VLV_GFX_CLK_FORCE_ON_BIT;
- val |= s->gtlc_survive & ~VLV_GFX_CLK_FORCE_ON_BIT;
- I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val);
-
- I915_WRITE(VLV_PMWGICZ, s->pmwgicz);
-
- /* Gunit-Display CZ domain, 0x182028-0x1821CF */
- I915_WRITE(VLV_GU_CTL0, s->gu_ctl0);
- I915_WRITE(VLV_GU_CTL1, s->gu_ctl1);
- I915_WRITE(VLV_PCBR, s->pcbr);
- I915_WRITE(VLV_GUNIT_CLOCK_GATE2, s->clock_gate_dis2);
-}
-
-static int vlv_wait_for_pw_status(struct drm_i915_private *i915,
- u32 mask, u32 val)
-{
- i915_reg_t reg = VLV_GTLC_PW_STATUS;
- u32 reg_value;
- int ret;
-
- /* The HW does not like us polling for PW_STATUS frequently, so
- * use the sleeping loop rather than risk the busy spin within
- * intel_wait_for_register().
- *
- * Transitioning between RC6 states should be at most 2ms (see
- * valleyview_enable_rps) so use a 3ms timeout.
- */
- ret = wait_for(((reg_value =
- intel_uncore_read_notrace(&i915->uncore, reg)) & mask)
- == val, 3);
-
- /* just trace the final value */
- trace_i915_reg_rw(false, reg, reg_value, sizeof(reg_value), true);
-
- return ret;
-}
-
-int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool force_on)
-{
- u32 val;
- int err;
-
- val = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
- val &= ~VLV_GFX_CLK_FORCE_ON_BIT;
- if (force_on)
- val |= VLV_GFX_CLK_FORCE_ON_BIT;
- I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val);
-
- if (!force_on)
- return 0;
-
- err = intel_wait_for_register(&dev_priv->uncore,
- VLV_GTLC_SURVIVABILITY_REG,
- VLV_GFX_CLK_STATUS_BIT,
- VLV_GFX_CLK_STATUS_BIT,
- 20);
- if (err)
- drm_err(&dev_priv->drm,
- "timeout waiting for GFX clock force-on (%08x)\n",
- I915_READ(VLV_GTLC_SURVIVABILITY_REG));
-
- return err;
-}
-
-static int vlv_allow_gt_wake(struct drm_i915_private *dev_priv, bool allow)
-{
- u32 mask;
- u32 val;
- int err;
-
- val = I915_READ(VLV_GTLC_WAKE_CTRL);
- val &= ~VLV_GTLC_ALLOWWAKEREQ;
- if (allow)
- val |= VLV_GTLC_ALLOWWAKEREQ;
- I915_WRITE(VLV_GTLC_WAKE_CTRL, val);
- POSTING_READ(VLV_GTLC_WAKE_CTRL);
-
- mask = VLV_GTLC_ALLOWWAKEACK;
- val = allow ? mask : 0;
-
- err = vlv_wait_for_pw_status(dev_priv, mask, val);
- if (err)
- drm_err(&dev_priv->drm, "timeout disabling GT waking\n");
-
- return err;
-}
-
-static void vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv,
- bool wait_for_on)
-{
- u32 mask;
- u32 val;
-
- mask = VLV_GTLC_PW_MEDIA_STATUS_MASK | VLV_GTLC_PW_RENDER_STATUS_MASK;
- val = wait_for_on ? mask : 0;
-
- /*
- * RC6 transitioning can be delayed up to 2 msec (see
- * valleyview_enable_rps), use 3 msec for safety.
- *
- * This can fail to turn off the rc6 if the GPU is stuck after a failed
- * reset and we are trying to force the machine to sleep.
- */
- if (vlv_wait_for_pw_status(dev_priv, mask, val))
- drm_dbg(&dev_priv->drm,
- "timeout waiting for GT wells to go %s\n",
- onoff(wait_for_on));
-}
-
-static void vlv_check_no_gt_access(struct drm_i915_private *dev_priv)
-{
- if (!(I915_READ(VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEERR))
- return;
-
- drm_dbg(&dev_priv->drm,
- "GT register access while GT waking disabled\n");
- I915_WRITE(VLV_GTLC_PW_STATUS, VLV_GTLC_ALLOWWAKEERR);
-}
-
-static int vlv_suspend_complete(struct drm_i915_private *dev_priv)
-{
- u32 mask;
- int err;
-
- /*
- * Bspec defines the following GT well on flags as debug only, so
- * don't treat them as hard failures.
- */
- vlv_wait_for_gt_wells(dev_priv, false);
-
- mask = VLV_GTLC_RENDER_CTX_EXISTS | VLV_GTLC_MEDIA_CTX_EXISTS;
- drm_WARN_ON(&dev_priv->drm,
- (I915_READ(VLV_GTLC_WAKE_CTRL) & mask) != mask);
-
- vlv_check_no_gt_access(dev_priv);
-
- err = vlv_force_gfx_clock(dev_priv, true);
- if (err)
- goto err1;
-
- err = vlv_allow_gt_wake(dev_priv, false);
- if (err)
- goto err2;
-
- vlv_save_gunit_s0ix_state(dev_priv);
-
- err = vlv_force_gfx_clock(dev_priv, false);
- if (err)
- goto err2;
-
- return 0;
-
-err2:
- /* For safety always re-enable waking and disable gfx clock forcing */
- vlv_allow_gt_wake(dev_priv, true);
-err1:
- vlv_force_gfx_clock(dev_priv, false);
-
- return err;
-}
-
-static int vlv_resume_prepare(struct drm_i915_private *dev_priv,
- bool rpm_resume)
-{
- int err;
- int ret;
-
- /*
- * If any of the steps fail just try to continue, that's the best we
- * can do at this point. Return the first error code (which will also
- * leave RPM permanently disabled).
- */
- ret = vlv_force_gfx_clock(dev_priv, true);
-
- vlv_restore_gunit_s0ix_state(dev_priv);
-
- err = vlv_allow_gt_wake(dev_priv, true);
- if (!ret)
- ret = err;
-
- err = vlv_force_gfx_clock(dev_priv, false);
- if (!ret)
- ret = err;
-
- vlv_check_no_gt_access(dev_priv);
-
- if (rpm_resume)
- intel_init_clock_gating(dev_priv);
-
- return ret;
-}
-
static int intel_runtime_suspend(struct device *kdev)
{
struct drm_i915_private *dev_priv = kdev_to_i915(kdev);
struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
- int ret = 0;
+ int ret;
if (drm_WARN_ON_ONCE(&dev_priv->drm, !HAS_RUNTIME_PM(dev_priv)))
return -ENODEV;
@@ -2531,9 +2060,7 @@ static int intel_runtime_suspend(struct device *kdev)
intel_display_power_suspend(dev_priv);
- if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
- ret = vlv_suspend_complete(dev_priv);
-
+ ret = vlv_suspend_complete(dev_priv);
if (ret) {
drm_err(&dev_priv->drm,
"Runtime suspend failed, disabling it (%d)\n", ret);
@@ -2595,7 +2122,7 @@ static int intel_runtime_resume(struct device *kdev)
{
struct drm_i915_private *dev_priv = kdev_to_i915(kdev);
struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
- int ret = 0;
+ int ret;
if (drm_WARN_ON_ONCE(&dev_priv->drm, !HAS_RUNTIME_PM(dev_priv)))
return -ENODEV;
@@ -2613,8 +2140,7 @@ static int intel_runtime_resume(struct device *kdev)
intel_display_power_resume(dev_priv);
- if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
- ret = vlv_resume_prepare(dev_priv, true);
+ ret = vlv_resume_prepare(dev_priv, true);
intel_uncore_runtime_resume(&dev_priv->uncore);
@@ -1755,8 +1755,6 @@ void i915_driver_remove(struct drm_i915_private *i915);
int i915_resume_switcheroo(struct drm_i915_private *i915);
int i915_suspend_switcheroo(struct drm_i915_private *i915, pm_message_t state);
-int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool on);
-
static inline bool intel_gvt_active(struct drm_i915_private *dev_priv)
{
return dev_priv->gvt;
new file mode 100644
@@ -0,0 +1,484 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2020 Intel Corporation
+ */
+
+#include <linux/kernel.h>
+
+#include <drm/drm_print.h>
+
+#include "i915_drv.h"
+#include "i915_reg.h"
+#include "i915_trace.h"
+#include "i915_utils.h"
+#include "intel_pm.h"
+#include "vlv_suspend.h"
+
+struct vlv_s0ix_state {
+ /* GAM */
+ u32 wr_watermark;
+ u32 gfx_prio_ctrl;
+ u32 arb_mode;
+ u32 gfx_pend_tlb0;
+ u32 gfx_pend_tlb1;
+ u32 lra_limits[GEN7_LRA_LIMITS_REG_NUM];
+ u32 media_max_req_count;
+ u32 gfx_max_req_count;
+ u32 render_hwsp;
+ u32 ecochk;
+ u32 bsd_hwsp;
+ u32 blt_hwsp;
+ u32 tlb_rd_addr;
+
+ /* MBC */
+ u32 g3dctl;
+ u32 gsckgctl;
+ u32 mbctl;
+
+ /* GCP */
+ u32 ucgctl1;
+ u32 ucgctl3;
+ u32 rcgctl1;
+ u32 rcgctl2;
+ u32 rstctl;
+ u32 misccpctl;
+
+ /* GPM */
+ u32 gfxpause;
+ u32 rpdeuhwtc;
+ u32 rpdeuc;
+ u32 ecobus;
+ u32 pwrdwnupctl;
+ u32 rp_down_timeout;
+ u32 rp_deucsw;
+ u32 rcubmabdtmr;
+ u32 rcedata;
+ u32 spare2gh;
+
+ /* Display 1 CZ domain */
+ u32 gt_imr;
+ u32 gt_ier;
+ u32 pm_imr;
+ u32 pm_ier;
+ u32 gt_scratch[GEN7_GT_SCRATCH_REG_NUM];
+
+ /* GT SA CZ domain */
+ u32 tilectl;
+ u32 gt_fifoctl;
+ u32 gtlc_wake_ctrl;
+ u32 gtlc_survive;
+ u32 pmwgicz;
+
+ /* Display 2 CZ domain */
+ u32 gu_ctl0;
+ u32 gu_ctl1;
+ u32 pcbr;
+ u32 clock_gate_dis2;
+};
+
+/*
+ * Save all Gunit registers that may be lost after a D3 and a subsequent
+ * S0i[R123] transition. The list of registers needing a save/restore is
+ * defined in the VLV2_S0IXRegs document. This documents marks all Gunit
+ * registers in the following way:
+ * - Driver: saved/restored by the driver
+ * - Punit : saved/restored by the Punit firmware
+ * - No, w/o marking: no need to save/restore, since the register is R/O or
+ * used internally by the HW in a way that doesn't depend
+ * keeping the content across a suspend/resume.
+ * - Debug : used for debugging
+ *
+ * We save/restore all registers marked with 'Driver', with the following
+ * exceptions:
+ * - Registers out of use, including also registers marked with 'Debug'.
+ * These have no effect on the driver's operation, so we don't save/restore
+ * them to reduce the overhead.
+ * - Registers that are fully setup by an initialization function called from
+ * the resume path. For example many clock gating and RPS/RC6 registers.
+ * - Registers that provide the right functionality with their reset defaults.
+ *
+ * TODO: Except for registers that based on the above 3 criteria can be safely
+ * ignored, we save/restore all others, practically treating the HW context as
+ * a black-box for the driver. Further investigation is needed to reduce the
+ * saved/restored registers even further, by following the same 3 criteria.
+ */
+static void vlv_save_gunit_s0ix_state(struct drm_i915_private *dev_priv)
+{
+ struct vlv_s0ix_state *s = dev_priv->vlv_s0ix_state;
+ int i;
+
+ if (!s)
+ return;
+
+ /* GAM 0x4000-0x4770 */
+ s->wr_watermark = I915_READ(GEN7_WR_WATERMARK);
+ s->gfx_prio_ctrl = I915_READ(GEN7_GFX_PRIO_CTRL);
+ s->arb_mode = I915_READ(ARB_MODE);
+ s->gfx_pend_tlb0 = I915_READ(GEN7_GFX_PEND_TLB0);
+ s->gfx_pend_tlb1 = I915_READ(GEN7_GFX_PEND_TLB1);
+
+ for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
+ s->lra_limits[i] = I915_READ(GEN7_LRA_LIMITS(i));
+
+ s->media_max_req_count = I915_READ(GEN7_MEDIA_MAX_REQ_COUNT);
+ s->gfx_max_req_count = I915_READ(GEN7_GFX_MAX_REQ_COUNT);
+
+ s->render_hwsp = I915_READ(RENDER_HWS_PGA_GEN7);
+ s->ecochk = I915_READ(GAM_ECOCHK);
+ s->bsd_hwsp = I915_READ(BSD_HWS_PGA_GEN7);
+ s->blt_hwsp = I915_READ(BLT_HWS_PGA_GEN7);
+
+ s->tlb_rd_addr = I915_READ(GEN7_TLB_RD_ADDR);
+
+ /* MBC 0x9024-0x91D0, 0x8500 */
+ s->g3dctl = I915_READ(VLV_G3DCTL);
+ s->gsckgctl = I915_READ(VLV_GSCKGCTL);
+ s->mbctl = I915_READ(GEN6_MBCTL);
+
+ /* GCP 0x9400-0x9424, 0x8100-0x810C */
+ s->ucgctl1 = I915_READ(GEN6_UCGCTL1);
+ s->ucgctl3 = I915_READ(GEN6_UCGCTL3);
+ s->rcgctl1 = I915_READ(GEN6_RCGCTL1);
+ s->rcgctl2 = I915_READ(GEN6_RCGCTL2);
+ s->rstctl = I915_READ(GEN6_RSTCTL);
+ s->misccpctl = I915_READ(GEN7_MISCCPCTL);
+
+ /* GPM 0xA000-0xAA84, 0x8000-0x80FC */
+ s->gfxpause = I915_READ(GEN6_GFXPAUSE);
+ s->rpdeuhwtc = I915_READ(GEN6_RPDEUHWTC);
+ s->rpdeuc = I915_READ(GEN6_RPDEUC);
+ s->ecobus = I915_READ(ECOBUS);
+ s->pwrdwnupctl = I915_READ(VLV_PWRDWNUPCTL);
+ s->rp_down_timeout = I915_READ(GEN6_RP_DOWN_TIMEOUT);
+ s->rp_deucsw = I915_READ(GEN6_RPDEUCSW);
+ s->rcubmabdtmr = I915_READ(GEN6_RCUBMABDTMR);
+ s->rcedata = I915_READ(VLV_RCEDATA);
+ s->spare2gh = I915_READ(VLV_SPAREG2H);
+
+ /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */
+ s->gt_imr = I915_READ(GTIMR);
+ s->gt_ier = I915_READ(GTIER);
+ s->pm_imr = I915_READ(GEN6_PMIMR);
+ s->pm_ier = I915_READ(GEN6_PMIER);
+
+ for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
+ s->gt_scratch[i] = I915_READ(GEN7_GT_SCRATCH(i));
+
+ /* GT SA CZ domain, 0x100000-0x138124 */
+ s->tilectl = I915_READ(TILECTL);
+ s->gt_fifoctl = I915_READ(GTFIFOCTL);
+ s->gtlc_wake_ctrl = I915_READ(VLV_GTLC_WAKE_CTRL);
+ s->gtlc_survive = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
+ s->pmwgicz = I915_READ(VLV_PMWGICZ);
+
+ /* Gunit-Display CZ domain, 0x182028-0x1821CF */
+ s->gu_ctl0 = I915_READ(VLV_GU_CTL0);
+ s->gu_ctl1 = I915_READ(VLV_GU_CTL1);
+ s->pcbr = I915_READ(VLV_PCBR);
+ s->clock_gate_dis2 = I915_READ(VLV_GUNIT_CLOCK_GATE2);
+
+ /*
+ * Not saving any of:
+ * DFT, 0x9800-0x9EC0
+ * SARB, 0xB000-0xB1FC
+ * GAC, 0x5208-0x524C, 0x14000-0x14C000
+ * PCI CFG
+ */
+}
+
+static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *dev_priv)
+{
+ struct vlv_s0ix_state *s = dev_priv->vlv_s0ix_state;
+ u32 val;
+ int i;
+
+ if (!s)
+ return;
+
+ /* GAM 0x4000-0x4770 */
+ I915_WRITE(GEN7_WR_WATERMARK, s->wr_watermark);
+ I915_WRITE(GEN7_GFX_PRIO_CTRL, s->gfx_prio_ctrl);
+ I915_WRITE(ARB_MODE, s->arb_mode | (0xffff << 16));
+ I915_WRITE(GEN7_GFX_PEND_TLB0, s->gfx_pend_tlb0);
+ I915_WRITE(GEN7_GFX_PEND_TLB1, s->gfx_pend_tlb1);
+
+ for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
+ I915_WRITE(GEN7_LRA_LIMITS(i), s->lra_limits[i]);
+
+ I915_WRITE(GEN7_MEDIA_MAX_REQ_COUNT, s->media_max_req_count);
+ I915_WRITE(GEN7_GFX_MAX_REQ_COUNT, s->gfx_max_req_count);
+
+ I915_WRITE(RENDER_HWS_PGA_GEN7, s->render_hwsp);
+ I915_WRITE(GAM_ECOCHK, s->ecochk);
+ I915_WRITE(BSD_HWS_PGA_GEN7, s->bsd_hwsp);
+ I915_WRITE(BLT_HWS_PGA_GEN7, s->blt_hwsp);
+
+ I915_WRITE(GEN7_TLB_RD_ADDR, s->tlb_rd_addr);
+
+ /* MBC 0x9024-0x91D0, 0x8500 */
+ I915_WRITE(VLV_G3DCTL, s->g3dctl);
+ I915_WRITE(VLV_GSCKGCTL, s->gsckgctl);
+ I915_WRITE(GEN6_MBCTL, s->mbctl);
+
+ /* GCP 0x9400-0x9424, 0x8100-0x810C */
+ I915_WRITE(GEN6_UCGCTL1, s->ucgctl1);
+ I915_WRITE(GEN6_UCGCTL3, s->ucgctl3);
+ I915_WRITE(GEN6_RCGCTL1, s->rcgctl1);
+ I915_WRITE(GEN6_RCGCTL2, s->rcgctl2);
+ I915_WRITE(GEN6_RSTCTL, s->rstctl);
+ I915_WRITE(GEN7_MISCCPCTL, s->misccpctl);
+
+ /* GPM 0xA000-0xAA84, 0x8000-0x80FC */
+ I915_WRITE(GEN6_GFXPAUSE, s->gfxpause);
+ I915_WRITE(GEN6_RPDEUHWTC, s->rpdeuhwtc);
+ I915_WRITE(GEN6_RPDEUC, s->rpdeuc);
+ I915_WRITE(ECOBUS, s->ecobus);
+ I915_WRITE(VLV_PWRDWNUPCTL, s->pwrdwnupctl);
+ I915_WRITE(GEN6_RP_DOWN_TIMEOUT,s->rp_down_timeout);
+ I915_WRITE(GEN6_RPDEUCSW, s->rp_deucsw);
+ I915_WRITE(GEN6_RCUBMABDTMR, s->rcubmabdtmr);
+ I915_WRITE(VLV_RCEDATA, s->rcedata);
+ I915_WRITE(VLV_SPAREG2H, s->spare2gh);
+
+ /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */
+ I915_WRITE(GTIMR, s->gt_imr);
+ I915_WRITE(GTIER, s->gt_ier);
+ I915_WRITE(GEN6_PMIMR, s->pm_imr);
+ I915_WRITE(GEN6_PMIER, s->pm_ier);
+
+ for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
+ I915_WRITE(GEN7_GT_SCRATCH(i), s->gt_scratch[i]);
+
+ /* GT SA CZ domain, 0x100000-0x138124 */
+ I915_WRITE(TILECTL, s->tilectl);
+ I915_WRITE(GTFIFOCTL, s->gt_fifoctl);
+ /*
+ * Preserve the GT allow wake and GFX force clock bit, they are not
+ * be restored, as they are used to control the s0ix suspend/resume
+ * sequence by the caller.
+ */
+ val = I915_READ(VLV_GTLC_WAKE_CTRL);
+ val &= VLV_GTLC_ALLOWWAKEREQ;
+ val |= s->gtlc_wake_ctrl & ~VLV_GTLC_ALLOWWAKEREQ;
+ I915_WRITE(VLV_GTLC_WAKE_CTRL, val);
+
+ val = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
+ val &= VLV_GFX_CLK_FORCE_ON_BIT;
+ val |= s->gtlc_survive & ~VLV_GFX_CLK_FORCE_ON_BIT;
+ I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val);
+
+ I915_WRITE(VLV_PMWGICZ, s->pmwgicz);
+
+ /* Gunit-Display CZ domain, 0x182028-0x1821CF */
+ I915_WRITE(VLV_GU_CTL0, s->gu_ctl0);
+ I915_WRITE(VLV_GU_CTL1, s->gu_ctl1);
+ I915_WRITE(VLV_PCBR, s->pcbr);
+ I915_WRITE(VLV_GUNIT_CLOCK_GATE2, s->clock_gate_dis2);
+}
+
+static int vlv_wait_for_pw_status(struct drm_i915_private *i915,
+ u32 mask, u32 val)
+{
+ i915_reg_t reg = VLV_GTLC_PW_STATUS;
+ u32 reg_value;
+ int ret;
+
+ /* The HW does not like us polling for PW_STATUS frequently, so
+ * use the sleeping loop rather than risk the busy spin within
+ * intel_wait_for_register().
+ *
+ * Transitioning between RC6 states should be at most 2ms (see
+ * valleyview_enable_rps) so use a 3ms timeout.
+ */
+ ret = wait_for(((reg_value =
+ intel_uncore_read_notrace(&i915->uncore, reg)) & mask)
+ == val, 3);
+
+ /* just trace the final value */
+ trace_i915_reg_rw(false, reg, reg_value, sizeof(reg_value), true);
+
+ return ret;
+}
+
+static int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool force_on)
+{
+ u32 val;
+ int err;
+
+ val = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
+ val &= ~VLV_GFX_CLK_FORCE_ON_BIT;
+ if (force_on)
+ val |= VLV_GFX_CLK_FORCE_ON_BIT;
+ I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val);
+
+ if (!force_on)
+ return 0;
+
+ err = intel_wait_for_register(&dev_priv->uncore,
+ VLV_GTLC_SURVIVABILITY_REG,
+ VLV_GFX_CLK_STATUS_BIT,
+ VLV_GFX_CLK_STATUS_BIT,
+ 20);
+ if (err)
+ drm_err(&dev_priv->drm,
+ "timeout waiting for GFX clock force-on (%08x)\n",
+ I915_READ(VLV_GTLC_SURVIVABILITY_REG));
+
+ return err;
+}
+
+static int vlv_allow_gt_wake(struct drm_i915_private *dev_priv, bool allow)
+{
+ u32 mask;
+ u32 val;
+ int err;
+
+ val = I915_READ(VLV_GTLC_WAKE_CTRL);
+ val &= ~VLV_GTLC_ALLOWWAKEREQ;
+ if (allow)
+ val |= VLV_GTLC_ALLOWWAKEREQ;
+ I915_WRITE(VLV_GTLC_WAKE_CTRL, val);
+ POSTING_READ(VLV_GTLC_WAKE_CTRL);
+
+ mask = VLV_GTLC_ALLOWWAKEACK;
+ val = allow ? mask : 0;
+
+ err = vlv_wait_for_pw_status(dev_priv, mask, val);
+ if (err)
+ drm_err(&dev_priv->drm, "timeout disabling GT waking\n");
+
+ return err;
+}
+
+static void vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv,
+ bool wait_for_on)
+{
+ u32 mask;
+ u32 val;
+
+ mask = VLV_GTLC_PW_MEDIA_STATUS_MASK | VLV_GTLC_PW_RENDER_STATUS_MASK;
+ val = wait_for_on ? mask : 0;
+
+ /*
+ * RC6 transitioning can be delayed up to 2 msec (see
+ * valleyview_enable_rps), use 3 msec for safety.
+ *
+ * This can fail to turn off the rc6 if the GPU is stuck after a failed
+ * reset and we are trying to force the machine to sleep.
+ */
+ if (vlv_wait_for_pw_status(dev_priv, mask, val))
+ drm_dbg(&dev_priv->drm,
+ "timeout waiting for GT wells to go %s\n",
+ onoff(wait_for_on));
+}
+
+static void vlv_check_no_gt_access(struct drm_i915_private *dev_priv)
+{
+ if (!(I915_READ(VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEERR))
+ return;
+
+ drm_dbg(&dev_priv->drm,
+ "GT register access while GT waking disabled\n");
+ I915_WRITE(VLV_GTLC_PW_STATUS, VLV_GTLC_ALLOWWAKEERR);
+}
+
+int vlv_suspend_complete(struct drm_i915_private *dev_priv)
+{
+ u32 mask;
+ int err;
+
+ if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
+ return 0;
+
+ /*
+ * Bspec defines the following GT well on flags as debug only, so
+ * don't treat them as hard failures.
+ */
+ vlv_wait_for_gt_wells(dev_priv, false);
+
+ mask = VLV_GTLC_RENDER_CTX_EXISTS | VLV_GTLC_MEDIA_CTX_EXISTS;
+ drm_WARN_ON(&dev_priv->drm,
+ (I915_READ(VLV_GTLC_WAKE_CTRL) & mask) != mask);
+
+ vlv_check_no_gt_access(dev_priv);
+
+ err = vlv_force_gfx_clock(dev_priv, true);
+ if (err)
+ goto err1;
+
+ err = vlv_allow_gt_wake(dev_priv, false);
+ if (err)
+ goto err2;
+
+ vlv_save_gunit_s0ix_state(dev_priv);
+
+ err = vlv_force_gfx_clock(dev_priv, false);
+ if (err)
+ goto err2;
+
+ return 0;
+
+err2:
+ /* For safety always re-enable waking and disable gfx clock forcing */
+ vlv_allow_gt_wake(dev_priv, true);
+err1:
+ vlv_force_gfx_clock(dev_priv, false);
+
+ return err;
+}
+
+int vlv_resume_prepare(struct drm_i915_private *dev_priv, bool rpm_resume)
+{
+ int err;
+ int ret;
+
+ if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
+ return 0;
+
+ /*
+ * If any of the steps fail just try to continue, that's the best we
+ * can do at this point. Return the first error code (which will also
+ * leave RPM permanently disabled).
+ */
+ ret = vlv_force_gfx_clock(dev_priv, true);
+
+ vlv_restore_gunit_s0ix_state(dev_priv);
+
+ err = vlv_allow_gt_wake(dev_priv, true);
+ if (!ret)
+ ret = err;
+
+ err = vlv_force_gfx_clock(dev_priv, false);
+ if (!ret)
+ ret = err;
+
+ vlv_check_no_gt_access(dev_priv);
+
+ if (rpm_resume)
+ intel_init_clock_gating(dev_priv);
+
+ return ret;
+}
+
+int vlv_suspend_init(struct drm_i915_private *i915)
+{
+ if (!IS_VALLEYVIEW(i915))
+ return 0;
+
+ /* we write all the values in the struct, so no need to zero it out */
+ i915->vlv_s0ix_state = kmalloc(sizeof(*i915->vlv_s0ix_state),
+ GFP_KERNEL);
+ if (!i915->vlv_s0ix_state)
+ return -ENOMEM;
+
+ return 0;
+}
+
+void vlv_suspend_cleanup(struct drm_i915_private *i915)
+{
+ if (!i915->vlv_s0ix_state)
+ return;
+
+ kfree(i915->vlv_s0ix_state);
+ i915->vlv_s0ix_state = NULL;
+}
new file mode 100644
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2020 Intel Corporation
+ */
+
+#ifndef __VLV_SUSPEND_H__
+#define __VLV_SUSPEND_H__
+
+#include <linux/types.h>
+
+struct drm_i915_private;
+
+int vlv_suspend_init(struct drm_i915_private *i915);
+void vlv_suspend_cleanup(struct drm_i915_private *i915);
+int vlv_suspend_complete(struct drm_i915_private *i915);
+int vlv_resume_prepare(struct drm_i915_private *i915, bool rpm_resume);
+
+#endif /* __VLV_SUSPEND_H__ */
i915_drv.c is a fairly big file, and having very specific vlv/chv suspend/resume code in it is a distraction. Split it out to a new vlv_suspend.[ch] file. Signed-off-by: Jani Nikula <jani.nikula@intel.com> --- drivers/gpu/drm/i915/Makefile | 3 +- drivers/gpu/drm/i915/i915_drv.c | 498 +---------------------------- drivers/gpu/drm/i915/i915_drv.h | 2 - drivers/gpu/drm/i915/vlv_suspend.c | 484 ++++++++++++++++++++++++++++ drivers/gpu/drm/i915/vlv_suspend.h | 18 ++ 5 files changed, 516 insertions(+), 489 deletions(-) create mode 100644 drivers/gpu/drm/i915/vlv_suspend.c create mode 100644 drivers/gpu/drm/i915/vlv_suspend.h