diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 3ac71db..74000f9 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -297,6 +297,7 @@ struct drm_i915_display_funcs {
 	int (*get_fifo_size)(struct drm_device *dev, int plane);
 	void (*update_wm)(struct drm_device *dev);
 	void (*update_sprite_wm)(struct drm_device *dev, int pipe,
+				 int plane, bool enabled, bool scaled,
 				 uint32_t sprite_width, int pixel_size);
 	void (*update_linetime_wm)(struct drm_device *dev, int pipe,
 				 struct drm_display_mode *mode);
@@ -955,7 +956,6 @@ typedef struct drm_i915_private {
 
 	/* overlay */
 	struct intel_overlay *overlay;
-	unsigned int sprite_scaling_enabled;
 
 	/* backlight */
 	struct {
@@ -1072,6 +1072,8 @@ typedef struct drm_i915_private {
 
 	u32 fdi_rx_config;
 
+	spinlock_t wm_lock;
+
 	struct i915_suspend_saved_registers regfile;
 
 	/* Old dri1 support infrastructure, beware the dragons ya fools entering
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 03a31be..a474a6d 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -1201,8 +1201,10 @@ static irqreturn_t ivybridge_irq_handler(int irq, void *arg)
 			intel_opregion_asle_intr(dev);
 
 		for (i = 0; i < 3; i++) {
-			if (de_iir & (DE_PIPEA_VBLANK_IVB << (5 * i)))
+			if (de_iir & (DE_PIPEA_VBLANK_IVB << (5 * i))) {
 				drm_handle_vblank(dev, i);
+				ilk_pipe_update_wm(dev, i);
+			}
 			if (de_iir & (DE_PLANEA_FLIP_DONE_IVB << (5 * i))) {
 				intel_prepare_page_flip(dev, i);
 				intel_finish_page_flip_plane(dev, i);
@@ -1297,11 +1299,15 @@ static irqreturn_t ironlake_irq_handler(int irq, void *arg)
 	if (de_iir & DE_GSE)
 		intel_opregion_asle_intr(dev);
 
-	if (de_iir & DE_PIPEA_VBLANK)
+	if (de_iir & DE_PIPEA_VBLANK) {
 		drm_handle_vblank(dev, 0);
+		ilk_pipe_update_wm(dev, 0);
+	}
 
-	if (de_iir & DE_PIPEB_VBLANK)
+	if (de_iir & DE_PIPEB_VBLANK) {
 		drm_handle_vblank(dev, 1);
+		ilk_pipe_update_wm(dev, 1);
+	}
 
 	if (de_iir & DE_POISON)
 		DRM_ERROR("Poison interrupt\n");
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index d84d694..cafed47 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -3100,6 +3100,14 @@
 #define SNB_READ_WM2_LATENCY()		SNB_LATENCY(SSKPD_WM2_SHIFT)
 #define SNB_READ_WM3_LATENCY()		SNB_LATENCY(SSKPD_WM3_SHIFT)
 
+#define HSW_LATENCY(offset, shift, mask)	(I915_READ(MCHBAR_MIRROR_BASE_SNB + SSKPD + (offset)) >> (shift) & (mask))
+#define HSW_READ_WM0_OLD_LATENCY()	HSW_LATENCY(0, 0, 0xf)
+#define HSW_READ_WM1_LATENCY()		HSW_LATENCY(0, 4, 0xff)
+#define HSW_READ_WM2_LATENCY()		HSW_LATENCY(0, 12, 0xff)
+#define HSW_READ_WM3_LATENCY()		HSW_LATENCY(0, 20, 0x1ff)
+#define HSW_READ_WM4_LATENCY()		HSW_LATENCY(4, 0, 0x1ff)
+#define HSW_READ_WM0_NEW_LATENCY()	HSW_LATENCY(4, 24, 0xff)
+
 /*
  * The two pipe frame counter registers are not synchronized, so
  * reading a stable value is somewhat tricky. The following code
@@ -3696,6 +3704,8 @@
 #define DISP_ARB_CTL	0x45000
 #define  DISP_TILE_SURFACE_SWIZZLING	(1<<13)
 #define  DISP_FBC_WM_DIS		(1<<15)
+#define DISP_ARB_CTL2	0x45004
+#define  DISP_DATA_BUFFER_PARTITIONING	(1<<6)
 #define GEN7_MSG_CTL	0x45010
 #define  WAIT_FOR_PCH_RESET_ACK		(1<<1)
 #define  WAIT_FOR_PCH_FLR_ACK		(1<<0)
@@ -4900,6 +4910,9 @@
 #define  LCPLL_CD2X_CLOCK_DISABLE	(1<<23)
 #define  LCPLL_CD_SOURCE_FCLK		(1<<21)
 
+#define WM_MISC				0x45260
+#define  WM_DATA_BUFFER_PARTITIONING	(1<<0)
+
 /* Pipe WM_LINETIME - watermark line time */
 #define PIPE_WM_LINETIME_A		0x45270
 #define PIPE_WM_LINETIME_B		0x45274
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index c0c4dfc..121b71a 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -3267,6 +3267,8 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc)
 
 	WARN_ON(!crtc->enabled);
 
+	intel_update_watermarks(dev);
+
 	if (intel_crtc->active)
 		return;
 
@@ -3275,8 +3277,6 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc)
 	intel_set_cpu_fifo_underrun_reporting(dev, pipe, true);
 	intel_set_pch_fifo_underrun_reporting(dev, pipe, true);
 
-	intel_update_watermarks(dev);
-
 	if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) {
 		temp = I915_READ(PCH_LVDS);
 		if ((temp & LVDS_PORT_EN) == 0)
@@ -3348,6 +3348,8 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
 
 	WARN_ON(!crtc->enabled);
 
+	intel_update_watermarks(dev);
+
 	if (intel_crtc->active)
 		return;
 
@@ -3357,8 +3359,6 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
 	if (intel_crtc->config.has_pch_encoder)
 		intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A, true);
 
-	intel_update_watermarks(dev);
-
 	if (intel_crtc->config.has_pch_encoder)
 		dev_priv->display.fdi_link_train(crtc);
 
@@ -3424,6 +3424,8 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc)
 	for_each_encoder_on_crtc(dev, crtc, encoder)
 		encoder->disable(encoder);
 
+	intel_update_watermarks(dev);
+
 	intel_crtc_wait_for_pending_flips(crtc);
 	drm_vblank_off(dev, pipe);
 	intel_crtc_update_cursor(crtc, false);
@@ -3482,7 +3484,6 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc)
 	ironlake_fdi_pll_disable(intel_crtc);
 
 	intel_crtc->active = false;
-	intel_update_watermarks(dev);
 
 	mutex_lock(&dev->struct_mutex);
 	intel_update_fbc(dev);
@@ -3505,6 +3506,8 @@ static void haswell_crtc_disable(struct drm_crtc *crtc)
 	for_each_encoder_on_crtc(dev, crtc, encoder)
 		encoder->disable(encoder);
 
+	intel_update_watermarks(dev);
+
 	intel_crtc_wait_for_pending_flips(crtc);
 	drm_vblank_off(dev, pipe);
 	intel_crtc_update_cursor(crtc, false);
@@ -3541,7 +3544,6 @@ static void haswell_crtc_disable(struct drm_crtc *crtc)
 	}
 
 	intel_crtc->active = false;
-	intel_update_watermarks(dev);
 
 	mutex_lock(&dev->struct_mutex);
 	intel_update_fbc(dev);
@@ -3649,11 +3651,12 @@ static void valleyview_crtc_enable(struct drm_crtc *crtc)
 
 	WARN_ON(!crtc->enabled);
 
+	intel_update_watermarks(dev);
+
 	if (intel_crtc->active)
 		return;
 
 	intel_crtc->active = true;
-	intel_update_watermarks(dev);
 
 	mutex_lock(&dev_priv->dpio_lock);
 
@@ -3698,11 +3701,12 @@ static void i9xx_crtc_enable(struct drm_crtc *crtc)
 
 	WARN_ON(!crtc->enabled);
 
+	intel_update_watermarks(dev);
+
 	if (intel_crtc->active)
 		return;
 
 	intel_crtc->active = true;
-	intel_update_watermarks(dev);
 
 	intel_enable_pll(dev_priv, pipe);
 
@@ -3786,7 +3790,6 @@ static void i9xx_crtc_disable(struct drm_crtc *crtc)
 
 	intel_crtc->active = false;
 	intel_update_fbc(dev);
-	intel_update_watermarks(dev);
 }
 
 static void i9xx_crtc_off(struct drm_crtc *crtc)
@@ -3844,6 +3847,9 @@ void intel_crtc_update_dpms(struct drm_crtc *crtc)
 	intel_crtc_update_sarea(crtc, enable);
 }
 
+static void
+intel_crtc_disable_plane_config(struct intel_crtc_config *pipe_config);
+
 static void intel_crtc_disable(struct drm_crtc *crtc)
 {
 	struct drm_device *dev = crtc->dev;
@@ -3851,11 +3857,18 @@ static void intel_crtc_disable(struct drm_crtc *crtc)
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 
+	intel_crtc_disable_plane_config(&intel_crtc->config);
+	intel_crtc->config.enabled = false;
+	DRM_DEBUG_KMS("Pipe %c pipe_config->enabled = %d\n",
+		      pipe_name(intel_crtc->pipe), intel_crtc->config.enabled);
+	intel_update_watermarks(dev);
+
 	/* crtc should still be enabled when we disable it. */
 	WARN_ON(!crtc->enabled);
 
 	intel_crtc->eld_vld = false;
 	dev_priv->display.crtc_disable(crtc);
+	ilk_pipe_update_wm(dev, intel_crtc->pipe);
 	intel_crtc_update_sarea(crtc, false);
 	dev_priv->display.off(crtc);
 
@@ -4133,6 +4146,10 @@ static int intel_crtc_compute_config(struct drm_crtc *crtc,
 	if (pipe_config->has_pch_encoder)
 		return ironlake_fdi_compute_config(to_intel_crtc(crtc), pipe_config);
 
+	pipe_config->enabled = true;
+	DRM_DEBUG_KMS("Pipe %c pipe_config->enabled = %d\n",
+		      pipe_name(to_intel_crtc(crtc)->pipe), pipe_config->enabled);
+
 	return 0;
 }
 
@@ -4992,6 +5009,10 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
 
 	intel_get_pipe_timings(crtc, pipe_config);
 
+	pipe_config->enabled = true;
+	DRM_DEBUG_KMS("get pipe %c: pipe_config->enabled = %d\n",
+		      pipe_name(crtc->pipe), pipe_config->enabled);
+
 	return true;
 }
 
@@ -5867,8 +5888,6 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
 
 	intel_update_watermarks(dev);
 
-	intel_update_linetime_watermarks(dev, pipe, adjusted_mode);
-
 	return ret;
 }
 
@@ -5911,6 +5930,10 @@ static bool ironlake_get_pipe_config(struct intel_crtc *crtc,
 
 	intel_get_pipe_timings(crtc, pipe_config);
 
+	pipe_config->enabled = true;
+	DRM_DEBUG_KMS("get pipe %c: pipe_config->enabled = %d\n",
+		      pipe_name(crtc->pipe), pipe_config->enabled);
+
 	return true;
 }
 
@@ -6019,8 +6042,6 @@ static int haswell_crtc_mode_set(struct drm_crtc *crtc,
 
 	intel_update_watermarks(dev);
 
-	intel_update_linetime_watermarks(dev, pipe, adjusted_mode);
-
 	return ret;
 }
 
@@ -6059,6 +6080,10 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc,
 
 	intel_get_pipe_timings(crtc, pipe_config);
 
+	pipe_config->enabled = true;
+	DRM_DEBUG_KMS("get pipe %c: pipe_config->enabled = %d\n",
+		      pipe_name(crtc->pipe), pipe_config->enabled);
+
 	return true;
 }
 
@@ -7778,6 +7803,59 @@ pipe_config_set_bpp(struct drm_crtc *crtc,
 	return bpp;
 }
 
+static void
+intel_crtc_compute_plane_config(const struct drm_crtc *crtc,
+				const struct drm_framebuffer *fb,
+				struct intel_crtc_config *pipe_config)
+{
+	pipe_config->primary.enabled = true;
+	pipe_config->primary.width = pipe_config->requested_mode.hdisplay;
+	pipe_config->primary.pixel_size = drm_format_plane_cpp(fb->pixel_format, 0);
+
+	pipe_config->cursor.enabled = true;
+	pipe_config->cursor.width = 64;
+	pipe_config->cursor.pixel_size = 4;
+
+	pipe_config->sprite[0].enabled = false;
+	pipe_config->sprite[0].scaled = false;
+	pipe_config->sprite[0].width = 0;
+	pipe_config->sprite[0].pixel_size = 0;
+}
+
+void
+intel_crtc_update_sprite_config(struct intel_crtc_config *pipe_config,
+				int plane, bool enabled, bool scaled,
+				unsigned int width, unsigned int pixel_size)
+{
+	if (!enabled) {
+		scaled = false;
+		width = 0;
+		pixel_size = 0;
+	}
+
+	pipe_config->sprite[plane].enabled = enabled;
+	pipe_config->sprite[plane].scaled = scaled;
+	pipe_config->sprite[plane].width = width;
+	pipe_config->sprite[plane].pixel_size = pixel_size;
+}
+
+static void
+intel_crtc_disable_plane_config(struct intel_crtc_config *pipe_config)
+{
+	pipe_config->primary.enabled = false;
+	pipe_config->primary.width = 0;
+	pipe_config->primary.pixel_size = 0;
+
+	pipe_config->cursor.enabled = false;
+	pipe_config->cursor.width = 0;
+	pipe_config->cursor.pixel_size = 0;
+
+	pipe_config->sprite[0].enabled = false;
+	pipe_config->sprite[0].scaled = false;
+	pipe_config->sprite[0].width = 0;
+	pipe_config->sprite[0].pixel_size = 0;
+}
+
 static struct intel_crtc_config *
 intel_modeset_pipe_config(struct drm_crtc *crtc,
 			  struct drm_framebuffer *fb,
@@ -7853,6 +7931,8 @@ encoder_retry:
 	DRM_DEBUG_KMS("plane bpp: %i, pipe bpp: %i, dithering: %i\n",
 		      plane_bpp, pipe_config->pipe_bpp, pipe_config->dither);
 
+	intel_crtc_compute_plane_config(crtc, fb, pipe_config);
+
 	return pipe_config;
 fail:
 	kfree(pipe_config);
@@ -8233,6 +8313,9 @@ static int __intel_set_mode(struct drm_crtc *crtc,
 	for_each_intel_crtc_masked(dev, prepare_pipes, intel_crtc) {
 		if (intel_crtc->base.enabled)
 			dev_priv->display.crtc_disable(&intel_crtc->base);
+		pipe_config->enabled = true;
+		DRM_DEBUG_KMS("Pipe %c pipe_config->enabled = %d\n",
+			      pipe_name(intel_crtc->pipe), pipe_config->enabled);
 	}
 
 	/* crtc->mode is already used by the ->mode_set callbacks, hence we need
@@ -9268,6 +9351,8 @@ void intel_modeset_init(struct drm_device *dev)
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	int i, j, ret;
 
+	spin_lock_init(&dev_priv->wm_lock);
+
 	drm_mode_config_init(dev);
 
 	dev->mode_config.min_width = 0;
diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h
index cb903d0..048b8f9 100644
--- a/drivers/gpu/drm/i915/intel_drv.h
+++ b/drivers/gpu/drm/i915/intel_drv.h
@@ -189,6 +189,13 @@ typedef struct dpll {
 	int	p;
 } intel_clock_t;
 
+struct intel_plane_config {
+	bool enabled;
+	bool scaled;
+	unsigned int width;
+	unsigned int pixel_size;
+};
+
 struct intel_crtc_config {
 	struct drm_display_mode requested_mode;
 	struct drm_display_mode adjusted_mode;
@@ -261,6 +268,29 @@ struct intel_crtc_config {
 	/* FDI configuration, only valid if has_pch_encoder is set. */
 	int fdi_lanes;
 	struct intel_link_m_n fdi_m_n;
+
+	bool enabled;
+	struct intel_plane_config primary;
+	struct intel_plane_config sprite[2];
+	struct intel_plane_config cursor;
+};
+
+struct intel_wm_level {
+	unsigned int primary;
+	unsigned int sprite;
+	unsigned int cursor;
+	unsigned int fbc;
+	unsigned int latency;
+	bool valid;
+};
+
+struct intel_pipe_wm {
+	struct intel_wm_level wm[5];
+	unsigned int linetime;
+	unsigned int ips_linetime;
+	bool pipe_enabled;
+	bool sprite_enabled;
+	bool sprite_scaled;
 };
 
 struct intel_crtc {
@@ -305,6 +335,12 @@ struct intel_crtc {
 	/* Access to these should be protected by dev_priv->irq_lock. */
 	bool cpu_fifo_underrun_disabled;
 	bool pch_fifo_underrun_disabled;
+
+	struct {
+		struct intel_pipe_wm active;
+		struct intel_pipe_wm pending;
+		bool dirty;
+	} wm;
 };
 
 struct intel_plane {
@@ -730,11 +766,10 @@ extern void intel_ddi_init(struct drm_device *dev, enum port port);
 
 /* For use by IVB LP watermark workaround in intel_sprite.c */
 extern void intel_update_watermarks(struct drm_device *dev);
-extern void intel_update_sprite_watermarks(struct drm_device *dev, int pipe,
-					   uint32_t sprite_width,
-					   int pixel_size);
-extern void intel_update_linetime_watermarks(struct drm_device *dev, int pipe,
-			 struct drm_display_mode *mode);
+extern void intel_update_sprite_watermarks(struct drm_device *dev,
+					   int pipe, int plane,
+					   bool enabled, bool scaled,
+					   uint32_t sprite_width, int pixel_size);
 
 extern unsigned long intel_gen4_compute_page_offset(int *x, int *y,
 						    unsigned int tiling_mode,
@@ -794,4 +829,10 @@ extern bool intel_set_pch_fifo_underrun_reporting(struct drm_device *dev,
 						 enum transcoder pch_transcoder,
 						 bool enable);
 
+extern void intel_crtc_update_sprite_config(struct intel_crtc_config *pipe_config,
+					    int plane, bool enabled, bool scaled,
+					    unsigned int width,
+					    unsigned int pixel_size);
+extern void ilk_pipe_update_wm(struct drm_device *dev, enum pipe pipe);
+
 #endif /* __INTEL_DRV_H__ */
diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
index 5f3e0ae..ff14bce 100644
--- a/drivers/gpu/drm/i915/intel_pm.c
+++ b/drivers/gpu/drm/i915/intel_pm.c
@@ -886,65 +886,6 @@ static const struct intel_watermark_params i830_wm_info = {
 	I830_FIFO_LINE_SIZE
 };
 
-static const struct intel_watermark_params ironlake_display_wm_info = {
-	ILK_DISPLAY_FIFO,
-	ILK_DISPLAY_MAXWM,
-	ILK_DISPLAY_DFTWM,
-	2,
-	ILK_FIFO_LINE_SIZE
-};
-static const struct intel_watermark_params ironlake_cursor_wm_info = {
-	ILK_CURSOR_FIFO,
-	ILK_CURSOR_MAXWM,
-	ILK_CURSOR_DFTWM,
-	2,
-	ILK_FIFO_LINE_SIZE
-};
-static const struct intel_watermark_params ironlake_display_srwm_info = {
-	ILK_DISPLAY_SR_FIFO,
-	ILK_DISPLAY_MAX_SRWM,
-	ILK_DISPLAY_DFT_SRWM,
-	2,
-	ILK_FIFO_LINE_SIZE
-};
-static const struct intel_watermark_params ironlake_cursor_srwm_info = {
-	ILK_CURSOR_SR_FIFO,
-	ILK_CURSOR_MAX_SRWM,
-	ILK_CURSOR_DFT_SRWM,
-	2,
-	ILK_FIFO_LINE_SIZE
-};
-
-static const struct intel_watermark_params sandybridge_display_wm_info = {
-	SNB_DISPLAY_FIFO,
-	SNB_DISPLAY_MAXWM,
-	SNB_DISPLAY_DFTWM,
-	2,
-	SNB_FIFO_LINE_SIZE
-};
-static const struct intel_watermark_params sandybridge_cursor_wm_info = {
-	SNB_CURSOR_FIFO,
-	SNB_CURSOR_MAXWM,
-	SNB_CURSOR_DFTWM,
-	2,
-	SNB_FIFO_LINE_SIZE
-};
-static const struct intel_watermark_params sandybridge_display_srwm_info = {
-	SNB_DISPLAY_SR_FIFO,
-	SNB_DISPLAY_MAX_SRWM,
-	SNB_DISPLAY_DFT_SRWM,
-	2,
-	SNB_FIFO_LINE_SIZE
-};
-static const struct intel_watermark_params sandybridge_cursor_srwm_info = {
-	SNB_CURSOR_SR_FIFO,
-	SNB_CURSOR_MAX_SRWM,
-	SNB_CURSOR_DFT_SRWM,
-	2,
-	SNB_FIFO_LINE_SIZE
-};
-
-
 /**
  * intel_calculate_wm - calculate watermark level
  * @clock_in_khz: pixel clock
@@ -1603,597 +1544,940 @@ static void i830_update_wm(struct drm_device *dev)
 	I915_WRITE(FW_BLC, fwater_lo);
 }
 
-#define ILK_LP0_PLANE_LATENCY		700
-#define ILK_LP0_CURSOR_LATENCY		1300
+#define ILK_LP0_PLANE_LATENCY		7
+#define ILK_LP0_CURSOR_LATENCY		13
 
-/*
- * Check the wm result.
- *
- * If any calculated watermark values is larger than the maximum value that
- * can be programmed into the associated watermark register, that watermark
- * must be disabled.
- */
-static bool ironlake_check_srwm(struct drm_device *dev, int level,
-				int fbc_wm, int display_wm, int cursor_wm,
-				const struct intel_watermark_params *display,
-				const struct intel_watermark_params *cursor)
+static void ilk_read_wm_latency(struct drm_i915_private *dev_priv,
+				int level,
+				unsigned int *plane,
+				unsigned int *cursor)
 {
-	struct drm_i915_private *dev_priv = dev->dev_private;
-
-	DRM_DEBUG_KMS("watermark %d: display plane %d, fbc lines %d,"
-		      " cursor %d\n", level, display_wm, fbc_wm, cursor_wm);
-
-	if (fbc_wm > SNB_FBC_MAX_SRWM) {
-		DRM_DEBUG_KMS("fbc watermark(%d) is too large(%d), disabling wm%d+\n",
-			      fbc_wm, SNB_FBC_MAX_SRWM, level);
-
-		/* fbc has it's own way to disable FBC WM */
-		I915_WRITE(DISP_ARB_CTL,
-			   I915_READ(DISP_ARB_CTL) | DISP_FBC_WM_DIS);
-		return false;
+	switch (level) {
+	case 0:
+		*plane = ILK_LP0_PLANE_LATENCY;
+		*cursor = ILK_LP0_CURSOR_LATENCY;
+		break;
+	case 1:
+		*plane = *cursor = ILK_READ_WM1_LATENCY();
+		break;
+	case 2:
+		*plane = *cursor = ILK_READ_WM2_LATENCY();
+		break;
+	default:
+		/*
+		 * WM3 is unsupported on ILK, probably because we
+		 * don't have latency data for that power state
+		 */
+		*plane = *cursor = 0;
+		break;
 	}
+}
 
-	if (display_wm > display->max_wm) {
-		DRM_DEBUG_KMS("display watermark(%d) is too large(%d), disabling wm%d+\n",
-			      display_wm, SNB_DISPLAY_MAX_SRWM, level);
-		return false;
+static void snb_read_wm_latency(struct drm_i915_private *dev_priv,
+				int level,
+				unsigned int *plane,
+				unsigned int *cursor)
+{
+	switch (level) {
+	case 0:
+		*plane = *cursor = SNB_READ_WM0_LATENCY();
+		break;
+	case 1:
+		*plane = *cursor = SNB_READ_WM1_LATENCY();
+		break;
+	case 2:
+		*plane = *cursor = SNB_READ_WM2_LATENCY();
+		break;
+	case 3:
+		*plane = *cursor = SNB_READ_WM3_LATENCY();
+		break;
+	default:
+		*plane = *cursor = 0;
+		break;
 	}
+}
 
-	if (cursor_wm > cursor->max_wm) {
-		DRM_DEBUG_KMS("cursor watermark(%d) is too large(%d), disabling wm%d+\n",
-			      cursor_wm, SNB_CURSOR_MAX_SRWM, level);
-		return false;
+static void hsw_read_wm_latency(struct drm_i915_private *dev_priv,
+				int level,
+				unsigned int *plane,
+				unsigned int *cursor)
+{
+	switch (level) {
+	case 0:
+		*plane = *cursor = HSW_READ_WM0_NEW_LATENCY();
+		if (*plane == 0)
+			*plane = *cursor = HSW_READ_WM0_OLD_LATENCY();
+		break;
+	case 1:
+		*plane = *cursor = HSW_READ_WM1_LATENCY();
+		break;
+	case 2:
+		*plane = *cursor = HSW_READ_WM2_LATENCY();
+		break;
+	case 3:
+		*plane = *cursor = HSW_READ_WM3_LATENCY();
+		break;
+	case 4:
+		*plane = *cursor = HSW_READ_WM4_LATENCY();
+		break;
+	default:
+		*plane = *cursor = 0;
+		break;
 	}
+}
 
-	if (!(fbc_wm || display_wm || cursor_wm)) {
-		DRM_DEBUG_KMS("latency %d is 0, disabling wm%d+\n", level, level);
-		return false;
-	}
+static void intel_read_wm_latency(struct drm_device *dev,
+				  int level,
+				  unsigned int *plane,
+				  unsigned int *cursor)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
 
-	return true;
+	if (INTEL_INFO(dev)->gen > 7 || IS_HASWELL(dev))
+		return hsw_read_wm_latency(dev_priv, level, plane, cursor);
+	else if (INTEL_INFO(dev)->gen >= 6)
+		return snb_read_wm_latency(dev_priv, level, plane, cursor);
+	else
+		return ilk_read_wm_latency(dev_priv, level, plane, cursor);
 }
 
-/*
- * Compute watermark values of WM[1-3],
- */
-static bool ironlake_compute_srwm(struct drm_device *dev, int level, int plane,
-				  int latency_ns,
-				  const struct intel_watermark_params *display,
-				  const struct intel_watermark_params *cursor,
-				  int *fbc_wm, int *display_wm, int *cursor_wm)
+#define ILK_GUARD_SIZE		2
+#define ILK_CACHELINE_SIZE	64
+
+static unsigned int
+ilk_compute_wm_large(const struct intel_crtc_config *pipe_config,
+		     const struct intel_plane_config *plane_config,
+		     unsigned int latency_ns)
 {
-	struct drm_crtc *crtc;
-	unsigned long line_time_us;
-	int hdisplay, htotal, pixel_size, clock;
-	int line_count, line_size;
-	int small, large;
-	int entries;
+	const struct drm_display_mode *mode = &pipe_config->requested_mode;
+	unsigned int htotal, clock;
+	unsigned int line_time_us, line_count;
+	unsigned int entries;
 
-	if (!latency_ns) {
-		*fbc_wm = *display_wm = *cursor_wm = 0;
-		return false;
-	}
+	if (!plane_config->enabled || !pipe_config->enabled)
+		return ILK_GUARD_SIZE;
 
-	crtc = intel_get_crtc_for_plane(dev, plane);
-	hdisplay = crtc->mode.hdisplay;
-	htotal = crtc->mode.htotal;
-	clock = crtc->mode.clock;
-	pixel_size = crtc->fb->bits_per_pixel / 8;
+	/* FIXME scaling and whatnot */
+	htotal = mode->htotal;
+	clock = mode->clock;
 
-	line_time_us = (htotal * 1000) / clock;
+	line_time_us = ((htotal * 1000) / clock);
 	line_count = (latency_ns / line_time_us + 1000) / 1000;
-	line_size = hdisplay * pixel_size;
+	entries = line_count * plane_config->width * plane_config->pixel_size;
+	entries = DIV_ROUND_UP(entries, ILK_CACHELINE_SIZE);
 
-	/* Use the minimum of the small and large buffer method for primary */
-	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
-	large = line_count * line_size;
+	return entries + ILK_GUARD_SIZE;
+}
 
-	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
-	*display_wm = entries + display->guard_size;
+static unsigned int
+ilk_compute_wm_small(const struct intel_crtc_config *pipe_config,
+		     const struct intel_plane_config *plane_config,
+		     unsigned int latency_ns)
+{
+	const struct drm_display_mode *mode = &pipe_config->requested_mode;
+	unsigned int clock;
+	unsigned int entries;
 
-	/*
-	 * Spec says:
-	 * FBC WM = ((Final Primary WM * 64) / number of bytes per line) + 2
-	 */
-	*fbc_wm = DIV_ROUND_UP(*display_wm * 64, line_size) + 2;
+	if (!plane_config->enabled || !pipe_config->enabled)
+		return ILK_GUARD_SIZE;
 
-	/* calculate the self-refresh watermark for display cursor */
-	entries = line_count * pixel_size * 64;
-	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
-	*cursor_wm = entries + cursor->guard_size;
+	/* FIXME scaling and whatnot */
+	clock = mode->clock;
 
-	return ironlake_check_srwm(dev, level,
-				   *fbc_wm, *display_wm, *cursor_wm,
-				   display, cursor);
+	entries = ((clock * plane_config->pixel_size / 1000) * latency_ns) / 1000;
+	entries = DIV_ROUND_UP(entries, ILK_CACHELINE_SIZE);
+
+	return entries + ILK_GUARD_SIZE;
 }
 
-static void ironlake_update_wm(struct drm_device *dev)
+static unsigned int
+ilk_compute_cursor_wm(struct drm_device *dev,
+		      int level,
+		      const struct intel_crtc_config *pipe_config,
+		      const struct intel_plane_config *plane_config,
+		      unsigned int latency_ns)
 {
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	int fbc_wm, plane_wm, cursor_wm;
-	unsigned int enabled;
+	/* WaDoubleCursorLP3Latency */
+	if (IS_IVYBRIDGE(dev) && level == 3)
+		latency_ns *= 2;
 
-	enabled = 0;
-	if (g4x_compute_wm0(dev, PIPE_A,
-			    &ironlake_display_wm_info,
-			    ILK_LP0_PLANE_LATENCY,
-			    &ironlake_cursor_wm_info,
-			    ILK_LP0_CURSOR_LATENCY,
-			    &plane_wm, &cursor_wm)) {
-		I915_WRITE(WM0_PIPEA_ILK,
-			   (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm);
-		DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
-			      " plane %d, " "cursor: %d\n",
-			      plane_wm, cursor_wm);
-		enabled |= 1 << PIPE_A;
-	}
+	DRM_DEBUG_KMS("cursor wm level %d: width %u, pixel_size = %u, latency %u\n",
+		      level, plane_config->width, plane_config->pixel_size, latency_ns);
 
-	if (g4x_compute_wm0(dev, PIPE_B,
-			    &ironlake_display_wm_info,
-			    ILK_LP0_PLANE_LATENCY,
-			    &ironlake_cursor_wm_info,
-			    ILK_LP0_CURSOR_LATENCY,
-			    &plane_wm, &cursor_wm)) {
-		I915_WRITE(WM0_PIPEB_ILK,
-			   (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm);
-		DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
-			      " plane %d, cursor: %d\n",
-			      plane_wm, cursor_wm);
-		enabled |= 1 << PIPE_B;
-	}
+	/* Use the large method for cursor plane */
+	return ilk_compute_wm_large(pipe_config, plane_config, latency_ns);
+}
+
+static unsigned int
+ilk_compute_primary_wm(int level,
+		       const struct intel_crtc_config *pipe_config,
+		       const struct intel_plane_config *plane_config,
+		       unsigned int latency_ns)
+{
+	unsigned int small, large = UINT_MAX;
+
+	DRM_DEBUG_KMS("primary wm level %d: width %u, pixel_size = %u, latency %u\n",
+		      level, plane_config->width, plane_config->pixel_size, latency_ns);
 
 	/*
-	 * Calculate and update the self-refresh watermark only when one
-	 * display plane is used.
+	 * Use the small method for primary plane WM0,
+	 * and the minimum of small and large methods for WM1+.
 	 */
-	I915_WRITE(WM3_LP_ILK, 0);
-	I915_WRITE(WM2_LP_ILK, 0);
-	I915_WRITE(WM1_LP_ILK, 0);
+	small = ilk_compute_wm_small(pipe_config, plane_config, latency_ns);
 
-	if (!single_plane_enabled(enabled))
-		return;
-	enabled = ffs(enabled) - 1;
-
-	/* WM1 */
-	if (!ironlake_compute_srwm(dev, 1, enabled,
-				   ILK_READ_WM1_LATENCY() * 500,
-				   &ironlake_display_srwm_info,
-				   &ironlake_cursor_srwm_info,
-				   &fbc_wm, &plane_wm, &cursor_wm))
-		return;
+	if (level > 0)
+		large = ilk_compute_wm_large(pipe_config, plane_config, latency_ns);
 
-	I915_WRITE(WM1_LP_ILK,
-		   WM1_LP_SR_EN |
-		   (ILK_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) |
-		   (fbc_wm << WM1_LP_FBC_SHIFT) |
-		   (plane_wm << WM1_LP_SR_SHIFT) |
-		   cursor_wm);
-
-	/* WM2 */
-	if (!ironlake_compute_srwm(dev, 2, enabled,
-				   ILK_READ_WM2_LATENCY() * 500,
-				   &ironlake_display_srwm_info,
-				   &ironlake_cursor_srwm_info,
-				   &fbc_wm, &plane_wm, &cursor_wm))
-		return;
+	return min(small, large);
+}
 
-	I915_WRITE(WM2_LP_ILK,
-		   WM2_LP_EN |
-		   (ILK_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) |
-		   (fbc_wm << WM1_LP_FBC_SHIFT) |
-		   (plane_wm << WM1_LP_SR_SHIFT) |
-		   cursor_wm);
+static unsigned int
+ilk_compute_sprite_wm(int level,
+		      const struct intel_crtc_config *pipe_config,
+		      const struct intel_plane_config *plane_config,
+		      unsigned int latency_ns)
+{
+	unsigned int small, large;
+
+	DRM_DEBUG_KMS("sprite wm level %d: width %u, pixel_size = %u, latency %u\n",
+		      level, plane_config->width, plane_config->pixel_size, latency_ns);
 
 	/*
-	 * WM3 is unsupported on ILK, probably because we don't have latency
-	 * data for that power state
+	 * Use the the minimum of small and larget methods for
+	 * sprite plane.
 	 */
+	small = ilk_compute_wm_small(pipe_config, plane_config, latency_ns);
+
+	large = ilk_compute_wm_large(pipe_config, plane_config, latency_ns);
+
+	return min(small, large);
 }
 
-static void sandybridge_update_wm(struct drm_device *dev)
+static unsigned int
+ilk_compute_fbc_wm(int level,
+		   const struct intel_crtc_config *pipe_config,
+		   const struct intel_plane_config *plane_config,
+		   unsigned int primary_wm)
 {
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	int latency = SNB_READ_WM0_LATENCY() * 100;	/* In unit 0.1us */
-	u32 val;
-	int fbc_wm, plane_wm, cursor_wm;
-	unsigned int enabled;
+	unsigned int line_size;
 
-	enabled = 0;
-	if (g4x_compute_wm0(dev, PIPE_A,
-			    &sandybridge_display_wm_info, latency,
-			    &sandybridge_cursor_wm_info, latency,
-			    &plane_wm, &cursor_wm)) {
-		val = I915_READ(WM0_PIPEA_ILK);
-		val &= ~(WM0_PIPE_PLANE_MASK | WM0_PIPE_CURSOR_MASK);
-		I915_WRITE(WM0_PIPEA_ILK, val |
-			   ((plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm));
-		DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
-			      " plane %d, " "cursor: %d\n",
-			      plane_wm, cursor_wm);
-		enabled |= 1 << PIPE_A;
-	}
+	if (level == 0)
+		return 0;
 
-	if (g4x_compute_wm0(dev, PIPE_B,
-			    &sandybridge_display_wm_info, latency,
-			    &sandybridge_cursor_wm_info, latency,
-			    &plane_wm, &cursor_wm)) {
-		val = I915_READ(WM0_PIPEB_ILK);
-		val &= ~(WM0_PIPE_PLANE_MASK | WM0_PIPE_CURSOR_MASK);
-		I915_WRITE(WM0_PIPEB_ILK, val |
-			   ((plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm));
-		DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
-			      " plane %d, cursor: %d\n",
-			      plane_wm, cursor_wm);
-		enabled |= 1 << PIPE_B;
-	}
+	DRM_DEBUG_KMS("fbc wm level %d: width %u, pixel_size = %u, primary_wm %u\n",
+		      level, plane_config->width, plane_config->pixel_size, primary_wm);
 
 	/*
-	 * Calculate and update the self-refresh watermark only when one
-	 * display plane is used.
-	 *
-	 * SNB support 3 levels of watermark.
-	 *
-	 * WM1/WM2/WM2 watermarks have to be enabled in the ascending order,
-	 * and disabled in the descending order
-	 *
+	 * Spec says:
+	 * FBC WM = ((Final Primary WM * 64) / number of bytes per line) + 2
 	 */
-	I915_WRITE(WM3_LP_ILK, 0);
-	I915_WRITE(WM2_LP_ILK, 0);
-	I915_WRITE(WM1_LP_ILK, 0);
+	line_size = plane_config->width * plane_config->pixel_size;
 
-	if (!single_plane_enabled(enabled) ||
-	    dev_priv->sprite_scaling_enabled)
-		return;
-	enabled = ffs(enabled) - 1;
-
-	/* WM1 */
-	if (!ironlake_compute_srwm(dev, 1, enabled,
-				   SNB_READ_WM1_LATENCY() * 500,
-				   &sandybridge_display_srwm_info,
-				   &sandybridge_cursor_srwm_info,
-				   &fbc_wm, &plane_wm, &cursor_wm))
-		return;
+	if (line_size == 0)
+		return 2;
 
-	I915_WRITE(WM1_LP_ILK,
-		   WM1_LP_SR_EN |
-		   (SNB_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) |
-		   (fbc_wm << WM1_LP_FBC_SHIFT) |
-		   (plane_wm << WM1_LP_SR_SHIFT) |
-		   cursor_wm);
-
-	/* WM2 */
-	if (!ironlake_compute_srwm(dev, 2, enabled,
-				   SNB_READ_WM2_LATENCY() * 500,
-				   &sandybridge_display_srwm_info,
-				   &sandybridge_cursor_srwm_info,
-				   &fbc_wm, &plane_wm, &cursor_wm))
-		return;
+	return DIV_ROUND_UP(primary_wm * 64, line_size) + 2;
+}
 
-	I915_WRITE(WM2_LP_ILK,
-		   WM2_LP_EN |
-		   (SNB_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) |
-		   (fbc_wm << WM1_LP_FBC_SHIFT) |
-		   (plane_wm << WM1_LP_SR_SHIFT) |
-		   cursor_wm);
-
-	/* WM3 */
-	if (!ironlake_compute_srwm(dev, 3, enabled,
-				   SNB_READ_WM3_LATENCY() * 500,
-				   &sandybridge_display_srwm_info,
-				   &sandybridge_cursor_srwm_info,
-				   &fbc_wm, &plane_wm, &cursor_wm))
-		return;
+static unsigned int
+hsw_compute_linetime_wm(const struct intel_crtc_config *pipe_config)
+{
+	const struct drm_display_mode *mode = &pipe_config->requested_mode;
+
+	if (!pipe_config->enabled)
+		return 0;
 
-	I915_WRITE(WM3_LP_ILK,
-		   WM3_LP_EN |
-		   (SNB_READ_WM3_LATENCY() << WM1_LP_LATENCY_SHIFT) |
-		   (fbc_wm << WM1_LP_FBC_SHIFT) |
-		   (plane_wm << WM1_LP_SR_SHIFT) |
-		   cursor_wm);
+	/* FIXME */
+	return ((mode->hdisplay * 1000) / mode->clock) * 8;
 }
 
-static void ivybridge_update_wm(struct drm_device *dev)
+static unsigned int
+hsw_compute_ips_linetime_wm(const struct intel_crtc_config *pipe_config)
 {
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	int latency = SNB_READ_WM0_LATENCY() * 100;	/* In unit 0.1us */
-	u32 val;
-	int fbc_wm, plane_wm, cursor_wm;
-	int ignore_fbc_wm, ignore_plane_wm, ignore_cursor_wm;
-	unsigned int enabled;
+	const struct drm_display_mode *mode = &pipe_config->requested_mode;
 
-	enabled = 0;
-	if (g4x_compute_wm0(dev, PIPE_A,
-			    &sandybridge_display_wm_info, latency,
-			    &sandybridge_cursor_wm_info, latency,
-			    &plane_wm, &cursor_wm)) {
-		val = I915_READ(WM0_PIPEA_ILK);
-		val &= ~(WM0_PIPE_PLANE_MASK | WM0_PIPE_CURSOR_MASK);
-		I915_WRITE(WM0_PIPEA_ILK, val |
-			   ((plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm));
-		DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
-			      " plane %d, " "cursor: %d\n",
-			      plane_wm, cursor_wm);
-		enabled |= 1 << PIPE_A;
-	}
+	if (!pipe_config->enabled)
+		return 0;
 
-	if (g4x_compute_wm0(dev, PIPE_B,
-			    &sandybridge_display_wm_info, latency,
-			    &sandybridge_cursor_wm_info, latency,
-			    &plane_wm, &cursor_wm)) {
-		val = I915_READ(WM0_PIPEB_ILK);
-		val &= ~(WM0_PIPE_PLANE_MASK | WM0_PIPE_CURSOR_MASK);
-		I915_WRITE(WM0_PIPEB_ILK, val |
-			   ((plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm));
-		DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
-			      " plane %d, cursor: %d\n",
-			      plane_wm, cursor_wm);
-		enabled |= 1 << PIPE_B;
-	}
+	/* FIXME */
+	return ((mode->hdisplay * 1000) / mode->clock) * 8;
+}
 
-	if (g4x_compute_wm0(dev, PIPE_C,
-			    &sandybridge_display_wm_info, latency,
-			    &sandybridge_cursor_wm_info, latency,
-			    &plane_wm, &cursor_wm)) {
-		val = I915_READ(WM0_PIPEC_IVB);
-		val &= ~(WM0_PIPE_PLANE_MASK | WM0_PIPE_CURSOR_MASK);
-		I915_WRITE(WM0_PIPEC_IVB, val |
-			   ((plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm));
-		DRM_DEBUG_KMS("FIFO watermarks For pipe C -"
-			      " plane %d, cursor: %d\n",
-			      plane_wm, cursor_wm);
-		enabled |= 1 << PIPE_C;
+static unsigned int ilk_display_fifo_size(const struct drm_device *dev)
+{
+	if (INTEL_INFO(dev)->gen >= 7)
+		return 768;
+	else
+		return 512; /* FIXME OK for ILK/SNB? */
+}
+
+static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
+				      int level,
+				      unsigned int num_active_pipes)
+{
+	/* HSW LP1+ watermarks w/ multiple pipes */
+	if (level > 0 && num_active_pipes > 1)
+		return 64;
+
+	/* othwewise just report max that registers can hold */
+	if (INTEL_INFO(dev)->gen >= 7)
+		return level == 0 ? 63 : 255;
+	else
+		return level == 0 ? 31 : 63;
+}
+
+static unsigned int ilk_fbc_wm_max(const struct drm_device *dev)
+{
+	/* max that registers can hold */
+	return 15;
+}
+
+static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
+				     int level,
+				     unsigned int num_active_pipes,
+				     bool sprite_enabled,
+				     bool ddb_partitioning,
+				     bool is_sprite)
+{
+	unsigned int fifo_size = ilk_display_fifo_size(dev);
+	unsigned int max;
+
+	/* HSW allows LP1+ watermarks even with multiple pipes */
+	if (level == 0 || num_active_pipes > 1)
+		fifo_size /= INTEL_INFO(dev)->num_pipes;
+
+	if (sprite_enabled) {
+		/* WM0 is always calculated with 1:1 split */
+		if (level > 0 && ddb_partitioning) {
+			if (is_sprite)
+				fifo_size *= 5;
+			fifo_size /= 6;
+		} else {
+			fifo_size /= 2;
+		}
 	}
 
+	/* clamp to max that the registers can hold */
 	/*
-	 * Calculate and update the self-refresh watermark only when one
-	 * display plane is used.
-	 *
-	 * SNB support 3 levels of watermark.
-	 *
-	 * WM1/WM2/WM2 watermarks have to be enabled in the ascending order,
-	 * and disabled in the descending order
-	 *
+	 * FIXME should confirm these as the spec or spreadsheets
+	 * don't always make sense wrt. sizes of the reg fields.
 	 */
-	I915_WRITE(WM3_LP_ILK, 0);
-	I915_WRITE(WM2_LP_ILK, 0);
-	I915_WRITE(WM1_LP_ILK, 0);
+	if (INTEL_INFO(dev)->gen >= 7)
+		max = level == 0 ? 127 : 1023;
+	else if (!is_sprite)
+		max = level == 0 ? 127 : 511;
+	else
+		max = level == 0 ? 63 : 255;
+
+	return min(fifo_size, max);
+}
+
+static bool ilk_check_wm(const struct drm_device *dev,
+			 int level,
+			 unsigned int num_active_pipes,
+			 bool sprite_enabled,
+			 bool ddb_partitioning,
+			 struct intel_wm_level *wm)
+{
+	unsigned int primary_max = ilk_plane_wm_max(dev, level,
+						    num_active_pipes,
+						    sprite_enabled,
+						    ddb_partitioning,
+						    false);
+	unsigned int sprite_max = ilk_plane_wm_max(dev, level,
+						   num_active_pipes,
+						   sprite_enabled,
+						   ddb_partitioning,
+						   true);
+	unsigned int cursor_max = ilk_cursor_wm_max(dev, level, num_active_pipes);
+
+	DRM_DEBUG_KMS("check WM level %d:\n"
+		      " primary %u (max %u)\n"
+		      " sprite %u (max %u)\n"
+		      " cursor %u (max %u)\n"
+		      " fbc %u (max %u)\n",
+		      level,
+		      wm->primary, primary_max,
+		      wm->sprite, sprite_max,
+		      wm->cursor, cursor_max,
+		      wm->fbc, ilk_fbc_wm_max(dev));
 
-	if (!single_plane_enabled(enabled) ||
-	    dev_priv->sprite_scaling_enabled)
-		return;
-	enabled = ffs(enabled) - 1;
-
-	/* WM1 */
-	if (!ironlake_compute_srwm(dev, 1, enabled,
-				   SNB_READ_WM1_LATENCY() * 500,
-				   &sandybridge_display_srwm_info,
-				   &sandybridge_cursor_srwm_info,
-				   &fbc_wm, &plane_wm, &cursor_wm))
-		return;
+	/*
+	 * HACK until we can pre-compute everything,
+	 * and thus fail gracefully if LP0 watermarks
+	 * are exceeded...
+	 */
+	if (level == 0) {
+		wm->primary = min(wm->primary, primary_max);
+		wm->sprite = min(wm->sprite, sprite_max);
+		wm->cursor = min(wm->cursor, cursor_max);
+	}
 
-	I915_WRITE(WM1_LP_ILK,
-		   WM1_LP_SR_EN |
-		   (SNB_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) |
-		   (fbc_wm << WM1_LP_FBC_SHIFT) |
-		   (plane_wm << WM1_LP_SR_SHIFT) |
-		   cursor_wm);
-
-	/* WM2 */
-	if (!ironlake_compute_srwm(dev, 2, enabled,
-				   SNB_READ_WM2_LATENCY() * 500,
-				   &sandybridge_display_srwm_info,
-				   &sandybridge_cursor_srwm_info,
-				   &fbc_wm, &plane_wm, &cursor_wm))
-		return;
+	if (wm->primary > primary_max)
+		wm->valid = false;
+
+	if (wm->sprite > sprite_max)
+		wm->valid = false;
+
+	if (wm->cursor > cursor_max)
+		wm->valid = false;
+
+	DRM_DEBUG_KMS("checked WM level %d: %svalid\n", level, wm->valid ? "" : "in");
+
+	return wm->valid;
+}
+
+static void ilk_compute_wm(struct drm_device *dev, int level,
+			   const struct intel_crtc_config *pipe_config,
+			   unsigned int plane_latency_ns,
+			   unsigned int cursor_latency_ns,
+			   struct intel_wm_level *wm)
+{
+	const struct intel_plane_config *primary = &pipe_config->primary;
+	const struct intel_plane_config *sprite = &pipe_config->sprite[0];
+	const struct intel_plane_config *cursor = &pipe_config->cursor;
+
+	DRM_DEBUG_KMS("WM%d: plane lateny %u ns, cursor latency %u ns\n",
+		      level, plane_latency_ns, cursor_latency_ns);
 
-	I915_WRITE(WM2_LP_ILK,
-		   WM2_LP_EN |
-		   (SNB_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) |
-		   (fbc_wm << WM1_LP_FBC_SHIFT) |
-		   (plane_wm << WM1_LP_SR_SHIFT) |
-		   cursor_wm);
-
-	/* WM3, note we have to correct the cursor latency */
-	if (!ironlake_compute_srwm(dev, 3, enabled,
-				   SNB_READ_WM3_LATENCY() * 500,
-				   &sandybridge_display_srwm_info,
-				   &sandybridge_cursor_srwm_info,
-				   &fbc_wm, &plane_wm, &ignore_cursor_wm) ||
-	    !ironlake_compute_srwm(dev, 3, enabled,
-				   2 * SNB_READ_WM3_LATENCY() * 500,
-				   &sandybridge_display_srwm_info,
-				   &sandybridge_cursor_srwm_info,
-				   &ignore_fbc_wm, &ignore_plane_wm, &cursor_wm))
+	if (plane_latency_ns == 0 || cursor_latency_ns == 0)
 		return;
 
-	I915_WRITE(WM3_LP_ILK,
-		   WM3_LP_EN |
-		   (SNB_READ_WM3_LATENCY() << WM1_LP_LATENCY_SHIFT) |
-		   (fbc_wm << WM1_LP_FBC_SHIFT) |
-		   (plane_wm << WM1_LP_SR_SHIFT) |
-		   cursor_wm);
+	wm->primary = ilk_compute_primary_wm(level, pipe_config,
+					     primary, plane_latency_ns);
+
+	wm->fbc = ilk_compute_fbc_wm(level, pipe_config,
+				     primary, wm->primary);
+
+	wm->cursor = ilk_compute_cursor_wm(dev, level, pipe_config,
+					   cursor, cursor_latency_ns);
+
+	wm->sprite = ilk_compute_sprite_wm(level, pipe_config,
+					   sprite, plane_latency_ns);
+
+	wm->valid = true;
+
+	DRM_DEBUG_KMS("WM%d: primary %u, sprite %u, cursor %u, fbc %u\n",
+		      level, wm->primary, wm->sprite, wm->cursor, wm->fbc);
 }
 
-static void
-haswell_update_linetime_wm(struct drm_device *dev, int pipe,
-				 struct drm_display_mode *mode)
+static unsigned int ilk_max_wm_level(const struct drm_device *dev,
+				     const struct intel_pipe_wm *pipe_wm)
+{
+	/* HSW: LP1+ watermarks allowed even with multiple pipes */
+	if (INTEL_INFO(dev)->gen > 7 || IS_HASWELL(dev))
+		return 4;
+
+	/* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
+	if (pipe_wm->sprite_scaled)
+		return 0;
+
+	/* ILK/SNB: LP2+ watermarks only w/o sprites */
+	if (INTEL_INFO(dev)->gen <= 6 && pipe_wm->sprite_enabled)
+		return 1;
+
+	return 3;
+}
+
+static bool ilk_compute_pipe_wm(struct drm_crtc *crtc,
+				unsigned int num_active_pipes,
+				const struct intel_crtc_config *pipe_config,
+				struct intel_pipe_wm *pipe_wm)
+{
+	struct drm_device *dev = crtc->dev;
+	unsigned int plane_latency, cursor_latency;
+	int level, max_level;
+
+	pipe_wm->pipe_enabled = pipe_config->enabled;
+	pipe_wm->sprite_enabled = pipe_config->sprite[0].enabled;
+	pipe_wm->sprite_scaled = pipe_config->sprite[0].scaled;
+
+	max_level = ilk_max_wm_level(dev, pipe_wm);
+
+	if (INTEL_INFO(dev)->gen > 7 || IS_HASWELL(dev)) {
+		pipe_wm->linetime = hsw_compute_linetime_wm(pipe_config);
+		pipe_wm->ips_linetime = hsw_compute_ips_linetime_wm(pipe_config);
+
+		/*
+		 * HACK until we can pre-compute everything,
+		 * and thus fail gracefully if linetime watermarks
+		 * are exceeded...
+		 */
+		if (pipe_wm->linetime > 511)
+			pipe_wm->linetime = 511;
+
+		if (pipe_wm->ips_linetime > 511)
+			pipe_wm->ips_linetime = 511;
+
+		/*
+		 * FIXME what's the correct course of action
+		 * if linetime watermarks are exceeded?
+		 */
+		if (pipe_wm->linetime > 511)
+			return false;
+
+		if (pipe_wm->ips_linetime > 511)
+			return false;
+	}
+
+	intel_read_wm_latency(dev, 0, &plane_latency, &cursor_latency);
+
+	ilk_compute_wm(dev, 0,
+		       pipe_config,
+		       plane_latency * 100,
+		       cursor_latency * 100,
+		       &pipe_wm->wm[0]);
+
+	if (!ilk_check_wm(dev, 0, num_active_pipes,
+			  pipe_wm->sprite_enabled, false,
+			  &pipe_wm->wm[0]))
+		return false;
+
+	for (level = 1; level <= max_level; level++) {
+		intel_read_wm_latency(dev, level, &plane_latency, &cursor_latency);
+
+		ilk_compute_wm(dev, level,
+			       pipe_config,
+			       plane_latency * 500,
+			       cursor_latency * 500,
+			       &pipe_wm->wm[level]);
+	}
+
+	DRM_DEBUG_KMS("pipe_enabled = %d, sprite_enabled = %d\n",
+		      pipe_wm->pipe_enabled, pipe_wm->sprite_enabled);
+
+	return true;
+}
+
+/*
+ * enable/disable the FBC WM.
+ */
+static void ilk_set_fbc_wm(struct drm_device *dev, bool fbc_wm_enabled)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	u32 temp;
+	uint32_t tmp;
 
-	temp = I915_READ(PIPE_WM_LINETIME(pipe));
-	temp &= ~PIPE_WM_LINETIME_MASK;
+	tmp = I915_READ(DISP_ARB_CTL);
+	if (fbc_wm_enabled)
+		tmp &= ~DISP_FBC_WM_DIS;
+	else
+		tmp |= DISP_FBC_WM_DIS;
+	I915_WRITE(DISP_ARB_CTL, tmp);
+}
 
-	/* The WM are computed with base on how long it takes to fill a single
-	 * row at the given clock rate, multiplied by 8.
-	 * */
-	temp |= PIPE_WM_LINETIME_TIME(
-		((mode->crtc_hdisplay * 1000) / mode->clock) * 8);
+/*
+ * Configure the data buffer partitioning.
+ * 1/2 + 1/2 or 1/6 + 5/6.
+ */
+static void ilk_set_ddb_partitionming(struct drm_device *dev, bool ddb_partitioning)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	uint32_t tmp;
 
-	/* IPS watermarks are only used by pipe A, and are ignored by
-	 * pipes B and C.  They are calculated similarly to the common
-	 * linetime values, except that we are using CD clock frequency
-	 * in MHz instead of pixel rate for the division.
-	 *
-	 * This is a placeholder for the IPS watermark calculation code.
-	 */
+	tmp = I915_READ(DISP_ARB_CTL2);
+	if (ddb_partitioning)
+		tmp |= DISP_DATA_BUFFER_PARTITIONING;
+	else
+		tmp &= ~DISP_DATA_BUFFER_PARTITIONING;
+	I915_WRITE(DISP_ARB_CTL2, tmp);
+}
 
-	I915_WRITE(PIPE_WM_LINETIME(pipe), temp);
+static void hsw_set_ddb_partitionming(struct drm_device *dev, bool ddb_partitioning)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	uint32_t tmp;
+
+	tmp = I915_READ(WM_MISC);
+	if (ddb_partitioning)
+		tmp |= WM_DATA_BUFFER_PARTITIONING;
+	else
+		tmp &= ~WM_DATA_BUFFER_PARTITIONING;
+	I915_WRITE(WM_MISC, tmp);
 }
 
-static bool
-sandybridge_compute_sprite_wm(struct drm_device *dev, int plane,
-			      uint32_t sprite_width, int pixel_size,
-			      const struct intel_watermark_params *display,
-			      int display_latency_ns, int *sprite_wm)
+/*
+ * Determine the number of active pipes.
+ */
+static unsigned int ilk_wm_num_active_pipes(struct drm_device *dev)
+
 {
-	struct drm_crtc *crtc;
-	int clock;
-	int entries, tlb_miss;
+	struct intel_crtc *intel_crtc;
+	unsigned int num_active_pipes = 0;
 
-	crtc = intel_get_crtc_for_plane(dev, plane);
-	if (!intel_crtc_active(crtc)) {
-		*sprite_wm = display->guard_size;
-		return false;
+	list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list, base.head) {
+		const struct intel_pipe_wm *pipe_wm = &intel_crtc->wm.active;
+
+		num_active_pipes += pipe_wm->pipe_enabled;
 	}
 
-	clock = crtc->mode.clock;
+	return num_active_pipes;
+}
 
-	/* Use the small buffer method to calculate the sprite watermark */
-	entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
-	tlb_miss = display->fifo_size*display->cacheline_size -
-		sprite_width * 8;
-	if (tlb_miss > 0)
-		entries += tlb_miss;
-	entries = DIV_ROUND_UP(entries, display->cacheline_size);
-	*sprite_wm = entries + display->guard_size;
-	if (*sprite_wm > (int)display->max_wm)
-		*sprite_wm = display->max_wm;
+/*
+ * Determine if even a single pipe has a sprite enabled.
+ */
+static bool ilk_wm_is_sprite_enabled(struct drm_device *dev)
 
-	return true;
+{
+	struct intel_crtc *intel_crtc;
+
+	list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list, base.head) {
+		const struct intel_pipe_wm *pipe_wm = &intel_crtc->wm.active;
+
+		if (!pipe_wm->pipe_enabled)
+			continue;
+
+		if (pipe_wm->sprite_enabled)
+			return true;
+	}
+
+	return false;
 }
 
-static bool
-sandybridge_compute_sprite_srwm(struct drm_device *dev, int plane,
-				uint32_t sprite_width, int pixel_size,
-				const struct intel_watermark_params *display,
-				int latency_ns, int *sprite_wm)
+/*
+ * Merge the watermarks from all active pipes for a specific level.
+ */
+static void ilk_merge_wm_level(struct drm_device *dev,
+			       int level,
+			       struct intel_wm_level *ret_wm)
 {
-	struct drm_crtc *crtc;
-	unsigned long line_time_us;
-	int clock;
-	int line_count, line_size;
-	int small, large;
-	int entries;
+	struct intel_crtc *intel_crtc;
 
-	if (!latency_ns) {
-		*sprite_wm = 0;
-		return false;
+	list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list, base.head) {
+		const struct intel_pipe_wm *pipe_wm = &intel_crtc->wm.active;
+		const struct intel_wm_level *wm = &pipe_wm->wm[level];
+
+		if (!pipe_wm->pipe_enabled)
+			continue;
+
+		if (!wm->valid) {
+			ret_wm->valid = false;
+			break;
+		}
+
+		DRM_DEBUG_KMS("Pipe %c WM level %d:\n"
+			      " primary = %u\n"
+			      " sprite  = %u\n"
+			      " cursor  = %u\n"
+			      " fbc     = %u\n",
+			      pipe_name(intel_crtc->pipe), level,
+			      wm->primary, wm->sprite, wm->cursor, wm->fbc);
+
+		ret_wm->valid = true;
+		ret_wm->primary = max(ret_wm->primary, wm->primary);
+		ret_wm->sprite = max(ret_wm->sprite, wm->sprite);
+		ret_wm->cursor = max(ret_wm->cursor, wm->cursor);
+		ret_wm->fbc = max(ret_wm->fbc, wm->fbc);
 	}
+}
 
-	crtc = intel_get_crtc_for_plane(dev, plane);
-	clock = crtc->mode.clock;
-	if (!clock) {
-		*sprite_wm = 0;
-		return false;
+/*
+ * Merge all low power watermarks for all active pipes.
+ */
+static void ilk_merge_wm(struct drm_device *dev,
+			 unsigned int num_active_pipes,
+			 bool sprite_enabled,
+			 bool ddb_partitioning,
+			 bool *fbc_wm_enabled,
+			 struct intel_pipe_wm *merged_wm)
+{
+	unsigned int fbc_max = ilk_fbc_wm_max(dev);
+	int level;
+
+	/* ILK: FBC WM must remain disabled */
+	*fbc_wm_enabled = INTEL_INFO(dev)->gen >= 6;
+
+	/* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
+	if (num_active_pipes > 1 &&
+	    (INTEL_INFO(dev)->gen <= 6 || IS_IVYBRIDGE(dev)))
+		return;
+
+	for (level = 1; level <= 4; level++) {
+		struct intel_wm_level *wm = &merged_wm->wm[level];
+
+		ilk_merge_wm_level(dev, level, wm);
+
+		if (!ilk_check_wm(dev, level, num_active_pipes,
+				  sprite_enabled, ddb_partitioning,
+				  wm))
+			break;
+
+		/*
+		 * If FBC WM is exceeded, disable just FBC WM
+		 * instead if the whole level.
+		 */
+		if (wm->fbc > fbc_max) {
+			*fbc_wm_enabled = false;
+			/*
+			 * Just to make sure we don't try to stuff
+			 * overly large values into the register.
+			 */
+			wm->fbc = fbc_max;
+		}
 	}
 
-	line_time_us = (sprite_width * 1000) / clock;
-	if (!line_time_us) {
-		*sprite_wm = 0;
-		return false;
+	/* ILK: LP2 must be disabled if FBC is enabled but FBC WM disabled. */
+	if (INTEL_INFO(dev)->gen == 5 && intel_fbc_enabled(dev) && !*fbc_wm_enabled) {
+		for (level = 2; level <= 4; level++) {
+			struct intel_wm_level *wm = &merged_wm->wm[level];
+
+			wm->valid = false;
+		}
 	}
+}
 
-	line_count = (latency_ns / line_time_us + 1000) / 1000;
-	line_size = sprite_width * pixel_size;
+static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level)
+{
+	unsigned int latency, unused;
 
-	/* Use the minimum of the small and large buffer method for primary */
-	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
-	large = line_count * line_size;
+	/*
+	 * FIXME should we always use level instead of latency on HSW?
+	 * At least there aren't enough bits in the LP WM registers
+	 * for large latency values.
+	 */
+	if (INTEL_INFO(dev)->gen > 7 || IS_HASWELL(dev))
+		return 2 * level;
 
-	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
-	*sprite_wm = entries + display->guard_size;
+	intel_read_wm_latency(dev, level, &latency, &unused);
 
-	return *sprite_wm > 0x3ff ? false : true;
+	return latency;
 }
 
-static void sandybridge_update_sprite_wm(struct drm_device *dev, int pipe,
-					 uint32_t sprite_width, int pixel_size)
+static void ilk_disable_lp_watermarks(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	int latency = SNB_READ_WM0_LATENCY() * 100;	/* In unit 0.1us */
-	u32 val;
-	int sprite_wm, reg;
-	int ret;
 
-	switch (pipe) {
-	case 0:
-		reg = WM0_PIPEA_ILK;
-		break;
-	case 1:
-		reg = WM0_PIPEB_ILK;
-		break;
-	case 2:
-		reg = WM0_PIPEC_IVB;
-		break;
-	default:
-		return; /* bad pipe */
+	/* disable LP1+ watermarks */
+	I915_WRITE(WM3_LP_ILK, 0);
+	I915_WRITE(WM2_LP_ILK, 0);
+	I915_WRITE(WM1_LP_ILK, 0);
+	if (INTEL_INFO(dev)->gen <= 6)
+		I915_WRITE(WM1S_LP_ILK, 0);
+}
+
+static void ilk_program_watermarks(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_crtc *crtc;
+	struct intel_pipe_wm merged_wm = {};
+	const struct intel_wm_level *wm;
+	unsigned int num_active_pipes;
+	bool fbc_wm_enabled;
+	bool ddb_partitioning;
+	bool sprite_enabled;
+	int wm_lp;
+
+	num_active_pipes = ilk_wm_num_active_pipes(dev);
+	sprite_enabled = ilk_wm_is_sprite_enabled(dev);
+	ddb_partitioning = false; /* FIXME */
+
+	DRM_DEBUG_KMS("WM: num_active_pipes = %u, sprite_enabled = %d, ddb_partitioning = %d\n",
+		      num_active_pipes, sprite_enabled, ddb_partitioning);
+
+	ilk_merge_wm(dev, num_active_pipes, sprite_enabled,
+		     ddb_partitioning, &fbc_wm_enabled, &merged_wm);
+
+	DRM_DEBUG_KMS("WM: fbc_wm_enabled = %d\n", fbc_wm_enabled);
+
+	/* disable LP1+ watermarks */
+	ilk_disable_lp_watermarks(dev);
+
+	/* program LP0 watermarks */
+	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+		static const unsigned int ilk_wm0_pipe_reg[] = {
+			[PIPE_A] = WM0_PIPEA_ILK,
+			[PIPE_B] = WM0_PIPEB_ILK,
+			[PIPE_C] = WM0_PIPEC_IVB,
+		};
+		struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+		const struct intel_pipe_wm *pipe_wm = &intel_crtc->wm.active;
+
+		wm = &pipe_wm->wm[0];
+
+		DRM_DEBUG_KMS("Pipe %c LP0:\n"
+			      " primary = %u\n"
+			      " sprite = %u\n"
+			      " cursor = %u\n",
+			      pipe_name(intel_crtc->pipe),
+			      wm->primary, wm->sprite, wm->cursor);
+
+		I915_WRITE(ilk_wm0_pipe_reg[intel_crtc->pipe],
+			   wm->primary << WM0_PIPE_PLANE_SHIFT |
+			   wm->sprite << WM0_PIPE_SPRITE_SHIFT |
+			   wm->cursor);
+
+		if (INTEL_INFO(dev)->gen > 7 || IS_HASWELL(dev))
+			I915_WRITE(PIPE_WM_LINETIME(intel_crtc->pipe),
+				   PIPE_WM_LINETIME_IPS_LINETIME(pipe_wm->ips_linetime) |
+				   PIPE_WM_LINETIME_TIME(pipe_wm->linetime));
 	}
 
-	ret = sandybridge_compute_sprite_wm(dev, pipe, sprite_width, pixel_size,
-					    &sandybridge_display_wm_info,
-					    latency, &sprite_wm);
-	if (!ret) {
-		DRM_DEBUG_KMS("failed to compute sprite wm for pipe %c\n",
-			      pipe_name(pipe));
-		return;
+	ilk_set_fbc_wm(dev, fbc_wm_enabled);
+
+	/* FIXME maybe need a bit more thought as to when/how partitioning is changed
+	 * IIRC it might be double buffered on vbl, so we may need to program it in
+	 * advance when we programe the registers for the plane(s).
+	 */
+	if (INTEL_INFO(dev)->gen > 7 || IS_HASWELL(dev))
+		hsw_set_ddb_partitionming(dev, ddb_partitioning);
+	else
+		ilk_set_ddb_partitionming(dev, ddb_partitioning);
+
+	/* program LP1+ watermarks */
+	for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
+		static const unsigned int ilk_wm_lp_reg[] = {
+			[1] = WM1_LP_ILK,
+			[2] = WM2_LP_ILK,
+			[3] = WM3_LP_ILK,
+		};
+		static const unsigned int ilk_wm_sprite_lp_reg[] = {
+			[1] = WM1S_LP_ILK,
+			[2] = WM2S_LP_IVB,
+			[3] = WM3S_LP_IVB,
+		};
+		/* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4  */
+		int level = wm_lp + (wm_lp >= 2 && merged_wm.wm[4].valid);
+
+		wm = &merged_wm.wm[level];
+		if (!wm->valid)
+			break;
+
+		DRM_DEBUG_KMS("LP%d (level %d):\n"
+			      " primary = %u\n"
+			      " sprite  = %u\n"
+			      " cursor  = %u\n"
+			      " fbc     = %u\n"
+			      " latency = %u\n",
+			      wm_lp, level, wm->primary, wm->sprite,
+			      wm->cursor, wm->fbc, ilk_wm_lp_latency(dev, level));
+
+		if (INTEL_INFO(dev)->gen >= 7)
+			I915_WRITE(ilk_wm_sprite_lp_reg[wm_lp],
+				   wm->sprite);
+		else if (wm_lp == 1)
+			I915_WRITE(WM1S_LP_ILK,
+				   (sprite_enabled ? WM1S_LP_EN : 0) |
+				   wm->sprite);
+
+		I915_WRITE(ilk_wm_lp_reg[wm_lp],
+			   WM1_LP_SR_EN |
+			   ilk_wm_lp_latency(dev, level) << WM1_LP_LATENCY_SHIFT |
+			   wm->fbc << WM1_LP_FBC_SHIFT |
+			   wm->primary << WM1_LP_SR_SHIFT |
+			   wm->cursor);
 	}
+}
 
-	val = I915_READ(reg);
-	val &= ~WM0_PIPE_SPRITE_MASK;
-	I915_WRITE(reg, val | (sprite_wm << WM0_PIPE_SPRITE_SHIFT));
-	DRM_DEBUG_KMS("sprite watermarks For pipe %c - %d\n", pipe_name(pipe), sprite_wm);
+/* Call from vblank irq */
+void ilk_pipe_update_wm(struct drm_device *dev, enum pipe pipe)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
+	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	unsigned long flags;
 
+	spin_lock_irqsave(&dev_priv->wm_lock, flags);
 
-	ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
-					      pixel_size,
-					      &sandybridge_display_srwm_info,
-					      SNB_READ_WM1_LATENCY() * 500,
-					      &sprite_wm);
-	if (!ret) {
-		DRM_DEBUG_KMS("failed to compute sprite lp1 wm on pipe %c\n",
-			      pipe_name(pipe));
-		return;
+	if (intel_crtc->wm.dirty) {
+		u32 pre, post;
+
+		drm_vblank_put(dev, intel_crtc->pipe);
+
+		intel_crtc->wm.active = intel_crtc->wm.pending;
+		intel_crtc->wm.dirty = false;
+
+		pre = I915_READ(PIPEDSL(intel_crtc->pipe));
+		ilk_program_watermarks(dev);
+		post = I915_READ(PIPEDSL(intel_crtc->pipe));
+
+		printk(KERN_CRIT "Update %u -> %u\n", pre, post);
 	}
-	I915_WRITE(WM1S_LP_ILK, sprite_wm);
 
-	/* Only IVB has two more LP watermarks for sprite */
-	if (!IS_IVYBRIDGE(dev))
-		return;
+	spin_unlock_irqrestore(&dev_priv->wm_lock, flags);
+}
 
-	ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
-					      pixel_size,
-					      &sandybridge_display_srwm_info,
-					      SNB_READ_WM2_LATENCY() * 500,
-					      &sprite_wm);
-	if (!ret) {
-		DRM_DEBUG_KMS("failed to compute sprite lp2 wm on pipe %c\n",
-			      pipe_name(pipe));
-		return;
+static void intel_update_wm(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_crtc *crtc;
+	struct intel_pipe_wm pipe_wm[I915_MAX_PIPES] = {};
+	unsigned int num_active_pipes = 0;
+	unsigned long flags;
+
+	DRM_DEBUG_KMS("begin\n");
+
+	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
+		num_active_pipes += to_intel_crtc(crtc)->config.enabled;
+
+	DRM_DEBUG_KMS("num_active_pipes = %u\n", num_active_pipes);
+
+	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+		struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+		const struct intel_crtc_config *pipe_config = &intel_crtc->config;
+
+		if (!ilk_compute_pipe_wm(&intel_crtc->base,
+					 num_active_pipes,
+					 pipe_config,
+					 &pipe_wm[intel_crtc->pipe]))
+			DRM_DEBUG_KMS("Pipe %c LP0 WM invalid\n", pipe_name(intel_crtc->pipe));
 	}
-	I915_WRITE(WM2S_LP_IVB, sprite_wm);
 
-	ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
-					      pixel_size,
-					      &sandybridge_display_srwm_info,
-					      SNB_READ_WM3_LATENCY() * 500,
-					      &sprite_wm);
-	if (!ret) {
-		DRM_DEBUG_KMS("failed to compute sprite lp3 wm on pipe %c\n",
-			      pipe_name(pipe));
-		return;
+	spin_lock_irqsave(&dev_priv->wm_lock, flags);
+
+	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+		struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+		int pipe = intel_crtc->pipe;
+
+		if (!memcmp(&intel_crtc->wm.pending,
+			    &pipe_wm[pipe], sizeof intel_crtc->wm.pending))
+			continue;
+
+		DRM_DEBUG_KMS("pipe %c WM dirty\n", pipe_name(pipe));
+
+		if (!intel_crtc->wm.dirty && drm_vblank_get(dev, pipe)) {
+			if (intel_crtc->active)
+				DRM_ERROR("can't update watermarks for pipe %c\n",
+					  pipe_name(pipe));
+			/* copy the new stuff over anyway. */
+			intel_crtc->wm.pending = pipe_wm[pipe];
+			continue;
+		}
+
+		/*
+		 * When going from no-scaling to scaling,
+		 * disable LP1+ watermarks ahead of time to avoid
+		 * underruns.
+		 */
+		/*
+		 * FIXME is this sufficient of do we need extra vbl waits?
+		 * Something like this is needed on IVB. Do we need this on ILK/SNB too?
+		 * We don't need to worry about multiple pipes here since only HSW supports
+		 * multi-pipe LP1+ watermarks but it doesn't support sprite scaling.
+		 */
+		if (!intel_crtc->wm.pending.sprite_scaled && pipe_wm[pipe].sprite_scaled)
+			ilk_disable_lp_watermarks(dev);
+
+		intel_crtc->wm.pending = pipe_wm[pipe];
+		intel_crtc->wm.dirty = true;
 	}
-	I915_WRITE(WM3S_LP_IVB, sprite_wm);
+
+	spin_unlock_irqrestore(&dev_priv->wm_lock, flags);
+
+	DRM_DEBUG_KMS("end\n");
+}
+
+static void intel_update_sprite_wm(struct drm_device *dev, int pipe,
+				   int plane, bool enabled, bool scaled,
+				   uint32_t sprite_width, int pixel_size)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
+	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+
+	intel_crtc_update_sprite_config(&intel_crtc->config,
+					plane, enabled, scaled,
+					sprite_width, pixel_size);
+	intel_update_wm(dev);
 }
 
 /**
@@ -2236,23 +2520,17 @@ void intel_update_watermarks(struct drm_device *dev)
 		dev_priv->display.update_wm(dev);
 }
 
-void intel_update_linetime_watermarks(struct drm_device *dev,
-		int pipe, struct drm_display_mode *mode)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-
-	if (dev_priv->display.update_linetime_wm)
-		dev_priv->display.update_linetime_wm(dev, pipe, mode);
-}
-
-void intel_update_sprite_watermarks(struct drm_device *dev, int pipe,
+void intel_update_sprite_watermarks(struct drm_device *dev,
+				    int pipe, int plane,
+				    bool enabled, bool scaled,
 				    uint32_t sprite_width, int pixel_size)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 
 	if (dev_priv->display.update_sprite_wm)
-		dev_priv->display.update_sprite_wm(dev, pipe, sprite_width,
-						   pixel_size);
+		dev_priv->display.update_sprite_wm(dev, pipe, plane,
+						   enabled, scaled,
+						   sprite_width, pixel_size);
 }
 
 static struct drm_i915_gem_object *
@@ -4431,9 +4709,10 @@ void intel_init_pm(struct drm_device *dev)
 	/* For FIFO watermark updates */
 	if (HAS_PCH_SPLIT(dev)) {
 		if (IS_GEN5(dev)) {
-			if (I915_READ(MLTR_ILK) & ILK_SRLT_MASK)
-				dev_priv->display.update_wm = ironlake_update_wm;
-			else {
+			if (I915_READ(MLTR_ILK) & ILK_SRLT_MASK) {
+				dev_priv->display.update_wm = intel_update_wm;
+				dev_priv->display.update_sprite_wm = intel_update_sprite_wm;
+			} else {
 				DRM_DEBUG_KMS("Failed to get proper latency. "
 					      "Disable CxSR\n");
 				dev_priv->display.update_wm = NULL;
@@ -4441,8 +4720,8 @@ void intel_init_pm(struct drm_device *dev)
 			dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
 		} else if (IS_GEN6(dev)) {
 			if (SNB_READ_WM0_LATENCY()) {
-				dev_priv->display.update_wm = sandybridge_update_wm;
-				dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
+				dev_priv->display.update_wm = intel_update_wm;
+				dev_priv->display.update_sprite_wm = intel_update_sprite_wm;
 			} else {
 				DRM_DEBUG_KMS("Failed to read display plane latency. "
 					      "Disable CxSR\n");
@@ -4451,8 +4730,8 @@ void intel_init_pm(struct drm_device *dev)
 			dev_priv->display.init_clock_gating = gen6_init_clock_gating;
 		} else if (IS_IVYBRIDGE(dev)) {
 			if (SNB_READ_WM0_LATENCY()) {
-				dev_priv->display.update_wm = ivybridge_update_wm;
-				dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
+				dev_priv->display.update_wm = intel_update_wm;
+				dev_priv->display.update_sprite_wm = intel_update_sprite_wm;
 			} else {
 				DRM_DEBUG_KMS("Failed to read display plane latency. "
 					      "Disable CxSR\n");
@@ -4461,9 +4740,8 @@ void intel_init_pm(struct drm_device *dev)
 			dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
 		} else if (IS_HASWELL(dev)) {
 			if (SNB_READ_WM0_LATENCY()) {
-				dev_priv->display.update_wm = sandybridge_update_wm;
-				dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
-				dev_priv->display.update_linetime_wm = haswell_update_linetime_wm;
+				dev_priv->display.update_wm = intel_update_wm;
+				dev_priv->display.update_sprite_wm = intel_update_sprite_wm;
 			} else {
 				DRM_DEBUG_KMS("Failed to read display plane latency. "
 					      "Disable CxSR\n");
diff --git a/drivers/gpu/drm/i915/intel_sprite.c b/drivers/gpu/drm/i915/intel_sprite.c
index 19b9cb9..426e7d5 100644
--- a/drivers/gpu/drm/i915/intel_sprite.c
+++ b/drivers/gpu/drm/i915/intel_sprite.c
@@ -114,8 +114,6 @@ vlv_update_plane(struct drm_plane *dplane, struct drm_framebuffer *fb,
 	crtc_w--;
 	crtc_h--;
 
-	intel_update_sprite_watermarks(dev, pipe, crtc_w, pixel_size);
-
 	I915_WRITE(SPSTRIDE(pipe, plane), fb->pitches[0]);
 	I915_WRITE(SPPOS(pipe, plane), (crtc_y << 16) | crtc_x);
 
@@ -219,7 +217,6 @@ ivb_update_plane(struct drm_plane *plane, struct drm_framebuffer *fb,
 	u32 sprctl, sprscale = 0;
 	unsigned long sprsurf_offset, linear_offset;
 	int pixel_size = drm_format_plane_cpp(fb->pixel_format, 0);
-	bool scaling_was_enabled = dev_priv->sprite_scaling_enabled;
 
 	sprctl = I915_READ(SPRCTL(pipe));
 
@@ -268,23 +265,13 @@ ivb_update_plane(struct drm_plane *plane, struct drm_framebuffer *fb,
 	crtc_w--;
 	crtc_h--;
 
-	intel_update_sprite_watermarks(dev, pipe, crtc_w, pixel_size);
-
 	/*
 	 * IVB workaround: must disable low power watermarks for at least
 	 * one frame before enabling scaling.  LP watermarks can be re-enabled
 	 * when scaling is disabled.
 	 */
-	if (crtc_w != src_w || crtc_h != src_h) {
-		dev_priv->sprite_scaling_enabled |= 1 << pipe;
-
-		if (!scaling_was_enabled) {
-			intel_update_watermarks(dev);
-			intel_wait_for_vblank(dev, pipe);
-		}
+	if (crtc_w != src_w || crtc_h != src_h)
 		sprscale = SPRITE_SCALE_ENABLE | (src_w << 16) | src_h;
-	} else
-		dev_priv->sprite_scaling_enabled &= ~(1 << pipe);
 
 	I915_WRITE(SPRSTRIDE(pipe), fb->pitches[0]);
 	I915_WRITE(SPRPOS(pipe), (crtc_y << 16) | crtc_x);
@@ -310,10 +297,6 @@ ivb_update_plane(struct drm_plane *plane, struct drm_framebuffer *fb,
 	I915_WRITE(SPRCTL(pipe), sprctl);
 	I915_MODIFY_DISPBASE(SPRSURF(pipe), obj->gtt_offset + sprsurf_offset);
 	POSTING_READ(SPRSURF(pipe));
-
-	/* potentially re-enable LP watermarks */
-	if (scaling_was_enabled && !dev_priv->sprite_scaling_enabled)
-		intel_update_watermarks(dev);
 }
 
 static void
@@ -323,7 +306,6 @@ ivb_disable_plane(struct drm_plane *plane)
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_plane *intel_plane = to_intel_plane(plane);
 	int pipe = intel_plane->pipe;
-	bool scaling_was_enabled = dev_priv->sprite_scaling_enabled;
 
 	I915_WRITE(SPRCTL(pipe), I915_READ(SPRCTL(pipe)) & ~SPRITE_ENABLE);
 	/* Can't leave the scaler enabled... */
@@ -332,12 +314,6 @@ ivb_disable_plane(struct drm_plane *plane)
 	/* Activate double buffered register update */
 	I915_MODIFY_DISPBASE(SPRSURF(pipe), 0);
 	POSTING_READ(SPRSURF(pipe));
-
-	dev_priv->sprite_scaling_enabled &= ~(1 << pipe);
-
-	/* potentially re-enable LP watermarks */
-	if (scaling_was_enabled && !dev_priv->sprite_scaling_enabled)
-		intel_update_watermarks(dev);
 }
 
 static int
@@ -453,8 +429,6 @@ ilk_update_plane(struct drm_plane *plane, struct drm_framebuffer *fb,
 	crtc_w--;
 	crtc_h--;
 
-	intel_update_sprite_watermarks(dev, pipe, crtc_w, pixel_size);
-
 	dvsscale = 0;
 	if (IS_GEN5(dev) || crtc_w != src_w || crtc_h != src_h)
 		dvsscale = DVS_SCALE_ENABLE | (src_w << 16) | src_h;
@@ -808,6 +782,11 @@ intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
 
 	intel_plane->obj = obj;
 
+	intel_update_sprite_watermarks(dev, pipe, intel_plane->plane,
+				       visible,
+				       src_w != crtc_w || src_h != crtc_h,
+				       src_w, pixel_size);
+
 	/*
 	 * Be sure to re-enable the primary before the sprite is no longer
 	 * covering it fully.
@@ -853,6 +832,11 @@ intel_disable_plane(struct drm_plane *plane)
 	struct intel_plane *intel_plane = to_intel_plane(plane);
 	int ret = 0;
 
+	intel_update_sprite_watermarks(dev,
+				       intel_plane->pipe,
+				       intel_plane->plane,
+				       false, false, 0, 0);
+
 	if (plane->crtc)
 		intel_enable_primary(plane->crtc);
 	intel_plane->disable_plane(plane);
