[2/2] v4l2-utils: fix overflow in cvt, gtf calculations
diff mbox

Message ID 1430736539-28469-3-git-send-email-prladdha@cisco.com
State New
Headers show

Commit Message

Prashant Laddha May 4, 2015, 10:48 a.m. UTC
Some of the intermediate calculations can exceed 32 bit signed range,
especially for higher resolutions and refresh rates. Type casting the
intermediate values to higher precision to avoid overflow.

Cc: Hans Verkuil <hans.verkuil@cisco.com>
Signed-off-by: Prashant Laddha <prladdha@cisco.com>
---
 utils/v4l2-ctl/v4l2-ctl-modes.cpp | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

Patch
diff mbox

diff --git a/utils/v4l2-ctl/v4l2-ctl-modes.cpp b/utils/v4l2-ctl/v4l2-ctl-modes.cpp
index 4689006..072763a 100644
--- a/utils/v4l2-ctl/v4l2-ctl-modes.cpp
+++ b/utils/v4l2-ctl/v4l2-ctl-modes.cpp
@@ -216,7 +216,7 @@  bool calc_cvt_modeline(int image_width, int image_height,
 		if (ideal_blank_duty_cycle < 20 * HV_FACTOR)
 			ideal_blank_duty_cycle = 20 * HV_FACTOR;
 
-		h_blank = active_h_pixel * ideal_blank_duty_cycle /
+		h_blank = active_h_pixel * (long long)ideal_blank_duty_cycle /
 			 (100 * HV_FACTOR - ideal_blank_duty_cycle);
 		h_blank -= h_blank % (2 * CVT_CELL_GRAN);
 
@@ -430,7 +430,6 @@  bool calc_gtf_modeline(int image_width, int image_height,
 	tmp2 = active_v_lines + GTF_MIN_PORCH + interlace;
 
 	h_period_est = tmp1 / (tmp2 * v_refresh);
-
 	v_sync_bp = GTF_MIN_VSYNC_BP * HV_FACTOR * 100 / h_period_est;
 	v_sync_bp = (v_sync_bp + 50) / 100;
 
@@ -444,7 +443,7 @@  bool calc_gtf_modeline(int image_width, int image_height,
 	v_refresh_est = (HV_FACTOR * (long long)1000000) /
 			(h_period_est * total_v_lines / HV_FACTOR);
 
-	h_period = (h_period_est * v_refresh_est) /
+	h_period = ((long long)h_period_est * v_refresh_est) /
 		   (v_refresh * HV_FACTOR);
 
 	if (!reduced_blanking)
@@ -455,7 +454,7 @@  bool calc_gtf_modeline(int image_width, int image_height,
 				      GTF_S_M_PRIME * h_period / 1000;
 
 
-	h_blank = active_h_pixel * ideal_blank_duty_cycle /
+	h_blank = active_h_pixel * (long long)ideal_blank_duty_cycle /
 			 (100 * HV_FACTOR - ideal_blank_duty_cycle);
 	h_blank = ((h_blank + GTF_CELL_GRAN) / (2 * GTF_CELL_GRAN))
 			  * (2 * GTF_CELL_GRAN);
@@ -467,7 +466,6 @@  bool calc_gtf_modeline(int image_width, int image_height,
 
 	h_fp = h_blank / 2 - h_sync;
 	h_bp = h_fp + h_sync;
-
 	pixel_clock = ((long long)total_h_pixel * HV_FACTOR * 1000000)
 					/ h_period;
 	/* Not sure if clock value needs to be truncated to multiple