diff --git a/src/gen-image.c b/src/gen-image.c
index 40773c8ef967..d2d4870a699a 100644
--- a/src/gen-image.c
+++ b/src/gen-image.c
@@ -205,6 +205,25 @@ static const struct format_info *format_by_name(const char *name)
 	return NULL;
 }
 
+static const struct format_info *format_by_type(enum format_type type)
+{
+	const char *name = "";
+
+	switch (type) {
+	case FORMAT_RGB:
+		name = "RGB24";
+		break;
+	case FORMAT_YUV:
+		name = "YUV24";
+		break;
+	case FORMAT_HSV:
+		name = "HSV24";
+		break;
+	}
+
+	return format_by_name(name);
+}
+
 /* -----------------------------------------------------------------------------
  * File I/O
  */
@@ -309,6 +328,13 @@ static void image_delete(struct image *image)
 	free(image);
 }
 
+static void image_move(struct image *src, struct image *dst)
+{
+	free(dst->data);
+	*dst = *src;
+	free(src);
+}
+
 /* -----------------------------------------------------------------------------
  * Image read and write
  */
@@ -707,9 +733,34 @@ static void image_format_yuv_planar(const struct image *input, struct image *out
  * Format conversion (as performed by the Renesas VSP HST, HSI, RPF and WPF)
  */
 
-static void colorspace_matrix(enum v4l2_ycbcr_encoding encoding,
-			      enum v4l2_quantization quantization,
-			      int (*matrix)[3][3])
+/* RGB to RGB (downsampling only) */
+static void image_convert_rgb_to_rgb(const struct image *input,
+				     struct image *output,
+				     const struct format_info *format,
+				     const struct csc_params *params)
+{
+	const uint8_t *idata = input->data;
+	uint8_t *odata = output->data;
+	unsigned int x;
+	unsigned int y;
+	uint8_t r, g, b;
+
+	for (y = 0; y < output->height; ++y) {
+		for (x = 0; x < output->width; ++x) {
+			r = *idata++ & (0xff << (8 - format->rgb.red.length));
+			g = *idata++ & (0xff << (8 - format->rgb.green.length));
+			b = *idata++ & (0xff << (8 - format->rgb.blue.length));
+			*odata++ = r;
+			*odata++ = g;
+			*odata++ = b;
+		}
+	}
+}
+
+/* RGB to YUV */
+static void csc_matrix(enum v4l2_ycbcr_encoding encoding,
+		       enum v4l2_quantization quantization,
+		       int (*matrix)[3][3])
 {
 	/*
 	 * The value of the coefficients has been reverse-engineered by
@@ -769,9 +820,8 @@ static void colorspace_matrix(enum v4l2_ycbcr_encoding encoding,
 		memcpy((*matrix)[i], (*m)[i], sizeof((*m)[i]));
 }
 
-static void colorspace_rgb2ycbcr(int m[3][3],
-				 enum v4l2_quantization quantization,
-				 const uint8_t rgb[3], uint8_t ycbcr[3])
+static void csc_rgb_to_yuv(int m[3][3], enum v4l2_quantization quantization,
+			   const uint8_t rgb[3], uint8_t ycbcr[3])
 {
 	bool full = quantization == V4L2_QUANTIZATION_FULL_RANGE;
 	int y_min = full ? 0 : 16;
@@ -798,10 +848,10 @@ static void colorspace_rgb2ycbcr(int m[3][3],
 	ycbcr[2] = CLAMP(cr, cbcr_min, cbcr_max);
 }
 
-static void image_colorspace_rgb_to_yuv(const struct image *input,
-					struct image *output,
-					const struct format_info *format,
-					const struct csc_params *params)
+static void image_convert_rgb_to_yuv(const struct image *input,
+				     struct image *output,
+				     const struct format_info *format,
+				     const struct csc_params *params)
 {
 	int matrix[3][3];
 	const uint8_t *idata = input->data;
@@ -809,12 +859,12 @@ static void image_colorspace_rgb_to_yuv(const struct image *input,
 	unsigned int x;
 	unsigned int y;
 
-	colorspace_matrix(params->encoding, params->quantization, &matrix);
+	csc_matrix(params->encoding, params->quantization, &matrix);
 
 	for (y = 0; y < output->height; ++y) {
 		for (x = 0; x < output->width; ++x) {
-			colorspace_rgb2ycbcr(matrix, params->quantization,
-					     &idata[3*x], &odata[3*x]);
+			csc_rgb_to_yuv(matrix, params->quantization,
+				       &idata[3*x], &odata[3*x]);
 		}
 		if (format->yuv.xsub == 2) {
 			for (x = 1; x < output->width - 1; x += 2) {
@@ -827,28 +877,7 @@ static void image_colorspace_rgb_to_yuv(const struct image *input,
 	}
 }
 
-static void image_convert_rgb_to_rgb(const struct image *input,
-				     struct image *output,
-				     const struct format_info *format)
-{
-	const uint8_t *idata = input->data;
-	uint8_t *odata = output->data;
-	unsigned int x;
-	unsigned int y;
-	uint8_t r, g, b;
-
-	for (y = 0; y < output->height; ++y) {
-		for (x = 0; x < output->width; ++x) {
-			r = *idata++ & (0xff << (8 - format->rgb.red.length));
-			g = *idata++ & (0xff << (8 - format->rgb.green.length));
-			b = *idata++ & (0xff << (8 - format->rgb.blue.length));
-			*odata++ = r;
-			*odata++ = g;
-			*odata++ = b;
-		}
-	}
-}
-
+/* RGB to HSV */
 #define K 4
 static uint8_t hst_calc_h(uint8_t r, uint8_t g, uint8_t b)
 {
@@ -944,9 +973,10 @@ static void hst_rgb_to_hsv(const uint8_t rgb[3], uint8_t hsv[3])
 	hsv[2] = hst_calc_v(rgb[0], rgb[1], rgb[2]);
 }
 
-static void image_rgb_to_hsv(const struct image *input,
-			     struct image *output,
-			     const struct csc_params *params)
+static void image_convert_rgb_to_hsv(const struct image *input,
+				     struct image *output,
+				     const struct format_info *format,
+				     const struct csc_params *params)
 {
 	const uint8_t *idata = input->data;
 	uint8_t *odata = output->data;
@@ -962,6 +992,78 @@ static void image_rgb_to_hsv(const struct image *input,
 	}
 }
 
+typedef void (*image_convert_func)(const struct image *input,
+				   struct image *output,
+				   const struct format_info *format,
+				   const struct csc_params *params);
+
+struct image_converter {
+	enum format_type input;
+	enum format_type output;
+	image_convert_func convert;
+};
+
+static const struct image_converter image_converters[] = {
+	{ FORMAT_RGB, FORMAT_HSV, image_convert_rgb_to_hsv },
+	{ FORMAT_RGB, FORMAT_RGB, image_convert_rgb_to_rgb },
+	{ FORMAT_RGB, FORMAT_YUV, image_convert_rgb_to_yuv },
+};
+
+static int image_convert(struct image *input, const struct format_info *format,
+			 const struct csc_params *params)
+{
+	const struct image_converter *converter;
+	const struct format_info *image_format;
+	struct image *output;
+	unsigned int i;
+
+	/*
+	 * Converters combines two tasks: they convert between 24-bit RGB, YUV
+	 * and HSV formats (as done by the VSP RPF and WPF CSC blocks for RGB
+	 * <-> YUV conversion, and the HSI and HST blocks for RGB <-> HSV
+	 * conversion), and they emulate the effects of reading downsampled
+	 * formats (RGB with less than 24bpp, or subsampled YUV). Combining
+	 * those operations makes the converter suitable to efficiently emulate
+	 *
+	 * - RPF unpacking
+	 * - RPF color space conversion
+	 * - HST and HSI
+	 * - WPF color space conversion
+	 *
+	 * The input image must be in a 24-bit format. The output format
+	 * parameter can be either a 24-bit format (when emulating RPF/WPF CSC,
+	 * HST or HSI), or a downsampled RGB or YUV format (when emulating RPF
+	 * unpacking). In all cases, the converter image will be in a 24-bit
+	 * format.
+	 */
+	image_format = format_by_type(format->type);
+
+	/* Find an appropriate converter. */
+	for (i = 0; i < ARRAY_SIZE(image_converters); ++i) {
+		converter = &image_converters[i];
+
+		if (converter->input == input->format->type &&
+		    converter->output == format->type)
+			break;
+	}
+
+	if (i == ARRAY_SIZE(image_converters)) {
+		printf("Conversion from %s to %s is not supported\n",
+		       input->format->name, format->name);
+		return -EINVAL;
+	}
+
+	output = image_new(image_format, input->width, input->height);
+	if (!output)
+		return -ENOMEM;
+
+	converter->convert(input, output, format, params);
+
+	image_move(output, input);
+
+	return 0;
+}
+
 /* -----------------------------------------------------------------------------
  * Image scaling
  */
@@ -1555,34 +1657,12 @@ static int process(const struct options *options)
 		input = cropped;
 	}
 
-	/* Convert colorspace */
-	if (options->input_format->type == FORMAT_YUV) {
-		struct image *yuv;
-
-		yuv = image_new(format_by_name("YUV24"), input->width,
-				input->height);
-		if (!yuv) {
-			ret = -ENOMEM;
+	/* Convert to the input format */
+	if (options->input_format != input->format) {
+		ret = image_convert(input, options->input_format,
+				    &options->csc_params);
+		if (ret)
 			goto done;
-		}
-
-		image_colorspace_rgb_to_yuv(input, yuv, options->input_format,
-					    &options->csc_params);
-		image_delete(input);
-		input = yuv;
-	} else if (options->input_format->rgb.bpp < 24) {
-		struct image *rgb;
-
-		rgb = image_new(format_by_name("RGB24"), input->width,
-				input->height);
-		if (!rgb) {
-			ret = -ENOMEM;
-			goto done;
-		}
-
-		image_convert_rgb_to_rgb(input, rgb, options->input_format);
-		image_delete(input);
-		input = rgb;
 	}
 
 	/* Scale */
@@ -1694,36 +1774,13 @@ static int process(const struct options *options)
 	}
 
 	/* Format the output */
-	if (input->format->type != options->output_format->type &&
-	    input->format->type != FORMAT_RGB) {
-		printf("Format conversion with non-RGB input not supported\n");
-		ret = -EINVAL;
-		goto done;
-	}
-
 	if (input->format->type != options->output_format->type) {
-		const struct format_info *format;
-		struct image *converted;
+		const struct format_info *format =
+			format_by_type(options->output_format->type);
 
-		if (options->output_format->type == FORMAT_YUV)
-			format = format_by_name("YUV24");
-		else
-			format = format_by_name("HSV24");
-
-		converted = image_new(format, input->width, input->height);
-		if (!converted) {
-			ret = -ENOMEM;
+		ret = image_convert(input, format, &options->csc_params);
+		if (ret)
 			goto done;
-		}
-
-		if (options->output_format->type == FORMAT_YUV)
-			image_colorspace_rgb_to_yuv(input, converted, format,
-						    &options->csc_params);
-		else
-			image_rgb_to_hsv(input, converted, &options->csc_params);
-
-		image_delete(input);
-		input = converted;
 	}
 
 	output = image_new(options->output_format, input->width, input->height);
