| Message ID | 20181123130209.11696-2-paul.kocialkowski@bootlin.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | HEVC/H.265 stateless support for V4L2 and Cedrus | expand |
On 11/23/18 14:02, Paul Kocialkowski wrote: > This introduces the required definitions for HEVC decoding support with > stateless VPUs. The controls associated to the HEVC slice format provide > the required meta-data for decoding slices extracted from the bitstream. > > This interface comes with the following limitations: > * No custom quantization matrices (scaling lists); > * Support for a single temporal layer only; > * No slice entry point offsets support; > * No conformance window support; > * No VUI parameters support; > * No support for SPS extensions: range, multilayer, 3d, scc, 4 bits; > * No support for PPS extensions: range, multilayer, 3d, scc, 4 bits. So if support for one or more of these items would have to be added, would that be just new controls, or would it affect existing controls? > > Signed-off-by: Paul Kocialkowski <paul.kocialkowski@bootlin.com> > --- > Documentation/media/uapi/v4l/biblio.rst | 9 + > .../media/uapi/v4l/extended-controls.rst | 417 ++++++++++++++++++ > .../media/uapi/v4l/pixfmt-compressed.rst | 15 + > .../media/uapi/v4l/vidioc-queryctrl.rst | 18 + > .../media/videodev2.h.rst.exceptions | 3 + > drivers/media/v4l2-core/v4l2-ctrls.c | 26 ++ > drivers/media/v4l2-core/v4l2-ioctl.c | 1 + > include/media/v4l2-ctrls.h | 6 + > include/uapi/linux/v4l2-controls.h | 155 +++++++ > include/uapi/linux/v4l2-controls.h.rej | 187 -------- Huh? .rej? > include/uapi/linux/videodev2.h | 7 + > 11 files changed, 657 insertions(+), 187 deletions(-) > delete mode 100644 include/uapi/linux/v4l2-controls.h.rej > > diff --git a/Documentation/media/uapi/v4l/biblio.rst b/Documentation/media/uapi/v4l/biblio.rst > index 73aeb7ce47d2..59a98feca3a1 100644 > --- a/Documentation/media/uapi/v4l/biblio.rst > +++ b/Documentation/media/uapi/v4l/biblio.rst > @@ -124,6 +124,15 @@ ITU H.264 > > :author: International Telecommunication Union (http://www.itu.ch) > > +.. _hevc: > + > +ITU H.265/HEVC > +============== > + > +:title: ITU-T Rec. H.265 | ISO/IEC 23008-2 "High Efficiency Video Coding" > + > +:author: International Telecommunication Union (http://www.itu.ch), International Organisation for Standardisation (http://www.iso.ch) > + > .. _jfif: > > JFIF > diff --git a/Documentation/media/uapi/v4l/extended-controls.rst b/Documentation/media/uapi/v4l/extended-controls.rst > index 87c0d151577f..906ff4f32634 100644 > --- a/Documentation/media/uapi/v4l/extended-controls.rst > +++ b/Documentation/media/uapi/v4l/extended-controls.rst > @@ -2038,6 +2038,423 @@ enum v4l2_mpeg_video_h264_hierarchical_coding_type - > - ``flags`` > - > > +.. _v4l2-mpeg-hevc: > + > +``V4L2_CID_MPEG_VIDEO_HEVC_SPS (struct)`` > + Specifies the Sequence Parameter Set fields (as extracted from the > + bitstream) for the associated HEVC slice data. > + These bitstream parameters are defined according to :ref:`hevc`. > + Refer to the specification for the documentation of these fields. Same as for h264: if possible, refer to the section(s) in the spec that deal with these fields. > + > +.. c:type:: v4l2_ctrl_hevc_sps > + > +.. cssclass:: longtable > + > +.. flat-table:: struct v4l2_ctrl_hevc_sps > + :header-rows: 0 > + :stub-columns: 0 > + :widths: 1 1 2 > + > + * - __u8 > + - ``chroma_format_idc`` > + - > + * - __u8 > + - ``separate_colour_plane_flag`` > + - > + * - __u16 > + - ``pic_width_in_luma_samples`` > + - > + * - __u16 > + - ``pic_height_in_luma_samples`` > + - > + * - __u8 > + - ``bit_depth_luma_minus8`` > + - > + * - __u8 > + - ``bit_depth_chroma_minus8`` > + - > + * - __u8 > + - ``log2_max_pic_order_cnt_lsb_minus4`` > + - > + * - __u8 > + - ``sps_max_dec_pic_buffering_minus1`` > + - > + * - __u8 > + - ``sps_max_num_reorder_pics`` > + - > + * - __u8 > + - ``sps_max_latency_increase_plus1`` > + - > + * - __u8 > + - ``log2_min_luma_coding_block_size_minus3`` > + - > + * - __u8 > + - ``log2_diff_max_min_luma_coding_block_size`` > + - > + * - __u8 > + - ``log2_min_luma_transform_block_size_minus2`` > + - > + * - __u8 > + - ``log2_diff_max_min_luma_transform_block_size`` > + - > + * - __u8 > + - ``max_transform_hierarchy_depth_inter`` > + - > + * - __u8 > + - ``max_transform_hierarchy_depth_intra`` > + - > + * - __u8 > + - ``scaling_list_enabled_flag`` > + - > + * - __u8 > + - ``amp_enabled_flag`` > + - > + * - __u8 > + - ``sample_adaptive_offset_enabled_flag`` > + - > + * - __u8 > + - ``pcm_enabled_flag`` > + - > + * - __u8 > + - ``pcm_sample_bit_depth_luma_minus1`` > + - > + * - __u8 > + - ``pcm_sample_bit_depth_chroma_minus1`` > + - > + * - __u8 > + - ``log2_min_pcm_luma_coding_block_size_minus3`` > + - > + * - __u8 > + - ``log2_diff_max_min_pcm_luma_coding_block_size`` > + - > + * - __u8 > + - ``pcm_loop_filter_disabled_flag`` > + - > + * - __u8 > + - ``num_short_term_ref_pic_sets`` > + - > + * - __u8 > + - ``long_term_ref_pics_present_flag`` > + - > + * - __u8 > + - ``num_long_term_ref_pics_sps`` > + - > + * - __u8 > + - ``sps_temporal_mvp_enabled_flag`` > + - > + * - __u8 > + - ``strong_intra_smoothing_enabled_flag`` > + - > + > +``V4L2_CID_MPEG_VIDEO_HEVC_PPS (struct)`` > + Specifies the Picture Parameter Set fields (as extracted from the > + bitstream) for the associated HEVC slice data. > + These bitstream parameters are defined according to :ref:`hevc`. > + Refer to the specification for the documentation of these fields. > + > +.. c:type:: v4l2_ctrl_hevc_pps > + > +.. cssclass:: longtable > + > +.. flat-table:: struct v4l2_ctrl_hevc_pps > + :header-rows: 0 > + :stub-columns: 0 > + :widths: 1 1 2 > + > + * - __u8 > + - ``dependent_slice_segment_flag`` > + - > + * - __u8 > + - ``output_flag_present_flag`` > + - > + * - __u8 > + - ``num_extra_slice_header_bits`` > + - > + * - __u8 > + - ``sign_data_hiding_enabled_flag`` > + - > + * - __u8 > + - ``cabac_init_present_flag`` > + - > + * - __s8 > + - ``init_qp_minus26`` > + - > + * - __u8 > + - ``constrained_intra_pred_flag`` > + - > + * - __u8 > + - ``transform_skip_enabled_flag`` > + - > + * - __u8 > + - ``cu_qp_delta_enabled_flag`` > + - > + * - __u8 > + - ``diff_cu_qp_delta_depth`` > + - > + * - __s8 > + - ``pps_cb_qp_offset`` > + - > + * - __s8 > + - ``pps_cr_qp_offset`` > + - > + * - __u8 > + - ``pps_slice_chroma_qp_offsets_present_flag`` > + - > + * - __u8 > + - ``weighted_pred_flag`` > + - > + * - __u8 > + - ``weighted_bipred_flag`` > + - > + * - __u8 > + - ``transquant_bypass_enabled_flag`` > + - > + * - __u8 > + - ``tiles_enabled_flag`` > + - > + * - __u8 > + - ``entropy_coding_sync_enabled_flag`` > + - > + * - __u8 > + - ``num_tile_columns_minus1`` > + - > + * - __u8 > + - ``num_tile_rows_minus1`` > + - > + * - __u8 > + - ``column_width_minus1[20]`` > + - > + * - __u8 > + - ``row_height_minus1[22]`` > + - > + * - __u8 > + - ``loop_filter_across_tiles_enabled_flag`` > + - > + * - __u8 > + - ``pps_loop_filter_across_slices_enabled_flag`` > + - > + * - __u8 > + - ``deblocking_filter_override_enabled_flag`` > + - > + * - __u8 > + - ``pps_disable_deblocking_filter_flag`` > + - > + * - __s8 > + - ``pps_beta_offset_div2`` > + - > + * - __s8 > + - ``pps_tc_offset_div2`` > + - > + * - __u8 > + - ``lists_modification_present_flag`` > + - > + * - __u8 > + - ``log2_parallel_merge_level_minus2`` > + - > + * - __u8 > + - ``slice_segment_header_extension_present_flag`` > + - > + > +``V4L2_CID_MPEG_VIDEO_HEVC_SLICE_PARAMS (struct)`` > + Specifies various slice-specific parameters, especially from the NAL unit > + header, general slice segment header and weighted prediction parameter > + parts of the bitstream. > + These bitstream parameters are defined according to :ref:`hevc`. > + Refer to the specification for the documentation of these fields. > + > +.. c:type:: v4l2_ctrl_hevc_slice_params > + > +.. cssclass:: longtable > + > +.. flat-table:: struct v4l2_ctrl_hevc_slice_params > + :header-rows: 0 > + :stub-columns: 0 > + :widths: 1 1 2 > + > + * - __u32 > + - ``bit_size`` > + - Size (in bits) of the current slice data. > + * - __u32 > + - ``data_bit_offset`` > + - Offset (in bits) to the video data in the current slice data. > + * - __u8 > + - ``nal_unit_type`` > + - > + * - __u8 > + - ``nuh_temporal_id_plus1`` > + - > + * - __u8 > + - ``slice_type`` > + - > + (V4L2_HEVC_SLICE_TYPE_I, V4L2_HEVC_SLICE_TYPE_P or > + V4L2_HEVC_SLICE_TYPE_B). > + * - __u8 > + - ``colour_plane_id`` > + - > + * - __u16 > + - ``slice_pic_order_cnt`` > + - > + * - __u8 > + - ``slice_sao_luma_flag`` > + - > + * - __u8 > + - ``slice_sao_chroma_flag`` > + - > + * - __u8 > + - ``slice_temporal_mvp_enabled_flag`` > + - > + * - __u8 > + - ``num_ref_idx_l0_active_minus1`` > + - > + * - __u8 > + - ``num_ref_idx_l1_active_minus1`` > + - > + * - __u8 > + - ``mvd_l1_zero_flag`` > + - > + * - __u8 > + - ``cabac_init_flag`` > + - > + * - __u8 > + - ``collocated_from_l0_flag`` > + - > + * - __u8 > + - ``collocated_ref_idx`` > + - > + * - __u8 > + - ``five_minus_max_num_merge_cand`` > + - > + * - __u8 > + - ``use_integer_mv_flag`` > + - > + * - __s8 > + - ``slice_qp_delta`` > + - > + * - __s8 > + - ``slice_cb_qp_offset`` > + - > + * - __s8 > + - ``slice_cr_qp_offset`` > + - > + * - __s8 > + - ``slice_act_y_qp_offset`` > + - > + * - __s8 > + - ``slice_act_cb_qp_offset`` > + - > + * - __s8 > + - ``slice_act_cr_qp_offset`` > + - > + * - __u8 > + - ``slice_deblocking_filter_disabled_flag`` > + - > + * - __s8 > + - ``slice_beta_offset_div2`` > + - > + * - __s8 > + - ``slice_tc_offset_div2`` > + - > + * - __u8 > + - ``slice_loop_filter_across_slices_enabled_flag`` > + - > + * - __u8 > + - ``pic_struct`` > + - > + * - struct :c:type:`v4l2_hevc_dpb_entry` > + - ``dpb[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]`` > + - The decoded picture buffer, for meta-data about reference frames. > + * - __u8 > + - ``num_active_dpb_entries`` > + - The number of entries in ``dpb``. > + * - __u8 > + - ``ref_idx_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]`` > + - The list of L0 reference elements as indices in the DPB. > + * - __u8 > + - ``ref_idx_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]`` > + - The list of L1 reference elements as indices in the DPB. > + * - __u8 > + - ``num_rps_poc_st_curr_before`` > + - The number of reference pictures in the short-term set that come before > + the current frame. > + * - __u8 > + - ``num_rps_poc_st_curr_after`` > + - The number of reference pictures in the short-term set that come after > + the current frame. > + * - __u8 > + - ``num_rps_poc_lt_curr`` > + - The number of reference pictures in the long-term set. > + * - struct :c:type:`v4l2_hevc_pred_weight_table` > + - ``pred_weight_table`` > + - The prediction weight coefficients for inter-picture prediction. > + > +.. c:type:: v4l2_hevc_dpb_entry > + > +.. cssclass:: longtable > + > +.. flat-table:: struct v4l2_hevc_dpb_entry > + :header-rows: 0 > + :stub-columns: 0 > + :widths: 1 1 2 > + > + * - __u32 > + - ``buffer_tag`` It's called 'tag' in v4l2_h264_dpb_entry. Probably a good idea to keep the same terminology. I have no preference, as long as the names are consistent. > + - The V4L2 buffer tag that matches the associated reference picture. > + * - __u8 > + - ``rps`` > + - The reference set for the reference frame > + (V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_BEFORE, > + V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_AFTER or > + V4L2_HEVC_DPB_ENTRY_RPS_LT_CURR) > + * - __u8 > + - ``field_pic`` > + - Whether the reference is a field picture or a frame. > + * - __u16 > + - ``pic_order_cnt[2]`` > + - The picture order count of the reference. Only the first element of the > + array is used for frame pictures, while the first element identifies the > + top field and the second the bottom field in field-coded pictures. > + > +.. c:type:: v4l2_hevc_pred_weight_table > + > +.. cssclass:: longtable > + > +.. flat-table:: struct v4l2_hevc_pred_weight_table > + :header-rows: 0 > + :stub-columns: 0 > + :widths: 1 1 2 > + > + * - __u8 > + - ``luma_log2_weight_denom`` > + - > + * - __s8 > + - ``delta_chroma_log2_weight_denom`` > + - > + * - __s8 > + - ``delta_luma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]`` > + - > + * - __s8 > + - ``luma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]`` > + - > + * - __s8 > + - ``delta_chroma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]`` > + - > + * - __s8 > + - ``chroma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]`` > + - > + * - __s8 > + - ``delta_luma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]`` > + - > + * - __s8 > + - ``luma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]`` > + - > + * - __s8 > + - ``delta_chroma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]`` > + - > + * - __s8 > + - ``chroma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]`` > + - > + > + > MFC 5.1 MPEG Controls > --------------------- > > diff --git a/Documentation/media/uapi/v4l/pixfmt-compressed.rst b/Documentation/media/uapi/v4l/pixfmt-compressed.rst > index f15fc1c8d479..572a43bfe9c9 100644 > --- a/Documentation/media/uapi/v4l/pixfmt-compressed.rst > +++ b/Documentation/media/uapi/v4l/pixfmt-compressed.rst > @@ -131,6 +131,21 @@ Compressed Formats > - ``V4L2_PIX_FMT_HEVC`` > - 'HEVC' > - HEVC/H.265 video elementary stream. > + * .. _V4L2-PIX-FMT-HEVC-SLICE: > + > + - ``V4L2_PIX_FMT_HEVC_SLICE`` > + - 'S265' > + - HEVC parsed slice data, as extracted from the HEVC bitstream. > + This format is adapted for stateless video decoders that implement a > + HEVC pipeline (using the :ref:`codec` and :ref:`media-request-api`). > + Metadata associated with the frame to decode is required to be passed > + through the following controls : > + * ``V4L2_CID_MPEG_VIDEO_HEVC_SPS`` > + * ``V4L2_CID_MPEG_VIDEO_HEVC_PPS`` > + * ``V4L2_CID_MPEG_VIDEO_HEVC_SLICE_PARAMS`` > + See the :ref:`associated Codec Control IDs <v4l2-mpeg-hevc>`. > + Buffers associated with this pixel format must contain the appropriate > + number of macroblocks to decode a full corresponding frame. > * .. _V4L2-PIX-FMT-FWHT: > > - ``V4L2_PIX_FMT_FWHT`` > diff --git a/Documentation/media/uapi/v4l/vidioc-queryctrl.rst b/Documentation/media/uapi/v4l/vidioc-queryctrl.rst > index 38a9c988124c..8e0cc836058d 100644 > --- a/Documentation/media/uapi/v4l/vidioc-queryctrl.rst > +++ b/Documentation/media/uapi/v4l/vidioc-queryctrl.rst > @@ -466,6 +466,24 @@ See also the examples in :ref:`control`. > - n/a > - A struct :c:type:`v4l2_ctrl_h264_decode_param`, containing H264 > decode parameters for stateless video decoders. > + * - ``V4L2_CTRL_TYPE_HEVC_SPS`` > + - n/a > + - n/a > + - n/a > + - A struct :c:type:`v4l2_ctrl_hevc_sps`, containing HEVC Sequence > + Parameter Set for stateless video decoders. > + * - ``V4L2_CTRL_TYPE_HEVC_PPS`` > + - n/a > + - n/a > + - n/a > + - A struct :c:type:`v4l2_ctrl_hevc_pps`, containing HEVC Picture > + Parameter Set for stateless video decoders. > + * - ``V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS`` > + - n/a > + - n/a > + - n/a > + - A struct :c:type:`v4l2_ctrl_hevc_slice_params`, containing HEVC > + slice parameters for stateless video decoders. > > .. tabularcolumns:: |p{6.6cm}|p{2.2cm}|p{8.7cm}| > > diff --git a/Documentation/media/videodev2.h.rst.exceptions b/Documentation/media/videodev2.h.rst.exceptions > index 99f1bd2bc44c..27978d8b18f5 100644 > --- a/Documentation/media/videodev2.h.rst.exceptions > +++ b/Documentation/media/videodev2.h.rst.exceptions > @@ -138,6 +138,9 @@ replace symbol V4L2_CTRL_TYPE_H264_PPS :c:type:`v4l2_ctrl_type` > replace symbol V4L2_CTRL_TYPE_H264_SCALING_MATRIX :c:type:`v4l2_ctrl_type` > replace symbol V4L2_CTRL_TYPE_H264_SLICE_PARAMS :c:type:`v4l2_ctrl_type` > replace symbol V4L2_CTRL_TYPE_H264_DECODE_PARAMS :c:type:`v4l2_ctrl_type` > +replace symbol V4L2_CTRL_TYPE_HEVC_SPS :c:type:`v4l2_ctrl_type` > +replace symbol V4L2_CTRL_TYPE_HEVC_PPS :c:type:`v4l2_ctrl_type` > +replace symbol V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS :c:type:`v4l2_ctrl_type` > > # V4L2 capability defines > replace define V4L2_CAP_VIDEO_CAPTURE device-capabilities > diff --git a/drivers/media/v4l2-core/v4l2-ctrls.c b/drivers/media/v4l2-core/v4l2-ctrls.c > index e96c453208e8..9af17815ecc3 100644 > --- a/drivers/media/v4l2-core/v4l2-ctrls.c > +++ b/drivers/media/v4l2-core/v4l2-ctrls.c > @@ -913,6 +913,9 @@ const char *v4l2_ctrl_get_name(u32 id) > case V4L2_CID_MPEG_VIDEO_HEVC_SIZE_OF_LENGTH_FIELD: return "HEVC Size of Length Field"; > case V4L2_CID_MPEG_VIDEO_REF_NUMBER_FOR_PFRAMES: return "Reference Frames for a P-Frame"; > case V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR: return "Prepend SPS and PPS to IDR"; > + case V4L2_CID_MPEG_VIDEO_HEVC_SPS: return "HEVC Sequence Parameter Set"; > + case V4L2_CID_MPEG_VIDEO_HEVC_PPS: return "HEVC Picture Parameter Set"; Hmm. The H264 control descriptions just say H264 SPS and PPS. I like this better. Can you update the H264 control descriptions? > + case V4L2_CID_MPEG_VIDEO_HEVC_SLICE_PARAMS: return "HEVC Slice Parameters"; > > /* CAMERA controls */ > /* Keep the order of the 'case's the same as in v4l2-controls.h! */ > @@ -1320,6 +1323,15 @@ void v4l2_ctrl_fill(u32 id, const char **name, enum v4l2_ctrl_type *type, > case V4L2_CID_MPEG_VIDEO_H264_DECODE_PARAMS: > *type = V4L2_CTRL_TYPE_H264_DECODE_PARAMS; > break; > + case V4L2_CID_MPEG_VIDEO_HEVC_SPS: > + *type = V4L2_CTRL_TYPE_HEVC_SPS; > + break; > + case V4L2_CID_MPEG_VIDEO_HEVC_PPS: > + *type = V4L2_CTRL_TYPE_HEVC_PPS; > + break; > + case V4L2_CID_MPEG_VIDEO_HEVC_SLICE_PARAMS: > + *type = V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS; > + break; > default: > *type = V4L2_CTRL_TYPE_INTEGER; > break; > @@ -1692,6 +1704,11 @@ static int std_validate(const struct v4l2_ctrl *ctrl, u32 idx, > case V4L2_CTRL_TYPE_H264_DECODE_PARAMS: > return 0; > > + case V4L2_CTRL_TYPE_HEVC_SPS: > + case V4L2_CTRL_TYPE_HEVC_PPS: > + case V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS: > + return 0; > + > default: > return -EINVAL; > } > @@ -2287,6 +2304,15 @@ static struct v4l2_ctrl *v4l2_ctrl_new(struct v4l2_ctrl_handler *hdl, > case V4L2_CTRL_TYPE_H264_DECODE_PARAMS: > elem_size = sizeof(struct v4l2_ctrl_h264_decode_param); > break; > + case V4L2_CTRL_TYPE_HEVC_SPS: > + elem_size = sizeof(struct v4l2_ctrl_hevc_sps); > + break; > + case V4L2_CTRL_TYPE_HEVC_PPS: > + elem_size = sizeof(struct v4l2_ctrl_hevc_pps); > + break; > + case V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS: > + elem_size = sizeof(struct v4l2_ctrl_hevc_slice_params); > + break; > default: > if (type < V4L2_CTRL_COMPOUND_TYPES) > elem_size = sizeof(s32); > diff --git a/drivers/media/v4l2-core/v4l2-ioctl.c b/drivers/media/v4l2-core/v4l2-ioctl.c > index aa63f1794272..7bec91c6effe 100644 > --- a/drivers/media/v4l2-core/v4l2-ioctl.c > +++ b/drivers/media/v4l2-core/v4l2-ioctl.c > @@ -1321,6 +1321,7 @@ static void v4l_fill_fmtdesc(struct v4l2_fmtdesc *fmt) > case V4L2_PIX_FMT_VP8: descr = "VP8"; break; > case V4L2_PIX_FMT_VP9: descr = "VP9"; break; > case V4L2_PIX_FMT_HEVC: descr = "HEVC"; break; /* aka H.265 */ > + case V4L2_PIX_FMT_HEVC_SLICE: descr = "HEVC Parsed Slice Data"; break; H264 calls it "H.264 Parsed Slice". Again, please pick one or the other. MPEG2 calls it Parsed Slice Data as well, so I'd say the H264 code should change. > case V4L2_PIX_FMT_FWHT: descr = "FWHT"; break; /* used in vicodec */ > case V4L2_PIX_FMT_CPIA1: descr = "GSPCA CPiA YUV"; break; > case V4L2_PIX_FMT_WNVA: descr = "WNVA"; break; > diff --git a/include/media/v4l2-ctrls.h b/include/media/v4l2-ctrls.h > index b4ca95710d2d..11664c5c3706 100644 > --- a/include/media/v4l2-ctrls.h > +++ b/include/media/v4l2-ctrls.h > @@ -48,6 +48,9 @@ struct poll_table_struct; > * @p_h264_scal_mtrx: Pointer to a struct v4l2_ctrl_h264_scaling_matrix. > * @p_h264_slice_param: Pointer to a struct v4l2_ctrl_h264_slice_param. > * @p_h264_decode_param: Pointer to a struct v4l2_ctrl_h264_decode_param. > + * @p_hevc_sps: Pointer to an HEVC sequence parameter set structure. > + * @p_hevc_pps: Pointer to an HEVC picture parameter set structure. > + * @p_hevc_slice_params Pointer to an HEVC slice parameters structure. > * @p: Pointer to a compound value. > */ > union v4l2_ctrl_ptr { > @@ -64,6 +67,9 @@ union v4l2_ctrl_ptr { > struct v4l2_ctrl_h264_scaling_matrix *p_h264_scal_mtrx; > struct v4l2_ctrl_h264_slice_param *p_h264_slice_param; > struct v4l2_ctrl_h264_decode_param *p_h264_decode_param; > + struct v4l2_ctrl_hevc_sps *p_hevc_sps; > + struct v4l2_ctrl_hevc_pps *p_hevc_pps; > + struct v4l2_ctrl_hevc_slice_params *p_hevc_slice_params; > void *p; > }; > > diff --git a/include/uapi/linux/v4l2-controls.h b/include/uapi/linux/v4l2-controls.h > index 628c0cdb51d9..5bbf63b2dad1 100644 > --- a/include/uapi/linux/v4l2-controls.h > +++ b/include/uapi/linux/v4l2-controls.h > @@ -709,6 +709,9 @@ enum v4l2_cid_mpeg_video_hevc_size_of_length_field { > #define V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L6_BR (V4L2_CID_MPEG_BASE + 642) > #define V4L2_CID_MPEG_VIDEO_REF_NUMBER_FOR_PFRAMES (V4L2_CID_MPEG_BASE + 643) > #define V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR (V4L2_CID_MPEG_BASE + 644) > +#define V4L2_CID_MPEG_VIDEO_HEVC_SPS (V4L2_CID_MPEG_BASE + 645) > +#define V4L2_CID_MPEG_VIDEO_HEVC_PPS (V4L2_CID_MPEG_BASE + 646) > +#define V4L2_CID_MPEG_VIDEO_HEVC_SLICE_PARAMS (V4L2_CID_MPEG_BASE + 647) > > /* MPEG-class control IDs specific to the CX2341x driver as defined by V4L2 */ > #define V4L2_CID_MPEG_CX2341X_BASE (V4L2_CTRL_CLASS_MPEG | 0x1000) > @@ -1324,4 +1327,156 @@ struct v4l2_ctrl_h264_decode_param { > struct v4l2_h264_dpb_entry dpb[16]; > }; > > +#define V4L2_HEVC_SLICE_TYPE_B 0 > +#define V4L2_HEVC_SLICE_TYPE_P 1 > +#define V4L2_HEVC_SLICE_TYPE_I 2 > + > +struct v4l2_ctrl_hevc_sps { > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Sequence parameter set */ > + __u8 chroma_format_idc; > + __u8 separate_colour_plane_flag; > + __u16 pic_width_in_luma_samples; > + __u16 pic_height_in_luma_samples; > + __u8 bit_depth_luma_minus8; > + __u8 bit_depth_chroma_minus8; > + __u8 log2_max_pic_order_cnt_lsb_minus4; > + __u8 sps_max_dec_pic_buffering_minus1; > + __u8 sps_max_num_reorder_pics; > + __u8 sps_max_latency_increase_plus1; > + __u8 log2_min_luma_coding_block_size_minus3; > + __u8 log2_diff_max_min_luma_coding_block_size; > + __u8 log2_min_luma_transform_block_size_minus2; > + __u8 log2_diff_max_min_luma_transform_block_size; > + __u8 max_transform_hierarchy_depth_inter; > + __u8 max_transform_hierarchy_depth_intra; > + __u8 scaling_list_enabled_flag; > + __u8 amp_enabled_flag; > + __u8 sample_adaptive_offset_enabled_flag; > + __u8 pcm_enabled_flag; > + __u8 pcm_sample_bit_depth_luma_minus1; > + __u8 pcm_sample_bit_depth_chroma_minus1; > + __u8 log2_min_pcm_luma_coding_block_size_minus3; > + __u8 log2_diff_max_min_pcm_luma_coding_block_size; > + __u8 pcm_loop_filter_disabled_flag; > + __u8 num_short_term_ref_pic_sets; > + __u8 long_term_ref_pics_present_flag; > + __u8 num_long_term_ref_pics_sps; > + __u8 sps_temporal_mvp_enabled_flag; > + __u8 strong_intra_smoothing_enabled_flag; Same comment as for H264: keep the structs 4 byte aligned and ensure that there are no holes. > +}; > + > +struct v4l2_ctrl_hevc_pps { > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Picture parameter set */ > + __u8 dependent_slice_segment_flag; > + __u8 output_flag_present_flag; > + __u8 num_extra_slice_header_bits; > + __u8 sign_data_hiding_enabled_flag; > + __u8 cabac_init_present_flag; > + __s8 init_qp_minus26; > + __u8 constrained_intra_pred_flag; > + __u8 transform_skip_enabled_flag; > + __u8 cu_qp_delta_enabled_flag; > + __u8 diff_cu_qp_delta_depth; > + __s8 pps_cb_qp_offset; > + __s8 pps_cr_qp_offset; > + __u8 pps_slice_chroma_qp_offsets_present_flag; > + __u8 weighted_pred_flag; > + __u8 weighted_bipred_flag; > + __u8 transquant_bypass_enabled_flag; > + __u8 tiles_enabled_flag; > + __u8 entropy_coding_sync_enabled_flag; > + __u8 num_tile_columns_minus1; > + __u8 num_tile_rows_minus1; > + __u8 column_width_minus1[20]; > + __u8 row_height_minus1[22]; > + __u8 loop_filter_across_tiles_enabled_flag; > + __u8 pps_loop_filter_across_slices_enabled_flag; > + __u8 deblocking_filter_override_enabled_flag; > + __u8 pps_disable_deblocking_filter_flag; > + __s8 pps_beta_offset_div2; > + __s8 pps_tc_offset_div2; > + __u8 lists_modification_present_flag; > + __u8 log2_parallel_merge_level_minus2; > + __u8 slice_segment_header_extension_present_flag; > +}; > + > +#define V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_BEFORE 0x01 > +#define V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_AFTER 0x02 > +#define V4L2_HEVC_DPB_ENTRY_RPS_LT_CURR 0x03 > + > +#define V4L2_HEVC_DPB_ENTRIES_NUM_MAX 16 > + > +struct v4l2_hevc_dpb_entry { > + __u32 buffer_tag; > + __u8 rps; > + __u8 field_pic; > + __u16 pic_order_cnt[2]; > +}; > + > +struct v4l2_hevc_pred_weight_table { > + __u8 luma_log2_weight_denom; > + __s8 delta_chroma_log2_weight_denom; > + > + __s8 delta_luma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > + __s8 luma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > + __s8 delta_chroma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > + __s8 chroma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > + > + __s8 delta_luma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > + __s8 luma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > + __s8 delta_chroma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > + __s8 chroma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > +}; > + > +struct v4l2_ctrl_hevc_slice_params { > + __u32 bit_size; > + __u32 data_bit_offset; > + > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: NAL unit header */ > + __u8 nal_unit_type; > + __u8 nuh_temporal_id_plus1; > + > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ > + __u8 slice_type; > + __u8 colour_plane_id; > + __u16 slice_pic_order_cnt; > + __u8 slice_sao_luma_flag; > + __u8 slice_sao_chroma_flag; > + __u8 slice_temporal_mvp_enabled_flag; > + __u8 num_ref_idx_l0_active_minus1; > + __u8 num_ref_idx_l1_active_minus1; > + __u8 mvd_l1_zero_flag; > + __u8 cabac_init_flag; > + __u8 collocated_from_l0_flag; > + __u8 collocated_ref_idx; > + __u8 five_minus_max_num_merge_cand; > + __u8 use_integer_mv_flag; > + __s8 slice_qp_delta; > + __s8 slice_cb_qp_offset; > + __s8 slice_cr_qp_offset; > + __s8 slice_act_y_qp_offset; > + __s8 slice_act_cb_qp_offset; > + __s8 slice_act_cr_qp_offset; > + __u8 slice_deblocking_filter_disabled_flag; > + __s8 slice_beta_offset_div2; > + __s8 slice_tc_offset_div2; > + __u8 slice_loop_filter_across_slices_enabled_flag; > + > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Picture timing SEI message */ > + __u8 pic_struct; > + > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ > + struct v4l2_hevc_dpb_entry dpb[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > + __u8 num_active_dpb_entries; > + __u8 ref_idx_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > + __u8 ref_idx_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > + > + __u8 num_rps_poc_st_curr_before; > + __u8 num_rps_poc_st_curr_after; > + __u8 num_rps_poc_lt_curr; > + > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Weighted prediction parameter */ > + struct v4l2_hevc_pred_weight_table pred_weight_table; > +}; > + > #endif > diff --git a/include/uapi/linux/v4l2-controls.h.rej b/include/uapi/linux/v4l2-controls.h.rej > deleted file mode 100644 > index 1fbb7bf8daa7..000000000000 > --- a/include/uapi/linux/v4l2-controls.h.rej > +++ /dev/null > @@ -1,187 +0,0 @@ > ---- include/uapi/linux/v4l2-controls.h > -+++ include/uapi/linux/v4l2-controls.h > -@@ -50,6 +50,8 @@ > - #ifndef __LINUX_V4L2_CONTROLS_H > - #define __LINUX_V4L2_CONTROLS_H > - > -+#include <linux/types.h> > -+ > - /* Control classes */ > - #define V4L2_CTRL_CLASS_USER 0x00980000 /* Old-style 'user' controls */ > - #define V4L2_CTRL_CLASS_MPEG 0x00990000 /* MPEG-compression controls */ > -@@ -534,6 +536,12 @@ enum v4l2_mpeg_video_h264_hierarchical_coding_type { > - }; > - #define V4L2_CID_MPEG_VIDEO_H264_HIERARCHICAL_CODING_LAYER (V4L2_CID_MPEG_BASE+381) > - #define V4L2_CID_MPEG_VIDEO_H264_HIERARCHICAL_CODING_LAYER_QP (V4L2_CID_MPEG_BASE+382) > -+#define V4L2_CID_MPEG_VIDEO_H264_SPS (V4L2_CID_MPEG_BASE+383) > -+#define V4L2_CID_MPEG_VIDEO_H264_PPS (V4L2_CID_MPEG_BASE+384) > -+#define V4L2_CID_MPEG_VIDEO_H264_SCALING_MATRIX (V4L2_CID_MPEG_BASE+385) > -+#define V4L2_CID_MPEG_VIDEO_H264_SLICE_PARAMS (V4L2_CID_MPEG_BASE+386) > -+#define V4L2_CID_MPEG_VIDEO_H264_DECODE_PARAMS (V4L2_CID_MPEG_BASE+387) > -+ > - #define V4L2_CID_MPEG_VIDEO_MPEG4_I_FRAME_QP (V4L2_CID_MPEG_BASE+400) > - #define V4L2_CID_MPEG_VIDEO_MPEG4_P_FRAME_QP (V4L2_CID_MPEG_BASE+401) > - #define V4L2_CID_MPEG_VIDEO_MPEG4_B_FRAME_QP (V4L2_CID_MPEG_BASE+402) > -@@ -1156,4 +1164,162 @@ struct v4l2_ctrl_mpeg2_quantization { > - __u8 chroma_non_intra_quantiser_matrix[64]; > - }; > - > -+/* Compounds controls */ > -+ > -+#define V4L2_H264_SPS_CONSTRAINT_SET0_FLAG 0x01 > -+#define V4L2_H264_SPS_CONSTRAINT_SET1_FLAG 0x02 > -+#define V4L2_H264_SPS_CONSTRAINT_SET2_FLAG 0x04 > -+#define V4L2_H264_SPS_CONSTRAINT_SET3_FLAG 0x08 > -+#define V4L2_H264_SPS_CONSTRAINT_SET4_FLAG 0x10 > -+#define V4L2_H264_SPS_CONSTRAINT_SET5_FLAG 0x20 > -+ > -+#define V4L2_H264_SPS_FLAG_SEPARATE_COLOUR_PLANE 0x01 > -+#define V4L2_H264_SPS_FLAG_QPPRIME_Y_ZERO_TRANSFORM_BYPASS 0x02 > -+#define V4L2_H264_SPS_FLAG_DELTA_PIC_ORDER_ALWAYS_ZERO 0x04 > -+#define V4L2_H264_SPS_FLAG_GAPS_IN_FRAME_NUM_VALUE_ALLOWED 0x08 > -+#define V4L2_H264_SPS_FLAG_FRAME_MBS_ONLY 0x10 > -+#define V4L2_H264_SPS_FLAG_MB_ADAPTIVE_FRAME_FIELD 0x20 > -+#define V4L2_H264_SPS_FLAG_DIRECT_8X8_INFERENCE 0x40 > -+ > -+struct v4l2_ctrl_h264_sps { > -+ __u8 profile_idc; > -+ __u8 constraint_set_flags; > -+ __u8 level_idc; > -+ __u8 seq_parameter_set_id; > -+ __u8 chroma_format_idc; > -+ __u8 bit_depth_luma_minus8; > -+ __u8 bit_depth_chroma_minus8; > -+ __u8 log2_max_frame_num_minus4; > -+ __u8 pic_order_cnt_type; > -+ __u8 log2_max_pic_order_cnt_lsb_minus4; > -+ __u8 max_num_ref_frames; > -+ __u8 num_ref_frames_in_pic_order_cnt_cycle; > -+ __s32 offset_for_ref_frame[255]; > -+ __s32 offset_for_non_ref_pic; > -+ __s32 offset_for_top_to_bottom_field; > -+ __u16 pic_width_in_mbs_minus1; > -+ __u16 pic_height_in_map_units_minus1; > -+ __u8 flags; > -+}; > -+ > -+#define V4L2_H264_PPS_FLAG_ENTROPY_CODING_MODE 0x0001 > -+#define V4L2_H264_PPS_FLAG_BOTTOM_FIELD_PIC_ORDER_IN_FRAME_PRESENT 0x0002 > -+#define V4L2_H264_PPS_FLAG_WEIGHTED_PRED 0x0004 > -+#define V4L2_H264_PPS_FLAG_DEBLOCKING_FILTER_CONTROL_PRESENT 0x0008 > -+#define V4L2_H264_PPS_FLAG_CONSTRAINED_INTRA_PRED 0x0010 > -+#define V4L2_H264_PPS_FLAG_REDUNDANT_PIC_CNT_PRESENT 0x0020 > -+#define V4L2_H264_PPS_FLAG_TRANSFORM_8X8_MODE 0x0040 > -+#define V4L2_H264_PPS_FLAG_PIC_SCALING_MATRIX_PRESENT 0x0080 > -+ > -+struct v4l2_ctrl_h264_pps { > -+ __u8 pic_parameter_set_id; > -+ __u8 seq_parameter_set_id; > -+ __u8 num_slice_groups_minus1; > -+ __u8 num_ref_idx_l0_default_active_minus1; > -+ __u8 num_ref_idx_l1_default_active_minus1; > -+ __u8 weighted_bipred_idc; > -+ __s8 pic_init_qp_minus26; > -+ __s8 pic_init_qs_minus26; > -+ __s8 chroma_qp_index_offset; > -+ __s8 second_chroma_qp_index_offset; > -+ __u8 flags; > -+}; > -+ > -+struct v4l2_ctrl_h264_scaling_matrix { > -+ __u8 scaling_list_4x4[6][16]; > -+ __u8 scaling_list_8x8[6][64]; > -+}; > -+ > -+struct v4l2_h264_weight_factors { > -+ __s8 luma_weight[32]; > -+ __s8 luma_offset[32]; > -+ __s8 chroma_weight[32][2]; > -+ __s8 chroma_offset[32][2]; > -+}; > -+ > -+struct v4l2_h264_pred_weight_table { > -+ __u8 luma_log2_weight_denom; > -+ __u8 chroma_log2_weight_denom; > -+ struct v4l2_h264_weight_factors weight_factors[2]; > -+}; > -+ > -+#define V4L2_H264_SLICE_TYPE_P 0 > -+#define V4L2_H264_SLICE_TYPE_B 1 > -+#define V4L2_H264_SLICE_TYPE_I 2 > -+#define V4L2_H264_SLICE_TYPE_SP 3 > -+#define V4L2_H264_SLICE_TYPE_SI 4 > -+ > -+#define V4L2_H264_SLICE_FLAG_FIELD_PIC 0x01 > -+#define V4L2_H264_SLICE_FLAG_BOTTOM_FIELD 0x02 > -+#define V4L2_H264_SLICE_FLAG_DIRECT_SPATIAL_MV_PRED 0x04 > -+#define V4L2_H264_SLICE_FLAG_SP_FOR_SWITCH 0x08 > -+ > -+struct v4l2_ctrl_h264_slice_param { > -+ /* Size in bytes, including header */ > -+ __u32 size; > -+ /* Offset in bits to slice_data() from the beginning of this slice. */ > -+ __u32 header_bit_size; > -+ > -+ __u16 first_mb_in_slice; > -+ __u8 slice_type; > -+ __u8 pic_parameter_set_id; > -+ __u8 colour_plane_id; > -+ __u16 frame_num; > -+ __u16 idr_pic_id; > -+ __u16 pic_order_cnt_lsb; > -+ __s32 delta_pic_order_cnt_bottom; > -+ __s32 delta_pic_order_cnt0; > -+ __s32 delta_pic_order_cnt1; > -+ __u8 redundant_pic_cnt; > -+ > -+ struct v4l2_h264_pred_weight_table pred_weight_table; > -+ /* Size in bits of dec_ref_pic_marking() syntax element. */ > -+ __u32 dec_ref_pic_marking_bit_size; > -+ /* Size in bits of pic order count syntax. */ > -+ __u32 pic_order_cnt_bit_size; > -+ > -+ __u8 cabac_init_idc; > -+ __s8 slice_qp_delta; > -+ __s8 slice_qs_delta; > -+ __u8 disable_deblocking_filter_idc; > -+ __s8 slice_alpha_c0_offset_div2; > -+ __s8 slice_beta_offset_div2; > -+ __u32 slice_group_change_cycle; > -+ > -+ __u8 num_ref_idx_l0_active_minus1; > -+ __u8 num_ref_idx_l1_active_minus1; > -+ /* Entries on each list are indices > -+ * into v4l2_ctrl_h264_decode_param.dpb[]. */ > -+ __u8 ref_pic_list0[32]; > -+ __u8 ref_pic_list1[32]; > -+ > -+ __u8 flags; > -+}; > -+ > -+#define V4L2_H264_DPB_ENTRY_FLAG_VALID 0x01 > -+#define V4L2_H264_DPB_ENTRY_FLAG_ACTIVE 0x02 > -+#define V4L2_H264_DPB_ENTRY_FLAG_LONG_TERM 0x04 > -+ > -+struct v4l2_h264_dpb_entry { > -+ __u32 tag; > -+ __u16 frame_num; > -+ __u16 pic_num; > -+ /* Note that field is indicated by v4l2_buffer.field */ > -+ __s32 top_field_order_cnt; > -+ __s32 bottom_field_order_cnt; > -+ __u8 flags; /* V4L2_H264_DPB_ENTRY_FLAG_* */ > -+}; > -+ > -+struct v4l2_ctrl_h264_decode_param { > -+ __u32 num_slices; > -+ __u8 idr_pic_flag; > -+ __u8 nal_ref_idc; > -+ __s32 top_field_order_cnt; > -+ __s32 bottom_field_order_cnt; > -+ __u8 ref_pic_list_p0[32]; > -+ __u8 ref_pic_list_b0[32]; > -+ __u8 ref_pic_list_b1[32]; > -+ struct v4l2_h264_dpb_entry dpb[16]; > -+}; > -+ > - #endif > diff --git a/include/uapi/linux/videodev2.h b/include/uapi/linux/videodev2.h > index dd028e0bf306..26f5bec9e988 100644 > --- a/include/uapi/linux/videodev2.h > +++ b/include/uapi/linux/videodev2.h > @@ -655,6 +655,7 @@ struct v4l2_pix_format { > #define V4L2_PIX_FMT_VP8 v4l2_fourcc('V', 'P', '8', '0') /* VP8 */ > #define V4L2_PIX_FMT_VP9 v4l2_fourcc('V', 'P', '9', '0') /* VP9 */ > #define V4L2_PIX_FMT_HEVC v4l2_fourcc('H', 'E', 'V', 'C') /* HEVC aka H.265 */ > +#define V4L2_PIX_FMT_HEVC_SLICE v4l2_fourcc('S', '2', '6', '5') /* HEVC parsed slices */ > #define V4L2_PIX_FMT_FWHT v4l2_fourcc('F', 'W', 'H', 'T') /* Fast Walsh Hadamard Transform (vicodec) */ > > /* Vendor-specific formats */ > @@ -1637,6 +1638,9 @@ struct v4l2_ext_control { > struct v4l2_ctrl_h264_scaling_matrix __user *p_h264_scal_mtrx; > struct v4l2_ctrl_h264_slice_param __user *p_h264_slice_param; > struct v4l2_ctrl_h264_decode_param __user *p_h264_decode_param; > + struct v4l2_ctrl_hevc_sps __user *p_hevc_sps; > + struct v4l2_ctrl_hevc_pps __user *p_hevc_pps; > + struct v4l2_ctrl_hevc_slice_params __user *p_hevc_slice_params; > void __user *ptr; > }; > } __attribute__ ((packed)); > @@ -1689,6 +1693,9 @@ enum v4l2_ctrl_type { > V4L2_CTRL_TYPE_H264_SCALING_MATRIX = 0x0107, > V4L2_CTRL_TYPE_H264_SLICE_PARAMS = 0x0108, > V4L2_CTRL_TYPE_H264_DECODE_PARAMS = 0x0109, > + V4L2_CTRL_TYPE_HEVC_SPS = 0x0110, > + V4L2_CTRL_TYPE_HEVC_PPS = 0x0111, > + V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS = 0x0112, > }; > > /* Used in the VIDIOC_QUERYCTRL ioctl for querying controls */ > Regards, Hans
Hi! Dne petek, 23. november 2018 ob 14:02:08 CET je Paul Kocialkowski napisal(a): > This introduces the required definitions for HEVC decoding support with > stateless VPUs. The controls associated to the HEVC slice format provide > the required meta-data for decoding slices extracted from the bitstream. > > This interface comes with the following limitations: > * No custom quantization matrices (scaling lists); > * Support for a single temporal layer only; > * No slice entry point offsets support; > * No conformance window support; > * No VUI parameters support; > * No support for SPS extensions: range, multilayer, 3d, scc, 4 bits; > * No support for PPS extensions: range, multilayer, 3d, scc, 4 bits. > > Signed-off-by: Paul Kocialkowski <paul.kocialkowski@bootlin.com> > --- <snip> > diff --git a/drivers/media/v4l2-core/v4l2-ctrls.c > b/drivers/media/v4l2-core/v4l2-ctrls.c index e96c453208e8..9af17815ecc3 > 100644 > --- a/drivers/media/v4l2-core/v4l2-ctrls.c > +++ b/drivers/media/v4l2-core/v4l2-ctrls.c > @@ -913,6 +913,9 @@ const char *v4l2_ctrl_get_name(u32 id) > case V4L2_CID_MPEG_VIDEO_HEVC_SIZE_OF_LENGTH_FIELD: return "HEVC Size of > Length Field"; case V4L2_CID_MPEG_VIDEO_REF_NUMBER_FOR_PFRAMES: return > "Reference Frames for a P-Frame"; case > V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR: return "Prepend SPS and PPS to > IDR"; + case V4L2_CID_MPEG_VIDEO_HEVC_SPS: return "HEVC Sequence > Parameter Set"; + case V4L2_CID_MPEG_VIDEO_HEVC_PPS: return "HEVC Picture > Parameter Set"; + case V4L2_CID_MPEG_VIDEO_HEVC_SLICE_PARAMS: return "HEVC > Slice Parameters"; > > /* CAMERA controls */ > /* Keep the order of the 'case's the same as in v4l2-controls.h! */ > @@ -1320,6 +1323,15 @@ void v4l2_ctrl_fill(u32 id, const char **name, enum > v4l2_ctrl_type *type, case V4L2_CID_MPEG_VIDEO_H264_DECODE_PARAMS: > *type = V4L2_CTRL_TYPE_H264_DECODE_PARAMS; > break; > + case V4L2_CID_MPEG_VIDEO_HEVC_SPS: > + *type = V4L2_CTRL_TYPE_HEVC_SPS; > + break; > + case V4L2_CID_MPEG_VIDEO_HEVC_PPS: > + *type = V4L2_CTRL_TYPE_HEVC_PPS; > + break; > + case V4L2_CID_MPEG_VIDEO_HEVC_SLICE_PARAMS: > + *type = V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS; > + break; > default: > *type = V4L2_CTRL_TYPE_INTEGER; > break; > @@ -1692,6 +1704,11 @@ static int std_validate(const struct v4l2_ctrl *ctrl, > u32 idx, case V4L2_CTRL_TYPE_H264_DECODE_PARAMS: > return 0; > > + case V4L2_CTRL_TYPE_HEVC_SPS: > + case V4L2_CTRL_TYPE_HEVC_PPS: > + case V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS: > + return 0; > + > default: > return -EINVAL; > } > @@ -2287,6 +2304,15 @@ static struct v4l2_ctrl *v4l2_ctrl_new(struct > v4l2_ctrl_handler *hdl, case V4L2_CTRL_TYPE_H264_DECODE_PARAMS: > elem_size = sizeof(struct v4l2_ctrl_h264_decode_param); > break; > + case V4L2_CTRL_TYPE_HEVC_SPS: > + elem_size = sizeof(struct v4l2_ctrl_hevc_sps); > + break; > + case V4L2_CTRL_TYPE_HEVC_PPS: > + elem_size = sizeof(struct v4l2_ctrl_hevc_pps); > + break; > + case V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS: > + elem_size = sizeof(struct v4l2_ctrl_hevc_slice_params); > + break; > default: > if (type < V4L2_CTRL_COMPOUND_TYPES) > elem_size = sizeof(s32); > diff --git a/drivers/media/v4l2-core/v4l2-ioctl.c > b/drivers/media/v4l2-core/v4l2-ioctl.c index aa63f1794272..7bec91c6effe > 100644 > --- a/drivers/media/v4l2-core/v4l2-ioctl.c > +++ b/drivers/media/v4l2-core/v4l2-ioctl.c > @@ -1321,6 +1321,7 @@ static void v4l_fill_fmtdesc(struct v4l2_fmtdesc *fmt) > case V4L2_PIX_FMT_VP8: descr = "VP8"; break; > case V4L2_PIX_FMT_VP9: descr = "VP9"; break; > case V4L2_PIX_FMT_HEVC: descr = "HEVC"; break; /* aka H.265 */ > + case V4L2_PIX_FMT_HEVC_SLICE: descr = "HEVC Parsed Slice Data"; break; > case V4L2_PIX_FMT_FWHT: descr = "FWHT"; break; /* used in vicodec */ > case V4L2_PIX_FMT_CPIA1: descr = "GSPCA CPiA YUV"; break; > case V4L2_PIX_FMT_WNVA: descr = "WNVA"; break; > diff --git a/include/media/v4l2-ctrls.h b/include/media/v4l2-ctrls.h > index b4ca95710d2d..11664c5c3706 100644 > --- a/include/media/v4l2-ctrls.h > +++ b/include/media/v4l2-ctrls.h > @@ -48,6 +48,9 @@ struct poll_table_struct; > * @p_h264_scal_mtrx: Pointer to a struct v4l2_ctrl_h264_scaling_matrix. > * @p_h264_slice_param: Pointer to a struct v4l2_ctrl_h264_slice_param. > * @p_h264_decode_param: Pointer to a struct v4l2_ctrl_h264_decode_param. > + * @p_hevc_sps: Pointer to an HEVC sequence parameter set structure. > + * @p_hevc_pps: Pointer to an HEVC picture parameter set structure. > + * @p_hevc_slice_params Pointer to an HEVC slice parameters structure. > * @p: Pointer to a compound value. > */ > union v4l2_ctrl_ptr { > @@ -64,6 +67,9 @@ union v4l2_ctrl_ptr { > struct v4l2_ctrl_h264_scaling_matrix *p_h264_scal_mtrx; > struct v4l2_ctrl_h264_slice_param *p_h264_slice_param; > struct v4l2_ctrl_h264_decode_param *p_h264_decode_param; > + struct v4l2_ctrl_hevc_sps *p_hevc_sps; > + struct v4l2_ctrl_hevc_pps *p_hevc_pps; > + struct v4l2_ctrl_hevc_slice_params *p_hevc_slice_params; > void *p; > }; > > diff --git a/include/uapi/linux/v4l2-controls.h > b/include/uapi/linux/v4l2-controls.h index 628c0cdb51d9..5bbf63b2dad1 > 100644 > --- a/include/uapi/linux/v4l2-controls.h > +++ b/include/uapi/linux/v4l2-controls.h > @@ -709,6 +709,9 @@ enum v4l2_cid_mpeg_video_hevc_size_of_length_field { > #define V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L6_BR (V4L2_CID_MPEG_BASE + > 642) #define V4L2_CID_MPEG_VIDEO_REF_NUMBER_FOR_PFRAMES (V4L2_CID_MPEG_BASE > + 643) #define > V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR (V4L2_CID_MPEG_BASE + 644) > +#define V4L2_CID_MPEG_VIDEO_HEVC_SPS (V4L2_CID_MPEG_BASE + 645) +#define > V4L2_CID_MPEG_VIDEO_HEVC_PPS (V4L2_CID_MPEG_BASE + 646) +#define > V4L2_CID_MPEG_VIDEO_HEVC_SLICE_PARAMS (V4L2_CID_MPEG_BASE + 647) > > /* MPEG-class control IDs specific to the CX2341x driver as defined by > V4L2 */ #define V4L2_CID_MPEG_CX2341X_BASE (V4L2_CTRL_CLASS_MPEG | > 0x1000) @@ -1324,4 +1327,156 @@ struct v4l2_ctrl_h264_decode_param { > struct v4l2_h264_dpb_entry dpb[16]; > }; > > +#define V4L2_HEVC_SLICE_TYPE_B 0 > +#define V4L2_HEVC_SLICE_TYPE_P 1 > +#define V4L2_HEVC_SLICE_TYPE_I 2 > + > +struct v4l2_ctrl_hevc_sps { > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Sequence parameter set */ > + __u8 chroma_format_idc; > + __u8 separate_colour_plane_flag; > + __u16 pic_width_in_luma_samples; > + __u16 pic_height_in_luma_samples; > + __u8 bit_depth_luma_minus8; > + __u8 bit_depth_chroma_minus8; > + __u8 log2_max_pic_order_cnt_lsb_minus4; > + __u8 sps_max_dec_pic_buffering_minus1; > + __u8 sps_max_num_reorder_pics; > + __u8 sps_max_latency_increase_plus1; > + __u8 log2_min_luma_coding_block_size_minus3; > + __u8 log2_diff_max_min_luma_coding_block_size; > + __u8 log2_min_luma_transform_block_size_minus2; > + __u8 log2_diff_max_min_luma_transform_block_size; > + __u8 max_transform_hierarchy_depth_inter; > + __u8 max_transform_hierarchy_depth_intra; > + __u8 scaling_list_enabled_flag; > + __u8 amp_enabled_flag; > + __u8 sample_adaptive_offset_enabled_flag; > + __u8 pcm_enabled_flag; > + __u8 pcm_sample_bit_depth_luma_minus1; > + __u8 pcm_sample_bit_depth_chroma_minus1; > + __u8 log2_min_pcm_luma_coding_block_size_minus3; > + __u8 log2_diff_max_min_pcm_luma_coding_block_size; > + __u8 pcm_loop_filter_disabled_flag; > + __u8 num_short_term_ref_pic_sets; > + __u8 long_term_ref_pics_present_flag; > + __u8 num_long_term_ref_pics_sps; > + __u8 sps_temporal_mvp_enabled_flag; > + __u8 strong_intra_smoothing_enabled_flag; > +}; > + > +struct v4l2_ctrl_hevc_pps { > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Picture parameter set */ > + __u8 dependent_slice_segment_flag; > + __u8 output_flag_present_flag; > + __u8 num_extra_slice_header_bits; > + __u8 sign_data_hiding_enabled_flag; > + __u8 cabac_init_present_flag; > + __s8 init_qp_minus26; > + __u8 constrained_intra_pred_flag; > + __u8 transform_skip_enabled_flag; > + __u8 cu_qp_delta_enabled_flag; > + __u8 diff_cu_qp_delta_depth; > + __s8 pps_cb_qp_offset; > + __s8 pps_cr_qp_offset; > + __u8 pps_slice_chroma_qp_offsets_present_flag; > + __u8 weighted_pred_flag; > + __u8 weighted_bipred_flag; > + __u8 transquant_bypass_enabled_flag; > + __u8 tiles_enabled_flag; > + __u8 entropy_coding_sync_enabled_flag; > + __u8 num_tile_columns_minus1; > + __u8 num_tile_rows_minus1; > + __u8 column_width_minus1[20]; > + __u8 row_height_minus1[22]; > + __u8 loop_filter_across_tiles_enabled_flag; > + __u8 pps_loop_filter_across_slices_enabled_flag; > + __u8 deblocking_filter_override_enabled_flag; > + __u8 pps_disable_deblocking_filter_flag; > + __s8 pps_beta_offset_div2; > + __s8 pps_tc_offset_div2; > + __u8 lists_modification_present_flag; > + __u8 log2_parallel_merge_level_minus2; > + __u8 slice_segment_header_extension_present_flag; > +}; Although scaling lists are not supported yet, I still think you should include "scaling_list_data_present_flag" here for the sake of completeness and you already included "scaling_list_enable_flag" in SPS. I didn't do any thorough review though, just noticed this bit. Best regards, Jernej > + > +#define V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_BEFORE 0x01 > +#define V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_AFTER 0x02 > +#define V4L2_HEVC_DPB_ENTRY_RPS_LT_CURR 0x03 > + > +#define V4L2_HEVC_DPB_ENTRIES_NUM_MAX 16 > + > +struct v4l2_hevc_dpb_entry { > + __u32 buffer_tag; > + __u8 rps; > + __u8 field_pic; > + __u16 pic_order_cnt[2]; > +}; > + > +struct v4l2_hevc_pred_weight_table { > + __u8 luma_log2_weight_denom; > + __s8 delta_chroma_log2_weight_denom; > + > + __s8 delta_luma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > + __s8 luma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > + __s8 delta_chroma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > + __s8 chroma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > + > + __s8 delta_luma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > + __s8 luma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > + __s8 delta_chroma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > + __s8 chroma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > +}; > + > +struct v4l2_ctrl_hevc_slice_params { > + __u32 bit_size; > + __u32 data_bit_offset; > + > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: NAL unit header */ > + __u8 nal_unit_type; > + __u8 nuh_temporal_id_plus1; > + > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ > + __u8 slice_type; > + __u8 colour_plane_id; > + __u16 slice_pic_order_cnt; > + __u8 slice_sao_luma_flag; > + __u8 slice_sao_chroma_flag; > + __u8 slice_temporal_mvp_enabled_flag; > + __u8 num_ref_idx_l0_active_minus1; > + __u8 num_ref_idx_l1_active_minus1; > + __u8 mvd_l1_zero_flag; > + __u8 cabac_init_flag; > + __u8 collocated_from_l0_flag; > + __u8 collocated_ref_idx; > + __u8 five_minus_max_num_merge_cand; > + __u8 use_integer_mv_flag; > + __s8 slice_qp_delta; > + __s8 slice_cb_qp_offset; > + __s8 slice_cr_qp_offset; > + __s8 slice_act_y_qp_offset; > + __s8 slice_act_cb_qp_offset; > + __s8 slice_act_cr_qp_offset; > + __u8 slice_deblocking_filter_disabled_flag; > + __s8 slice_beta_offset_div2; > + __s8 slice_tc_offset_div2; > + __u8 slice_loop_filter_across_slices_enabled_flag; > + > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Picture timing SEI message */ > + __u8 pic_struct; > + > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ > + struct v4l2_hevc_dpb_entry dpb[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > + __u8 num_active_dpb_entries; > + __u8 ref_idx_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > + __u8 ref_idx_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > + > + __u8 num_rps_poc_st_curr_before; > + __u8 num_rps_poc_st_curr_after; > + __u8 num_rps_poc_lt_curr; > + > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Weighted prediction parameter */ > + struct v4l2_hevc_pred_weight_table pred_weight_table; > +}; > + > #endif
Hi, On Wed, 2018-12-05 at 21:59 +0100, Jernej Škrabec wrote: > Hi! > > Dne petek, 23. november 2018 ob 14:02:08 CET je Paul Kocialkowski napisal(a): > > This introduces the required definitions for HEVC decoding support with > > stateless VPUs. The controls associated to the HEVC slice format provide > > the required meta-data for decoding slices extracted from the bitstream. > > > > This interface comes with the following limitations: > > * No custom quantization matrices (scaling lists); > > * Support for a single temporal layer only; > > * No slice entry point offsets support; > > * No conformance window support; > > * No VUI parameters support; > > * No support for SPS extensions: range, multilayer, 3d, scc, 4 bits; > > * No support for PPS extensions: range, multilayer, 3d, scc, 4 bits. > > > > Signed-off-by: Paul Kocialkowski <paul.kocialkowski@bootlin.com> > > --- > > <snip> > > > diff --git a/drivers/media/v4l2-core/v4l2-ctrls.c > > b/drivers/media/v4l2-core/v4l2-ctrls.c index e96c453208e8..9af17815ecc3 > > 100644 > > --- a/drivers/media/v4l2-core/v4l2-ctrls.c > > +++ b/drivers/media/v4l2-core/v4l2-ctrls.c > > @@ -913,6 +913,9 @@ const char *v4l2_ctrl_get_name(u32 id) > > case V4L2_CID_MPEG_VIDEO_HEVC_SIZE_OF_LENGTH_FIELD: return "HEVC Size of > > Length Field"; case V4L2_CID_MPEG_VIDEO_REF_NUMBER_FOR_PFRAMES: return > > "Reference Frames for a P-Frame"; case > > V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR: return "Prepend SPS and PPS > to > > IDR"; + case V4L2_CID_MPEG_VIDEO_HEVC_SPS: return "HEVC Sequence > > Parameter Set"; + case V4L2_CID_MPEG_VIDEO_HEVC_PPS: return "HEVC > Picture > > Parameter Set"; + case V4L2_CID_MPEG_VIDEO_HEVC_SLICE_PARAMS: return > "HEVC > > Slice Parameters"; > > > > /* CAMERA controls */ > > /* Keep the order of the 'case's the same as in v4l2-controls.h! */ > > @@ -1320,6 +1323,15 @@ void v4l2_ctrl_fill(u32 id, const char **name, enum > > v4l2_ctrl_type *type, case V4L2_CID_MPEG_VIDEO_H264_DECODE_PARAMS: > > *type = V4L2_CTRL_TYPE_H264_DECODE_PARAMS; > > break; > > + case V4L2_CID_MPEG_VIDEO_HEVC_SPS: > > + *type = V4L2_CTRL_TYPE_HEVC_SPS; > > + break; > > + case V4L2_CID_MPEG_VIDEO_HEVC_PPS: > > + *type = V4L2_CTRL_TYPE_HEVC_PPS; > > + break; > > + case V4L2_CID_MPEG_VIDEO_HEVC_SLICE_PARAMS: > > + *type = V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS; > > + break; > > default: > > *type = V4L2_CTRL_TYPE_INTEGER; > > break; > > @@ -1692,6 +1704,11 @@ static int std_validate(const struct v4l2_ctrl *ctrl, > > u32 idx, case V4L2_CTRL_TYPE_H264_DECODE_PARAMS: > > return 0; > > > > + case V4L2_CTRL_TYPE_HEVC_SPS: > > + case V4L2_CTRL_TYPE_HEVC_PPS: > > + case V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS: > > + return 0; > > + > > default: > > return -EINVAL; > > } > > @@ -2287,6 +2304,15 @@ static struct v4l2_ctrl *v4l2_ctrl_new(struct > > v4l2_ctrl_handler *hdl, case V4L2_CTRL_TYPE_H264_DECODE_PARAMS: > > elem_size = sizeof(struct v4l2_ctrl_h264_decode_param); > > break; > > + case V4L2_CTRL_TYPE_HEVC_SPS: > > + elem_size = sizeof(struct v4l2_ctrl_hevc_sps); > > + break; > > + case V4L2_CTRL_TYPE_HEVC_PPS: > > + elem_size = sizeof(struct v4l2_ctrl_hevc_pps); > > + break; > > + case V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS: > > + elem_size = sizeof(struct v4l2_ctrl_hevc_slice_params); > > + break; > > default: > > if (type < V4L2_CTRL_COMPOUND_TYPES) > > elem_size = sizeof(s32); > > diff --git a/drivers/media/v4l2-core/v4l2-ioctl.c > > b/drivers/media/v4l2-core/v4l2-ioctl.c index aa63f1794272..7bec91c6effe > > 100644 > > --- a/drivers/media/v4l2-core/v4l2-ioctl.c > > +++ b/drivers/media/v4l2-core/v4l2-ioctl.c > > @@ -1321,6 +1321,7 @@ static void v4l_fill_fmtdesc(struct v4l2_fmtdesc *fmt) > > case V4L2_PIX_FMT_VP8: descr = "VP8"; break; > > case V4L2_PIX_FMT_VP9: descr = "VP9"; break; > > case V4L2_PIX_FMT_HEVC: descr = "HEVC"; break; /* aka H.265 */ > > + case V4L2_PIX_FMT_HEVC_SLICE: descr = "HEVC Parsed Slice Data"; break; > > case V4L2_PIX_FMT_FWHT: descr = "FWHT"; break; /* used in vicodec */ > > case V4L2_PIX_FMT_CPIA1: descr = "GSPCA CPiA YUV"; break; > > case V4L2_PIX_FMT_WNVA: descr = "WNVA"; break; > > diff --git a/include/media/v4l2-ctrls.h b/include/media/v4l2-ctrls.h > > index b4ca95710d2d..11664c5c3706 100644 > > --- a/include/media/v4l2-ctrls.h > > +++ b/include/media/v4l2-ctrls.h > > @@ -48,6 +48,9 @@ struct poll_table_struct; > > * @p_h264_scal_mtrx: Pointer to a struct > v4l2_ctrl_h264_scaling_matrix. > > * @p_h264_slice_param: Pointer to a struct v4l2_ctrl_h264_slice_param. > > * @p_h264_decode_param: Pointer to a struct v4l2_ctrl_h264_decode_param. > > + * @p_hevc_sps: Pointer to an HEVC sequence parameter set structure. > > + * @p_hevc_pps: Pointer to an HEVC picture parameter set structure. > > + * @p_hevc_slice_params Pointer to an HEVC slice parameters structure. > > * @p: Pointer to a compound value. > > */ > > union v4l2_ctrl_ptr { > > @@ -64,6 +67,9 @@ union v4l2_ctrl_ptr { > > struct v4l2_ctrl_h264_scaling_matrix *p_h264_scal_mtrx; > > struct v4l2_ctrl_h264_slice_param *p_h264_slice_param; > > struct v4l2_ctrl_h264_decode_param *p_h264_decode_param; > > + struct v4l2_ctrl_hevc_sps *p_hevc_sps; > > + struct v4l2_ctrl_hevc_pps *p_hevc_pps; > > + struct v4l2_ctrl_hevc_slice_params *p_hevc_slice_params; > > void *p; > > }; > > > > diff --git a/include/uapi/linux/v4l2-controls.h > > b/include/uapi/linux/v4l2-controls.h index 628c0cdb51d9..5bbf63b2dad1 > > 100644 > > --- a/include/uapi/linux/v4l2-controls.h > > +++ b/include/uapi/linux/v4l2-controls.h > > @@ -709,6 +709,9 @@ enum v4l2_cid_mpeg_video_hevc_size_of_length_field { > > #define V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L6_BR (V4L2_CID_MPEG_BASE + > > 642) #define V4L2_CID_MPEG_VIDEO_REF_NUMBER_FOR_PFRAMES (V4L2_CID_MPEG_BASE > > + 643) #define > > V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR (V4L2_CID_MPEG_BASE + 644) > > +#define V4L2_CID_MPEG_VIDEO_HEVC_SPS (V4L2_CID_MPEG_BASE + 645) > +#define > > V4L2_CID_MPEG_VIDEO_HEVC_PPS (V4L2_CID_MPEG_BASE + 646) +#define > > V4L2_CID_MPEG_VIDEO_HEVC_SLICE_PARAMS (V4L2_CID_MPEG_BASE + 647) > > > > /* MPEG-class control IDs specific to the CX2341x driver as defined by > > V4L2 */ #define V4L2_CID_MPEG_CX2341X_BASE (V4L2_CTRL_CLASS_MPEG | > > 0x1000) @@ -1324,4 +1327,156 @@ struct v4l2_ctrl_h264_decode_param { > > struct v4l2_h264_dpb_entry dpb[16]; > > }; > > > > +#define V4L2_HEVC_SLICE_TYPE_B 0 > > +#define V4L2_HEVC_SLICE_TYPE_P 1 > > +#define V4L2_HEVC_SLICE_TYPE_I 2 > > + > > +struct v4l2_ctrl_hevc_sps { > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Sequence parameter set */ > > + __u8 chroma_format_idc; > > + __u8 separate_colour_plane_flag; > > + __u16 pic_width_in_luma_samples; > > + __u16 pic_height_in_luma_samples; > > + __u8 bit_depth_luma_minus8; > > + __u8 bit_depth_chroma_minus8; > > + __u8 log2_max_pic_order_cnt_lsb_minus4; > > + __u8 sps_max_dec_pic_buffering_minus1; > > + __u8 sps_max_num_reorder_pics; > > + __u8 sps_max_latency_increase_plus1; > > + __u8 log2_min_luma_coding_block_size_minus3; > > + __u8 log2_diff_max_min_luma_coding_block_size; > > + __u8 log2_min_luma_transform_block_size_minus2; > > + __u8 log2_diff_max_min_luma_transform_block_size; > > + __u8 max_transform_hierarchy_depth_inter; > > + __u8 max_transform_hierarchy_depth_intra; > > + __u8 scaling_list_enabled_flag; > > + __u8 amp_enabled_flag; > > + __u8 sample_adaptive_offset_enabled_flag; > > + __u8 pcm_enabled_flag; > > + __u8 pcm_sample_bit_depth_luma_minus1; > > + __u8 pcm_sample_bit_depth_chroma_minus1; > > + __u8 log2_min_pcm_luma_coding_block_size_minus3; > > + __u8 log2_diff_max_min_pcm_luma_coding_block_size; > > + __u8 pcm_loop_filter_disabled_flag; > > + __u8 num_short_term_ref_pic_sets; > > + __u8 long_term_ref_pics_present_flag; > > + __u8 num_long_term_ref_pics_sps; > > + __u8 sps_temporal_mvp_enabled_flag; > > + __u8 strong_intra_smoothing_enabled_flag; > > +}; > > + > > +struct v4l2_ctrl_hevc_pps { > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Picture parameter set */ > > + __u8 dependent_slice_segment_flag; > > + __u8 output_flag_present_flag; > > + __u8 num_extra_slice_header_bits; > > + __u8 sign_data_hiding_enabled_flag; > > + __u8 cabac_init_present_flag; > > + __s8 init_qp_minus26; > > + __u8 constrained_intra_pred_flag; > > + __u8 transform_skip_enabled_flag; > > + __u8 cu_qp_delta_enabled_flag; > > + __u8 diff_cu_qp_delta_depth; > > + __s8 pps_cb_qp_offset; > > + __s8 pps_cr_qp_offset; > > + __u8 pps_slice_chroma_qp_offsets_present_flag; > > + __u8 weighted_pred_flag; > > + __u8 weighted_bipred_flag; > > + __u8 transquant_bypass_enabled_flag; > > + __u8 tiles_enabled_flag; > > + __u8 entropy_coding_sync_enabled_flag; > > + __u8 num_tile_columns_minus1; > > + __u8 num_tile_rows_minus1; > > + __u8 column_width_minus1[20]; > > + __u8 row_height_minus1[22]; > > + __u8 loop_filter_across_tiles_enabled_flag; > > + __u8 pps_loop_filter_across_slices_enabled_flag; > > + __u8 deblocking_filter_override_enabled_flag; > > + __u8 pps_disable_deblocking_filter_flag; > > + __s8 pps_beta_offset_div2; > > + __s8 pps_tc_offset_div2; > > + __u8 lists_modification_present_flag; > > + __u8 log2_parallel_merge_level_minus2; > > + __u8 slice_segment_header_extension_present_flag; > > +}; > > Although scaling lists are not supported yet, I still think you should include > "scaling_list_data_present_flag" here for the sake of completeness and you > already included "scaling_list_enable_flag" in SPS. > > I didn't do any thorough review though, just noticed this bit. Thanks for suggestion! I decided to discard these "scaling_list_data_present_flag" fields because I think it's best to have a dedicated control for the scaling list (like in the current H.264 proposal). With a dedicated control, scaling lists are no longer attached to either the PPS or SPS so I don't think it makes sense to have "scaling_list_data_present_flag" fields in these structures. Drivers can just infer whether custom scaling lists are used or not with the presence of the optional control and they don't need to know if it was originally extracted from the PPS or SPS. Does that make sense to you? Cheers, Paul > Best regards, > Jernej > > > + > > +#define V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_BEFORE 0x01 > > +#define V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_AFTER 0x02 > > +#define V4L2_HEVC_DPB_ENTRY_RPS_LT_CURR 0x03 > > + > > +#define V4L2_HEVC_DPB_ENTRIES_NUM_MAX 16 > > + > > +struct v4l2_hevc_dpb_entry { > > + __u32 buffer_tag; > > + __u8 rps; > > + __u8 field_pic; > > + __u16 pic_order_cnt[2]; > > +}; > > + > > +struct v4l2_hevc_pred_weight_table { > > + __u8 luma_log2_weight_denom; > > + __s8 delta_chroma_log2_weight_denom; > > + > > + __s8 delta_luma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > + __s8 luma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > + __s8 delta_chroma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > > + __s8 chroma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > > + > > + __s8 delta_luma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > + __s8 luma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > + __s8 delta_chroma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > > + __s8 chroma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > > +}; > > + > > +struct v4l2_ctrl_hevc_slice_params { > > + __u32 bit_size; > > + __u32 data_bit_offset; > > + > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: NAL unit header */ > > + __u8 nal_unit_type; > > + __u8 nuh_temporal_id_plus1; > > + > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ > > + __u8 slice_type; > > + __u8 colour_plane_id; > > + __u16 slice_pic_order_cnt; > > + __u8 slice_sao_luma_flag; > > + __u8 slice_sao_chroma_flag; > > + __u8 slice_temporal_mvp_enabled_flag; > > + __u8 num_ref_idx_l0_active_minus1; > > + __u8 num_ref_idx_l1_active_minus1; > > + __u8 mvd_l1_zero_flag; > > + __u8 cabac_init_flag; > > + __u8 collocated_from_l0_flag; > > + __u8 collocated_ref_idx; > > + __u8 five_minus_max_num_merge_cand; > > + __u8 use_integer_mv_flag; > > + __s8 slice_qp_delta; > > + __s8 slice_cb_qp_offset; > > + __s8 slice_cr_qp_offset; > > + __s8 slice_act_y_qp_offset; > > + __s8 slice_act_cb_qp_offset; > > + __s8 slice_act_cr_qp_offset; > > + __u8 slice_deblocking_filter_disabled_flag; > > + __s8 slice_beta_offset_div2; > > + __s8 slice_tc_offset_div2; > > + __u8 slice_loop_filter_across_slices_enabled_flag; > > + > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Picture timing SEI message */ > > + __u8 pic_struct; > > + > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ > > + struct v4l2_hevc_dpb_entry dpb[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > + __u8 num_active_dpb_entries; > > + __u8 ref_idx_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > + __u8 ref_idx_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > + > > + __u8 num_rps_poc_st_curr_before; > > + __u8 num_rps_poc_st_curr_after; > > + __u8 num_rps_poc_lt_curr; > > + > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Weighted prediction parameter */ > > + struct v4l2_hevc_pred_weight_table pred_weight_table; > > +}; > > + > > #endif > >
On 12/12/18 8:51 PM, Paul Kocialkowski wrote: > Hi, > > On Wed, 2018-12-05 at 21:59 +0100, Jernej Škrabec wrote: > >>> + >>> +#define V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_BEFORE 0x01 >>> +#define V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_AFTER 0x02 >>> +#define V4L2_HEVC_DPB_ENTRY_RPS_LT_CURR 0x03 >>> + >>> +#define V4L2_HEVC_DPB_ENTRIES_NUM_MAX 16 >>> + >>> +struct v4l2_hevc_dpb_entry { >>> + __u32 buffer_tag; >>> + __u8 rps; >>> + __u8 field_pic; >>> + __u16 pic_order_cnt[2]; >>> +}; Please add a property for reference index, if that rps is not used for this, some device would request that(not the rockchip one). And Rockchip's VDPU1 and VDPU2 for AVC would request a similar property. Adding another buffer_tag for referring the memory of the motion vectors for each frames. Or a better method is add a meta data to echo picture buffer, since the picture output is just the same as the original, display won't care whether the motion vectors are written the button of picture or somewhere else. >>> + >>> +struct v4l2_hevc_pred_weight_table { >>> + __u8 luma_log2_weight_denom; >>> + __s8 delta_chroma_log2_weight_denom; >>> + >>> + __s8 delta_luma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>> + __s8 luma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>> + __s8 delta_chroma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; >>> + __s8 chroma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; >>> + >>> + __s8 delta_luma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>> + __s8 luma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>> + __s8 delta_chroma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; >>> + __s8 chroma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; >>> +}; >>> + Those properties I think are not necessary are applying for the Rockchip's device, may not work for the others. >>> +struct v4l2_ctrl_hevc_slice_params { >>> + __u32 bit_size; >>> + __u32 data_bit_offset; >>> + >>> + /* ISO/IEC 23008-2, ITU-T Rec. H.265: NAL unit header */ >>> + __u8 nal_unit_type; >>> + __u8 nuh_temporal_id_plus1; >>> + >>> + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ >>> + __u8 slice_type; >>> + __u8 colour_plane_id; ---------------------------------------------------------------------------- >>> + __u16 slice_pic_order_cnt; >>> + __u8 slice_sao_luma_flag; >>> + __u8 slice_sao_chroma_flag; >>> + __u8 slice_temporal_mvp_enabled_flag; >>> + __u8 num_ref_idx_l0_active_minus1; >>> + __u8 num_ref_idx_l1_active_minus1; Rockchip's decoder doesn't use this part. >>> + __u8 mvd_l1_zero_flag; >>> + __u8 cabac_init_flag; >>> + __u8 collocated_from_l0_flag; >>> + __u8 collocated_ref_idx; >>> + __u8 five_minus_max_num_merge_cand; >>> + __u8 use_integer_mv_flag; >>> + __s8 slice_qp_delta; >>> + __s8 slice_cb_qp_offset; >>> + __s8 slice_cr_qp_offset; >>> + __s8 slice_act_y_qp_offset; >>> + __s8 slice_act_cb_qp_offset; >>> + __s8 slice_act_cr_qp_offset; >>> + __u8 slice_deblocking_filter_disabled_flag; >>> + __s8 slice_beta_offset_div2; >>> + __s8 slice_tc_offset_div2; >>> + __u8 slice_loop_filter_across_slices_enabled_flag; >>> + >>> + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Picture timing SEI message */ >>> + __u8 pic_struct; I think the decoder doesn't care about this, it is used for display. >>> + >>> + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ >>> + struct v4l2_hevc_dpb_entry dpb[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>> + __u8 num_active_dpb_entries; >>> + __u8 ref_idx_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>> + __u8 ref_idx_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>> + >>> + __u8 num_rps_poc_st_curr_before; >>> + __u8 num_rps_poc_st_curr_after; >>> + __u8 num_rps_poc_lt_curr; >>> + >>> + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Weighted prediction parameter */ >>> + struct v4l2_hevc_pred_weight_table pred_weight_table; >>> +}; >>> + >>> #endif >>
Hi, On Mon, 2019-01-07 at 11:49 +0800, Randy Li wrote: > On 12/12/18 8:51 PM, Paul Kocialkowski wrote: > > Hi, > > > > On Wed, 2018-12-05 at 21:59 +0100, Jernej Škrabec wrote: > > > > > > + > > > > +#define V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_BEFORE 0x01 > > > > +#define V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_AFTER 0x02 > > > > +#define V4L2_HEVC_DPB_ENTRY_RPS_LT_CURR 0x03 > > > > + > > > > +#define V4L2_HEVC_DPB_ENTRIES_NUM_MAX 16 > > > > + > > > > +struct v4l2_hevc_dpb_entry { > > > > + __u32 buffer_tag; > > > > + __u8 rps; > > > > + __u8 field_pic; > > > > + __u16 pic_order_cnt[2]; > > > > +}; > > Please add a property for reference index, if that rps is not used for > this, some device would request that(not the rockchip one). And > Rockchip's VDPU1 and VDPU2 for AVC would request a similar property. What exactly is that reference index? Is it a bitstream element or something deduced from the bitstream? > Adding another buffer_tag for referring the memory of the motion vectors > for each frames. Or a better method is add a meta data to echo picture > buffer, since the picture output is just the same as the original, > display won't care whether the motion vectors are written the button of > picture or somewhere else. The motion vectors are passed as part of the raw bitstream data, in the slices. Is there a case where the motion vectors are coded differently? > > > > + > > > > +struct v4l2_hevc_pred_weight_table { > > > > + __u8 luma_log2_weight_denom; > > > > + __s8 delta_chroma_log2_weight_denom; > > > > + > > > > + __s8 delta_luma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > + __s8 luma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > + __s8 delta_chroma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > > > > + __s8 chroma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > > > > + > > > > + __s8 delta_luma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > + __s8 luma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > + __s8 delta_chroma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > > > > + __s8 chroma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > > > > +}; > > > > + > Those properties I think are not necessary are applying for the > Rockchip's device, may not work for the others. Yes, it's possible that some of the elements are not necessary for some decoders. What we want is to cover all the elements that might be required for a decoder. > > > > +struct v4l2_ctrl_hevc_slice_params { > > > > + __u32 bit_size; > > > > + __u32 data_bit_offset; > > > > + > > > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: NAL unit header */ > > > > + __u8 nal_unit_type; > > > > + __u8 nuh_temporal_id_plus1; > > > > + > > > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ > > > > + __u8 slice_type; > > > > + __u8 colour_plane_id; > ---------------------------------------------------------------------------- > > > > + __u16 slice_pic_order_cnt; > > > > + __u8 slice_sao_luma_flag; > > > > + __u8 slice_sao_chroma_flag; > > > > + __u8 slice_temporal_mvp_enabled_flag; > > > > + __u8 num_ref_idx_l0_active_minus1; > > > > + __u8 num_ref_idx_l1_active_minus1; > Rockchip's decoder doesn't use this part. > > > > + __u8 mvd_l1_zero_flag; > > > > + __u8 cabac_init_flag; > > > > + __u8 collocated_from_l0_flag; > > > > + __u8 collocated_ref_idx; > > > > + __u8 five_minus_max_num_merge_cand; > > > > + __u8 use_integer_mv_flag; > > > > + __s8 slice_qp_delta; > > > > + __s8 slice_cb_qp_offset; > > > > + __s8 slice_cr_qp_offset; > > > > + __s8 slice_act_y_qp_offset; > > > > + __s8 slice_act_cb_qp_offset; > > > > + __s8 slice_act_cr_qp_offset; > > > > + __u8 slice_deblocking_filter_disabled_flag; > > > > + __s8 slice_beta_offset_div2; > > > > + __s8 slice_tc_offset_div2; > > > > + __u8 slice_loop_filter_across_slices_enabled_flag; > > > > + > > > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Picture timing SEI message */ > > > > + __u8 pic_struct; > I think the decoder doesn't care about this, it is used for display. The purpose of this field is to indicate whether the current picture is a progressive frame or an interlaced field picture, which is useful for decoding. At least our decoder has a register field to indicate frame/top field/bottom field, so we certainly need to keep the info around. Looking at the spec and the ffmpeg implementation, it looks like this flag of the bitstream is the usual way to report field coding. Cheers, Paul > > > > + > > > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ > > > > + struct v4l2_hevc_dpb_entry dpb[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > + __u8 num_active_dpb_entries; > > > > + __u8 ref_idx_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > + __u8 ref_idx_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > + > > > > + __u8 num_rps_poc_st_curr_before; > > > > + __u8 num_rps_poc_st_curr_after; > > > > + __u8 num_rps_poc_lt_curr; > > > > + > > > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Weighted prediction parameter */ > > > > + struct v4l2_hevc_pred_weight_table pred_weight_table; > > > > +}; > > > > + > > > > #endif
Sent from my iPad > On Jan 7, 2019, at 5:57 PM, Paul Kocialkowski <paul.kocialkowski@bootlin.com> wrote: > > Hi, > >> On Mon, 2019-01-07 at 11:49 +0800, Randy Li wrote: >>> On 12/12/18 8:51 PM, Paul Kocialkowski wrote: >>> Hi, >>> >>> On Wed, 2018-12-05 at 21:59 +0100, Jernej Škrabec wrote: >>> >>>>> + >>>>> +#define V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_BEFORE 0x01 >>>>> +#define V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_AFTER 0x02 >>>>> +#define V4L2_HEVC_DPB_ENTRY_RPS_LT_CURR 0x03 >>>>> + >>>>> +#define V4L2_HEVC_DPB_ENTRIES_NUM_MAX 16 >>>>> + >>>>> +struct v4l2_hevc_dpb_entry { >>>>> + __u32 buffer_tag; >>>>> + __u8 rps; >>>>> + __u8 field_pic; >>>>> + __u16 pic_order_cnt[2]; >>>>> +}; >> >> Please add a property for reference index, if that rps is not used for >> this, some device would request that(not the rockchip one). And >> Rockchip's VDPU1 and VDPU2 for AVC would request a similar property. > > What exactly is that reference index? Is it a bitstream element or > something deduced from the bitstream? > picture order count(POC) for HEVC and frame_num in AVC. I think it is the number used in list0(P slice and B slice) and list1(B slice). >> Adding another buffer_tag for referring the memory of the motion vectors >> for each frames. Or a better method is add a meta data to echo picture >> buffer, since the picture output is just the same as the original, >> display won't care whether the motion vectors are written the button of >> picture or somewhere else. > > The motion vectors are passed as part of the raw bitstream data, in the > slices. Is there a case where the motion vectors are coded differently? No, it is an additional cache for decoder, even FFmpeg having such data, I think allwinner must output it into somewhere. > >>>>> + >>>>> +struct v4l2_hevc_pred_weight_table { >>>>> + __u8 luma_log2_weight_denom; >>>>> + __s8 delta_chroma_log2_weight_denom; >>>>> + >>>>> + __s8 delta_luma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>> + __s8 luma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>> + __s8 delta_chroma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; >>>>> + __s8 chroma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; >>>>> + >>>>> + __s8 delta_luma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>> + __s8 luma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>> + __s8 delta_chroma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; >>>>> + __s8 chroma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; >>>>> +}; >>>>> + >> Those properties I think are not necessary are applying for the >> Rockchip's device, may not work for the others. > > Yes, it's possible that some of the elements are not necessary for some > decoders. What we want is to cover all the elements that might be > required for a decoder. I wonder whether allwinner need that, those sao flag usually ignored by decoder in design. But more is better than less, it is hard to extend a v4l2 structure in the future, maybe a new HEVC profile would bring a new property, it is still too early for HEVC. > >>>>> +struct v4l2_ctrl_hevc_slice_params { >>>>> + __u32 bit_size; >>>>> + __u32 data_bit_offset; >>>>> + >>>>> + /* ISO/IEC 23008-2, ITU-T Rec. H.265: NAL unit header */ >>>>> + __u8 nal_unit_type; >>>>> + __u8 nuh_temporal_id_plus1; >>>>> + >>>>> + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ >>>>> + __u8 slice_type; >>>>> + __u8 colour_plane_id; >> ---------------------------------------------------------------------------- >>>>> + __u16 slice_pic_order_cnt; >>>>> + __u8 slice_sao_luma_flag; >>>>> + __u8 slice_sao_chroma_flag; >>>>> + __u8 slice_temporal_mvp_enabled_flag; >>>>> + __u8 num_ref_idx_l0_active_minus1; >>>>> + __u8 num_ref_idx_l1_active_minus1; >> Rockchip's decoder doesn't use this part. >>>>> + __u8 mvd_l1_zero_flag; >>>>> + __u8 cabac_init_flag; >>>>> + __u8 collocated_from_l0_flag; >>>>> + __u8 collocated_ref_idx; >>>>> + __u8 five_minus_max_num_merge_cand; >>>>> + __u8 use_integer_mv_flag; >>>>> + __s8 slice_qp_delta; >>>>> + __s8 slice_cb_qp_offset; >>>>> + __s8 slice_cr_qp_offset; >>>>> + __s8 slice_act_y_qp_offset; >>>>> + __s8 slice_act_cb_qp_offset; >>>>> + __s8 slice_act_cr_qp_offset; >>>>> + __u8 slice_deblocking_filter_disabled_flag; >>>>> + __s8 slice_beta_offset_div2; >>>>> + __s8 slice_tc_offset_div2; >>>>> + __u8 slice_loop_filter_across_slices_enabled_flag; >>>>> + >>>>> + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Picture timing SEI message */ >>>>> + __u8 pic_struct; >> I think the decoder doesn't care about this, it is used for display. > > The purpose of this field is to indicate whether the current picture is > a progressive frame or an interlaced field picture, which is useful for > decoding. > > At least our decoder has a register field to indicate frame/top > field/bottom field, so we certainly need to keep the info around. > Looking at the spec and the ffmpeg implementation, it looks like this > flag of the bitstream is the usual way to report field coding. It depends whether the decoder cares about scan type or more, I wonder prefer general_interlaced_source_flag for just scan type, it would be better than reading another SEL. > > Cheers, > > Paul Randy “ayaka” LI > >>>>> + >>>>> + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ >>>>> + struct v4l2_hevc_dpb_entry dpb[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>> + __u8 num_active_dpb_entries; >>>>> + __u8 ref_idx_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>> + __u8 ref_idx_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>> + >>>>> + __u8 num_rps_poc_st_curr_before; >>>>> + __u8 num_rps_poc_st_curr_after; >>>>> + __u8 num_rps_poc_lt_curr; >>>>> + >>>>> + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Weighted prediction parameter */ >>>>> + struct v4l2_hevc_pred_weight_table pred_weight_table; >>>>> +}; >>>>> + >>>>> #endif > -- > Paul Kocialkowski, Bootlin (formerly Free Electrons) > Embedded Linux and kernel engineering > https://bootlin.com >
Hi, On Tue, 2019-01-08 at 09:16 +0800, Ayaka wrote: > > Sent from my iPad > > > On Jan 7, 2019, at 5:57 PM, Paul Kocialkowski <paul.kocialkowski@bootlin.com> wrote: > > > > Hi, > > > > > On Mon, 2019-01-07 at 11:49 +0800, Randy Li wrote: > > > > On 12/12/18 8:51 PM, Paul Kocialkowski wrote: > > > > Hi, > > > > > > > > On Wed, 2018-12-05 at 21:59 +0100, Jernej Škrabec wrote: > > > > > > > > > > + > > > > > > +#define V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_BEFORE 0x01 > > > > > > +#define V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_AFTER 0x02 > > > > > > +#define V4L2_HEVC_DPB_ENTRY_RPS_LT_CURR 0x03 > > > > > > + > > > > > > +#define V4L2_HEVC_DPB_ENTRIES_NUM_MAX 16 > > > > > > + > > > > > > +struct v4l2_hevc_dpb_entry { > > > > > > + __u32 buffer_tag; > > > > > > + __u8 rps; > > > > > > + __u8 field_pic; > > > > > > + __u16 pic_order_cnt[2]; > > > > > > +}; > > > > > > Please add a property for reference index, if that rps is not used for > > > this, some device would request that(not the rockchip one). And > > > Rockchip's VDPU1 and VDPU2 for AVC would request a similar property. > > > > What exactly is that reference index? Is it a bitstream element or > > something deduced from the bitstream? > > > picture order count(POC) for HEVC and frame_num in AVC. I think it is > the number used in list0(P slice and B slice) and list1(B slice). The picture order count is already the last field of the DPB entry structure. There is one for each field picture. > > > Adding another buffer_tag for referring the memory of the motion vectors > > > for each frames. Or a better method is add a meta data to echo picture > > > buffer, since the picture output is just the same as the original, > > > display won't care whether the motion vectors are written the button of > > > picture or somewhere else. > > > > The motion vectors are passed as part of the raw bitstream data, in the > > slices. Is there a case where the motion vectors are coded differently? > No, it is an additional cache for decoder, even FFmpeg having such > data, I think allwinner must output it into somewhere. Ah yes I see what you mean! This is handled internally by our driver and not exposed to userspace. I don't think it would be a good idea to expose this cache or request that userspace allocates it like a video buffer. > > > > > > + > > > > > > +struct v4l2_hevc_pred_weight_table { > > > > > > + __u8 luma_log2_weight_denom; > > > > > > + __s8 delta_chroma_log2_weight_denom; > > > > > > + > > > > > > + __s8 delta_luma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > > > + __s8 luma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > > > + __s8 delta_chroma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > > > > > > + __s8 chroma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > > > > > > + > > > > > > + __s8 delta_luma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > > > + __s8 luma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > > > + __s8 delta_chroma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > > > > > > + __s8 chroma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > > > > > > +}; > > > > > > + > > > Those properties I think are not necessary are applying for the > > > Rockchip's device, may not work for the others. > > > > Yes, it's possible that some of the elements are not necessary for some > > decoders. What we want is to cover all the elements that might be > > required for a decoder. > I wonder whether allwinner need that, those sao flag usually ignored > by decoder in design. But more is better than less, it is hard to > extend a v4l2 structure in the future, maybe a new HEVC profile > would bring a new property, it is still too early for HEVC. Yes this is used by our decoder. The idea is to have all the basic bitstream elements in the structures (even if some decoders don't use them all) and add others for extension as separate controls later. > > > > > > +struct v4l2_ctrl_hevc_slice_params { > > > > > > + __u32 bit_size; > > > > > > + __u32 data_bit_offset; > > > > > > + > > > > > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: NAL unit header */ > > > > > > + __u8 nal_unit_type; > > > > > > + __u8 nuh_temporal_id_plus1; > > > > > > + > > > > > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ > > > > > > + __u8 slice_type; > > > > > > + __u8 colour_plane_id; > > > ---------------------------------------------------------------------------- > > > > > > + __u16 slice_pic_order_cnt; > > > > > > + __u8 slice_sao_luma_flag; > > > > > > + __u8 slice_sao_chroma_flag; > > > > > > + __u8 slice_temporal_mvp_enabled_flag; > > > > > > + __u8 num_ref_idx_l0_active_minus1; > > > > > > + __u8 num_ref_idx_l1_active_minus1; > > > Rockchip's decoder doesn't use this part. > > > > > > + __u8 mvd_l1_zero_flag; > > > > > > + __u8 cabac_init_flag; > > > > > > + __u8 collocated_from_l0_flag; > > > > > > + __u8 collocated_ref_idx; > > > > > > + __u8 five_minus_max_num_merge_cand; > > > > > > + __u8 use_integer_mv_flag; > > > > > > + __s8 slice_qp_delta; > > > > > > + __s8 slice_cb_qp_offset; > > > > > > + __s8 slice_cr_qp_offset; > > > > > > + __s8 slice_act_y_qp_offset; > > > > > > + __s8 slice_act_cb_qp_offset; > > > > > > + __s8 slice_act_cr_qp_offset; > > > > > > + __u8 slice_deblocking_filter_disabled_flag; > > > > > > + __s8 slice_beta_offset_div2; > > > > > > + __s8 slice_tc_offset_div2; > > > > > > + __u8 slice_loop_filter_across_slices_enabled_flag; > > > > > > + > > > > > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Picture timing SEI message */ > > > > > > + __u8 pic_struct; > > > I think the decoder doesn't care about this, it is used for display. > > > > The purpose of this field is to indicate whether the current picture is > > a progressive frame or an interlaced field picture, which is useful for > > decoding. > > > > At least our decoder has a register field to indicate frame/top > > field/bottom field, so we certainly need to keep the info around. > > Looking at the spec and the ffmpeg implementation, it looks like this > > flag of the bitstream is the usual way to report field coding. > It depends whether the decoder cares about scan type or more, I > wonder prefer general_interlaced_source_flag for just scan type, it > would be better than reading another SEL. Well we still need a way to indicate if the current data is top or bottom field for interlaced. I don't think that knowing that the whole video is interlaced would be precise enough. Cheers, Paul > > > > > > + > > > > > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ > > > > > > + struct v4l2_hevc_dpb_entry dpb[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > > > + __u8 num_active_dpb_entries; > > > > > > + __u8 ref_idx_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > > > + __u8 ref_idx_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > > > + > > > > > > + __u8 num_rps_poc_st_curr_before; > > > > > > + __u8 num_rps_poc_st_curr_after; > > > > > > + __u8 num_rps_poc_lt_curr; > > > > > > + > > > > > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Weighted prediction parameter */ > > > > > > + struct v4l2_hevc_pred_weight_table pred_weight_table; > > > > > > +}; > > > > > > + > > > > > > #endif > > -- > > Paul Kocialkowski, Bootlin (formerly Free Electrons) > > Embedded Linux and kernel engineering > > https://bootlin.com > >
Sent from my iPad > On Jan 8, 2019, at 4:38 PM, Paul Kocialkowski <paul.kocialkowski@bootlin.com> wrote: > > Hi, > >> On Tue, 2019-01-08 at 09:16 +0800, Ayaka wrote: >> >> Sent from my iPad >> >>> On Jan 7, 2019, at 5:57 PM, Paul Kocialkowski <paul.kocialkowski@bootlin.com> wrote: >>> >>> Hi, >>> >>>>> On Mon, 2019-01-07 at 11:49 +0800, Randy Li wrote: >>>>> On 12/12/18 8:51 PM, Paul Kocialkowski wrote: >>>>> Hi, >>>>> >>>>> On Wed, 2018-12-05 at 21:59 +0100, Jernej Škrabec wrote: >>>>> >>>>>>> + >>>>>>> +#define V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_BEFORE 0x01 >>>>>>> +#define V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_AFTER 0x02 >>>>>>> +#define V4L2_HEVC_DPB_ENTRY_RPS_LT_CURR 0x03 >>>>>>> + >>>>>>> +#define V4L2_HEVC_DPB_ENTRIES_NUM_MAX 16 >>>>>>> + >>>>>>> +struct v4l2_hevc_dpb_entry { >>>>>>> + __u32 buffer_tag; >>>>>>> + __u8 rps; >>>>>>> + __u8 field_pic; >>>>>>> + __u16 pic_order_cnt[2]; >>>>>>> +}; >>>> >>>> Please add a property for reference index, if that rps is not used for >>>> this, some device would request that(not the rockchip one). And >>>> Rockchip's VDPU1 and VDPU2 for AVC would request a similar property. >>> >>> What exactly is that reference index? Is it a bitstream element or >>> something deduced from the bitstream? >>> >> picture order count(POC) for HEVC and frame_num in AVC. I think it is >> the number used in list0(P slice and B slice) and list1(B slice). > > The picture order count is already the last field of the DPB entry > structure. There is one for each field picture. As we are not sure whether there is a field coded slice or CTU, I would hold this part and else about the field. > >>>> Adding another buffer_tag for referring the memory of the motion vectors >>>> for each frames. Or a better method is add a meta data to echo picture >>>> buffer, since the picture output is just the same as the original, >>>> display won't care whether the motion vectors are written the button of >>>> picture or somewhere else. >>> >>> The motion vectors are passed as part of the raw bitstream data, in the >>> slices. Is there a case where the motion vectors are coded differently? >> No, it is an additional cache for decoder, even FFmpeg having such >> data, I think allwinner must output it into somewhere. > > Ah yes I see what you mean! This is handled internally by our driver > and not exposed to userspace. I don't think it would be a good idea to > expose this cache or request that userspace allocates it like a video > buffer. > No, usually the driver should allocate, as the user space have no idea on size of each devices. But for advantage user, application can fix a broken picture with a proper data or analysis a object motion from that. So I would suggest attaching this information to a picture buffer as a meta data. >>>>>>> + >>>>>>> +struct v4l2_hevc_pred_weight_table { >>>>>>> + __u8 luma_log2_weight_denom; >>>>>>> + __s8 delta_chroma_log2_weight_denom; >>>>>>> + >>>>>>> + __s8 delta_luma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>>>> + __s8 luma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>>>> + __s8 delta_chroma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; >>>>>>> + __s8 chroma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; >>>>>>> + >>>>>>> + __s8 delta_luma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>>>> + __s8 luma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>>>> + __s8 delta_chroma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; >>>>>>> + __s8 chroma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; >>>>>>> +}; >>>>>>> + >>>> Those properties I think are not necessary are applying for the >>>> Rockchip's device, may not work for the others. >>> >>> Yes, it's possible that some of the elements are not necessary for some >>> decoders. What we want is to cover all the elements that might be >>> required for a decoder. >> I wonder whether allwinner need that, those sao flag usually ignored >> by decoder in design. But more is better than less, it is hard to >> extend a v4l2 structure in the future, maybe a new HEVC profile >> would bring a new property, it is still too early for HEVC. > > Yes this is used by our decoder. The idea is to have all the basic > bitstream elements in the structures (even if some decoders don't use > them all) and add others for extension as separate controls later. > >>>>>>> +struct v4l2_ctrl_hevc_slice_params { >>>>>>> + __u32 bit_size; >>>>>>> + __u32 data_bit_offset; >>>>>>> + >>>>>>> + /* ISO/IEC 23008-2, ITU-T Rec. H.265: NAL unit header */ >>>>>>> + __u8 nal_unit_type; >>>>>>> + __u8 nuh_temporal_id_plus1; >>>>>>> + >>>>>>> + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ >>>>>>> + __u8 slice_type; >>>>>>> + __u8 colour_plane_id; >>>> ---------------------------------------------------------------------------- >>>>>>> + __u16 slice_pic_order_cnt; >>>>>>> + __u8 slice_sao_luma_flag; >>>>>>> + __u8 slice_sao_chroma_flag; >>>>>>> + __u8 slice_temporal_mvp_enabled_flag; >>>>>>> + __u8 num_ref_idx_l0_active_minus1; >>>>>>> + __u8 num_ref_idx_l1_active_minus1; >>>> Rockchip's decoder doesn't use this part. >>>>>>> + __u8 mvd_l1_zero_flag; >>>>>>> + __u8 cabac_init_flag; >>>>>>> + __u8 collocated_from_l0_flag; >>>>>>> + __u8 collocated_ref_idx; >>>>>>> + __u8 five_minus_max_num_merge_cand; >>>>>>> + __u8 use_integer_mv_flag; >>>>>>> + __s8 slice_qp_delta; >>>>>>> + __s8 slice_cb_qp_offset; >>>>>>> + __s8 slice_cr_qp_offset; >>>>>>> + __s8 slice_act_y_qp_offset; >>>>>>> + __s8 slice_act_cb_qp_offset; >>>>>>> + __s8 slice_act_cr_qp_offset; >>>>>>> + __u8 slice_deblocking_filter_disabled_flag; >>>>>>> + __s8 slice_beta_offset_div2; >>>>>>> + __s8 slice_tc_offset_div2; >>>>>>> + __u8 slice_loop_filter_across_slices_enabled_flag; >>>>>>> + >>>>>>> + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Picture timing SEI message */ >>>>>>> + __u8 pic_struct; >>>> I think the decoder doesn't care about this, it is used for display. >>> >>> The purpose of this field is to indicate whether the current picture is >>> a progressive frame or an interlaced field picture, which is useful for >>> decoding. >>> >>> At least our decoder has a register field to indicate frame/top >>> field/bottom field, so we certainly need to keep the info around. >>> Looking at the spec and the ffmpeg implementation, it looks like this >>> flag of the bitstream is the usual way to report field coding. >> It depends whether the decoder cares about scan type or more, I >> wonder prefer general_interlaced_source_flag for just scan type, it >> would be better than reading another SEL. > > Well we still need a way to indicate if the current data is top or > bottom field for interlaced. I don't think that knowing that the whole > video is interlaced would be precise enough. > > Cheers, > > Paul > >>>>>>> + >>>>>>> + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ >>>>>>> + struct v4l2_hevc_dpb_entry dpb[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>>>> + __u8 num_active_dpb_entries; >>>>>>> + __u8 ref_idx_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>>>> + __u8 ref_idx_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>>>> + >>>>>>> + __u8 num_rps_poc_st_curr_before; >>>>>>> + __u8 num_rps_poc_st_curr_after; >>>>>>> + __u8 num_rps_poc_lt_curr; >>>>>>> + >>>>>>> + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Weighted prediction parameter */ >>>>>>> + struct v4l2_hevc_pred_weight_table pred_weight_table; >>>>>>> +}; >>>>>>> + >>>>>>> #endif >>> -- >>> Paul Kocialkowski, Bootlin (formerly Free Electrons) >>> Embedded Linux and kernel engineering >>> https://bootlin.com >>> > -- > Paul Kocialkowski, Bootlin (formerly Free Electrons) > Embedded Linux and kernel engineering > https://bootlin.com >
I forget a important thing, for the rkvdec and rk hevc decoder, it would requests cabac table, scaling list, picture parameter set and reference picture storing in one or various of DMA buffers. I am not talking about the data been parsed, the decoder would requests a raw data. For the pps and rps, it is possible to reuse the slice header, just let the decoder know the offset from the bitstream bufer, I would suggest to add three properties(with sps) for them. But I think we need a method to mark a OUTPUT side buffer for those aux data. On 1/8/19 6:00 PM, Ayaka wrote: > > Sent from my iPad > >> On Jan 8, 2019, at 4:38 PM, Paul Kocialkowski <paul.kocialkowski@bootlin.com> wrote: >> >> Hi, >> >>> On Tue, 2019-01-08 at 09:16 +0800, Ayaka wrote: >>> >>> Sent from my iPad >>> >>>> On Jan 7, 2019, at 5:57 PM, Paul Kocialkowski <paul.kocialkowski@bootlin.com> wrote: >>>> >>>> Hi, >>>> >>>>>> On Mon, 2019-01-07 at 11:49 +0800, Randy Li wrote: >>>>>> On 12/12/18 8:51 PM, Paul Kocialkowski wrote: >>>>>> Hi, >>>>>> >>>>>> On Wed, 2018-12-05 at 21:59 +0100, Jernej Škrabec wrote: >>>>>> >>>>>>>> + >>>>>>>> +#define V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_BEFORE 0x01 >>>>>>>> +#define V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_AFTER 0x02 >>>>>>>> +#define V4L2_HEVC_DPB_ENTRY_RPS_LT_CURR 0x03 >>>>>>>> + >>>>>>>> +#define V4L2_HEVC_DPB_ENTRIES_NUM_MAX 16 >>>>>>>> + >>>>>>>> +struct v4l2_hevc_dpb_entry { >>>>>>>> + __u32 buffer_tag; >>>>>>>> + __u8 rps; >>>>>>>> + __u8 field_pic; >>>>>>>> + __u16 pic_order_cnt[2]; >>>>>>>> +}; >>>>> Please add a property for reference index, if that rps is not used for >>>>> this, some device would request that(not the rockchip one). And >>>>> Rockchip's VDPU1 and VDPU2 for AVC would request a similar property. >>>> What exactly is that reference index? Is it a bitstream element or >>>> something deduced from the bitstream? >>>> >>> picture order count(POC) for HEVC and frame_num in AVC. I think it is >>> the number used in list0(P slice and B slice) and list1(B slice). >> The picture order count is already the last field of the DPB entry >> structure. There is one for each field picture. > As we are not sure whether there is a field coded slice or CTU, I would hold this part and else about the field. >>>>> Adding another buffer_tag for referring the memory of the motion vectors >>>>> for each frames. Or a better method is add a meta data to echo picture >>>>> buffer, since the picture output is just the same as the original, >>>>> display won't care whether the motion vectors are written the button of >>>>> picture or somewhere else. >>>> The motion vectors are passed as part of the raw bitstream data, in the >>>> slices. Is there a case where the motion vectors are coded differently? >>> No, it is an additional cache for decoder, even FFmpeg having such >>> data, I think allwinner must output it into somewhere. >> Ah yes I see what you mean! This is handled internally by our driver >> and not exposed to userspace. I don't think it would be a good idea to >> expose this cache or request that userspace allocates it like a video >> buffer. >> > No, usually the driver should allocate, as the user space have no idea on size of each devices. > But for advantage user, application can fix a broken picture with a proper data or analysis a object motion from that. > So I would suggest attaching this information to a picture buffer as a meta data. >>>>>>>> + >>>>>>>> +struct v4l2_hevc_pred_weight_table { >>>>>>>> + __u8 luma_log2_weight_denom; >>>>>>>> + __s8 delta_chroma_log2_weight_denom; >>>>>>>> + >>>>>>>> + __s8 delta_luma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>>>>> + __s8 luma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>>>>> + __s8 delta_chroma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; >>>>>>>> + __s8 chroma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; >>>>>>>> + >>>>>>>> + __s8 delta_luma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>>>>> + __s8 luma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>>>>> + __s8 delta_chroma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; >>>>>>>> + __s8 chroma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; >>>>>>>> +}; >>>>>>>> + >>>>> Those properties I think are not necessary are applying for the >>>>> Rockchip's device, may not work for the others. >>>> Yes, it's possible that some of the elements are not necessary for some >>>> decoders. What we want is to cover all the elements that might be >>>> required for a decoder. >>> I wonder whether allwinner need that, those sao flag usually ignored >>> by decoder in design. But more is better than less, it is hard to >>> extend a v4l2 structure in the future, maybe a new HEVC profile >>> would bring a new property, it is still too early for HEVC. >> Yes this is used by our decoder. The idea is to have all the basic >> bitstream elements in the structures (even if some decoders don't use >> them all) and add others for extension as separate controls later. >> >>>>>>>> +struct v4l2_ctrl_hevc_slice_params { >>>>>>>> + __u32 bit_size; >>>>>>>> + __u32 data_bit_offset; >>>>>>>> + >>>>>>>> + /* ISO/IEC 23008-2, ITU-T Rec. H.265: NAL unit header */ >>>>>>>> + __u8 nal_unit_type; >>>>>>>> + __u8 nuh_temporal_id_plus1; >>>>>>>> + >>>>>>>> + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ >>>>>>>> + __u8 slice_type; >>>>>>>> + __u8 colour_plane_id; >>>>> ---------------------------------------------------------------------------- >>>>>>>> + __u16 slice_pic_order_cnt; >>>>>>>> + __u8 slice_sao_luma_flag; >>>>>>>> + __u8 slice_sao_chroma_flag; >>>>>>>> + __u8 slice_temporal_mvp_enabled_flag; >>>>>>>> + __u8 num_ref_idx_l0_active_minus1; >>>>>>>> + __u8 num_ref_idx_l1_active_minus1; >>>>> Rockchip's decoder doesn't use this part. >>>>>>>> + __u8 mvd_l1_zero_flag; >>>>>>>> + __u8 cabac_init_flag; >>>>>>>> + __u8 collocated_from_l0_flag; >>>>>>>> + __u8 collocated_ref_idx; >>>>>>>> + __u8 five_minus_max_num_merge_cand; >>>>>>>> + __u8 use_integer_mv_flag; >>>>>>>> + __s8 slice_qp_delta; >>>>>>>> + __s8 slice_cb_qp_offset; >>>>>>>> + __s8 slice_cr_qp_offset; >>>>>>>> + __s8 slice_act_y_qp_offset; >>>>>>>> + __s8 slice_act_cb_qp_offset; >>>>>>>> + __s8 slice_act_cr_qp_offset; >>>>>>>> + __u8 slice_deblocking_filter_disabled_flag; >>>>>>>> + __s8 slice_beta_offset_div2; >>>>>>>> + __s8 slice_tc_offset_div2; >>>>>>>> + __u8 slice_loop_filter_across_slices_enabled_flag; >>>>>>>> + >>>>>>>> + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Picture timing SEI message */ >>>>>>>> + __u8 pic_struct; >>>>> I think the decoder doesn't care about this, it is used for display. >>>> The purpose of this field is to indicate whether the current picture is >>>> a progressive frame or an interlaced field picture, which is useful for >>>> decoding. >>>> >>>> At least our decoder has a register field to indicate frame/top >>>> field/bottom field, so we certainly need to keep the info around. >>>> Looking at the spec and the ffmpeg implementation, it looks like this >>>> flag of the bitstream is the usual way to report field coding. >>> It depends whether the decoder cares about scan type or more, I >>> wonder prefer general_interlaced_source_flag for just scan type, it >>> would be better than reading another SEL. >> Well we still need a way to indicate if the current data is top or >> bottom field for interlaced. I don't think that knowing that the whole >> video is interlaced would be precise enough. >> >> Cheers, >> >> Paul >> >>>>>>>> + >>>>>>>> + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ >>>>>>>> + struct v4l2_hevc_dpb_entry dpb[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>>>>> + __u8 num_active_dpb_entries; >>>>>>>> + __u8 ref_idx_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>>>>> + __u8 ref_idx_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>>>>> + >>>>>>>> + __u8 num_rps_poc_st_curr_before; >>>>>>>> + __u8 num_rps_poc_st_curr_after; >>>>>>>> + __u8 num_rps_poc_lt_curr; >>>>>>>> + >>>>>>>> + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Weighted prediction parameter */ >>>>>>>> + struct v4l2_hevc_pred_weight_table pred_weight_table; >>>>>>>> +}; >>>>>>>> + >>>>>>>> #endif >>>> -- >>>> Paul Kocialkowski, Bootlin (formerly Free Electrons) >>>> Embedded Linux and kernel engineering >>>> https://bootlin.com >>>> >> -- >> Paul Kocialkowski, Bootlin (formerly Free Electrons) >> Embedded Linux and kernel engineering >> https://bootlin.com >>
Hi, On Thu, 2019-01-10 at 21:32 +0800, ayaka wrote: > I forget a important thing, for the rkvdec and rk hevc decoder, it would > requests cabac table, scaling list, picture parameter set and reference > picture storing in one or various of DMA buffers. I am not talking about > the data been parsed, the decoder would requests a raw data. > > For the pps and rps, it is possible to reuse the slice header, just let > the decoder know the offset from the bitstream bufer, I would suggest to > add three properties(with sps) for them. But I think we need a method to > mark a OUTPUT side buffer for those aux data. I'm quite confused about the hardware implementation then. From what you're saying, it seems that it takes the raw bitstream elements rather than parsed elements. Is it really a stateless implementation? The stateless implementation was designed with the idea that only the raw slice data should be passed in bitstream form to the decoder. For H.264, it seems that some decoders also need the slice header in raw bitstream form (because they take the full slice NAL unit), see the discussions in this thread: media: docs-rst: Document m2m stateless video decoder interface Can you detail exactly what the rockchip decoder absolutely needs in raw bitstream format? Cheers, Paul > On 1/8/19 6:00 PM, Ayaka wrote: > > Sent from my iPad > > > > > On Jan 8, 2019, at 4:38 PM, Paul Kocialkowski <paul.kocialkowski@bootlin.com> wrote: > > > > > > Hi, > > > > > > > On Tue, 2019-01-08 at 09:16 +0800, Ayaka wrote: > > > > > > > > Sent from my iPad > > > > > > > > > On Jan 7, 2019, at 5:57 PM, Paul Kocialkowski <paul.kocialkowski@bootlin.com> wrote: > > > > > > > > > > Hi, > > > > > > > > > > > > On Mon, 2019-01-07 at 11:49 +0800, Randy Li wrote: > > > > > > > On 12/12/18 8:51 PM, Paul Kocialkowski wrote: > > > > > > > Hi, > > > > > > > > > > > > > > On Wed, 2018-12-05 at 21:59 +0100, Jernej Škrabec wrote: > > > > > > > > > > > > > > > > + > > > > > > > > > +#define V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_BEFORE 0x01 > > > > > > > > > +#define V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_AFTER 0x02 > > > > > > > > > +#define V4L2_HEVC_DPB_ENTRY_RPS_LT_CURR 0x03 > > > > > > > > > + > > > > > > > > > +#define V4L2_HEVC_DPB_ENTRIES_NUM_MAX 16 > > > > > > > > > + > > > > > > > > > +struct v4l2_hevc_dpb_entry { > > > > > > > > > + __u32 buffer_tag; > > > > > > > > > + __u8 rps; > > > > > > > > > + __u8 field_pic; > > > > > > > > > + __u16 pic_order_cnt[2]; > > > > > > > > > +}; > > > > > > Please add a property for reference index, if that rps is not used for > > > > > > this, some device would request that(not the rockchip one). And > > > > > > Rockchip's VDPU1 and VDPU2 for AVC would request a similar property. > > > > > What exactly is that reference index? Is it a bitstream element or > > > > > something deduced from the bitstream? > > > > > > > > > picture order count(POC) for HEVC and frame_num in AVC. I think it is > > > > the number used in list0(P slice and B slice) and list1(B slice). > > > The picture order count is already the last field of the DPB entry > > > structure. There is one for each field picture. > > As we are not sure whether there is a field coded slice or CTU, I would hold this part and else about the field. > > > > > > Adding another buffer_tag for referring the memory of the motion vectors > > > > > > for each frames. Or a better method is add a meta data to echo picture > > > > > > buffer, since the picture output is just the same as the original, > > > > > > display won't care whether the motion vectors are written the button of > > > > > > picture or somewhere else. > > > > > The motion vectors are passed as part of the raw bitstream data, in the > > > > > slices. Is there a case where the motion vectors are coded differently? > > > > No, it is an additional cache for decoder, even FFmpeg having such > > > > data, I think allwinner must output it into somewhere. > > > Ah yes I see what you mean! This is handled internally by our driver > > > and not exposed to userspace. I don't think it would be a good idea to > > > expose this cache or request that userspace allocates it like a video > > > buffer. > > > > > No, usually the driver should allocate, as the user space have no idea on size of each devices. > > But for advantage user, application can fix a broken picture with a proper data or analysis a object motion from that. > > So I would suggest attaching this information to a picture buffer as a meta data. > > > > > > > > > + > > > > > > > > > +struct v4l2_hevc_pred_weight_table { > > > > > > > > > + __u8 luma_log2_weight_denom; > > > > > > > > > + __s8 delta_chroma_log2_weight_denom; > > > > > > > > > + > > > > > > > > > + __s8 delta_luma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > > > > > > + __s8 luma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > > > > > > + __s8 delta_chroma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > > > > > > > > > + __s8 chroma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > > > > > > > > > + > > > > > > > > > + __s8 delta_luma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > > > > > > + __s8 luma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > > > > > > + __s8 delta_chroma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > > > > > > > > > + __s8 chroma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > > > > > > > > > +}; > > > > > > > > > + > > > > > > Those properties I think are not necessary are applying for the > > > > > > Rockchip's device, may not work for the others. > > > > > Yes, it's possible that some of the elements are not necessary for some > > > > > decoders. What we want is to cover all the elements that might be > > > > > required for a decoder. > > > > I wonder whether allwinner need that, those sao flag usually ignored > > > > by decoder in design. But more is better than less, it is hard to > > > > extend a v4l2 structure in the future, maybe a new HEVC profile > > > > would bring a new property, it is still too early for HEVC. > > > Yes this is used by our decoder. The idea is to have all the basic > > > bitstream elements in the structures (even if some decoders don't use > > > them all) and add others for extension as separate controls later. > > > > > > > > > > > > +struct v4l2_ctrl_hevc_slice_params { > > > > > > > > > + __u32 bit_size; > > > > > > > > > + __u32 data_bit_offset; > > > > > > > > > + > > > > > > > > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: NAL unit header */ > > > > > > > > > + __u8 nal_unit_type; > > > > > > > > > + __u8 nuh_temporal_id_plus1; > > > > > > > > > + > > > > > > > > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ > > > > > > > > > + __u8 slice_type; > > > > > > > > > + __u8 colour_plane_id; > > > > > > ---------------------------------------------------------------------------- > > > > > > > > > + __u16 slice_pic_order_cnt; > > > > > > > > > + __u8 slice_sao_luma_flag; > > > > > > > > > + __u8 slice_sao_chroma_flag; > > > > > > > > > + __u8 slice_temporal_mvp_enabled_flag; > > > > > > > > > + __u8 num_ref_idx_l0_active_minus1; > > > > > > > > > + __u8 num_ref_idx_l1_active_minus1; > > > > > > Rockchip's decoder doesn't use this part. > > > > > > > > > + __u8 mvd_l1_zero_flag; > > > > > > > > > + __u8 cabac_init_flag; > > > > > > > > > + __u8 collocated_from_l0_flag; > > > > > > > > > + __u8 collocated_ref_idx; > > > > > > > > > + __u8 five_minus_max_num_merge_cand; > > > > > > > > > + __u8 use_integer_mv_flag; > > > > > > > > > + __s8 slice_qp_delta; > > > > > > > > > + __s8 slice_cb_qp_offset; > > > > > > > > > + __s8 slice_cr_qp_offset; > > > > > > > > > + __s8 slice_act_y_qp_offset; > > > > > > > > > + __s8 slice_act_cb_qp_offset; > > > > > > > > > + __s8 slice_act_cr_qp_offset; > > > > > > > > > + __u8 slice_deblocking_filter_disabled_flag; > > > > > > > > > + __s8 slice_beta_offset_div2; > > > > > > > > > + __s8 slice_tc_offset_div2; > > > > > > > > > + __u8 slice_loop_filter_across_slices_enabled_flag; > > > > > > > > > + > > > > > > > > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Picture timing SEI message */ > > > > > > > > > + __u8 pic_struct; > > > > > > I think the decoder doesn't care about this, it is used for display. > > > > > The purpose of this field is to indicate whether the current picture is > > > > > a progressive frame or an interlaced field picture, which is useful for > > > > > decoding. > > > > > > > > > > At least our decoder has a register field to indicate frame/top > > > > > field/bottom field, so we certainly need to keep the info around. > > > > > Looking at the spec and the ffmpeg implementation, it looks like this > > > > > flag of the bitstream is the usual way to report field coding. > > > > It depends whether the decoder cares about scan type or more, I > > > > wonder prefer general_interlaced_source_flag for just scan type, it > > > > would be better than reading another SEL. > > > Well we still need a way to indicate if the current data is top or > > > bottom field for interlaced. I don't think that knowing that the whole > > > video is interlaced would be precise enough. > > > > > > Cheers, > > > > > > Paul > > > > > > > > > > > > + > > > > > > > > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ > > > > > > > > > + struct v4l2_hevc_dpb_entry dpb[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > > > > > > + __u8 num_active_dpb_entries; > > > > > > > > > + __u8 ref_idx_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > > > > > > + __u8 ref_idx_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > > > > > > + > > > > > > > > > + __u8 num_rps_poc_st_curr_before; > > > > > > > > > + __u8 num_rps_poc_st_curr_after; > > > > > > > > > + __u8 num_rps_poc_lt_curr; > > > > > > > > > + > > > > > > > > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Weighted prediction parameter */ > > > > > > > > > + struct v4l2_hevc_pred_weight_table pred_weight_table; > > > > > > > > > +}; > > > > > > > > > + > > > > > > > > > #endif > > > > > -- > > > > > Paul Kocialkowski, Bootlin (formerly Free Electrons) > > > > > Embedded Linux and kernel engineering > > > > > https://bootlin.com > > > > > > > > -- > > > Paul Kocialkowski, Bootlin (formerly Free Electrons) > > > Embedded Linux and kernel engineering > > > https://bootlin.com > > >
Hi, On Tue, 2019-01-08 at 18:00 +0800, Ayaka wrote: > > Sent from my iPad > > > On Jan 8, 2019, at 4:38 PM, Paul Kocialkowski <paul.kocialkowski@bootlin.com> wrote: > > > > Hi, > > > > > On Tue, 2019-01-08 at 09:16 +0800, Ayaka wrote: > > > > > > Sent from my iPad > > > > > > > On Jan 7, 2019, at 5:57 PM, Paul Kocialkowski <paul.kocialkowski@bootlin.com> wrote: > > > > > > > > Hi, > > > > > > > > > > On Mon, 2019-01-07 at 11:49 +0800, Randy Li wrote: > > > > > > On 12/12/18 8:51 PM, Paul Kocialkowski wrote: > > > > > > Hi, > > > > > > > > > > > > On Wed, 2018-12-05 at 21:59 +0100, Jernej Škrabec wrote: > > > > > > > > > > > > > > + > > > > > > > > +#define V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_BEFORE 0x01 > > > > > > > > +#define V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_AFTER 0x02 > > > > > > > > +#define V4L2_HEVC_DPB_ENTRY_RPS_LT_CURR 0x03 > > > > > > > > + > > > > > > > > +#define V4L2_HEVC_DPB_ENTRIES_NUM_MAX 16 > > > > > > > > + > > > > > > > > +struct v4l2_hevc_dpb_entry { > > > > > > > > + __u32 buffer_tag; > > > > > > > > + __u8 rps; > > > > > > > > + __u8 field_pic; > > > > > > > > + __u16 pic_order_cnt[2]; > > > > > > > > +}; > > > > > > > > > > Please add a property for reference index, if that rps is not used for > > > > > this, some device would request that(not the rockchip one). And > > > > > Rockchip's VDPU1 and VDPU2 for AVC would request a similar property. > > > > > > > > What exactly is that reference index? Is it a bitstream element or > > > > something deduced from the bitstream? > > > > > > > picture order count(POC) for HEVC and frame_num in AVC. I think it is > > > the number used in list0(P slice and B slice) and list1(B slice). > > > > The picture order count is already the last field of the DPB entry > > structure. There is one for each field picture. > As we are not sure whether there is a field coded slice or CTU, I > would hold this part and else about the field. I'm not sure what you meant here, sorry. > > > > > Adding another buffer_tag for referring the memory of the motion vectors > > > > > for each frames. Or a better method is add a meta data to echo picture > > > > > buffer, since the picture output is just the same as the original, > > > > > display won't care whether the motion vectors are written the button of > > > > > picture or somewhere else. > > > > > > > > The motion vectors are passed as part of the raw bitstream data, in the > > > > slices. Is there a case where the motion vectors are coded differently? > > > No, it is an additional cache for decoder, even FFmpeg having such > > > data, I think allwinner must output it into somewhere. > > > > Ah yes I see what you mean! This is handled internally by our driver > > and not exposed to userspace. I don't think it would be a good idea to > > expose this cache or request that userspace allocates it like a video > > buffer. > > > No, usually the driver should allocate, as the user space have no > idea on size of each devices. > But for advantage user, application can fix a broken picture with a > proper data or analysis a object motion from that. > So I would suggest attaching this information to a picture buffer as > a meta data. Right, the driver will allocate chunks of memory for the decoding metadata used by the hardware decoder. Well, I don't think V4L2 has any mechanism to expose this data for now and since it's very specific to the hardware implementation, I guess the interest in having that is generally pretty low. That's maybe something that could be added later if someone wants to work on it, but I think we are better off keeping this metadata hidden by the driver for now. > > > > > > > > + > > > > > > > > +struct v4l2_hevc_pred_weight_table { > > > > > > > > + __u8 luma_log2_weight_denom; > > > > > > > > + __s8 delta_chroma_log2_weight_denom; > > > > > > > > + > > > > > > > > + __s8 delta_luma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > > > > > + __s8 luma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > > > > > + __s8 delta_chroma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > > > > > > > > + __s8 chroma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > > > > > > > > + > > > > > > > > + __s8 delta_luma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > > > > > + __s8 luma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > > > > > + __s8 delta_chroma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > > > > > > > > + __s8 chroma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > > > > > > > > +}; > > > > > > > > + > > > > > Those properties I think are not necessary are applying for the > > > > > Rockchip's device, may not work for the others. > > > > > > > > Yes, it's possible that some of the elements are not necessary for some > > > > decoders. What we want is to cover all the elements that might be > > > > required for a decoder. > > > I wonder whether allwinner need that, those sao flag usually ignored > > > by decoder in design. But more is better than less, it is hard to > > > extend a v4l2 structure in the future, maybe a new HEVC profile > > > would bring a new property, it is still too early for HEVC. > > > > Yes this is used by our decoder. The idea is to have all the basic > > bitstream elements in the structures (even if some decoders don't use > > them all) and add others for extension as separate controls later. > > > > > > > > > > +struct v4l2_ctrl_hevc_slice_params { > > > > > > > > + __u32 bit_size; > > > > > > > > + __u32 data_bit_offset; > > > > > > > > + > > > > > > > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: NAL unit header */ > > > > > > > > + __u8 nal_unit_type; > > > > > > > > + __u8 nuh_temporal_id_plus1; > > > > > > > > + > > > > > > > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ > > > > > > > > + __u8 slice_type; > > > > > > > > + __u8 colour_plane_id; > > > > > ---------------------------------------------------------------------------- > > > > > > > > + __u16 slice_pic_order_cnt; > > > > > > > > + __u8 slice_sao_luma_flag; > > > > > > > > + __u8 slice_sao_chroma_flag; > > > > > > > > + __u8 slice_temporal_mvp_enabled_flag; > > > > > > > > + __u8 num_ref_idx_l0_active_minus1; > > > > > > > > + __u8 num_ref_idx_l1_active_minus1; > > > > > Rockchip's decoder doesn't use this part. > > > > > > > > + __u8 mvd_l1_zero_flag; > > > > > > > > + __u8 cabac_init_flag; > > > > > > > > + __u8 collocated_from_l0_flag; > > > > > > > > + __u8 collocated_ref_idx; > > > > > > > > + __u8 five_minus_max_num_merge_cand; > > > > > > > > + __u8 use_integer_mv_flag; > > > > > > > > + __s8 slice_qp_delta; > > > > > > > > + __s8 slice_cb_qp_offset; > > > > > > > > + __s8 slice_cr_qp_offset; > > > > > > > > + __s8 slice_act_y_qp_offset; > > > > > > > > + __s8 slice_act_cb_qp_offset; > > > > > > > > + __s8 slice_act_cr_qp_offset; > > > > > > > > + __u8 slice_deblocking_filter_disabled_flag; > > > > > > > > + __s8 slice_beta_offset_div2; > > > > > > > > + __s8 slice_tc_offset_div2; > > > > > > > > + __u8 slice_loop_filter_across_slices_enabled_flag; > > > > > > > > + > > > > > > > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Picture timing SEI message */ > > > > > > > > + __u8 pic_struct; > > > > > I think the decoder doesn't care about this, it is used for display. > > > > > > > > The purpose of this field is to indicate whether the current picture is > > > > a progressive frame or an interlaced field picture, which is useful for > > > > decoding. > > > > > > > > At least our decoder has a register field to indicate frame/top > > > > field/bottom field, so we certainly need to keep the info around. > > > > Looking at the spec and the ffmpeg implementation, it looks like this > > > > flag of the bitstream is the usual way to report field coding. > > > It depends whether the decoder cares about scan type or more, I > > > wonder prefer general_interlaced_source_flag for just scan type, it > > > would be better than reading another SEL. > > > > Well we still need a way to indicate if the current data is top or > > bottom field for interlaced. I don't think that knowing that the whole > > video is interlaced would be precise enough. > > > > Cheers, > > > > Paul > > > > > > > > > > + > > > > > > > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ > > > > > > > > + struct v4l2_hevc_dpb_entry dpb[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > > > > > + __u8 num_active_dpb_entries; > > > > > > > > + __u8 ref_idx_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > > > > > + __u8 ref_idx_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > > > > > + > > > > > > > > + __u8 num_rps_poc_st_curr_before; > > > > > > > > + __u8 num_rps_poc_st_curr_after; > > > > > > > > + __u8 num_rps_poc_lt_curr; > > > > > > > > + > > > > > > > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Weighted prediction parameter */ > > > > > > > > + struct v4l2_hevc_pred_weight_table pred_weight_table; > > > > > > > > +}; > > > > > > > > + > > > > > > > > #endif > > > > -- > > > > Paul Kocialkowski, Bootlin (formerly Free Electrons) > > > > Embedded Linux and kernel engineering > > > > https://bootlin.com > > > > > > -- > > Paul Kocialkowski, Bootlin (formerly Free Electrons) > > Embedded Linux and kernel engineering > > https://bootlin.com > >
Sent from my iPad > On Jan 24, 2019, at 6:36 PM, Paul Kocialkowski <paul.kocialkowski@bootlin.com> wrote: > > Hi, > >> On Tue, 2019-01-08 at 18:00 +0800, Ayaka wrote: >> >> Sent from my iPad >> >>> On Jan 8, 2019, at 4:38 PM, Paul Kocialkowski <paul.kocialkowski@bootlin.com> wrote: >>> >>> Hi, >>> >>>> On Tue, 2019-01-08 at 09:16 +0800, Ayaka wrote: >>>> >>>> Sent from my iPad >>>> >>>>> On Jan 7, 2019, at 5:57 PM, Paul Kocialkowski <paul.kocialkowski@bootlin.com> wrote: >>>>> >>>>> Hi, >>>>> >>>>>>> On Mon, 2019-01-07 at 11:49 +0800, Randy Li wrote: >>>>>>> On 12/12/18 8:51 PM, Paul Kocialkowski wrote: >>>>>>> Hi, >>>>>>> >>>>>>> On Wed, 2018-12-05 at 21:59 +0100, Jernej Škrabec wrote: >>>>>>> >>>>>>>>> + >>>>>>>>> +#define V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_BEFORE 0x01 >>>>>>>>> +#define V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_AFTER 0x02 >>>>>>>>> +#define V4L2_HEVC_DPB_ENTRY_RPS_LT_CURR 0x03 >>>>>>>>> + >>>>>>>>> +#define V4L2_HEVC_DPB_ENTRIES_NUM_MAX 16 >>>>>>>>> + >>>>>>>>> +struct v4l2_hevc_dpb_entry { >>>>>>>>> + __u32 buffer_tag; >>>>>>>>> + __u8 rps; >>>>>>>>> + __u8 field_pic; >>>>>>>>> + __u16 pic_order_cnt[2]; >>>>>>>>> +}; >>>>>> >>>>>> Please add a property for reference index, if that rps is not used for >>>>>> this, some device would request that(not the rockchip one). And >>>>>> Rockchip's VDPU1 and VDPU2 for AVC would request a similar property. >>>>> >>>>> What exactly is that reference index? Is it a bitstream element or >>>>> something deduced from the bitstream? >>>>> >>>> picture order count(POC) for HEVC and frame_num in AVC. I think it is >>>> the number used in list0(P slice and B slice) and list1(B slice). >>> >>> The picture order count is already the last field of the DPB entry >>> structure. There is one for each field picture. >> As we are not sure whether there is a field coded slice or CTU, I >> would hold this part and else about the field. > > I'm not sure what you meant here, sorry. As we talked in IRC, I am not sure the field coded picture is supported in HEVC. And I don’t why there would be two pic order cnt, a picture can only be used a short term or a long term reference at one picture decoding > >>>>>> Adding another buffer_tag for referring the memory of the motion vectors >>>>>> for each frames. Or a better method is add a meta data to echo picture >>>>>> buffer, since the picture output is just the same as the original, >>>>>> display won't care whether the motion vectors are written the button of >>>>>> picture or somewhere else. >>>>> >>>>> The motion vectors are passed as part of the raw bitstream data, in the >>>>> slices. Is there a case where the motion vectors are coded differently? >>>> No, it is an additional cache for decoder, even FFmpeg having such >>>> data, I think allwinner must output it into somewhere. >>> >>> Ah yes I see what you mean! This is handled internally by our driver >>> and not exposed to userspace. I don't think it would be a good idea to >>> expose this cache or request that userspace allocates it like a video >>> buffer. >>> >> No, usually the driver should allocate, as the user space have no >> idea on size of each devices. >> But for advantage user, application can fix a broken picture with a >> proper data or analysis a object motion from that. >> So I would suggest attaching this information to a picture buffer as >> a meta data. > > Right, the driver will allocate chunks of memory for the decoding > metadata used by the hardware decoder. > > Well, I don't think V4L2 has any mechanism to expose this data for now > and since it's very specific to the hardware implementation, I guess > the interest in having that is generally pretty low. > > That's maybe something that could be added later if someone wants to > work on it, but I think we are better off keeping this metadata hidden > by the driver for now. I am writing a V4l2 driver for rockchip based on the previous vendor driver I sent to mail list. I think I would offer a better way to describe the meta after that. But it need both work in derives and userspace, it would cost some times. > >>>>>>>>> + >>>>>>>>> +struct v4l2_hevc_pred_weight_table { >>>>>>>>> + __u8 luma_log2_weight_denom; >>>>>>>>> + __s8 delta_chroma_log2_weight_denom; >>>>>>>>> + >>>>>>>>> + __s8 delta_luma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>>>>>> + __s8 luma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>>>>>> + __s8 delta_chroma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; >>>>>>>>> + __s8 chroma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; >>>>>>>>> + >>>>>>>>> + __s8 delta_luma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>>>>>> + __s8 luma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>>>>>> + __s8 delta_chroma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; >>>>>>>>> + __s8 chroma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; >>>>>>>>> +}; >>>>>>>>> + >>>>>> Those properties I think are not necessary are applying for the >>>>>> Rockchip's device, may not work for the others. >>>>> >>>>> Yes, it's possible that some of the elements are not necessary for some >>>>> decoders. What we want is to cover all the elements that might be >>>>> required for a decoder. >>>> I wonder whether allwinner need that, those sao flag usually ignored >>>> by decoder in design. But more is better than less, it is hard to >>>> extend a v4l2 structure in the future, maybe a new HEVC profile >>>> would bring a new property, it is still too early for HEVC. >>> >>> Yes this is used by our decoder. The idea is to have all the basic >>> bitstream elements in the structures (even if some decoders don't use >>> them all) and add others for extension as separate controls later. >>> >>>>>>>>> +struct v4l2_ctrl_hevc_slice_params { >>>>>>>>> + __u32 bit_size; >>>>>>>>> + __u32 data_bit_offset; >>>>>>>>> + >>>>>>>>> + /* ISO/IEC 23008-2, ITU-T Rec. H.265: NAL unit header */ >>>>>>>>> + __u8 nal_unit_type; >>>>>>>>> + __u8 nuh_temporal_id_plus1; >>>>>>>>> + >>>>>>>>> + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ >>>>>>>>> + __u8 slice_type; >>>>>>>>> + __u8 colour_plane_id; >>>>>> ---------------------------------------------------------------------------- >>>>>>>>> + __u16 slice_pic_order_cnt; >>>>>>>>> + __u8 slice_sao_luma_flag; >>>>>>>>> + __u8 slice_sao_chroma_flag; >>>>>>>>> + __u8 slice_temporal_mvp_enabled_flag; >>>>>>>>> + __u8 num_ref_idx_l0_active_minus1; >>>>>>>>> + __u8 num_ref_idx_l1_active_minus1; >>>>>> Rockchip's decoder doesn't use this part. >>>>>>>>> + __u8 mvd_l1_zero_flag; >>>>>>>>> + __u8 cabac_init_flag; >>>>>>>>> + __u8 collocated_from_l0_flag; >>>>>>>>> + __u8 collocated_ref_idx; >>>>>>>>> + __u8 five_minus_max_num_merge_cand; >>>>>>>>> + __u8 use_integer_mv_flag; >>>>>>>>> + __s8 slice_qp_delta; >>>>>>>>> + __s8 slice_cb_qp_offset; >>>>>>>>> + __s8 slice_cr_qp_offset; >>>>>>>>> + __s8 slice_act_y_qp_offset; >>>>>>>>> + __s8 slice_act_cb_qp_offset; >>>>>>>>> + __s8 slice_act_cr_qp_offset; >>>>>>>>> + __u8 slice_deblocking_filter_disabled_flag; >>>>>>>>> + __s8 slice_beta_offset_div2; >>>>>>>>> + __s8 slice_tc_offset_div2; >>>>>>>>> + __u8 slice_loop_filter_across_slices_enabled_flag; >>>>>>>>> + >>>>>>>>> + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Picture timing SEI message */ >>>>>>>>> + __u8 pic_struct; >>>>>> I think the decoder doesn't care about this, it is used for display. >>>>> >>>>> The purpose of this field is to indicate whether the current picture is >>>>> a progressive frame or an interlaced field picture, which is useful for >>>>> decoding. >>>>> >>>>> At least our decoder has a register field to indicate frame/top >>>>> field/bottom field, so we certainly need to keep the info around. >>>>> Looking at the spec and the ffmpeg implementation, it looks like this >>>>> flag of the bitstream is the usual way to report field coding. >>>> It depends whether the decoder cares about scan type or more, I >>>> wonder prefer general_interlaced_source_flag for just scan type, it >>>> would be better than reading another SEL. >>> >>> Well we still need a way to indicate if the current data is top or >>> bottom field for interlaced. I don't think that knowing that the whole >>> video is interlaced would be precise enough. >>> >>> Cheers, >>> >>> Paul >>> >>>>>>>>> + >>>>>>>>> + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ >>>>>>>>> + struct v4l2_hevc_dpb_entry dpb[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>>>>>> + __u8 num_active_dpb_entries; >>>>>>>>> + __u8 ref_idx_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>>>>>> + __u8 ref_idx_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>>>>>> + >>>>>>>>> + __u8 num_rps_poc_st_curr_before; >>>>>>>>> + __u8 num_rps_poc_st_curr_after; >>>>>>>>> + __u8 num_rps_poc_lt_curr; >>>>>>>>> + >>>>>>>>> + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Weighted prediction parameter */ >>>>>>>>> + struct v4l2_hevc_pred_weight_table pred_weight_table; >>>>>>>>> +}; >>>>>>>>> + >>>>>>>>> #endif >>>>> -- >>>>> Paul Kocialkowski, Bootlin (formerly Free Electrons) >>>>> Embedded Linux and kernel engineering >>>>> https://bootlin.com >>>>> >>> -- >>> Paul Kocialkowski, Bootlin (formerly Free Electrons) >>> Embedded Linux and kernel engineering >>> https://bootlin.com >>> > -- > Paul Kocialkowski, Bootlin > Embedded Linux and kernel engineering > https://bootlin.com >
Sent from my iPad > On Jan 24, 2019, at 6:27 PM, Paul Kocialkowski <paul.kocialkowski@bootlin.com> wrote: > > Hi, > >> On Thu, 2019-01-10 at 21:32 +0800, ayaka wrote: >> I forget a important thing, for the rkvdec and rk hevc decoder, it would >> requests cabac table, scaling list, picture parameter set and reference >> picture storing in one or various of DMA buffers. I am not talking about >> the data been parsed, the decoder would requests a raw data. >> >> For the pps and rps, it is possible to reuse the slice header, just let >> the decoder know the offset from the bitstream bufer, I would suggest to >> add three properties(with sps) for them. But I think we need a method to >> mark a OUTPUT side buffer for those aux data. > > I'm quite confused about the hardware implementation then. From what > you're saying, it seems that it takes the raw bitstream elements rather > than parsed elements. Is it really a stateless implementation? > > The stateless implementation was designed with the idea that only the > raw slice data should be passed in bitstream form to the decoder. For > H.264, it seems that some decoders also need the slice header in raw > bitstream form (because they take the full slice NAL unit), see the > discussions in this thread: > media: docs-rst: Document m2m stateless video decoder interface Stateless just mean it won’t track the previous result, but I don’t think you can define what a date the hardware would need. Even you just build a dpb for the decoder, it is still stateless, but parsing less or more data from the bitstream doesn’t stop a decoder become a stateless decoder. > > Can you detail exactly what the rockchip decoder absolutely needs in > raw bitstream format? > > Cheers, > > Paul > >>> On 1/8/19 6:00 PM, Ayaka wrote: >>> Sent from my iPad >>> >>>> On Jan 8, 2019, at 4:38 PM, Paul Kocialkowski <paul.kocialkowski@bootlin.com> wrote: >>>> >>>> Hi, >>>> >>>>> On Tue, 2019-01-08 at 09:16 +0800, Ayaka wrote: >>>>> >>>>> Sent from my iPad >>>>> >>>>>> On Jan 7, 2019, at 5:57 PM, Paul Kocialkowski <paul.kocialkowski@bootlin.com> wrote: >>>>>> >>>>>> Hi, >>>>>> >>>>>>>> On Mon, 2019-01-07 at 11:49 +0800, Randy Li wrote: >>>>>>>> On 12/12/18 8:51 PM, Paul Kocialkowski wrote: >>>>>>>> Hi, >>>>>>>> >>>>>>>> On Wed, 2018-12-05 at 21:59 +0100, Jernej Škrabec wrote: >>>>>>>> >>>>>>>>>> + >>>>>>>>>> +#define V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_BEFORE 0x01 >>>>>>>>>> +#define V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_AFTER 0x02 >>>>>>>>>> +#define V4L2_HEVC_DPB_ENTRY_RPS_LT_CURR 0x03 >>>>>>>>>> + >>>>>>>>>> +#define V4L2_HEVC_DPB_ENTRIES_NUM_MAX 16 >>>>>>>>>> + >>>>>>>>>> +struct v4l2_hevc_dpb_entry { >>>>>>>>>> + __u32 buffer_tag; >>>>>>>>>> + __u8 rps; >>>>>>>>>> + __u8 field_pic; >>>>>>>>>> + __u16 pic_order_cnt[2]; >>>>>>>>>> +}; >>>>>>> Please add a property for reference index, if that rps is not used for >>>>>>> this, some device would request that(not the rockchip one). And >>>>>>> Rockchip's VDPU1 and VDPU2 for AVC would request a similar property. >>>>>> What exactly is that reference index? Is it a bitstream element or >>>>>> something deduced from the bitstream? >>>>>> >>>>> picture order count(POC) for HEVC and frame_num in AVC. I think it is >>>>> the number used in list0(P slice and B slice) and list1(B slice). >>>> The picture order count is already the last field of the DPB entry >>>> structure. There is one for each field picture. >>> As we are not sure whether there is a field coded slice or CTU, I would hold this part and else about the field. >>>>>>> Adding another buffer_tag for referring the memory of the motion vectors >>>>>>> for each frames. Or a better method is add a meta data to echo picture >>>>>>> buffer, since the picture output is just the same as the original, >>>>>>> display won't care whether the motion vectors are written the button of >>>>>>> picture or somewhere else. >>>>>> The motion vectors are passed as part of the raw bitstream data, in the >>>>>> slices. Is there a case where the motion vectors are coded differently? >>>>> No, it is an additional cache for decoder, even FFmpeg having such >>>>> data, I think allwinner must output it into somewhere. >>>> Ah yes I see what you mean! This is handled internally by our driver >>>> and not exposed to userspace. I don't think it would be a good idea to >>>> expose this cache or request that userspace allocates it like a video >>>> buffer. >>>> >>> No, usually the driver should allocate, as the user space have no idea on size of each devices. >>> But for advantage user, application can fix a broken picture with a proper data or analysis a object motion from that. >>> So I would suggest attaching this information to a picture buffer as a meta data. >>>>>>>>>> + >>>>>>>>>> +struct v4l2_hevc_pred_weight_table { >>>>>>>>>> + __u8 luma_log2_weight_denom; >>>>>>>>>> + __s8 delta_chroma_log2_weight_denom; >>>>>>>>>> + >>>>>>>>>> + __s8 delta_luma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>>>>>>> + __s8 luma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>>>>>>> + __s8 delta_chroma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; >>>>>>>>>> + __s8 chroma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; >>>>>>>>>> + >>>>>>>>>> + __s8 delta_luma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>>>>>>> + __s8 luma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>>>>>>> + __s8 delta_chroma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; >>>>>>>>>> + __s8 chroma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; >>>>>>>>>> +}; >>>>>>>>>> + >>>>>>> Those properties I think are not necessary are applying for the >>>>>>> Rockchip's device, may not work for the others. >>>>>> Yes, it's possible that some of the elements are not necessary for some >>>>>> decoders. What we want is to cover all the elements that might be >>>>>> required for a decoder. >>>>> I wonder whether allwinner need that, those sao flag usually ignored >>>>> by decoder in design. But more is better than less, it is hard to >>>>> extend a v4l2 structure in the future, maybe a new HEVC profile >>>>> would bring a new property, it is still too early for HEVC. >>>> Yes this is used by our decoder. The idea is to have all the basic >>>> bitstream elements in the structures (even if some decoders don't use >>>> them all) and add others for extension as separate controls later. >>>> >>>>>>>>>> +struct v4l2_ctrl_hevc_slice_params { >>>>>>>>>> + __u32 bit_size; >>>>>>>>>> + __u32 data_bit_offset; >>>>>>>>>> + >>>>>>>>>> + /* ISO/IEC 23008-2, ITU-T Rec. H.265: NAL unit header */ >>>>>>>>>> + __u8 nal_unit_type; >>>>>>>>>> + __u8 nuh_temporal_id_plus1; >>>>>>>>>> + >>>>>>>>>> + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ >>>>>>>>>> + __u8 slice_type; >>>>>>>>>> + __u8 colour_plane_id; >>>>>>> ---------------------------------------------------------------------------- >>>>>>>>>> + __u16 slice_pic_order_cnt; >>>>>>>>>> + __u8 slice_sao_luma_flag; >>>>>>>>>> + __u8 slice_sao_chroma_flag; >>>>>>>>>> + __u8 slice_temporal_mvp_enabled_flag; >>>>>>>>>> + __u8 num_ref_idx_l0_active_minus1; >>>>>>>>>> + __u8 num_ref_idx_l1_active_minus1; >>>>>>> Rockchip's decoder doesn't use this part. >>>>>>>>>> + __u8 mvd_l1_zero_flag; >>>>>>>>>> + __u8 cabac_init_flag; >>>>>>>>>> + __u8 collocated_from_l0_flag; >>>>>>>>>> + __u8 collocated_ref_idx; >>>>>>>>>> + __u8 five_minus_max_num_merge_cand; >>>>>>>>>> + __u8 use_integer_mv_flag; >>>>>>>>>> + __s8 slice_qp_delta; >>>>>>>>>> + __s8 slice_cb_qp_offset; >>>>>>>>>> + __s8 slice_cr_qp_offset; >>>>>>>>>> + __s8 slice_act_y_qp_offset; >>>>>>>>>> + __s8 slice_act_cb_qp_offset; >>>>>>>>>> + __s8 slice_act_cr_qp_offset; >>>>>>>>>> + __u8 slice_deblocking_filter_disabled_flag; >>>>>>>>>> + __s8 slice_beta_offset_div2; >>>>>>>>>> + __s8 slice_tc_offset_div2; >>>>>>>>>> + __u8 slice_loop_filter_across_slices_enabled_flag; >>>>>>>>>> + >>>>>>>>>> + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Picture timing SEI message */ >>>>>>>>>> + __u8 pic_struct; >>>>>>> I think the decoder doesn't care about this, it is used for display. >>>>>> The purpose of this field is to indicate whether the current picture is >>>>>> a progressive frame or an interlaced field picture, which is useful for >>>>>> decoding. >>>>>> >>>>>> At least our decoder has a register field to indicate frame/top >>>>>> field/bottom field, so we certainly need to keep the info around. >>>>>> Looking at the spec and the ffmpeg implementation, it looks like this >>>>>> flag of the bitstream is the usual way to report field coding. >>>>> It depends whether the decoder cares about scan type or more, I >>>>> wonder prefer general_interlaced_source_flag for just scan type, it >>>>> would be better than reading another SEL. >>>> Well we still need a way to indicate if the current data is top or >>>> bottom field for interlaced. I don't think that knowing that the whole >>>> video is interlaced would be precise enough. >>>> >>>> Cheers, >>>> >>>> Paul >>>> >>>>>>>>>> + >>>>>>>>>> + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ >>>>>>>>>> + struct v4l2_hevc_dpb_entry dpb[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>>>>>>> + __u8 num_active_dpb_entries; >>>>>>>>>> + __u8 ref_idx_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>>>>>>> + __u8 ref_idx_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; >>>>>>>>>> + >>>>>>>>>> + __u8 num_rps_poc_st_curr_before; >>>>>>>>>> + __u8 num_rps_poc_st_curr_after; >>>>>>>>>> + __u8 num_rps_poc_lt_curr; >>>>>>>>>> + >>>>>>>>>> + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Weighted prediction parameter */ >>>>>>>>>> + struct v4l2_hevc_pred_weight_table pred_weight_table; >>>>>>>>>> +}; >>>>>>>>>> + >>>>>>>>>> #endif >>>>>> -- >>>>>> Paul Kocialkowski, Bootlin (formerly Free Electrons) >>>>>> Embedded Linux and kernel engineering >>>>>> https://bootlin.com >>>>>> >>>> -- >>>> Paul Kocialkowski, Bootlin (formerly Free Electrons) >>>> Embedded Linux and kernel engineering >>>> https://bootlin.com >>>> > -- > Paul Kocialkowski, Bootlin > Embedded Linux and kernel engineering > https://bootlin.com >
Hi, On Thu, 2019-01-24 at 20:23 +0800, Ayaka wrote: > > Sent from my iPad > > > On Jan 24, 2019, at 6:27 PM, Paul Kocialkowski <paul.kocialkowski@bootlin.com> wrote: > > > > Hi, > > > > > On Thu, 2019-01-10 at 21:32 +0800, ayaka wrote: > > > I forget a important thing, for the rkvdec and rk hevc decoder, it would > > > requests cabac table, scaling list, picture parameter set and reference > > > picture storing in one or various of DMA buffers. I am not talking about > > > the data been parsed, the decoder would requests a raw data. > > > > > > For the pps and rps, it is possible to reuse the slice header, just let > > > the decoder know the offset from the bitstream bufer, I would suggest to > > > add three properties(with sps) for them. But I think we need a method to > > > mark a OUTPUT side buffer for those aux data. > > > > I'm quite confused about the hardware implementation then. From what > > you're saying, it seems that it takes the raw bitstream elements rather > > than parsed elements. Is it really a stateless implementation? > > > > The stateless implementation was designed with the idea that only the > > raw slice data should be passed in bitstream form to the decoder. For > > H.264, it seems that some decoders also need the slice header in raw > > bitstream form (because they take the full slice NAL unit), see the > > discussions in this thread: > > media: docs-rst: Document m2m stateless video decoder interface > > Stateless just mean it won’t track the previous result, but I don’t > think you can define what a date the hardware would need. Even you > just build a dpb for the decoder, it is still stateless, but parsing > less or more data from the bitstream doesn’t stop a decoder become a > stateless decoder. Yes fair enough, the format in which the hardware decoder takes the bitstream parameters does not make it stateless or stateful per-se. It's just that stateless decoders should have no particular reason for parsing the bitstream on their own since the hardware can be designed with registers for each relevant bitstream element to configure the decoding pipeline. That's how GPU-based decoder implementations are implemented (VAAPI/VDPAU/NVDEC, etc). So the format we have agreed on so far for the stateless interface is to pass parsed elements via v4l2 control structures. If the hardware can only work by parsing the bitstream itself, I'm not sure what the best solution would be. Reconstructing the bitstream in the kernel is a pretty bad option, but so is parsing in the kernel or having the data both in parsed and raw forms. Do you see another possibility? Cheers, Paul > > Can you detail exactly what the rockchip decoder absolutely needs in > > raw bitstream format? > > > > Cheers, > > > > Paul > > > > > > On 1/8/19 6:00 PM, Ayaka wrote: > > > > Sent from my iPad > > > > > > > > > On Jan 8, 2019, at 4:38 PM, Paul Kocialkowski <paul.kocialkowski@bootlin.com> wrote: > > > > > > > > > > Hi, > > > > > > > > > > > On Tue, 2019-01-08 at 09:16 +0800, Ayaka wrote: > > > > > > > > > > > > Sent from my iPad > > > > > > > > > > > > > On Jan 7, 2019, at 5:57 PM, Paul Kocialkowski <paul.kocialkowski@bootlin.com> wrote: > > > > > > > > > > > > > > Hi, > > > > > > > > > > > > > > > > On Mon, 2019-01-07 at 11:49 +0800, Randy Li wrote: > > > > > > > > > On 12/12/18 8:51 PM, Paul Kocialkowski wrote: > > > > > > > > > Hi, > > > > > > > > > > > > > > > > > > On Wed, 2018-12-05 at 21:59 +0100, Jernej Škrabec wrote: > > > > > > > > > > > > > > > > > > > > + > > > > > > > > > > > +#define V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_BEFORE 0x01 > > > > > > > > > > > +#define V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_AFTER 0x02 > > > > > > > > > > > +#define V4L2_HEVC_DPB_ENTRY_RPS_LT_CURR 0x03 > > > > > > > > > > > + > > > > > > > > > > > +#define V4L2_HEVC_DPB_ENTRIES_NUM_MAX 16 > > > > > > > > > > > + > > > > > > > > > > > +struct v4l2_hevc_dpb_entry { > > > > > > > > > > > + __u32 buffer_tag; > > > > > > > > > > > + __u8 rps; > > > > > > > > > > > + __u8 field_pic; > > > > > > > > > > > + __u16 pic_order_cnt[2]; > > > > > > > > > > > +}; > > > > > > > > Please add a property for reference index, if that rps is not used for > > > > > > > > this, some device would request that(not the rockchip one). And > > > > > > > > Rockchip's VDPU1 and VDPU2 for AVC would request a similar property. > > > > > > > What exactly is that reference index? Is it a bitstream element or > > > > > > > something deduced from the bitstream? > > > > > > > > > > > > > picture order count(POC) for HEVC and frame_num in AVC. I think it is > > > > > > the number used in list0(P slice and B slice) and list1(B slice). > > > > > The picture order count is already the last field of the DPB entry > > > > > structure. There is one for each field picture. > > > > As we are not sure whether there is a field coded slice or CTU, I would hold this part and else about the field. > > > > > > > > Adding another buffer_tag for referring the memory of the motion vectors > > > > > > > > for each frames. Or a better method is add a meta data to echo picture > > > > > > > > buffer, since the picture output is just the same as the original, > > > > > > > > display won't care whether the motion vectors are written the button of > > > > > > > > picture or somewhere else. > > > > > > > The motion vectors are passed as part of the raw bitstream data, in the > > > > > > > slices. Is there a case where the motion vectors are coded differently? > > > > > > No, it is an additional cache for decoder, even FFmpeg having such > > > > > > data, I think allwinner must output it into somewhere. > > > > > Ah yes I see what you mean! This is handled internally by our driver > > > > > and not exposed to userspace. I don't think it would be a good idea to > > > > > expose this cache or request that userspace allocates it like a video > > > > > buffer. > > > > > > > > > No, usually the driver should allocate, as the user space have no idea on size of each devices. > > > > But for advantage user, application can fix a broken picture with a proper data or analysis a object motion from that. > > > > So I would suggest attaching this information to a picture buffer as a meta data. > > > > > > > > > > > + > > > > > > > > > > > +struct v4l2_hevc_pred_weight_table { > > > > > > > > > > > + __u8 luma_log2_weight_denom; > > > > > > > > > > > + __s8 delta_chroma_log2_weight_denom; > > > > > > > > > > > + > > > > > > > > > > > + __s8 delta_luma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > > > > > > > > + __s8 luma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > > > > > > > > + __s8 delta_chroma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > > > > > > > > > > > + __s8 chroma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > > > > > > > > > > > + > > > > > > > > > > > + __s8 delta_luma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > > > > > > > > + __s8 luma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > > > > > > > > + __s8 delta_chroma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > > > > > > > > > > > + __s8 chroma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; > > > > > > > > > > > +}; > > > > > > > > > > > + > > > > > > > > Those properties I think are not necessary are applying for the > > > > > > > > Rockchip's device, may not work for the others. > > > > > > > Yes, it's possible that some of the elements are not necessary for some > > > > > > > decoders. What we want is to cover all the elements that might be > > > > > > > required for a decoder. > > > > > > I wonder whether allwinner need that, those sao flag usually ignored > > > > > > by decoder in design. But more is better than less, it is hard to > > > > > > extend a v4l2 structure in the future, maybe a new HEVC profile > > > > > > would bring a new property, it is still too early for HEVC. > > > > > Yes this is used by our decoder. The idea is to have all the basic > > > > > bitstream elements in the structures (even if some decoders don't use > > > > > them all) and add others for extension as separate controls later. > > > > > > > > > > > > > > > > +struct v4l2_ctrl_hevc_slice_params { > > > > > > > > > > > + __u32 bit_size; > > > > > > > > > > > + __u32 data_bit_offset; > > > > > > > > > > > + > > > > > > > > > > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: NAL unit header */ > > > > > > > > > > > + __u8 nal_unit_type; > > > > > > > > > > > + __u8 nuh_temporal_id_plus1; > > > > > > > > > > > + > > > > > > > > > > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ > > > > > > > > > > > + __u8 slice_type; > > > > > > > > > > > + __u8 colour_plane_id; > > > > > > > > ---------------------------------------------------------------------------- > > > > > > > > > > > + __u16 slice_pic_order_cnt; > > > > > > > > > > > + __u8 slice_sao_luma_flag; > > > > > > > > > > > + __u8 slice_sao_chroma_flag; > > > > > > > > > > > + __u8 slice_temporal_mvp_enabled_flag; > > > > > > > > > > > + __u8 num_ref_idx_l0_active_minus1; > > > > > > > > > > > + __u8 num_ref_idx_l1_active_minus1; > > > > > > > > Rockchip's decoder doesn't use this part. > > > > > > > > > > > + __u8 mvd_l1_zero_flag; > > > > > > > > > > > + __u8 cabac_init_flag; > > > > > > > > > > > + __u8 collocated_from_l0_flag; > > > > > > > > > > > + __u8 collocated_ref_idx; > > > > > > > > > > > + __u8 five_minus_max_num_merge_cand; > > > > > > > > > > > + __u8 use_integer_mv_flag; > > > > > > > > > > > + __s8 slice_qp_delta; > > > > > > > > > > > + __s8 slice_cb_qp_offset; > > > > > > > > > > > + __s8 slice_cr_qp_offset; > > > > > > > > > > > + __s8 slice_act_y_qp_offset; > > > > > > > > > > > + __s8 slice_act_cb_qp_offset; > > > > > > > > > > > + __s8 slice_act_cr_qp_offset; > > > > > > > > > > > + __u8 slice_deblocking_filter_disabled_flag; > > > > > > > > > > > + __s8 slice_beta_offset_div2; > > > > > > > > > > > + __s8 slice_tc_offset_div2; > > > > > > > > > > > + __u8 slice_loop_filter_across_slices_enabled_flag; > > > > > > > > > > > + > > > > > > > > > > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Picture timing SEI message */ > > > > > > > > > > > + __u8 pic_struct; > > > > > > > > I think the decoder doesn't care about this, it is used for display. > > > > > > > The purpose of this field is to indicate whether the current picture is > > > > > > > a progressive frame or an interlaced field picture, which is useful for > > > > > > > decoding. > > > > > > > > > > > > > > At least our decoder has a register field to indicate frame/top > > > > > > > field/bottom field, so we certainly need to keep the info around. > > > > > > > Looking at the spec and the ffmpeg implementation, it looks like this > > > > > > > flag of the bitstream is the usual way to report field coding. > > > > > > It depends whether the decoder cares about scan type or more, I > > > > > > wonder prefer general_interlaced_source_flag for just scan type, it > > > > > > would be better than reading another SEL. > > > > > Well we still need a way to indicate if the current data is top or > > > > > bottom field for interlaced. I don't think that knowing that the whole > > > > > video is interlaced would be precise enough. > > > > > > > > > > Cheers, > > > > > > > > > > Paul > > > > > > > > > > > > > > > > + > > > > > > > > > > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ > > > > > > > > > > > + struct v4l2_hevc_dpb_entry dpb[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > > > > > > > > + __u8 num_active_dpb_entries; > > > > > > > > > > > + __u8 ref_idx_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > > > > > > > > + __u8 ref_idx_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; > > > > > > > > > > > + > > > > > > > > > > > + __u8 num_rps_poc_st_curr_before; > > > > > > > > > > > + __u8 num_rps_poc_st_curr_after; > > > > > > > > > > > + __u8 num_rps_poc_lt_curr; > > > > > > > > > > > + > > > > > > > > > > > + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Weighted prediction parameter */ > > > > > > > > > > > + struct v4l2_hevc_pred_weight_table pred_weight_table; > > > > > > > > > > > +}; > > > > > > > > > > > + > > > > > > > > > > > #endif > > > > > > > -- > > > > > > > Paul Kocialkowski, Bootlin (formerly Free Electrons) > > > > > > > Embedded Linux and kernel engineering > > > > > > > https://bootlin.com > > > > > > > > > > > > -- > > > > > Paul Kocialkowski, Bootlin (formerly Free Electrons) > > > > > Embedded Linux and kernel engineering > > > > > https://bootlin.com > > > > > > > -- > > Paul Kocialkowski, Bootlin > > Embedded Linux and kernel engineering > > https://bootlin.com > >
On Fri, Jan 25, 2019 at 10:04 PM Paul Kocialkowski <paul.kocialkowski@bootlin.com> wrote: > > Hi, > > On Thu, 2019-01-24 at 20:23 +0800, Ayaka wrote: > > > > Sent from my iPad > > > > > On Jan 24, 2019, at 6:27 PM, Paul Kocialkowski <paul.kocialkowski@bootlin.com> wrote: > > > > > > Hi, > > > > > > > On Thu, 2019-01-10 at 21:32 +0800, ayaka wrote: > > > > I forget a important thing, for the rkvdec and rk hevc decoder, it would > > > > requests cabac table, scaling list, picture parameter set and reference > > > > picture storing in one or various of DMA buffers. I am not talking about > > > > the data been parsed, the decoder would requests a raw data. > > > > > > > > For the pps and rps, it is possible to reuse the slice header, just let > > > > the decoder know the offset from the bitstream bufer, I would suggest to > > > > add three properties(with sps) for them. But I think we need a method to > > > > mark a OUTPUT side buffer for those aux data. > > > > > > I'm quite confused about the hardware implementation then. From what > > > you're saying, it seems that it takes the raw bitstream elements rather > > > than parsed elements. Is it really a stateless implementation? > > > > > > The stateless implementation was designed with the idea that only the > > > raw slice data should be passed in bitstream form to the decoder. For > > > H.264, it seems that some decoders also need the slice header in raw > > > bitstream form (because they take the full slice NAL unit), see the > > > discussions in this thread: > > > media: docs-rst: Document m2m stateless video decoder interface > > > > Stateless just mean it won’t track the previous result, but I don’t > > think you can define what a date the hardware would need. Even you > > just build a dpb for the decoder, it is still stateless, but parsing > > less or more data from the bitstream doesn’t stop a decoder become a > > stateless decoder. > > Yes fair enough, the format in which the hardware decoder takes the > bitstream parameters does not make it stateless or stateful per-se. > It's just that stateless decoders should have no particular reason for > parsing the bitstream on their own since the hardware can be designed > with registers for each relevant bitstream element to configure the > decoding pipeline. That's how GPU-based decoder implementations are > implemented (VAAPI/VDPAU/NVDEC, etc). > > So the format we have agreed on so far for the stateless interface is > to pass parsed elements via v4l2 control structures. > > If the hardware can only work by parsing the bitstream itself, I'm not > sure what the best solution would be. Reconstructing the bitstream in > the kernel is a pretty bad option, but so is parsing in the kernel or > having the data both in parsed and raw forms. Do you see another > possibility? Is reconstructing the bitstream so bad? The v4l2 controls provide a generic interface to an encoded format which the driver needs to convert into a sequence that the hardware can understand. Typically this is done by populating hardware-specific structures. Can't we consider that in this specific instance, the hardware-specific structure just happens to be identical to the original bitstream format? I agree that this is not strictly optimal for that particular hardware, but such is the cost of abstractions, and in this specific case I don't believe the cost would be particularly high?
On Tue, Jan 29, 2019 at 04:44:35PM +0900, Alexandre Courbot wrote: > On Fri, Jan 25, 2019 at 10:04 PM Paul Kocialkowski > > On Thu, 2019-01-24 at 20:23 +0800, Ayaka wrote: > > > > > > Sent from my iPad > > > > > > > On Jan 24, 2019, at 6:27 PM, Paul Kocialkowski <paul.kocialkowski@bootlin.com> wrote: > > > > > > > > Hi, > > > > > > > > > On Thu, 2019-01-10 at 21:32 +0800, ayaka wrote: > > > > > I forget a important thing, for the rkvdec and rk hevc decoder, it would > > > > > requests cabac table, scaling list, picture parameter set and reference > > > > > picture storing in one or various of DMA buffers. I am not talking about > > > > > the data been parsed, the decoder would requests a raw data. > > > > > > > > > > For the pps and rps, it is possible to reuse the slice header, just let > > > > > the decoder know the offset from the bitstream bufer, I would suggest to > > > > > add three properties(with sps) for them. But I think we need a method to > > > > > mark a OUTPUT side buffer for those aux data. > > > > > > > > I'm quite confused about the hardware implementation then. From what > > > > you're saying, it seems that it takes the raw bitstream elements rather > > > > than parsed elements. Is it really a stateless implementation? > > > > > > > > The stateless implementation was designed with the idea that only the > > > > raw slice data should be passed in bitstream form to the decoder. For > > > > H.264, it seems that some decoders also need the slice header in raw > > > > bitstream form (because they take the full slice NAL unit), see the > > > > discussions in this thread: > > > > media: docs-rst: Document m2m stateless video decoder interface > > > > > > Stateless just mean it won’t track the previous result, but I don’t > > > think you can define what a date the hardware would need. Even you > > > just build a dpb for the decoder, it is still stateless, but parsing > > > less or more data from the bitstream doesn’t stop a decoder become a > > > stateless decoder. > > > > Yes fair enough, the format in which the hardware decoder takes the > > bitstream parameters does not make it stateless or stateful per-se. > > It's just that stateless decoders should have no particular reason for > > parsing the bitstream on their own since the hardware can be designed > > with registers for each relevant bitstream element to configure the > > decoding pipeline. That's how GPU-based decoder implementations are > > implemented (VAAPI/VDPAU/NVDEC, etc). > > > > So the format we have agreed on so far for the stateless interface is > > to pass parsed elements via v4l2 control structures. > > > > If the hardware can only work by parsing the bitstream itself, I'm not > > sure what the best solution would be. Reconstructing the bitstream in > > the kernel is a pretty bad option, but so is parsing in the kernel or > > having the data both in parsed and raw forms. Do you see another > > possibility? > > Is reconstructing the bitstream so bad? The v4l2 controls provide a > generic interface to an encoded format which the driver needs to > convert into a sequence that the hardware can understand. Typically > this is done by populating hardware-specific structures. Can't we > consider that in this specific instance, the hardware-specific > structure just happens to be identical to the original bitstream > format? > > I agree that this is not strictly optimal for that particular > hardware, but such is the cost of abstractions, and in this specific > case I don't believe the cost would be particularly high? I mean, that argument can be made for the rockchip driver as well. If reconstructing the bitstream is something we can do, and if we don't care about being suboptimal for one particular hardware, then why the rockchip driver doesn't just recreate the bitstream from that API? After all, this is just a hardware specific header that happens to be identical to the original bitstream format Maxime
On Tue, Jan 29, 2019 at 5:09 PM Maxime Ripard <maxime.ripard@bootlin.com> wrote: > > On Tue, Jan 29, 2019 at 04:44:35PM +0900, Alexandre Courbot wrote: > > On Fri, Jan 25, 2019 at 10:04 PM Paul Kocialkowski > > > On Thu, 2019-01-24 at 20:23 +0800, Ayaka wrote: > > > > > > > > Sent from my iPad > > > > > > > > > On Jan 24, 2019, at 6:27 PM, Paul Kocialkowski <paul.kocialkowski@bootlin.com> wrote: > > > > > > > > > > Hi, > > > > > > > > > > > On Thu, 2019-01-10 at 21:32 +0800, ayaka wrote: > > > > > > I forget a important thing, for the rkvdec and rk hevc decoder, it would > > > > > > requests cabac table, scaling list, picture parameter set and reference > > > > > > picture storing in one or various of DMA buffers. I am not talking about > > > > > > the data been parsed, the decoder would requests a raw data. > > > > > > > > > > > > For the pps and rps, it is possible to reuse the slice header, just let > > > > > > the decoder know the offset from the bitstream bufer, I would suggest to > > > > > > add three properties(with sps) for them. But I think we need a method to > > > > > > mark a OUTPUT side buffer for those aux data. > > > > > > > > > > I'm quite confused about the hardware implementation then. From what > > > > > you're saying, it seems that it takes the raw bitstream elements rather > > > > > than parsed elements. Is it really a stateless implementation? > > > > > > > > > > The stateless implementation was designed with the idea that only the > > > > > raw slice data should be passed in bitstream form to the decoder. For > > > > > H.264, it seems that some decoders also need the slice header in raw > > > > > bitstream form (because they take the full slice NAL unit), see the > > > > > discussions in this thread: > > > > > media: docs-rst: Document m2m stateless video decoder interface > > > > > > > > Stateless just mean it won’t track the previous result, but I don’t > > > > think you can define what a date the hardware would need. Even you > > > > just build a dpb for the decoder, it is still stateless, but parsing > > > > less or more data from the bitstream doesn’t stop a decoder become a > > > > stateless decoder. > > > > > > Yes fair enough, the format in which the hardware decoder takes the > > > bitstream parameters does not make it stateless or stateful per-se. > > > It's just that stateless decoders should have no particular reason for > > > parsing the bitstream on their own since the hardware can be designed > > > with registers for each relevant bitstream element to configure the > > > decoding pipeline. That's how GPU-based decoder implementations are > > > implemented (VAAPI/VDPAU/NVDEC, etc). > > > > > > So the format we have agreed on so far for the stateless interface is > > > to pass parsed elements via v4l2 control structures. > > > > > > If the hardware can only work by parsing the bitstream itself, I'm not > > > sure what the best solution would be. Reconstructing the bitstream in > > > the kernel is a pretty bad option, but so is parsing in the kernel or > > > having the data both in parsed and raw forms. Do you see another > > > possibility? > > > > Is reconstructing the bitstream so bad? The v4l2 controls provide a > > generic interface to an encoded format which the driver needs to > > convert into a sequence that the hardware can understand. Typically > > this is done by populating hardware-specific structures. Can't we > > consider that in this specific instance, the hardware-specific > > structure just happens to be identical to the original bitstream > > format? > > > > I agree that this is not strictly optimal for that particular > > hardware, but such is the cost of abstractions, and in this specific > > case I don't believe the cost would be particularly high? > > I mean, that argument can be made for the rockchip driver as well. If > reconstructing the bitstream is something we can do, and if we don't > care about being suboptimal for one particular hardware, then why the > rockchip driver doesn't just recreate the bitstream from that API? > > After all, this is just a hardware specific header that happens to be > identical to the original bitstream format I think in another thread (about H.264 I believe), we realized that it could be a good idea to just include the Slice NAL units in the Annex.B format in the buffers and that should work for all the hardware we could think of (given offsets to particular parts inside of the buffer). Wouldn't something similar work here for HEVC? I don't really get the meaning of "raw" for "cabac table, scaling list, picture parameter set and reference picture", since those are parts of the bitstream, which needs to be parsed to obtain those. Best regards, Tomasz
Le mardi 29 janvier 2019 à 16:44 +0900, Alexandre Courbot a écrit : > On Fri, Jan 25, 2019 at 10:04 PM Paul Kocialkowski > <paul.kocialkowski@bootlin.com> wrote: > > Hi, > > > > On Thu, 2019-01-24 at 20:23 +0800, Ayaka wrote: > > > Sent from my iPad > > > > > > > On Jan 24, 2019, at 6:27 PM, Paul Kocialkowski <paul.kocialkowski@bootlin.com> wrote: > > > > > > > > Hi, > > > > > > > > > On Thu, 2019-01-10 at 21:32 +0800, ayaka wrote: > > > > > I forget a important thing, for the rkvdec and rk hevc decoder, it would > > > > > requests cabac table, scaling list, picture parameter set and reference > > > > > picture storing in one or various of DMA buffers. I am not talking about > > > > > the data been parsed, the decoder would requests a raw data. > > > > > > > > > > For the pps and rps, it is possible to reuse the slice header, just let > > > > > the decoder know the offset from the bitstream bufer, I would suggest to > > > > > add three properties(with sps) for them. But I think we need a method to > > > > > mark a OUTPUT side buffer for those aux data. > > > > > > > > I'm quite confused about the hardware implementation then. From what > > > > you're saying, it seems that it takes the raw bitstream elements rather > > > > than parsed elements. Is it really a stateless implementation? > > > > > > > > The stateless implementation was designed with the idea that only the > > > > raw slice data should be passed in bitstream form to the decoder. For > > > > H.264, it seems that some decoders also need the slice header in raw > > > > bitstream form (because they take the full slice NAL unit), see the > > > > discussions in this thread: > > > > media: docs-rst: Document m2m stateless video decoder interface > > > > > > Stateless just mean it won’t track the previous result, but I don’t > > > think you can define what a date the hardware would need. Even you > > > just build a dpb for the decoder, it is still stateless, but parsing > > > less or more data from the bitstream doesn’t stop a decoder become a > > > stateless decoder. > > > > Yes fair enough, the format in which the hardware decoder takes the > > bitstream parameters does not make it stateless or stateful per-se. > > It's just that stateless decoders should have no particular reason for > > parsing the bitstream on their own since the hardware can be designed > > with registers for each relevant bitstream element to configure the > > decoding pipeline. That's how GPU-based decoder implementations are > > implemented (VAAPI/VDPAU/NVDEC, etc). > > > > So the format we have agreed on so far for the stateless interface is > > to pass parsed elements via v4l2 control structures. > > > > If the hardware can only work by parsing the bitstream itself, I'm not > > sure what the best solution would be. Reconstructing the bitstream in > > the kernel is a pretty bad option, but so is parsing in the kernel or > > having the data both in parsed and raw forms. Do you see another > > possibility? > > Is reconstructing the bitstream so bad? The v4l2 controls provide a > generic interface to an encoded format which the driver needs to > convert into a sequence that the hardware can understand. Typically > this is done by populating hardware-specific structures. Can't we > consider that in this specific instance, the hardware-specific > structure just happens to be identical to the original bitstream > format? At maximum allowed bitrate for let's say HEVC (940MB/s iirc), yes, it would be really really bad. In GStreamer project we have discussed for a while (but have never done anything about) adding the ability through a bitmask to select which part of the stream need to be parsed, as parsing itself was causing some overhead. Maybe similar thing applies, though as per our new design, it's the fourcc that dictate the driver behaviour, we'd need yet another fourcc for drivers that wants the full bitstream (which seems odd if you have already parsed everything, I think this need some clarification). > > I agree that this is not strictly optimal for that particular > hardware, but such is the cost of abstractions, and in this specific > case I don't believe the cost would be particularly high?
On Wed, Jan 30, 2019 at 6:41 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote: > > Le mardi 29 janvier 2019 à 16:44 +0900, Alexandre Courbot a écrit : > > On Fri, Jan 25, 2019 at 10:04 PM Paul Kocialkowski > > <paul.kocialkowski@bootlin.com> wrote: > > > Hi, > > > > > > On Thu, 2019-01-24 at 20:23 +0800, Ayaka wrote: > > > > Sent from my iPad > > > > > > > > > On Jan 24, 2019, at 6:27 PM, Paul Kocialkowski <paul.kocialkowski@bootlin.com> wrote: > > > > > > > > > > Hi, > > > > > > > > > > > On Thu, 2019-01-10 at 21:32 +0800, ayaka wrote: > > > > > > I forget a important thing, for the rkvdec and rk hevc decoder, it would > > > > > > requests cabac table, scaling list, picture parameter set and reference > > > > > > picture storing in one or various of DMA buffers. I am not talking about > > > > > > the data been parsed, the decoder would requests a raw data. > > > > > > > > > > > > For the pps and rps, it is possible to reuse the slice header, just let > > > > > > the decoder know the offset from the bitstream bufer, I would suggest to > > > > > > add three properties(with sps) for them. But I think we need a method to > > > > > > mark a OUTPUT side buffer for those aux data. > > > > > > > > > > I'm quite confused about the hardware implementation then. From what > > > > > you're saying, it seems that it takes the raw bitstream elements rather > > > > > than parsed elements. Is it really a stateless implementation? > > > > > > > > > > The stateless implementation was designed with the idea that only the > > > > > raw slice data should be passed in bitstream form to the decoder. For > > > > > H.264, it seems that some decoders also need the slice header in raw > > > > > bitstream form (because they take the full slice NAL unit), see the > > > > > discussions in this thread: > > > > > media: docs-rst: Document m2m stateless video decoder interface > > > > > > > > Stateless just mean it won’t track the previous result, but I don’t > > > > think you can define what a date the hardware would need. Even you > > > > just build a dpb for the decoder, it is still stateless, but parsing > > > > less or more data from the bitstream doesn’t stop a decoder become a > > > > stateless decoder. > > > > > > Yes fair enough, the format in which the hardware decoder takes the > > > bitstream parameters does not make it stateless or stateful per-se. > > > It's just that stateless decoders should have no particular reason for > > > parsing the bitstream on their own since the hardware can be designed > > > with registers for each relevant bitstream element to configure the > > > decoding pipeline. That's how GPU-based decoder implementations are > > > implemented (VAAPI/VDPAU/NVDEC, etc). > > > > > > So the format we have agreed on so far for the stateless interface is > > > to pass parsed elements via v4l2 control structures. > > > > > > If the hardware can only work by parsing the bitstream itself, I'm not > > > sure what the best solution would be. Reconstructing the bitstream in > > > the kernel is a pretty bad option, but so is parsing in the kernel or > > > having the data both in parsed and raw forms. Do you see another > > > possibility? > > > > Is reconstructing the bitstream so bad? The v4l2 controls provide a > > generic interface to an encoded format which the driver needs to > > convert into a sequence that the hardware can understand. Typically > > this is done by populating hardware-specific structures. Can't we > > consider that in this specific instance, the hardware-specific > > structure just happens to be identical to the original bitstream > > format? > > At maximum allowed bitrate for let's say HEVC (940MB/s iirc), yes, it > would be really really bad. In GStreamer project we have discussed for > a while (but have never done anything about) adding the ability through > a bitmask to select which part of the stream need to be parsed, as > parsing itself was causing some overhead. Maybe similar thing applies, > though as per our new design, it's the fourcc that dictate the driver > behaviour, we'd need yet another fourcc for drivers that wants the full > bitstream (which seems odd if you have already parsed everything, I > think this need some clarification). Note that I am not proposing to rebuild the *entire* bitstream in-kernel. What I am saying is that if the hardware interprets some structures (like SPS/PPS) in their raw format, this raw format could be reconstructed from the structures passed by userspace at negligible cost. Such manipulation would only happen on a small amount of data. Exposing finer-grained driver requirements through a bitmask may deserve more exploring. Maybe we could end with a spectrum of capabilities that would allow us to cover the range from fully stateless to fully stateful IPs more smoothly. Right now we have two specifications that only consider the extremes of that range.
On Wed, Jan 30, 2019 at 11:29 AM Alexandre Courbot <acourbot@chromium.org> wrote: > > On Wed, Jan 30, 2019 at 6:41 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote: > > > > Le mardi 29 janvier 2019 à 16:44 +0900, Alexandre Courbot a écrit : > > > On Fri, Jan 25, 2019 at 10:04 PM Paul Kocialkowski > > > <paul.kocialkowski@bootlin.com> wrote: > > > > Hi, > > > > > > > > On Thu, 2019-01-24 at 20:23 +0800, Ayaka wrote: > > > > > Sent from my iPad > > > > > > > > > > > On Jan 24, 2019, at 6:27 PM, Paul Kocialkowski <paul.kocialkowski@bootlin.com> wrote: > > > > > > > > > > > > Hi, > > > > > > > > > > > > > On Thu, 2019-01-10 at 21:32 +0800, ayaka wrote: > > > > > > > I forget a important thing, for the rkvdec and rk hevc decoder, it would > > > > > > > requests cabac table, scaling list, picture parameter set and reference > > > > > > > picture storing in one or various of DMA buffers. I am not talking about > > > > > > > the data been parsed, the decoder would requests a raw data. > > > > > > > > > > > > > > For the pps and rps, it is possible to reuse the slice header, just let > > > > > > > the decoder know the offset from the bitstream bufer, I would suggest to > > > > > > > add three properties(with sps) for them. But I think we need a method to > > > > > > > mark a OUTPUT side buffer for those aux data. > > > > > > > > > > > > I'm quite confused about the hardware implementation then. From what > > > > > > you're saying, it seems that it takes the raw bitstream elements rather > > > > > > than parsed elements. Is it really a stateless implementation? > > > > > > > > > > > > The stateless implementation was designed with the idea that only the > > > > > > raw slice data should be passed in bitstream form to the decoder. For > > > > > > H.264, it seems that some decoders also need the slice header in raw > > > > > > bitstream form (because they take the full slice NAL unit), see the > > > > > > discussions in this thread: > > > > > > media: docs-rst: Document m2m stateless video decoder interface > > > > > > > > > > Stateless just mean it won’t track the previous result, but I don’t > > > > > think you can define what a date the hardware would need. Even you > > > > > just build a dpb for the decoder, it is still stateless, but parsing > > > > > less or more data from the bitstream doesn’t stop a decoder become a > > > > > stateless decoder. > > > > > > > > Yes fair enough, the format in which the hardware decoder takes the > > > > bitstream parameters does not make it stateless or stateful per-se. > > > > It's just that stateless decoders should have no particular reason for > > > > parsing the bitstream on their own since the hardware can be designed > > > > with registers for each relevant bitstream element to configure the > > > > decoding pipeline. That's how GPU-based decoder implementations are > > > > implemented (VAAPI/VDPAU/NVDEC, etc). > > > > > > > > So the format we have agreed on so far for the stateless interface is > > > > to pass parsed elements via v4l2 control structures. > > > > > > > > If the hardware can only work by parsing the bitstream itself, I'm not > > > > sure what the best solution would be. Reconstructing the bitstream in > > > > the kernel is a pretty bad option, but so is parsing in the kernel or > > > > having the data both in parsed and raw forms. Do you see another > > > > possibility? > > > > > > Is reconstructing the bitstream so bad? The v4l2 controls provide a > > > generic interface to an encoded format which the driver needs to > > > convert into a sequence that the hardware can understand. Typically > > > this is done by populating hardware-specific structures. Can't we > > > consider that in this specific instance, the hardware-specific > > > structure just happens to be identical to the original bitstream > > > format? > > > > At maximum allowed bitrate for let's say HEVC (940MB/s iirc), yes, it > > would be really really bad. In GStreamer project we have discussed for > > a while (but have never done anything about) adding the ability through > > a bitmask to select which part of the stream need to be parsed, as > > parsing itself was causing some overhead. Maybe similar thing applies, > > though as per our new design, it's the fourcc that dictate the driver > > behaviour, we'd need yet another fourcc for drivers that wants the full > > bitstream (which seems odd if you have already parsed everything, I > > think this need some clarification). > > Note that I am not proposing to rebuild the *entire* bitstream > in-kernel. What I am saying is that if the hardware interprets some > structures (like SPS/PPS) in their raw format, this raw format could > be reconstructed from the structures passed by userspace at negligible > cost. Such manipulation would only happen on a small amount of data. > > Exposing finer-grained driver requirements through a bitmask may > deserve more exploring. Maybe we could end with a spectrum of > capabilities that would allow us to cover the range from fully > stateless to fully stateful IPs more smoothly. Right now we have two > specifications that only consider the extremes of that range. I gave it a bit more thought and if we combine what Nicolas suggested about the bitmask control with the userspace providing the full bitstream in the OUTPUT buffers, split into some logical units and "tagged" with their type (e.g. SPS, PPS, slice, etc.), we could potentially get an interface that would work for any kind of decoder I can think of, actually eliminating the boundary between stateful and stateless decoders. For example, a fully stateful decoder would have the bitmask control set to 0 and accept data from all the OUTPUT buffers as they come. A decoder that doesn't do any parsing on its own would have all the valid bits in the bitmask set and ignore the data in OUTPUT buffers tagged as any kind of metadata. And then, we could have any cases in between, including stateful decoders which just can't parse the stream on their own, but still manage anything else themselves, or stateless ones which can parse parts of the stream, like the rk3399 vdec can parse the H.264 slice headers on its own. That could potentially let us completely eliminate the distinction between the stateful and stateless interfaces and just have one that covers both. Thoughts? Best regards, Tomasz
Sent from my iPad > On Jan 30, 2019, at 11:35 AM, Tomasz Figa <tfiga@chromium.org> wrote: > > On Wed, Jan 30, 2019 at 11:29 AM Alexandre Courbot > <acourbot@chromium.org> wrote: >> >>> On Wed, Jan 30, 2019 at 6:41 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote: >>> >>>> Le mardi 29 janvier 2019 à 16:44 +0900, Alexandre Courbot a écrit : >>>> On Fri, Jan 25, 2019 at 10:04 PM Paul Kocialkowski >>>> <paul.kocialkowski@bootlin.com> wrote: >>>>> Hi, >>>>> >>>>>> On Thu, 2019-01-24 at 20:23 +0800, Ayaka wrote: >>>>>> Sent from my iPad >>>>>> >>>>>>> On Jan 24, 2019, at 6:27 PM, Paul Kocialkowski <paul.kocialkowski@bootlin.com> wrote: >>>>>>> >>>>>>> Hi, >>>>>>> >>>>>>>> On Thu, 2019-01-10 at 21:32 +0800, ayaka wrote: >>>>>>>> I forget a important thing, for the rkvdec and rk hevc decoder, it would >>>>>>>> requests cabac table, scaling list, picture parameter set and reference >>>>>>>> picture storing in one or various of DMA buffers. I am not talking about >>>>>>>> the data been parsed, the decoder would requests a raw data. >>>>>>>> >>>>>>>> For the pps and rps, it is possible to reuse the slice header, just let >>>>>>>> the decoder know the offset from the bitstream bufer, I would suggest to >>>>>>>> add three properties(with sps) for them. But I think we need a method to >>>>>>>> mark a OUTPUT side buffer for those aux data. >>>>>>> >>>>>>> I'm quite confused about the hardware implementation then. From what >>>>>>> you're saying, it seems that it takes the raw bitstream elements rather >>>>>>> than parsed elements. Is it really a stateless implementation? >>>>>>> >>>>>>> The stateless implementation was designed with the idea that only the >>>>>>> raw slice data should be passed in bitstream form to the decoder. For >>>>>>> H.264, it seems that some decoders also need the slice header in raw >>>>>>> bitstream form (because they take the full slice NAL unit), see the >>>>>>> discussions in this thread: >>>>>>> media: docs-rst: Document m2m stateless video decoder interface >>>>>> >>>>>> Stateless just mean it won’t track the previous result, but I don’t >>>>>> think you can define what a date the hardware would need. Even you >>>>>> just build a dpb for the decoder, it is still stateless, but parsing >>>>>> less or more data from the bitstream doesn’t stop a decoder become a >>>>>> stateless decoder. >>>>> >>>>> Yes fair enough, the format in which the hardware decoder takes the >>>>> bitstream parameters does not make it stateless or stateful per-se. >>>>> It's just that stateless decoders should have no particular reason for >>>>> parsing the bitstream on their own since the hardware can be designed >>>>> with registers for each relevant bitstream element to configure the >>>>> decoding pipeline. That's how GPU-based decoder implementations are >>>>> implemented (VAAPI/VDPAU/NVDEC, etc). >>>>> >>>>> So the format we have agreed on so far for the stateless interface is >>>>> to pass parsed elements via v4l2 control structures. >>>>> >>>>> If the hardware can only work by parsing the bitstream itself, I'm not >>>>> sure what the best solution would be. Reconstructing the bitstream in >>>>> the kernel is a pretty bad option, but so is parsing in the kernel or >>>>> having the data both in parsed and raw forms. Do you see another >>>>> possibility? >>>> >>>> Is reconstructing the bitstream so bad? The v4l2 controls provide a >>>> generic interface to an encoded format which the driver needs to >>>> convert into a sequence that the hardware can understand. Typically >>>> this is done by populating hardware-specific structures. Can't we >>>> consider that in this specific instance, the hardware-specific >>>> structure just happens to be identical to the original bitstream >>>> format? >>> >>> At maximum allowed bitrate for let's say HEVC (940MB/s iirc), yes, it >>> would be really really bad. In GStreamer project we have discussed for >>> a while (but have never done anything about) adding the ability through >>> a bitmask to select which part of the stream need to be parsed, as >>> parsing itself was causing some overhead. Maybe similar thing applies, >>> though as per our new design, it's the fourcc that dictate the driver >>> behaviour, we'd need yet another fourcc for drivers that wants the full >>> bitstream (which seems odd if you have already parsed everything, I >>> think this need some clarification). >> >> Note that I am not proposing to rebuild the *entire* bitstream >> in-kernel. What I am saying is that if the hardware interprets some >> structures (like SPS/PPS) in their raw format, this raw format could >> be reconstructed from the structures passed by userspace at negligible >> cost. Such manipulation would only happen on a small amount of data. >> >> Exposing finer-grained driver requirements through a bitmask may >> deserve more exploring. Maybe we could end with a spectrum of >> capabilities that would allow us to cover the range from fully >> stateless to fully stateful IPs more smoothly. Right now we have two >> specifications that only consider the extremes of that range. > > I gave it a bit more thought and if we combine what Nicolas suggested > about the bitmask control with the userspace providing the full > bitstream in the OUTPUT buffers, split into some logical units and > "tagged" with their type (e.g. SPS, PPS, slice, etc.), we could > potentially get an interface that would work for any kind of decoder I > can think of, actually eliminating the boundary between stateful and > stateless decoders. I agree with this idea, that is what I want calling memory region description while I am still struggling with userspace to post my driver demo. > > For example, a fully stateful decoder would have the bitmask control > set to 0 and accept data from all the OUTPUT buffers as they come. A > decoder that doesn't do any parsing on its own would have all the > valid bits in the bitmask set and ignore the data in OUTPUT buffers > tagged as any kind of metadata. And then, we could have any cases in > between, including stateful decoders which just can't parse the stream > on their own, but still manage anything else themselves, or stateless > ones which can parse parts of the stream, like the rk3399 vdec can > parse the H.264 slice headers on its own. > Actually not, the rkvdec and rkhevc can parse most but not all syntax sections. Besides the vp9 decoder of rkvdec won’t parse most of the syntax. I talked to some rockchip staff about the performance problem of reconstruction bitstream after yesterday arguing with tfiga at IRC yesterday. Although 1ms looks small to those decoder which can decode a picture of a UHD 4K HEVC videos in 9ms, it is enough for 60fps. But how about a higher frame rate like 120fps or 240fps and when it comes to 8K which is used in Japan broadcast. I would bring more detail in the FOSDEM 2019, I may stay at graphics devroom at Saturday. > That could potentially let us completely eliminate the distinction > between the stateful and stateless interfaces and just have one that > covers both. > > Thoughts? > > Best regards, > Tomasz
Sent from my iPad > On Jan 30, 2019, at 5:41 AM, Nicolas Dufresne <nicolas@ndufresne.ca> wrote: > >> Le mardi 29 janvier 2019 à 16:44 +0900, Alexandre Courbot a écrit : >> On Fri, Jan 25, 2019 at 10:04 PM Paul Kocialkowski >> <paul.kocialkowski@bootlin.com> wrote: >>> Hi, >>> >>>> On Thu, 2019-01-24 at 20:23 +0800, Ayaka wrote: >>>> Sent from my iPad >>>> >>>>> On Jan 24, 2019, at 6:27 PM, Paul Kocialkowski <paul.kocialkowski@bootlin.com> wrote: >>>>> >>>>> Hi, >>>>> >>>>>> On Thu, 2019-01-10 at 21:32 +0800, ayaka wrote: >>>>>> I forget a important thing, for the rkvdec and rk hevc decoder, it would >>>>>> requests cabac table, scaling list, picture parameter set and reference >>>>>> picture storing in one or various of DMA buffers. I am not talking about >>>>>> the data been parsed, the decoder would requests a raw data. >>>>>> >>>>>> For the pps and rps, it is possible to reuse the slice header, just let >>>>>> the decoder know the offset from the bitstream bufer, I would suggest to >>>>>> add three properties(with sps) for them. But I think we need a method to >>>>>> mark a OUTPUT side buffer for those aux data. >>>>> >>>>> I'm quite confused about the hardware implementation then. From what >>>>> you're saying, it seems that it takes the raw bitstream elements rather >>>>> than parsed elements. Is it really a stateless implementation? >>>>> >>>>> The stateless implementation was designed with the idea that only the >>>>> raw slice data should be passed in bitstream form to the decoder. For >>>>> H.264, it seems that some decoders also need the slice header in raw >>>>> bitstream form (because they take the full slice NAL unit), see the >>>>> discussions in this thread: >>>>> media: docs-rst: Document m2m stateless video decoder interface >>>> >>>> Stateless just mean it won’t track the previous result, but I don’t >>>> think you can define what a date the hardware would need. Even you >>>> just build a dpb for the decoder, it is still stateless, but parsing >>>> less or more data from the bitstream doesn’t stop a decoder become a >>>> stateless decoder. >>> >>> Yes fair enough, the format in which the hardware decoder takes the >>> bitstream parameters does not make it stateless or stateful per-se. >>> It's just that stateless decoders should have no particular reason for >>> parsing the bitstream on their own since the hardware can be designed >>> with registers for each relevant bitstream element to configure the >>> decoding pipeline. That's how GPU-based decoder implementations are >>> implemented (VAAPI/VDPAU/NVDEC, etc). >>> >>> So the format we have agreed on so far for the stateless interface is >>> to pass parsed elements via v4l2 control structures. >>> >>> If the hardware can only work by parsing the bitstream itself, I'm not >>> sure what the best solution would be. Reconstructing the bitstream in >>> the kernel is a pretty bad option, but so is parsing in the kernel or >>> having the data both in parsed and raw forms. Do you see another >>> possibility? >> >> Is reconstructing the bitstream so bad? The v4l2 controls provide a >> generic interface to an encoded format which the driver needs to >> convert into a sequence that the hardware can understand. Typically >> this is done by populating hardware-specific structures. Can't we >> consider that in this specific instance, the hardware-specific >> structure just happens to be identical to the original bitstream >> format? > > At maximum allowed bitrate for let's say HEVC (940MB/s iirc), yes, it Lucky, most of hardware won’t be able to processing such a big buffer. General speaking, the register is 24bits for stream length in bytes. > would be really really bad. In GStreamer project we have discussed for > a while (but have never done anything about) adding the ability through > a bitmask to select which part of the stream need to be parsed, as > parsing itself was causing some overhead. Maybe similar thing applies, > though as per our new design, it's the fourcc that dictate the driver > behaviour, we'd need yet another fourcc for drivers that wants the full > bitstream (which seems odd if you have already parsed everything, I > think this need some clarification). > >> >> I agree that this is not strictly optimal for that particular >> hardware, but such is the cost of abstractions, and in this specific >> case I don't believe the cost would be particularly high?
On Wed, Jan 30, 2019 at 3:28 PM Ayaka <ayaka@soulik.info> wrote: > > > > Sent from my iPad > > > On Jan 30, 2019, at 11:35 AM, Tomasz Figa <tfiga@chromium.org> wrote: > > > > On Wed, Jan 30, 2019 at 11:29 AM Alexandre Courbot > > <acourbot@chromium.org> wrote: > >> > >>> On Wed, Jan 30, 2019 at 6:41 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote: > >>> > >>>> Le mardi 29 janvier 2019 à 16:44 +0900, Alexandre Courbot a écrit : > >>>> On Fri, Jan 25, 2019 at 10:04 PM Paul Kocialkowski > >>>> <paul.kocialkowski@bootlin.com> wrote: > >>>>> Hi, > >>>>> > >>>>>> On Thu, 2019-01-24 at 20:23 +0800, Ayaka wrote: > >>>>>> Sent from my iPad > >>>>>> > >>>>>>> On Jan 24, 2019, at 6:27 PM, Paul Kocialkowski <paul.kocialkowski@bootlin.com> wrote: > >>>>>>> > >>>>>>> Hi, > >>>>>>> > >>>>>>>> On Thu, 2019-01-10 at 21:32 +0800, ayaka wrote: > >>>>>>>> I forget a important thing, for the rkvdec and rk hevc decoder, it would > >>>>>>>> requests cabac table, scaling list, picture parameter set and reference > >>>>>>>> picture storing in one or various of DMA buffers. I am not talking about > >>>>>>>> the data been parsed, the decoder would requests a raw data. > >>>>>>>> > >>>>>>>> For the pps and rps, it is possible to reuse the slice header, just let > >>>>>>>> the decoder know the offset from the bitstream bufer, I would suggest to > >>>>>>>> add three properties(with sps) for them. But I think we need a method to > >>>>>>>> mark a OUTPUT side buffer for those aux data. > >>>>>>> > >>>>>>> I'm quite confused about the hardware implementation then. From what > >>>>>>> you're saying, it seems that it takes the raw bitstream elements rather > >>>>>>> than parsed elements. Is it really a stateless implementation? > >>>>>>> > >>>>>>> The stateless implementation was designed with the idea that only the > >>>>>>> raw slice data should be passed in bitstream form to the decoder. For > >>>>>>> H.264, it seems that some decoders also need the slice header in raw > >>>>>>> bitstream form (because they take the full slice NAL unit), see the > >>>>>>> discussions in this thread: > >>>>>>> media: docs-rst: Document m2m stateless video decoder interface > >>>>>> > >>>>>> Stateless just mean it won’t track the previous result, but I don’t > >>>>>> think you can define what a date the hardware would need. Even you > >>>>>> just build a dpb for the decoder, it is still stateless, but parsing > >>>>>> less or more data from the bitstream doesn’t stop a decoder become a > >>>>>> stateless decoder. > >>>>> > >>>>> Yes fair enough, the format in which the hardware decoder takes the > >>>>> bitstream parameters does not make it stateless or stateful per-se. > >>>>> It's just that stateless decoders should have no particular reason for > >>>>> parsing the bitstream on their own since the hardware can be designed > >>>>> with registers for each relevant bitstream element to configure the > >>>>> decoding pipeline. That's how GPU-based decoder implementations are > >>>>> implemented (VAAPI/VDPAU/NVDEC, etc). > >>>>> > >>>>> So the format we have agreed on so far for the stateless interface is > >>>>> to pass parsed elements via v4l2 control structures. > >>>>> > >>>>> If the hardware can only work by parsing the bitstream itself, I'm not > >>>>> sure what the best solution would be. Reconstructing the bitstream in > >>>>> the kernel is a pretty bad option, but so is parsing in the kernel or > >>>>> having the data both in parsed and raw forms. Do you see another > >>>>> possibility? > >>>> > >>>> Is reconstructing the bitstream so bad? The v4l2 controls provide a > >>>> generic interface to an encoded format which the driver needs to > >>>> convert into a sequence that the hardware can understand. Typically > >>>> this is done by populating hardware-specific structures. Can't we > >>>> consider that in this specific instance, the hardware-specific > >>>> structure just happens to be identical to the original bitstream > >>>> format? > >>> > >>> At maximum allowed bitrate for let's say HEVC (940MB/s iirc), yes, it > >>> would be really really bad. In GStreamer project we have discussed for > >>> a while (but have never done anything about) adding the ability through > >>> a bitmask to select which part of the stream need to be parsed, as > >>> parsing itself was causing some overhead. Maybe similar thing applies, > >>> though as per our new design, it's the fourcc that dictate the driver > >>> behaviour, we'd need yet another fourcc for drivers that wants the full > >>> bitstream (which seems odd if you have already parsed everything, I > >>> think this need some clarification). > >> > >> Note that I am not proposing to rebuild the *entire* bitstream > >> in-kernel. What I am saying is that if the hardware interprets some > >> structures (like SPS/PPS) in their raw format, this raw format could > >> be reconstructed from the structures passed by userspace at negligible > >> cost. Such manipulation would only happen on a small amount of data. > >> > >> Exposing finer-grained driver requirements through a bitmask may > >> deserve more exploring. Maybe we could end with a spectrum of > >> capabilities that would allow us to cover the range from fully > >> stateless to fully stateful IPs more smoothly. Right now we have two > >> specifications that only consider the extremes of that range. > > > > I gave it a bit more thought and if we combine what Nicolas suggested > > about the bitmask control with the userspace providing the full > > bitstream in the OUTPUT buffers, split into some logical units and > > "tagged" with their type (e.g. SPS, PPS, slice, etc.), we could > > potentially get an interface that would work for any kind of decoder I > > can think of, actually eliminating the boundary between stateful and > > stateless decoders. > I agree with this idea, that is what I want calling memory region description while I am still struggling with userspace to post my driver demo. > > > > For example, a fully stateful decoder would have the bitmask control > > set to 0 and accept data from all the OUTPUT buffers as they come. A > > decoder that doesn't do any parsing on its own would have all the > > valid bits in the bitmask set and ignore the data in OUTPUT buffers > > tagged as any kind of metadata. And then, we could have any cases in > > between, including stateful decoders which just can't parse the stream > > on their own, but still manage anything else themselves, or stateless > > ones which can parse parts of the stream, like the rk3399 vdec can > > parse the H.264 slice headers on its own. > > > Actually not, the rkvdec and rkhevc can parse most but not all syntax sections. > Besides the vp9 decoder of rkvdec won’t parse most of the syntax. > > I talked to some rockchip staff about the performance problem of reconstruction bitstream after yesterday arguing with tfiga at IRC yesterday. Although 1ms looks small to those decoder which can decode a picture of a UHD 4K HEVC videos in 9ms, it is enough for 60fps. But how about a higher frame rate like 120fps or 240fps and when it comes to 8K which is used in Japan broadcast. 1 ms for a 500 MHz CPU (which is quite slow these days) is 500k cycles. We don't have to reconstruct the whole bitstream, just the parsed metadata and also we don't get a new PPS or SPS every frame. Not sure where you have this 1 ms from. Most of the difference between our structures and the bitstream is that the latter is packed and could be variable length. We actually have some sample bitstream assembly code for the rockchip encoder: https://chromium.googlesource.com/chromiumos/third_party/libv4lplugins/+/5e6034258146af6be973fb6a5bb6b9d6e7489437/libv4l-rockchip_v2/libvepu/h264e/h264e.c#148 https://chromium.googlesource.com/chromiumos/third_party/libv4lplugins/+/5e6034258146af6be973fb6a5bb6b9d6e7489437/libv4l-rockchip_v2/libvepu/streams.c#36 Disassembling the stream_put_bits() gives 36 thumb2 instructions, including 23 for the loop for each byte that is written. stream_write_ue() is a bit more complicated, but in the worst case it ends up with 4 calls to stream_put_bits(), each at most spanning 4 bytes for simplicity. Let's count the operations for SPS then: (1) stream_put_bits() spanning 1 byte: 33 times (2) stream_put_bits() spanning up to 3 bytes: 4 times (3) stream_write_ue() up to 31 bits: 19 times Adding it together: (1) 33 * 36 + (2) 4 * (36 + 2 * 23) + (3) 19 * (4 * (36 + 3 * 23)) = 1188 + 328 + 7980 = 9496 ~= 10k instructions The code above doesn't seem to contain any expensive instructions, like division, so for a modern pipelined out of order core (e.g. A53), it could be safe to assume 1 instruction per cycle. At 500 MHz that gives you 20 usecs. SPS is the most complex header and for H.264 we just do PPS and some slice headers. Let's round it up a bit and we could have around 100 usecs for the complete frame metadata. > > I would bring more detail in the FOSDEM 2019, I may stay at graphics devroom at Saturday. > > That could potentially let us completely eliminate the distinction > > between the stateful and stateless interfaces and just have one that > > covers both. > > > > Thoughts? Any thoughts on my proposal to make the interface more flexible? Any specific examples of issues that we could encounter that would prevent it from working efficiently with Rockchip (or other) hardware? Best regards, Tomasz
On Wed, Jan 30, 2019 at 12:35:41PM +0900, Tomasz Figa wrote: > On Wed, Jan 30, 2019 at 11:29 AM Alexandre Courbot > <acourbot@chromium.org> wrote: > > > > On Wed, Jan 30, 2019 at 6:41 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote: > > > > > > Le mardi 29 janvier 2019 à 16:44 +0900, Alexandre Courbot a écrit : > > > > On Fri, Jan 25, 2019 at 10:04 PM Paul Kocialkowski > > > > <paul.kocialkowski@bootlin.com> wrote: > > > > > Hi, > > > > > > > > > > On Thu, 2019-01-24 at 20:23 +0800, Ayaka wrote: > > > > > > Sent from my iPad > > > > > > > > > > > > > On Jan 24, 2019, at 6:27 PM, Paul Kocialkowski <paul.kocialkowski@bootlin.com> wrote: > > > > > > > > > > > > > > Hi, > > > > > > > > > > > > > > > On Thu, 2019-01-10 at 21:32 +0800, ayaka wrote: > > > > > > > > I forget a important thing, for the rkvdec and rk hevc decoder, it would > > > > > > > > requests cabac table, scaling list, picture parameter set and reference > > > > > > > > picture storing in one or various of DMA buffers. I am not talking about > > > > > > > > the data been parsed, the decoder would requests a raw data. > > > > > > > > > > > > > > > > For the pps and rps, it is possible to reuse the slice header, just let > > > > > > > > the decoder know the offset from the bitstream bufer, I would suggest to > > > > > > > > add three properties(with sps) for them. But I think we need a method to > > > > > > > > mark a OUTPUT side buffer for those aux data. > > > > > > > > > > > > > > I'm quite confused about the hardware implementation then. From what > > > > > > > you're saying, it seems that it takes the raw bitstream elements rather > > > > > > > than parsed elements. Is it really a stateless implementation? > > > > > > > > > > > > > > The stateless implementation was designed with the idea that only the > > > > > > > raw slice data should be passed in bitstream form to the decoder. For > > > > > > > H.264, it seems that some decoders also need the slice header in raw > > > > > > > bitstream form (because they take the full slice NAL unit), see the > > > > > > > discussions in this thread: > > > > > > > media: docs-rst: Document m2m stateless video decoder interface > > > > > > > > > > > > Stateless just mean it won’t track the previous result, but I don’t > > > > > > think you can define what a date the hardware would need. Even you > > > > > > just build a dpb for the decoder, it is still stateless, but parsing > > > > > > less or more data from the bitstream doesn’t stop a decoder become a > > > > > > stateless decoder. > > > > > > > > > > Yes fair enough, the format in which the hardware decoder takes the > > > > > bitstream parameters does not make it stateless or stateful per-se. > > > > > It's just that stateless decoders should have no particular reason for > > > > > parsing the bitstream on their own since the hardware can be designed > > > > > with registers for each relevant bitstream element to configure the > > > > > decoding pipeline. That's how GPU-based decoder implementations are > > > > > implemented (VAAPI/VDPAU/NVDEC, etc). > > > > > > > > > > So the format we have agreed on so far for the stateless interface is > > > > > to pass parsed elements via v4l2 control structures. > > > > > > > > > > If the hardware can only work by parsing the bitstream itself, I'm not > > > > > sure what the best solution would be. Reconstructing the bitstream in > > > > > the kernel is a pretty bad option, but so is parsing in the kernel or > > > > > having the data both in parsed and raw forms. Do you see another > > > > > possibility? > > > > > > > > Is reconstructing the bitstream so bad? The v4l2 controls provide a > > > > generic interface to an encoded format which the driver needs to > > > > convert into a sequence that the hardware can understand. Typically > > > > this is done by populating hardware-specific structures. Can't we > > > > consider that in this specific instance, the hardware-specific > > > > structure just happens to be identical to the original bitstream > > > > format? > > > > > > At maximum allowed bitrate for let's say HEVC (940MB/s iirc), yes, it > > > would be really really bad. In GStreamer project we have discussed for > > > a while (but have never done anything about) adding the ability through > > > a bitmask to select which part of the stream need to be parsed, as > > > parsing itself was causing some overhead. Maybe similar thing applies, > > > though as per our new design, it's the fourcc that dictate the driver > > > behaviour, we'd need yet another fourcc for drivers that wants the full > > > bitstream (which seems odd if you have already parsed everything, I > > > think this need some clarification). > > > > Note that I am not proposing to rebuild the *entire* bitstream > > in-kernel. What I am saying is that if the hardware interprets some > > structures (like SPS/PPS) in their raw format, this raw format could > > be reconstructed from the structures passed by userspace at negligible > > cost. Such manipulation would only happen on a small amount of data. > > > > Exposing finer-grained driver requirements through a bitmask may > > deserve more exploring. Maybe we could end with a spectrum of > > capabilities that would allow us to cover the range from fully > > stateless to fully stateful IPs more smoothly. Right now we have two > > specifications that only consider the extremes of that range. > > I gave it a bit more thought and if we combine what Nicolas suggested > about the bitmask control with the userspace providing the full > bitstream in the OUTPUT buffers, split into some logical units and > "tagged" with their type (e.g. SPS, PPS, slice, etc.), we could > potentially get an interface that would work for any kind of decoder I > can think of, actually eliminating the boundary between stateful and > stateless decoders. > > For example, a fully stateful decoder would have the bitmask control > set to 0 and accept data from all the OUTPUT buffers as they come. A > decoder that doesn't do any parsing on its own would have all the > valid bits in the bitmask set and ignore the data in OUTPUT buffers > tagged as any kind of metadata. And then, we could have any cases in > between, including stateful decoders which just can't parse the stream > on their own, but still manage anything else themselves, or stateless > ones which can parse parts of the stream, like the rk3399 vdec can > parse the H.264 slice headers on its own. > > That could potentially let us completely eliminate the distinction > between the stateful and stateless interfaces and just have one that > covers both. > > Thoughts? If we have to provide the whole bitstream in the buffers, then it entirely breaks the sole software stack we have running and working currently, for a use case and a driver that hasn't seen a single line of code. Seriously, this is a *private* API that we did that way so that we can change it and only make it public. Why not do just that? Maxime
Sent from my iPad > On Jan 30, 2019, at 3:17 PM, Tomasz Figa <tfiga@chromium.org> wrote: > >> On Wed, Jan 30, 2019 at 3:28 PM Ayaka <ayaka@soulik.info> wrote: >> >> >> >> Sent from my iPad >> >>> On Jan 30, 2019, at 11:35 AM, Tomasz Figa <tfiga@chromium.org> wrote: >>> >>> On Wed, Jan 30, 2019 at 11:29 AM Alexandre Courbot >>> <acourbot@chromium.org> wrote: >>>> >>>>>> On Wed, Jan 30, 2019 at 6:41 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote: >>>>>> >>>>>> Le mardi 29 janvier 2019 à 16:44 +0900, Alexandre Courbot a écrit : >>>>>> On Fri, Jan 25, 2019 at 10:04 PM Paul Kocialkowski >>>>>> <paul.kocialkowski@bootlin.com> wrote: >>>>>>> Hi, >>>>>>> >>>>>>>> On Thu, 2019-01-24 at 20:23 +0800, Ayaka wrote: >>>>>>>> Sent from my iPad >>>>>>>> >>>>>>>>> On Jan 24, 2019, at 6:27 PM, Paul Kocialkowski <paul.kocialkowski@bootlin.com> wrote: >>>>>>>>> >>>>>>>>> Hi, >>>>>>>>> >>>>>>>>>> On Thu, 2019-01-10 at 21:32 +0800, ayaka wrote: >>>>>>>>>> I forget a important thing, for the rkvdec and rk hevc decoder, it would >>>>>>>>>> requests cabac table, scaling list, picture parameter set and reference >>>>>>>>>> picture storing in one or various of DMA buffers. I am not talking about >>>>>>>>>> the data been parsed, the decoder would requests a raw data. >>>>>>>>>> >>>>>>>>>> For the pps and rps, it is possible to reuse the slice header, just let >>>>>>>>>> the decoder know the offset from the bitstream bufer, I would suggest to >>>>>>>>>> add three properties(with sps) for them. But I think we need a method to >>>>>>>>>> mark a OUTPUT side buffer for those aux data. >>>>>>>>> >>>>>>>>> I'm quite confused about the hardware implementation then. From what >>>>>>>>> you're saying, it seems that it takes the raw bitstream elements rather >>>>>>>>> than parsed elements. Is it really a stateless implementation? >>>>>>>>> >>>>>>>>> The stateless implementation was designed with the idea that only the >>>>>>>>> raw slice data should be passed in bitstream form to the decoder. For >>>>>>>>> H.264, it seems that some decoders also need the slice header in raw >>>>>>>>> bitstream form (because they take the full slice NAL unit), see the >>>>>>>>> discussions in this thread: >>>>>>>>> media: docs-rst: Document m2m stateless video decoder interface >>>>>>>> >>>>>>>> Stateless just mean it won’t track the previous result, but I don’t >>>>>>>> think you can define what a date the hardware would need. Even you >>>>>>>> just build a dpb for the decoder, it is still stateless, but parsing >>>>>>>> less or more data from the bitstream doesn’t stop a decoder become a >>>>>>>> stateless decoder. >>>>>>> >>>>>>> Yes fair enough, the format in which the hardware decoder takes the >>>>>>> bitstream parameters does not make it stateless or stateful per-se. >>>>>>> It's just that stateless decoders should have no particular reason for >>>>>>> parsing the bitstream on their own since the hardware can be designed >>>>>>> with registers for each relevant bitstream element to configure the >>>>>>> decoding pipeline. That's how GPU-based decoder implementations are >>>>>>> implemented (VAAPI/VDPAU/NVDEC, etc). >>>>>>> >>>>>>> So the format we have agreed on so far for the stateless interface is >>>>>>> to pass parsed elements via v4l2 control structures. >>>>>>> >>>>>>> If the hardware can only work by parsing the bitstream itself, I'm not >>>>>>> sure what the best solution would be. Reconstructing the bitstream in >>>>>>> the kernel is a pretty bad option, but so is parsing in the kernel or >>>>>>> having the data both in parsed and raw forms. Do you see another >>>>>>> possibility? >>>>>> >>>>>> Is reconstructing the bitstream so bad? The v4l2 controls provide a >>>>>> generic interface to an encoded format which the driver needs to >>>>>> convert into a sequence that the hardware can understand. Typically >>>>>> this is done by populating hardware-specific structures. Can't we >>>>>> consider that in this specific instance, the hardware-specific >>>>>> structure just happens to be identical to the original bitstream >>>>>> format? >>>>> >>>>> At maximum allowed bitrate for let's say HEVC (940MB/s iirc), yes, it >>>>> would be really really bad. In GStreamer project we have discussed for >>>>> a while (but have never done anything about) adding the ability through >>>>> a bitmask to select which part of the stream need to be parsed, as >>>>> parsing itself was causing some overhead. Maybe similar thing applies, >>>>> though as per our new design, it's the fourcc that dictate the driver >>>>> behaviour, we'd need yet another fourcc for drivers that wants the full >>>>> bitstream (which seems odd if you have already parsed everything, I >>>>> think this need some clarification). >>>> >>>> Note that I am not proposing to rebuild the *entire* bitstream >>>> in-kernel. What I am saying is that if the hardware interprets some >>>> structures (like SPS/PPS) in their raw format, this raw format could >>>> be reconstructed from the structures passed by userspace at negligible >>>> cost. Such manipulation would only happen on a small amount of data. >>>> >>>> Exposing finer-grained driver requirements through a bitmask may >>>> deserve more exploring. Maybe we could end with a spectrum of >>>> capabilities that would allow us to cover the range from fully >>>> stateless to fully stateful IPs more smoothly. Right now we have two >>>> specifications that only consider the extremes of that range. >>> >>> I gave it a bit more thought and if we combine what Nicolas suggested >>> about the bitmask control with the userspace providing the full >>> bitstream in the OUTPUT buffers, split into some logical units and >>> "tagged" with their type (e.g. SPS, PPS, slice, etc.), we could >>> potentially get an interface that would work for any kind of decoder I >>> can think of, actually eliminating the boundary between stateful and >>> stateless decoders. >> I agree with this idea, that is what I want calling memory region description while I am still struggling with userspace to post my driver demo. >>> >>> For example, a fully stateful decoder would have the bitmask control >>> set to 0 and accept data from all the OUTPUT buffers as they come. A >>> decoder that doesn't do any parsing on its own would have all the >>> valid bits in the bitmask set and ignore the data in OUTPUT buffers >>> tagged as any kind of metadata. And then, we could have any cases in >>> between, including stateful decoders which just can't parse the stream >>> on their own, but still manage anything else themselves, or stateless >>> ones which can parse parts of the stream, like the rk3399 vdec can >>> parse the H.264 slice headers on its own. >>> >> Actually not, the rkvdec and rkhevc can parse most but not all syntax sections. >> Besides the vp9 decoder of rkvdec won’t parse most of the syntax. >> >> I talked to some rockchip staff about the performance problem of reconstruction bitstream after yesterday arguing with tfiga at IRC yesterday. Although 1ms looks small to those decoder which can decode a picture of a UHD 4K HEVC videos in 9ms, it is enough for 60fps. But how about a higher frame rate like 120fps or 240fps and when it comes to 8K which is used in Japan broadcast. > > 1 ms for a 500 MHz CPU (which is quite slow these days) is 500k > cycles. We don't have to reconstruct the whole bitstream, just the > parsed metadata and also we don't get a new PPS or SPS every frame. > Not sure where you have this 1 ms from. Most of the difference between > our structures and the bitstream is that the latter is packed and > could be variable length. You told me that number yesterday. > > We actually have some sample bitstream assembly code for the rockchip encoder: > > https://chromium.googlesource.com/chromiumos/third_party/libv4lplugins/+/5e6034258146af6be973fb6a5bb6b9d6e7489437/libv4l-rockchip_v2/libvepu/h264e/h264e.c#148 > https://chromium.googlesource.com/chromiumos/third_party/libv4lplugins/+/5e6034258146af6be973fb6a5bb6b9d6e7489437/libv4l-rockchip_v2/libvepu/streams.c#36 > > Disassembling the stream_put_bits() gives 36 thumb2 instructions, > including 23 for the loop for each byte that is written. > stream_write_ue() is a bit more complicated, but in the worst case it > ends up with 4 calls to stream_put_bits(), each at most spanning 4 > bytes for simplicity. > > Let's count the operations for SPS then: > (1) stream_put_bits() spanning 1 byte: 33 times > (2) stream_put_bits() spanning up to 3 bytes: 4 times > (3) stream_write_ue() up to 31 bits: 19 times > > Adding it together: > (1) 33 * 36 + > (2) 4 * (36 + 2 * 23) + > (3) 19 * (4 * (36 + 3 * 23)) = > > 1188 + 328 + 7980 = 9496 ~= 10k instructions > > The code above doesn't seem to contain any expensive instructions, > like division, so for a modern pipelined out of order core (e.g. A53), > it could be safe to assume 1 instruction per cycle. At 500 MHz that > gives you 20 usecs. > > SPS is the most complex header and for H.264 we just do PPS and some > slice headers. Let's round it up a bit and we could have around 100 > usecs for the complete frame metadata. The scaling list (cabac table) address need to fill the pps header, which would request mapping and unmapping a 4K memory echo of times. And you ever think of multiple session at the same time. Besides, have you tired the Android CTS test? There is a test that would do resolution change every 5 frames which would request to construct a new sps and pps for H.264. It is the Android CTS make me have such of thought. Anyway the problem here is that v4l2 driver need to wait the previous picture is done to generate the register for the next pictures. Which leading more idle time of the device. >> >> I would bring more detail in the FOSDEM 2019, I may stay at graphics devroom at Saturday. >>> That could potentially let us completely eliminate the distinction >>> between the stateful and stateless interfaces and just have one that >>> covers both. >>> >>> Thoughts? > > Any thoughts on my proposal to make the interface more flexible? Any > specific examples of issues that we could encounter that would prevent > it from working efficiently with Rockchip (or other) hardware? > > Best regards, > Tomasz
diff --git a/Documentation/media/uapi/v4l/biblio.rst b/Documentation/media/uapi/v4l/biblio.rst index 73aeb7ce47d2..59a98feca3a1 100644 --- a/Documentation/media/uapi/v4l/biblio.rst +++ b/Documentation/media/uapi/v4l/biblio.rst @@ -124,6 +124,15 @@ ITU H.264 :author: International Telecommunication Union (http://www.itu.ch) +.. _hevc: + +ITU H.265/HEVC +============== + +:title: ITU-T Rec. H.265 | ISO/IEC 23008-2 "High Efficiency Video Coding" + +:author: International Telecommunication Union (http://www.itu.ch), International Organisation for Standardisation (http://www.iso.ch) + .. _jfif: JFIF diff --git a/Documentation/media/uapi/v4l/extended-controls.rst b/Documentation/media/uapi/v4l/extended-controls.rst index 87c0d151577f..906ff4f32634 100644 --- a/Documentation/media/uapi/v4l/extended-controls.rst +++ b/Documentation/media/uapi/v4l/extended-controls.rst @@ -2038,6 +2038,423 @@ enum v4l2_mpeg_video_h264_hierarchical_coding_type - - ``flags`` - +.. _v4l2-mpeg-hevc: + +``V4L2_CID_MPEG_VIDEO_HEVC_SPS (struct)`` + Specifies the Sequence Parameter Set fields (as extracted from the + bitstream) for the associated HEVC slice data. + These bitstream parameters are defined according to :ref:`hevc`. + Refer to the specification for the documentation of these fields. + +.. c:type:: v4l2_ctrl_hevc_sps + +.. cssclass:: longtable + +.. flat-table:: struct v4l2_ctrl_hevc_sps + :header-rows: 0 + :stub-columns: 0 + :widths: 1 1 2 + + * - __u8 + - ``chroma_format_idc`` + - + * - __u8 + - ``separate_colour_plane_flag`` + - + * - __u16 + - ``pic_width_in_luma_samples`` + - + * - __u16 + - ``pic_height_in_luma_samples`` + - + * - __u8 + - ``bit_depth_luma_minus8`` + - + * - __u8 + - ``bit_depth_chroma_minus8`` + - + * - __u8 + - ``log2_max_pic_order_cnt_lsb_minus4`` + - + * - __u8 + - ``sps_max_dec_pic_buffering_minus1`` + - + * - __u8 + - ``sps_max_num_reorder_pics`` + - + * - __u8 + - ``sps_max_latency_increase_plus1`` + - + * - __u8 + - ``log2_min_luma_coding_block_size_minus3`` + - + * - __u8 + - ``log2_diff_max_min_luma_coding_block_size`` + - + * - __u8 + - ``log2_min_luma_transform_block_size_minus2`` + - + * - __u8 + - ``log2_diff_max_min_luma_transform_block_size`` + - + * - __u8 + - ``max_transform_hierarchy_depth_inter`` + - + * - __u8 + - ``max_transform_hierarchy_depth_intra`` + - + * - __u8 + - ``scaling_list_enabled_flag`` + - + * - __u8 + - ``amp_enabled_flag`` + - + * - __u8 + - ``sample_adaptive_offset_enabled_flag`` + - + * - __u8 + - ``pcm_enabled_flag`` + - + * - __u8 + - ``pcm_sample_bit_depth_luma_minus1`` + - + * - __u8 + - ``pcm_sample_bit_depth_chroma_minus1`` + - + * - __u8 + - ``log2_min_pcm_luma_coding_block_size_minus3`` + - + * - __u8 + - ``log2_diff_max_min_pcm_luma_coding_block_size`` + - + * - __u8 + - ``pcm_loop_filter_disabled_flag`` + - + * - __u8 + - ``num_short_term_ref_pic_sets`` + - + * - __u8 + - ``long_term_ref_pics_present_flag`` + - + * - __u8 + - ``num_long_term_ref_pics_sps`` + - + * - __u8 + - ``sps_temporal_mvp_enabled_flag`` + - + * - __u8 + - ``strong_intra_smoothing_enabled_flag`` + - + +``V4L2_CID_MPEG_VIDEO_HEVC_PPS (struct)`` + Specifies the Picture Parameter Set fields (as extracted from the + bitstream) for the associated HEVC slice data. + These bitstream parameters are defined according to :ref:`hevc`. + Refer to the specification for the documentation of these fields. + +.. c:type:: v4l2_ctrl_hevc_pps + +.. cssclass:: longtable + +.. flat-table:: struct v4l2_ctrl_hevc_pps + :header-rows: 0 + :stub-columns: 0 + :widths: 1 1 2 + + * - __u8 + - ``dependent_slice_segment_flag`` + - + * - __u8 + - ``output_flag_present_flag`` + - + * - __u8 + - ``num_extra_slice_header_bits`` + - + * - __u8 + - ``sign_data_hiding_enabled_flag`` + - + * - __u8 + - ``cabac_init_present_flag`` + - + * - __s8 + - ``init_qp_minus26`` + - + * - __u8 + - ``constrained_intra_pred_flag`` + - + * - __u8 + - ``transform_skip_enabled_flag`` + - + * - __u8 + - ``cu_qp_delta_enabled_flag`` + - + * - __u8 + - ``diff_cu_qp_delta_depth`` + - + * - __s8 + - ``pps_cb_qp_offset`` + - + * - __s8 + - ``pps_cr_qp_offset`` + - + * - __u8 + - ``pps_slice_chroma_qp_offsets_present_flag`` + - + * - __u8 + - ``weighted_pred_flag`` + - + * - __u8 + - ``weighted_bipred_flag`` + - + * - __u8 + - ``transquant_bypass_enabled_flag`` + - + * - __u8 + - ``tiles_enabled_flag`` + - + * - __u8 + - ``entropy_coding_sync_enabled_flag`` + - + * - __u8 + - ``num_tile_columns_minus1`` + - + * - __u8 + - ``num_tile_rows_minus1`` + - + * - __u8 + - ``column_width_minus1[20]`` + - + * - __u8 + - ``row_height_minus1[22]`` + - + * - __u8 + - ``loop_filter_across_tiles_enabled_flag`` + - + * - __u8 + - ``pps_loop_filter_across_slices_enabled_flag`` + - + * - __u8 + - ``deblocking_filter_override_enabled_flag`` + - + * - __u8 + - ``pps_disable_deblocking_filter_flag`` + - + * - __s8 + - ``pps_beta_offset_div2`` + - + * - __s8 + - ``pps_tc_offset_div2`` + - + * - __u8 + - ``lists_modification_present_flag`` + - + * - __u8 + - ``log2_parallel_merge_level_minus2`` + - + * - __u8 + - ``slice_segment_header_extension_present_flag`` + - + +``V4L2_CID_MPEG_VIDEO_HEVC_SLICE_PARAMS (struct)`` + Specifies various slice-specific parameters, especially from the NAL unit + header, general slice segment header and weighted prediction parameter + parts of the bitstream. + These bitstream parameters are defined according to :ref:`hevc`. + Refer to the specification for the documentation of these fields. + +.. c:type:: v4l2_ctrl_hevc_slice_params + +.. cssclass:: longtable + +.. flat-table:: struct v4l2_ctrl_hevc_slice_params + :header-rows: 0 + :stub-columns: 0 + :widths: 1 1 2 + + * - __u32 + - ``bit_size`` + - Size (in bits) of the current slice data. + * - __u32 + - ``data_bit_offset`` + - Offset (in bits) to the video data in the current slice data. + * - __u8 + - ``nal_unit_type`` + - + * - __u8 + - ``nuh_temporal_id_plus1`` + - + * - __u8 + - ``slice_type`` + - + (V4L2_HEVC_SLICE_TYPE_I, V4L2_HEVC_SLICE_TYPE_P or + V4L2_HEVC_SLICE_TYPE_B). + * - __u8 + - ``colour_plane_id`` + - + * - __u16 + - ``slice_pic_order_cnt`` + - + * - __u8 + - ``slice_sao_luma_flag`` + - + * - __u8 + - ``slice_sao_chroma_flag`` + - + * - __u8 + - ``slice_temporal_mvp_enabled_flag`` + - + * - __u8 + - ``num_ref_idx_l0_active_minus1`` + - + * - __u8 + - ``num_ref_idx_l1_active_minus1`` + - + * - __u8 + - ``mvd_l1_zero_flag`` + - + * - __u8 + - ``cabac_init_flag`` + - + * - __u8 + - ``collocated_from_l0_flag`` + - + * - __u8 + - ``collocated_ref_idx`` + - + * - __u8 + - ``five_minus_max_num_merge_cand`` + - + * - __u8 + - ``use_integer_mv_flag`` + - + * - __s8 + - ``slice_qp_delta`` + - + * - __s8 + - ``slice_cb_qp_offset`` + - + * - __s8 + - ``slice_cr_qp_offset`` + - + * - __s8 + - ``slice_act_y_qp_offset`` + - + * - __s8 + - ``slice_act_cb_qp_offset`` + - + * - __s8 + - ``slice_act_cr_qp_offset`` + - + * - __u8 + - ``slice_deblocking_filter_disabled_flag`` + - + * - __s8 + - ``slice_beta_offset_div2`` + - + * - __s8 + - ``slice_tc_offset_div2`` + - + * - __u8 + - ``slice_loop_filter_across_slices_enabled_flag`` + - + * - __u8 + - ``pic_struct`` + - + * - struct :c:type:`v4l2_hevc_dpb_entry` + - ``dpb[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]`` + - The decoded picture buffer, for meta-data about reference frames. + * - __u8 + - ``num_active_dpb_entries`` + - The number of entries in ``dpb``. + * - __u8 + - ``ref_idx_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]`` + - The list of L0 reference elements as indices in the DPB. + * - __u8 + - ``ref_idx_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]`` + - The list of L1 reference elements as indices in the DPB. + * - __u8 + - ``num_rps_poc_st_curr_before`` + - The number of reference pictures in the short-term set that come before + the current frame. + * - __u8 + - ``num_rps_poc_st_curr_after`` + - The number of reference pictures in the short-term set that come after + the current frame. + * - __u8 + - ``num_rps_poc_lt_curr`` + - The number of reference pictures in the long-term set. + * - struct :c:type:`v4l2_hevc_pred_weight_table` + - ``pred_weight_table`` + - The prediction weight coefficients for inter-picture prediction. + +.. c:type:: v4l2_hevc_dpb_entry + +.. cssclass:: longtable + +.. flat-table:: struct v4l2_hevc_dpb_entry + :header-rows: 0 + :stub-columns: 0 + :widths: 1 1 2 + + * - __u32 + - ``buffer_tag`` + - The V4L2 buffer tag that matches the associated reference picture. + * - __u8 + - ``rps`` + - The reference set for the reference frame + (V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_BEFORE, + V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_AFTER or + V4L2_HEVC_DPB_ENTRY_RPS_LT_CURR) + * - __u8 + - ``field_pic`` + - Whether the reference is a field picture or a frame. + * - __u16 + - ``pic_order_cnt[2]`` + - The picture order count of the reference. Only the first element of the + array is used for frame pictures, while the first element identifies the + top field and the second the bottom field in field-coded pictures. + +.. c:type:: v4l2_hevc_pred_weight_table + +.. cssclass:: longtable + +.. flat-table:: struct v4l2_hevc_pred_weight_table + :header-rows: 0 + :stub-columns: 0 + :widths: 1 1 2 + + * - __u8 + - ``luma_log2_weight_denom`` + - + * - __s8 + - ``delta_chroma_log2_weight_denom`` + - + * - __s8 + - ``delta_luma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]`` + - + * - __s8 + - ``luma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]`` + - + * - __s8 + - ``delta_chroma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]`` + - + * - __s8 + - ``chroma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]`` + - + * - __s8 + - ``delta_luma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]`` + - + * - __s8 + - ``luma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]`` + - + * - __s8 + - ``delta_chroma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]`` + - + * - __s8 + - ``chroma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]`` + - + + MFC 5.1 MPEG Controls --------------------- diff --git a/Documentation/media/uapi/v4l/pixfmt-compressed.rst b/Documentation/media/uapi/v4l/pixfmt-compressed.rst index f15fc1c8d479..572a43bfe9c9 100644 --- a/Documentation/media/uapi/v4l/pixfmt-compressed.rst +++ b/Documentation/media/uapi/v4l/pixfmt-compressed.rst @@ -131,6 +131,21 @@ Compressed Formats - ``V4L2_PIX_FMT_HEVC`` - 'HEVC' - HEVC/H.265 video elementary stream. + * .. _V4L2-PIX-FMT-HEVC-SLICE: + + - ``V4L2_PIX_FMT_HEVC_SLICE`` + - 'S265' + - HEVC parsed slice data, as extracted from the HEVC bitstream. + This format is adapted for stateless video decoders that implement a + HEVC pipeline (using the :ref:`codec` and :ref:`media-request-api`). + Metadata associated with the frame to decode is required to be passed + through the following controls : + * ``V4L2_CID_MPEG_VIDEO_HEVC_SPS`` + * ``V4L2_CID_MPEG_VIDEO_HEVC_PPS`` + * ``V4L2_CID_MPEG_VIDEO_HEVC_SLICE_PARAMS`` + See the :ref:`associated Codec Control IDs <v4l2-mpeg-hevc>`. + Buffers associated with this pixel format must contain the appropriate + number of macroblocks to decode a full corresponding frame. * .. _V4L2-PIX-FMT-FWHT: - ``V4L2_PIX_FMT_FWHT`` diff --git a/Documentation/media/uapi/v4l/vidioc-queryctrl.rst b/Documentation/media/uapi/v4l/vidioc-queryctrl.rst index 38a9c988124c..8e0cc836058d 100644 --- a/Documentation/media/uapi/v4l/vidioc-queryctrl.rst +++ b/Documentation/media/uapi/v4l/vidioc-queryctrl.rst @@ -466,6 +466,24 @@ See also the examples in :ref:`control`. - n/a - A struct :c:type:`v4l2_ctrl_h264_decode_param`, containing H264 decode parameters for stateless video decoders. + * - ``V4L2_CTRL_TYPE_HEVC_SPS`` + - n/a + - n/a + - n/a + - A struct :c:type:`v4l2_ctrl_hevc_sps`, containing HEVC Sequence + Parameter Set for stateless video decoders. + * - ``V4L2_CTRL_TYPE_HEVC_PPS`` + - n/a + - n/a + - n/a + - A struct :c:type:`v4l2_ctrl_hevc_pps`, containing HEVC Picture + Parameter Set for stateless video decoders. + * - ``V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS`` + - n/a + - n/a + - n/a + - A struct :c:type:`v4l2_ctrl_hevc_slice_params`, containing HEVC + slice parameters for stateless video decoders. .. tabularcolumns:: |p{6.6cm}|p{2.2cm}|p{8.7cm}| diff --git a/Documentation/media/videodev2.h.rst.exceptions b/Documentation/media/videodev2.h.rst.exceptions index 99f1bd2bc44c..27978d8b18f5 100644 --- a/Documentation/media/videodev2.h.rst.exceptions +++ b/Documentation/media/videodev2.h.rst.exceptions @@ -138,6 +138,9 @@ replace symbol V4L2_CTRL_TYPE_H264_PPS :c:type:`v4l2_ctrl_type` replace symbol V4L2_CTRL_TYPE_H264_SCALING_MATRIX :c:type:`v4l2_ctrl_type` replace symbol V4L2_CTRL_TYPE_H264_SLICE_PARAMS :c:type:`v4l2_ctrl_type` replace symbol V4L2_CTRL_TYPE_H264_DECODE_PARAMS :c:type:`v4l2_ctrl_type` +replace symbol V4L2_CTRL_TYPE_HEVC_SPS :c:type:`v4l2_ctrl_type` +replace symbol V4L2_CTRL_TYPE_HEVC_PPS :c:type:`v4l2_ctrl_type` +replace symbol V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS :c:type:`v4l2_ctrl_type` # V4L2 capability defines replace define V4L2_CAP_VIDEO_CAPTURE device-capabilities diff --git a/drivers/media/v4l2-core/v4l2-ctrls.c b/drivers/media/v4l2-core/v4l2-ctrls.c index e96c453208e8..9af17815ecc3 100644 --- a/drivers/media/v4l2-core/v4l2-ctrls.c +++ b/drivers/media/v4l2-core/v4l2-ctrls.c @@ -913,6 +913,9 @@ const char *v4l2_ctrl_get_name(u32 id) case V4L2_CID_MPEG_VIDEO_HEVC_SIZE_OF_LENGTH_FIELD: return "HEVC Size of Length Field"; case V4L2_CID_MPEG_VIDEO_REF_NUMBER_FOR_PFRAMES: return "Reference Frames for a P-Frame"; case V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR: return "Prepend SPS and PPS to IDR"; + case V4L2_CID_MPEG_VIDEO_HEVC_SPS: return "HEVC Sequence Parameter Set"; + case V4L2_CID_MPEG_VIDEO_HEVC_PPS: return "HEVC Picture Parameter Set"; + case V4L2_CID_MPEG_VIDEO_HEVC_SLICE_PARAMS: return "HEVC Slice Parameters"; /* CAMERA controls */ /* Keep the order of the 'case's the same as in v4l2-controls.h! */ @@ -1320,6 +1323,15 @@ void v4l2_ctrl_fill(u32 id, const char **name, enum v4l2_ctrl_type *type, case V4L2_CID_MPEG_VIDEO_H264_DECODE_PARAMS: *type = V4L2_CTRL_TYPE_H264_DECODE_PARAMS; break; + case V4L2_CID_MPEG_VIDEO_HEVC_SPS: + *type = V4L2_CTRL_TYPE_HEVC_SPS; + break; + case V4L2_CID_MPEG_VIDEO_HEVC_PPS: + *type = V4L2_CTRL_TYPE_HEVC_PPS; + break; + case V4L2_CID_MPEG_VIDEO_HEVC_SLICE_PARAMS: + *type = V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS; + break; default: *type = V4L2_CTRL_TYPE_INTEGER; break; @@ -1692,6 +1704,11 @@ static int std_validate(const struct v4l2_ctrl *ctrl, u32 idx, case V4L2_CTRL_TYPE_H264_DECODE_PARAMS: return 0; + case V4L2_CTRL_TYPE_HEVC_SPS: + case V4L2_CTRL_TYPE_HEVC_PPS: + case V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS: + return 0; + default: return -EINVAL; } @@ -2287,6 +2304,15 @@ static struct v4l2_ctrl *v4l2_ctrl_new(struct v4l2_ctrl_handler *hdl, case V4L2_CTRL_TYPE_H264_DECODE_PARAMS: elem_size = sizeof(struct v4l2_ctrl_h264_decode_param); break; + case V4L2_CTRL_TYPE_HEVC_SPS: + elem_size = sizeof(struct v4l2_ctrl_hevc_sps); + break; + case V4L2_CTRL_TYPE_HEVC_PPS: + elem_size = sizeof(struct v4l2_ctrl_hevc_pps); + break; + case V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS: + elem_size = sizeof(struct v4l2_ctrl_hevc_slice_params); + break; default: if (type < V4L2_CTRL_COMPOUND_TYPES) elem_size = sizeof(s32); diff --git a/drivers/media/v4l2-core/v4l2-ioctl.c b/drivers/media/v4l2-core/v4l2-ioctl.c index aa63f1794272..7bec91c6effe 100644 --- a/drivers/media/v4l2-core/v4l2-ioctl.c +++ b/drivers/media/v4l2-core/v4l2-ioctl.c @@ -1321,6 +1321,7 @@ static void v4l_fill_fmtdesc(struct v4l2_fmtdesc *fmt) case V4L2_PIX_FMT_VP8: descr = "VP8"; break; case V4L2_PIX_FMT_VP9: descr = "VP9"; break; case V4L2_PIX_FMT_HEVC: descr = "HEVC"; break; /* aka H.265 */ + case V4L2_PIX_FMT_HEVC_SLICE: descr = "HEVC Parsed Slice Data"; break; case V4L2_PIX_FMT_FWHT: descr = "FWHT"; break; /* used in vicodec */ case V4L2_PIX_FMT_CPIA1: descr = "GSPCA CPiA YUV"; break; case V4L2_PIX_FMT_WNVA: descr = "WNVA"; break; diff --git a/include/media/v4l2-ctrls.h b/include/media/v4l2-ctrls.h index b4ca95710d2d..11664c5c3706 100644 --- a/include/media/v4l2-ctrls.h +++ b/include/media/v4l2-ctrls.h @@ -48,6 +48,9 @@ struct poll_table_struct; * @p_h264_scal_mtrx: Pointer to a struct v4l2_ctrl_h264_scaling_matrix. * @p_h264_slice_param: Pointer to a struct v4l2_ctrl_h264_slice_param. * @p_h264_decode_param: Pointer to a struct v4l2_ctrl_h264_decode_param. + * @p_hevc_sps: Pointer to an HEVC sequence parameter set structure. + * @p_hevc_pps: Pointer to an HEVC picture parameter set structure. + * @p_hevc_slice_params Pointer to an HEVC slice parameters structure. * @p: Pointer to a compound value. */ union v4l2_ctrl_ptr { @@ -64,6 +67,9 @@ union v4l2_ctrl_ptr { struct v4l2_ctrl_h264_scaling_matrix *p_h264_scal_mtrx; struct v4l2_ctrl_h264_slice_param *p_h264_slice_param; struct v4l2_ctrl_h264_decode_param *p_h264_decode_param; + struct v4l2_ctrl_hevc_sps *p_hevc_sps; + struct v4l2_ctrl_hevc_pps *p_hevc_pps; + struct v4l2_ctrl_hevc_slice_params *p_hevc_slice_params; void *p; }; diff --git a/include/uapi/linux/v4l2-controls.h b/include/uapi/linux/v4l2-controls.h index 628c0cdb51d9..5bbf63b2dad1 100644 --- a/include/uapi/linux/v4l2-controls.h +++ b/include/uapi/linux/v4l2-controls.h @@ -709,6 +709,9 @@ enum v4l2_cid_mpeg_video_hevc_size_of_length_field { #define V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L6_BR (V4L2_CID_MPEG_BASE + 642) #define V4L2_CID_MPEG_VIDEO_REF_NUMBER_FOR_PFRAMES (V4L2_CID_MPEG_BASE + 643) #define V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR (V4L2_CID_MPEG_BASE + 644) +#define V4L2_CID_MPEG_VIDEO_HEVC_SPS (V4L2_CID_MPEG_BASE + 645) +#define V4L2_CID_MPEG_VIDEO_HEVC_PPS (V4L2_CID_MPEG_BASE + 646) +#define V4L2_CID_MPEG_VIDEO_HEVC_SLICE_PARAMS (V4L2_CID_MPEG_BASE + 647) /* MPEG-class control IDs specific to the CX2341x driver as defined by V4L2 */ #define V4L2_CID_MPEG_CX2341X_BASE (V4L2_CTRL_CLASS_MPEG | 0x1000) @@ -1324,4 +1327,156 @@ struct v4l2_ctrl_h264_decode_param { struct v4l2_h264_dpb_entry dpb[16]; }; +#define V4L2_HEVC_SLICE_TYPE_B 0 +#define V4L2_HEVC_SLICE_TYPE_P 1 +#define V4L2_HEVC_SLICE_TYPE_I 2 + +struct v4l2_ctrl_hevc_sps { + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Sequence parameter set */ + __u8 chroma_format_idc; + __u8 separate_colour_plane_flag; + __u16 pic_width_in_luma_samples; + __u16 pic_height_in_luma_samples; + __u8 bit_depth_luma_minus8; + __u8 bit_depth_chroma_minus8; + __u8 log2_max_pic_order_cnt_lsb_minus4; + __u8 sps_max_dec_pic_buffering_minus1; + __u8 sps_max_num_reorder_pics; + __u8 sps_max_latency_increase_plus1; + __u8 log2_min_luma_coding_block_size_minus3; + __u8 log2_diff_max_min_luma_coding_block_size; + __u8 log2_min_luma_transform_block_size_minus2; + __u8 log2_diff_max_min_luma_transform_block_size; + __u8 max_transform_hierarchy_depth_inter; + __u8 max_transform_hierarchy_depth_intra; + __u8 scaling_list_enabled_flag; + __u8 amp_enabled_flag; + __u8 sample_adaptive_offset_enabled_flag; + __u8 pcm_enabled_flag; + __u8 pcm_sample_bit_depth_luma_minus1; + __u8 pcm_sample_bit_depth_chroma_minus1; + __u8 log2_min_pcm_luma_coding_block_size_minus3; + __u8 log2_diff_max_min_pcm_luma_coding_block_size; + __u8 pcm_loop_filter_disabled_flag; + __u8 num_short_term_ref_pic_sets; + __u8 long_term_ref_pics_present_flag; + __u8 num_long_term_ref_pics_sps; + __u8 sps_temporal_mvp_enabled_flag; + __u8 strong_intra_smoothing_enabled_flag; +}; + +struct v4l2_ctrl_hevc_pps { + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Picture parameter set */ + __u8 dependent_slice_segment_flag; + __u8 output_flag_present_flag; + __u8 num_extra_slice_header_bits; + __u8 sign_data_hiding_enabled_flag; + __u8 cabac_init_present_flag; + __s8 init_qp_minus26; + __u8 constrained_intra_pred_flag; + __u8 transform_skip_enabled_flag; + __u8 cu_qp_delta_enabled_flag; + __u8 diff_cu_qp_delta_depth; + __s8 pps_cb_qp_offset; + __s8 pps_cr_qp_offset; + __u8 pps_slice_chroma_qp_offsets_present_flag; + __u8 weighted_pred_flag; + __u8 weighted_bipred_flag; + __u8 transquant_bypass_enabled_flag; + __u8 tiles_enabled_flag; + __u8 entropy_coding_sync_enabled_flag; + __u8 num_tile_columns_minus1; + __u8 num_tile_rows_minus1; + __u8 column_width_minus1[20]; + __u8 row_height_minus1[22]; + __u8 loop_filter_across_tiles_enabled_flag; + __u8 pps_loop_filter_across_slices_enabled_flag; + __u8 deblocking_filter_override_enabled_flag; + __u8 pps_disable_deblocking_filter_flag; + __s8 pps_beta_offset_div2; + __s8 pps_tc_offset_div2; + __u8 lists_modification_present_flag; + __u8 log2_parallel_merge_level_minus2; + __u8 slice_segment_header_extension_present_flag; +}; + +#define V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_BEFORE 0x01 +#define V4L2_HEVC_DPB_ENTRY_RPS_ST_CURR_AFTER 0x02 +#define V4L2_HEVC_DPB_ENTRY_RPS_LT_CURR 0x03 + +#define V4L2_HEVC_DPB_ENTRIES_NUM_MAX 16 + +struct v4l2_hevc_dpb_entry { + __u32 buffer_tag; + __u8 rps; + __u8 field_pic; + __u16 pic_order_cnt[2]; +}; + +struct v4l2_hevc_pred_weight_table { + __u8 luma_log2_weight_denom; + __s8 delta_chroma_log2_weight_denom; + + __s8 delta_luma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; + __s8 luma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; + __s8 delta_chroma_weight_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; + __s8 chroma_offset_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; + + __s8 delta_luma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; + __s8 luma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; + __s8 delta_chroma_weight_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; + __s8 chroma_offset_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2]; +}; + +struct v4l2_ctrl_hevc_slice_params { + __u32 bit_size; + __u32 data_bit_offset; + + /* ISO/IEC 23008-2, ITU-T Rec. H.265: NAL unit header */ + __u8 nal_unit_type; + __u8 nuh_temporal_id_plus1; + + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ + __u8 slice_type; + __u8 colour_plane_id; + __u16 slice_pic_order_cnt; + __u8 slice_sao_luma_flag; + __u8 slice_sao_chroma_flag; + __u8 slice_temporal_mvp_enabled_flag; + __u8 num_ref_idx_l0_active_minus1; + __u8 num_ref_idx_l1_active_minus1; + __u8 mvd_l1_zero_flag; + __u8 cabac_init_flag; + __u8 collocated_from_l0_flag; + __u8 collocated_ref_idx; + __u8 five_minus_max_num_merge_cand; + __u8 use_integer_mv_flag; + __s8 slice_qp_delta; + __s8 slice_cb_qp_offset; + __s8 slice_cr_qp_offset; + __s8 slice_act_y_qp_offset; + __s8 slice_act_cb_qp_offset; + __s8 slice_act_cr_qp_offset; + __u8 slice_deblocking_filter_disabled_flag; + __s8 slice_beta_offset_div2; + __s8 slice_tc_offset_div2; + __u8 slice_loop_filter_across_slices_enabled_flag; + + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Picture timing SEI message */ + __u8 pic_struct; + + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ + struct v4l2_hevc_dpb_entry dpb[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; + __u8 num_active_dpb_entries; + __u8 ref_idx_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; + __u8 ref_idx_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; + + __u8 num_rps_poc_st_curr_before; + __u8 num_rps_poc_st_curr_after; + __u8 num_rps_poc_lt_curr; + + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Weighted prediction parameter */ + struct v4l2_hevc_pred_weight_table pred_weight_table; +}; + #endif diff --git a/include/uapi/linux/v4l2-controls.h.rej b/include/uapi/linux/v4l2-controls.h.rej deleted file mode 100644 index 1fbb7bf8daa7..000000000000 --- a/include/uapi/linux/v4l2-controls.h.rej +++ /dev/null @@ -1,187 +0,0 @@ ---- include/uapi/linux/v4l2-controls.h -+++ include/uapi/linux/v4l2-controls.h -@@ -50,6 +50,8 @@ - #ifndef __LINUX_V4L2_CONTROLS_H - #define __LINUX_V4L2_CONTROLS_H - -+#include <linux/types.h> -+ - /* Control classes */ - #define V4L2_CTRL_CLASS_USER 0x00980000 /* Old-style 'user' controls */ - #define V4L2_CTRL_CLASS_MPEG 0x00990000 /* MPEG-compression controls */ -@@ -534,6 +536,12 @@ enum v4l2_mpeg_video_h264_hierarchical_coding_type { - }; - #define V4L2_CID_MPEG_VIDEO_H264_HIERARCHICAL_CODING_LAYER (V4L2_CID_MPEG_BASE+381) - #define V4L2_CID_MPEG_VIDEO_H264_HIERARCHICAL_CODING_LAYER_QP (V4L2_CID_MPEG_BASE+382) -+#define V4L2_CID_MPEG_VIDEO_H264_SPS (V4L2_CID_MPEG_BASE+383) -+#define V4L2_CID_MPEG_VIDEO_H264_PPS (V4L2_CID_MPEG_BASE+384) -+#define V4L2_CID_MPEG_VIDEO_H264_SCALING_MATRIX (V4L2_CID_MPEG_BASE+385) -+#define V4L2_CID_MPEG_VIDEO_H264_SLICE_PARAMS (V4L2_CID_MPEG_BASE+386) -+#define V4L2_CID_MPEG_VIDEO_H264_DECODE_PARAMS (V4L2_CID_MPEG_BASE+387) -+ - #define V4L2_CID_MPEG_VIDEO_MPEG4_I_FRAME_QP (V4L2_CID_MPEG_BASE+400) - #define V4L2_CID_MPEG_VIDEO_MPEG4_P_FRAME_QP (V4L2_CID_MPEG_BASE+401) - #define V4L2_CID_MPEG_VIDEO_MPEG4_B_FRAME_QP (V4L2_CID_MPEG_BASE+402) -@@ -1156,4 +1164,162 @@ struct v4l2_ctrl_mpeg2_quantization { - __u8 chroma_non_intra_quantiser_matrix[64]; - }; - -+/* Compounds controls */ -+ -+#define V4L2_H264_SPS_CONSTRAINT_SET0_FLAG 0x01 -+#define V4L2_H264_SPS_CONSTRAINT_SET1_FLAG 0x02 -+#define V4L2_H264_SPS_CONSTRAINT_SET2_FLAG 0x04 -+#define V4L2_H264_SPS_CONSTRAINT_SET3_FLAG 0x08 -+#define V4L2_H264_SPS_CONSTRAINT_SET4_FLAG 0x10 -+#define V4L2_H264_SPS_CONSTRAINT_SET5_FLAG 0x20 -+ -+#define V4L2_H264_SPS_FLAG_SEPARATE_COLOUR_PLANE 0x01 -+#define V4L2_H264_SPS_FLAG_QPPRIME_Y_ZERO_TRANSFORM_BYPASS 0x02 -+#define V4L2_H264_SPS_FLAG_DELTA_PIC_ORDER_ALWAYS_ZERO 0x04 -+#define V4L2_H264_SPS_FLAG_GAPS_IN_FRAME_NUM_VALUE_ALLOWED 0x08 -+#define V4L2_H264_SPS_FLAG_FRAME_MBS_ONLY 0x10 -+#define V4L2_H264_SPS_FLAG_MB_ADAPTIVE_FRAME_FIELD 0x20 -+#define V4L2_H264_SPS_FLAG_DIRECT_8X8_INFERENCE 0x40 -+ -+struct v4l2_ctrl_h264_sps { -+ __u8 profile_idc; -+ __u8 constraint_set_flags; -+ __u8 level_idc; -+ __u8 seq_parameter_set_id; -+ __u8 chroma_format_idc; -+ __u8 bit_depth_luma_minus8; -+ __u8 bit_depth_chroma_minus8; -+ __u8 log2_max_frame_num_minus4; -+ __u8 pic_order_cnt_type; -+ __u8 log2_max_pic_order_cnt_lsb_minus4; -+ __u8 max_num_ref_frames; -+ __u8 num_ref_frames_in_pic_order_cnt_cycle; -+ __s32 offset_for_ref_frame[255]; -+ __s32 offset_for_non_ref_pic; -+ __s32 offset_for_top_to_bottom_field; -+ __u16 pic_width_in_mbs_minus1; -+ __u16 pic_height_in_map_units_minus1; -+ __u8 flags; -+}; -+ -+#define V4L2_H264_PPS_FLAG_ENTROPY_CODING_MODE 0x0001 -+#define V4L2_H264_PPS_FLAG_BOTTOM_FIELD_PIC_ORDER_IN_FRAME_PRESENT 0x0002 -+#define V4L2_H264_PPS_FLAG_WEIGHTED_PRED 0x0004 -+#define V4L2_H264_PPS_FLAG_DEBLOCKING_FILTER_CONTROL_PRESENT 0x0008 -+#define V4L2_H264_PPS_FLAG_CONSTRAINED_INTRA_PRED 0x0010 -+#define V4L2_H264_PPS_FLAG_REDUNDANT_PIC_CNT_PRESENT 0x0020 -+#define V4L2_H264_PPS_FLAG_TRANSFORM_8X8_MODE 0x0040 -+#define V4L2_H264_PPS_FLAG_PIC_SCALING_MATRIX_PRESENT 0x0080 -+ -+struct v4l2_ctrl_h264_pps { -+ __u8 pic_parameter_set_id; -+ __u8 seq_parameter_set_id; -+ __u8 num_slice_groups_minus1; -+ __u8 num_ref_idx_l0_default_active_minus1; -+ __u8 num_ref_idx_l1_default_active_minus1; -+ __u8 weighted_bipred_idc; -+ __s8 pic_init_qp_minus26; -+ __s8 pic_init_qs_minus26; -+ __s8 chroma_qp_index_offset; -+ __s8 second_chroma_qp_index_offset; -+ __u8 flags; -+}; -+ -+struct v4l2_ctrl_h264_scaling_matrix { -+ __u8 scaling_list_4x4[6][16]; -+ __u8 scaling_list_8x8[6][64]; -+}; -+ -+struct v4l2_h264_weight_factors { -+ __s8 luma_weight[32]; -+ __s8 luma_offset[32]; -+ __s8 chroma_weight[32][2]; -+ __s8 chroma_offset[32][2]; -+}; -+ -+struct v4l2_h264_pred_weight_table { -+ __u8 luma_log2_weight_denom; -+ __u8 chroma_log2_weight_denom; -+ struct v4l2_h264_weight_factors weight_factors[2]; -+}; -+ -+#define V4L2_H264_SLICE_TYPE_P 0 -+#define V4L2_H264_SLICE_TYPE_B 1 -+#define V4L2_H264_SLICE_TYPE_I 2 -+#define V4L2_H264_SLICE_TYPE_SP 3 -+#define V4L2_H264_SLICE_TYPE_SI 4 -+ -+#define V4L2_H264_SLICE_FLAG_FIELD_PIC 0x01 -+#define V4L2_H264_SLICE_FLAG_BOTTOM_FIELD 0x02 -+#define V4L2_H264_SLICE_FLAG_DIRECT_SPATIAL_MV_PRED 0x04 -+#define V4L2_H264_SLICE_FLAG_SP_FOR_SWITCH 0x08 -+ -+struct v4l2_ctrl_h264_slice_param { -+ /* Size in bytes, including header */ -+ __u32 size; -+ /* Offset in bits to slice_data() from the beginning of this slice. */ -+ __u32 header_bit_size; -+ -+ __u16 first_mb_in_slice; -+ __u8 slice_type; -+ __u8 pic_parameter_set_id; -+ __u8 colour_plane_id; -+ __u16 frame_num; -+ __u16 idr_pic_id; -+ __u16 pic_order_cnt_lsb; -+ __s32 delta_pic_order_cnt_bottom; -+ __s32 delta_pic_order_cnt0; -+ __s32 delta_pic_order_cnt1; -+ __u8 redundant_pic_cnt; -+ -+ struct v4l2_h264_pred_weight_table pred_weight_table; -+ /* Size in bits of dec_ref_pic_marking() syntax element. */ -+ __u32 dec_ref_pic_marking_bit_size; -+ /* Size in bits of pic order count syntax. */ -+ __u32 pic_order_cnt_bit_size; -+ -+ __u8 cabac_init_idc; -+ __s8 slice_qp_delta; -+ __s8 slice_qs_delta; -+ __u8 disable_deblocking_filter_idc; -+ __s8 slice_alpha_c0_offset_div2; -+ __s8 slice_beta_offset_div2; -+ __u32 slice_group_change_cycle; -+ -+ __u8 num_ref_idx_l0_active_minus1; -+ __u8 num_ref_idx_l1_active_minus1; -+ /* Entries on each list are indices -+ * into v4l2_ctrl_h264_decode_param.dpb[]. */ -+ __u8 ref_pic_list0[32]; -+ __u8 ref_pic_list1[32]; -+ -+ __u8 flags; -+}; -+ -+#define V4L2_H264_DPB_ENTRY_FLAG_VALID 0x01 -+#define V4L2_H264_DPB_ENTRY_FLAG_ACTIVE 0x02 -+#define V4L2_H264_DPB_ENTRY_FLAG_LONG_TERM 0x04 -+ -+struct v4l2_h264_dpb_entry { -+ __u32 tag; -+ __u16 frame_num; -+ __u16 pic_num; -+ /* Note that field is indicated by v4l2_buffer.field */ -+ __s32 top_field_order_cnt; -+ __s32 bottom_field_order_cnt; -+ __u8 flags; /* V4L2_H264_DPB_ENTRY_FLAG_* */ -+}; -+ -+struct v4l2_ctrl_h264_decode_param { -+ __u32 num_slices; -+ __u8 idr_pic_flag; -+ __u8 nal_ref_idc; -+ __s32 top_field_order_cnt; -+ __s32 bottom_field_order_cnt; -+ __u8 ref_pic_list_p0[32]; -+ __u8 ref_pic_list_b0[32]; -+ __u8 ref_pic_list_b1[32]; -+ struct v4l2_h264_dpb_entry dpb[16]; -+}; -+ - #endif diff --git a/include/uapi/linux/videodev2.h b/include/uapi/linux/videodev2.h index dd028e0bf306..26f5bec9e988 100644 --- a/include/uapi/linux/videodev2.h +++ b/include/uapi/linux/videodev2.h @@ -655,6 +655,7 @@ struct v4l2_pix_format { #define V4L2_PIX_FMT_VP8 v4l2_fourcc('V', 'P', '8', '0') /* VP8 */ #define V4L2_PIX_FMT_VP9 v4l2_fourcc('V', 'P', '9', '0') /* VP9 */ #define V4L2_PIX_FMT_HEVC v4l2_fourcc('H', 'E', 'V', 'C') /* HEVC aka H.265 */ +#define V4L2_PIX_FMT_HEVC_SLICE v4l2_fourcc('S', '2', '6', '5') /* HEVC parsed slices */ #define V4L2_PIX_FMT_FWHT v4l2_fourcc('F', 'W', 'H', 'T') /* Fast Walsh Hadamard Transform (vicodec) */ /* Vendor-specific formats */ @@ -1637,6 +1638,9 @@ struct v4l2_ext_control { struct v4l2_ctrl_h264_scaling_matrix __user *p_h264_scal_mtrx; struct v4l2_ctrl_h264_slice_param __user *p_h264_slice_param; struct v4l2_ctrl_h264_decode_param __user *p_h264_decode_param; + struct v4l2_ctrl_hevc_sps __user *p_hevc_sps; + struct v4l2_ctrl_hevc_pps __user *p_hevc_pps; + struct v4l2_ctrl_hevc_slice_params __user *p_hevc_slice_params; void __user *ptr; }; } __attribute__ ((packed)); @@ -1689,6 +1693,9 @@ enum v4l2_ctrl_type { V4L2_CTRL_TYPE_H264_SCALING_MATRIX = 0x0107, V4L2_CTRL_TYPE_H264_SLICE_PARAMS = 0x0108, V4L2_CTRL_TYPE_H264_DECODE_PARAMS = 0x0109, + V4L2_CTRL_TYPE_HEVC_SPS = 0x0110, + V4L2_CTRL_TYPE_HEVC_PPS = 0x0111, + V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS = 0x0112, }; /* Used in the VIDIOC_QUERYCTRL ioctl for querying controls */
This introduces the required definitions for HEVC decoding support with stateless VPUs. The controls associated to the HEVC slice format provide the required meta-data for decoding slices extracted from the bitstream. This interface comes with the following limitations: * No custom quantization matrices (scaling lists); * Support for a single temporal layer only; * No slice entry point offsets support; * No conformance window support; * No VUI parameters support; * No support for SPS extensions: range, multilayer, 3d, scc, 4 bits; * No support for PPS extensions: range, multilayer, 3d, scc, 4 bits. Signed-off-by: Paul Kocialkowski <paul.kocialkowski@bootlin.com> --- Documentation/media/uapi/v4l/biblio.rst | 9 + .../media/uapi/v4l/extended-controls.rst | 417 ++++++++++++++++++ .../media/uapi/v4l/pixfmt-compressed.rst | 15 + .../media/uapi/v4l/vidioc-queryctrl.rst | 18 + .../media/videodev2.h.rst.exceptions | 3 + drivers/media/v4l2-core/v4l2-ctrls.c | 26 ++ drivers/media/v4l2-core/v4l2-ioctl.c | 1 + include/media/v4l2-ctrls.h | 6 + include/uapi/linux/v4l2-controls.h | 155 +++++++ include/uapi/linux/v4l2-controls.h.rej | 187 -------- include/uapi/linux/videodev2.h | 7 + 11 files changed, 657 insertions(+), 187 deletions(-) delete mode 100644 include/uapi/linux/v4l2-controls.h.rej