From e9ca2a03db7856021f443cccbe0cab6c3a95133b Mon Sep 17 00:00:00 2001 From: James Darnley Date: Wed, 4 Jul 2018 18:14:43 +0200 Subject: [PATCH 01/11] avcdec: add initial callback for draw_horiz_band --- lib/upipe-av/upipe_avcodec_decode.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/upipe-av/upipe_avcodec_decode.c b/lib/upipe-av/upipe_avcodec_decode.c index e1263a0b8..ffa9e927f 100644 --- a/lib/upipe-av/upipe_avcodec_decode.c +++ b/lib/upipe-av/upipe_avcodec_decode.c @@ -209,6 +209,9 @@ static int upipe_avcdec_check(struct upipe *upipe, struct uref *flow_format) static void upipe_av_uref_pic_free(void *opaque, uint8_t *data); +static void draw_horiz_band(AVCodecContext *avctx, const AVFrame *frame, + int offset[AV_NUM_DATA_POINTERS], int y, int type, int height); + /* Documentation from libavcodec.h (get_buffer) : * The function will set AVFrame.data[], AVFrame.linesize[]. * AVFrame.extended_data[] must also be set, but it should be the same as @@ -999,6 +1002,16 @@ static void upipe_avcdec_output_sub(struct upipe *upipe, AVSubtitle *sub, return; } +static void draw_horiz_band(AVCodecContext *avctx, const AVFrame *frame, + int offset[AV_NUM_DATA_POINTERS], int y, int type, int height) +{ + struct upipe *upipe = avctx->opaque; +#if 1 + upipe_notice_va(upipe, "draw_horiz_band y: %d, height: %d, type: %d", + y, height, type); +#endif +} + /** @internal @This outputs video frames. * * @param upipe description structure of the pipe @@ -1457,6 +1470,7 @@ static int upipe_avcdec_set_flow_def(struct upipe *upipe, struct uref *flow_def) return UBASE_ERR_EXTERNAL; } + upipe_avcdec->context->draw_horiz_band = draw_horiz_band; upipe_avcdec->context->codec = codec; upipe_avcdec->context->opaque = upipe; if (extradata_alloc != NULL) { From 3b695b6082201d61ef845caaca586ea560f430d5 Mon Sep 17 00:00:00 2001 From: James Darnley Date: Thu, 5 Jul 2018 11:37:44 +0200 Subject: [PATCH 02/11] avcdec: duplicate code to set output properties and flowdef --- lib/upipe-av/upipe_avcodec_decode.c | 71 +++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/lib/upipe-av/upipe_avcodec_decode.c b/lib/upipe-av/upipe_avcodec_decode.c index ffa9e927f..b18afa235 100644 --- a/lib/upipe-av/upipe_avcodec_decode.c +++ b/lib/upipe-av/upipe_avcodec_decode.c @@ -1002,6 +1002,77 @@ static void upipe_avcdec_output_sub(struct upipe *upipe, AVSubtitle *sub, return; } +static int set_output_pic_properties(struct upipe *upipe, struct uref *uref, + AVCodecContext *context, const AVFrame *frame) +{ + struct upipe_avcdec *upipe_avcdec = upipe_avcdec_from_upipe(upipe); + struct uref *flow_def_attr = uref_from_uchain(uref->uchain.next); + + /* Resize the picture (was allocated too big). */ + if (unlikely(!ubase_check(uref_pic_resize(uref, 0, 0, frame->width, frame->height)))) { + upipe_warn_va(upipe, "couldn't resize picture to %dx%d", + frame->width, frame->height); + return UBASE_ERR_EXTERNAL; + } + + /* Duplicate uref because it is freed in _release, because the ubuf + * is still in use by avcodec. */ + uref = uref_dup(uref); + if (unlikely(uref == NULL)) { + return UBASE_ERR_ALLOC; + } + + UBASE_RETURN(uref_pic_set_tf(uref)) + UBASE_RETURN(uref_pic_set_bf(uref)) + if (!frame->interlaced_frame) + UBASE_RETURN(uref_pic_set_progressive(uref)) + else if (frame->top_field_first) + UBASE_RETURN(uref_pic_set_tff(uref)) + + if (context->time_base.den) + UBASE_RETURN(uref_clock_set_duration(uref, + (uint64_t)(2 + frame->repeat_pict) * context->ticks_per_frame * + UCLOCK_FREQ * context->time_base.num / + (2 * context->time_base.den))) + + if (frame->key_frame) + uref_pic_set_key(uref); + +#if 0 + side_data = av_frame_get_side_data(frame, AV_FRAME_DATA_AFD); + if (side_data && side_data->size == 1) + uref_pic_set_afd(uref, side_data->data[0]); + + side_data = av_frame_get_side_data(frame, AV_FRAME_DATA_A53_CC); + if (side_data) + uref_pic_set_cea_708(uref, side_data->data, side_data->size); +#endif + + /* various time-related attributes */ + upipe_avcdec_set_time_attributes(upipe, uref); + + //uref_h26x_delete_nal_offsets(uref); + + /* Find out if flow def attributes have changed. */ + if (!upipe_avcdec_check_flow_def_attr(upipe, flow_def_attr)) { + /* Make a copy as flow_def_attr is still used by _release. */ + flow_def_attr = uref_dup(flow_def_attr); + if (unlikely(flow_def_attr == NULL)) { + uref_free(uref); + return UBASE_ERR_ALLOC; + } + struct uref *flow_def = + upipe_avcdec_store_flow_def_attr(upipe, flow_def_attr); + if (flow_def != NULL) { + uref_block_flow_clear_format(flow_def); + uref_flow_delete_headers(flow_def); + upipe_avcdec_store_flow_def(upipe, flow_def); + } + } + + return UBASE_ERR_NONE; +} + static void draw_horiz_band(AVCodecContext *avctx, const AVFrame *frame, int offset[AV_NUM_DATA_POINTERS], int y, int type, int height) { From d898257073e067d72b4c2c9bb3e69f59a9c5434c Mon Sep 17 00:00:00 2001 From: James Darnley Date: Thu, 5 Jul 2018 11:46:55 +0200 Subject: [PATCH 03/11] avcdec: use draw_horiz_band to output chunks --- lib/upipe-av/upipe_avcodec_decode.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/lib/upipe-av/upipe_avcodec_decode.c b/lib/upipe-av/upipe_avcodec_decode.c index b18afa235..ab5035fe4 100644 --- a/lib/upipe-av/upipe_avcodec_decode.c +++ b/lib/upipe-av/upipe_avcodec_decode.c @@ -1076,11 +1076,38 @@ static int set_output_pic_properties(struct upipe *upipe, struct uref *uref, static void draw_horiz_band(AVCodecContext *avctx, const AVFrame *frame, int offset[AV_NUM_DATA_POINTERS], int y, int type, int height) { + /* TODO: handle errors, clocks. */ + struct upipe *upipe = avctx->opaque; #if 1 upipe_notice_va(upipe, "draw_horiz_band y: %d, height: %d, type: %d", y, height, type); #endif + + struct uref *uref = frame->opaque; + if (!ubase_check(set_output_pic_properties(upipe, uref, avctx, frame))) { + upipe_err(upipe, "set_output_pic_properties failed"); + return; + } + + struct ubuf *ubuf_chunk = ubuf_dup(uref->ubuf); + if (!ubuf_chunk) + return; + if (!ubase_check(ubuf_pic_resize(ubuf_chunk, 0, y, -1, height))) + return; + struct uref *uref_chunk = uref_fork(uref, ubuf_chunk); + if (!uref_chunk) + return; + uref_pic_set_vposition(uref_chunk, y); + +#if 1 + struct upipe_avcdec *upipe_avcdec = upipe_avcdec_from_upipe(upipe); + upipe_avcdec_output(upipe, uref_chunk, NULL); +#else + uref_free(uref_chunk); +#endif + + upipe_notice(upipe, "draw_horiz_band succeeded"); } /** @internal @This outputs video frames. From 43c35e09e273e2fb39519e57dd6bd31c4487f916 Mon Sep 17 00:00:00 2001 From: James Darnley Date: Thu, 5 Jul 2018 14:03:32 +0200 Subject: [PATCH 04/11] avcdec: remove logging and dead code in draw_horiz_band --- lib/upipe-av/upipe_avcodec_decode.c | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/lib/upipe-av/upipe_avcodec_decode.c b/lib/upipe-av/upipe_avcodec_decode.c index ab5035fe4..c82a6a552 100644 --- a/lib/upipe-av/upipe_avcodec_decode.c +++ b/lib/upipe-av/upipe_avcodec_decode.c @@ -1079,11 +1079,6 @@ static void draw_horiz_band(AVCodecContext *avctx, const AVFrame *frame, /* TODO: handle errors, clocks. */ struct upipe *upipe = avctx->opaque; -#if 1 - upipe_notice_va(upipe, "draw_horiz_band y: %d, height: %d, type: %d", - y, height, type); -#endif - struct uref *uref = frame->opaque; if (!ubase_check(set_output_pic_properties(upipe, uref, avctx, frame))) { upipe_err(upipe, "set_output_pic_properties failed"); @@ -1100,14 +1095,8 @@ static void draw_horiz_band(AVCodecContext *avctx, const AVFrame *frame, return; uref_pic_set_vposition(uref_chunk, y); -#if 1 struct upipe_avcdec *upipe_avcdec = upipe_avcdec_from_upipe(upipe); upipe_avcdec_output(upipe, uref_chunk, NULL); -#else - uref_free(uref_chunk); -#endif - - upipe_notice(upipe, "draw_horiz_band succeeded"); } /** @internal @This outputs video frames. From 95795d8c52dc6852dceb571b9e82cae6f2b15d6b Mon Sep 17 00:00:00 2001 From: James Darnley Date: Fri, 6 Jul 2018 13:45:52 +0200 Subject: [PATCH 05/11] avcdec: don't lose duplicated uref in set_output_pic_properties --- lib/upipe-av/upipe_avcodec_decode.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/upipe-av/upipe_avcodec_decode.c b/lib/upipe-av/upipe_avcodec_decode.c index c82a6a552..ad6f68892 100644 --- a/lib/upipe-av/upipe_avcodec_decode.c +++ b/lib/upipe-av/upipe_avcodec_decode.c @@ -1002,10 +1002,11 @@ static void upipe_avcdec_output_sub(struct upipe *upipe, AVSubtitle *sub, return; } -static int set_output_pic_properties(struct upipe *upipe, struct uref *uref, +static int set_output_pic_properties(struct upipe *upipe, struct uref **uref_input, AVCodecContext *context, const AVFrame *frame) { struct upipe_avcdec *upipe_avcdec = upipe_avcdec_from_upipe(upipe); + struct uref *uref = *uref_input; struct uref *flow_def_attr = uref_from_uchain(uref->uchain.next); /* Resize the picture (was allocated too big). */ @@ -1070,6 +1071,7 @@ static int set_output_pic_properties(struct upipe *upipe, struct uref *uref, } } + *uref_input = uref; return UBASE_ERR_NONE; } @@ -1080,7 +1082,7 @@ static void draw_horiz_band(AVCodecContext *avctx, const AVFrame *frame, struct upipe *upipe = avctx->opaque; struct uref *uref = frame->opaque; - if (!ubase_check(set_output_pic_properties(upipe, uref, avctx, frame))) { + if (!ubase_check(set_output_pic_properties(upipe, &uref, avctx, frame))) { upipe_err(upipe, "set_output_pic_properties failed"); return; } From 70032fc319cc1837b247a8958da5948c69bfa1ec Mon Sep 17 00:00:00 2001 From: James Darnley Date: Fri, 6 Jul 2018 13:49:40 +0200 Subject: [PATCH 06/11] avcdec: don't duplicate things again in draw_horiz_band --- lib/upipe-av/upipe_avcodec_decode.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/lib/upipe-av/upipe_avcodec_decode.c b/lib/upipe-av/upipe_avcodec_decode.c index ad6f68892..82ef66539 100644 --- a/lib/upipe-av/upipe_avcodec_decode.c +++ b/lib/upipe-av/upipe_avcodec_decode.c @@ -1087,18 +1087,12 @@ static void draw_horiz_band(AVCodecContext *avctx, const AVFrame *frame, return; } - struct ubuf *ubuf_chunk = ubuf_dup(uref->ubuf); - if (!ubuf_chunk) + if (!ubase_check(ubuf_pic_resize(uref->ubuf, 0, y, -1, height))) return; - if (!ubase_check(ubuf_pic_resize(ubuf_chunk, 0, y, -1, height))) - return; - struct uref *uref_chunk = uref_fork(uref, ubuf_chunk); - if (!uref_chunk) - return; - uref_pic_set_vposition(uref_chunk, y); + uref_pic_set_vposition(uref, y); struct upipe_avcdec *upipe_avcdec = upipe_avcdec_from_upipe(upipe); - upipe_avcdec_output(upipe, uref_chunk, NULL); + upipe_avcdec_output(upipe, uref, NULL); } /** @internal @This outputs video frames. From 0cd3ebd4db34410559bffd5b32e94c950ac68651 Mon Sep 17 00:00:00 2001 From: James Darnley Date: Fri, 6 Jul 2018 14:23:45 +0200 Subject: [PATCH 07/11] avcdec: adjust timestamps on draw_horiz_band urefs --- lib/upipe-av/upipe_avcodec_decode.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/upipe-av/upipe_avcodec_decode.c b/lib/upipe-av/upipe_avcodec_decode.c index 82ef66539..37ca0c596 100644 --- a/lib/upipe-av/upipe_avcodec_decode.c +++ b/lib/upipe-av/upipe_avcodec_decode.c @@ -1078,7 +1078,7 @@ static int set_output_pic_properties(struct upipe *upipe, struct uref **uref_inp static void draw_horiz_band(AVCodecContext *avctx, const AVFrame *frame, int offset[AV_NUM_DATA_POINTERS], int y, int type, int height) { - /* TODO: handle errors, clocks. */ + /* TODO: handle errors. */ struct upipe *upipe = avctx->opaque; struct uref *uref = frame->opaque; @@ -1090,6 +1090,13 @@ static void draw_horiz_band(AVCodecContext *avctx, const AVFrame *frame, if (!ubase_check(ubuf_pic_resize(uref->ubuf, 0, y, -1, height))) return; uref_pic_set_vposition(uref, y); + uint64_t duration; + if (ubase_check(uref_clock_get_duration(uref, &duration))) { + uref_clock_set_duration(uref, height * duration / frame->height); + uref_clock_add_date_sys(uref, y * duration / frame->height); + uref_clock_add_date_prog(uref, y * duration / frame->height); + uref_clock_add_date_orig(uref, y * duration / frame->height); + } struct upipe_avcdec *upipe_avcdec = upipe_avcdec_from_upipe(upipe); upipe_avcdec_output(upipe, uref, NULL); From 35e77c4309cbf6482aa09954518947bd55d45539 Mon Sep 17 00:00:00 2001 From: James Darnley Date: Mon, 9 Jul 2018 17:33:53 +0200 Subject: [PATCH 08/11] avcdec: use the type argument in draw_horiz_band to set progressive and field attributes Also don't set the same ones elsewhere. --- lib/upipe-av/upipe_avcodec_decode.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/upipe-av/upipe_avcodec_decode.c b/lib/upipe-av/upipe_avcodec_decode.c index 37ca0c596..a6e0a7a44 100644 --- a/lib/upipe-av/upipe_avcodec_decode.c +++ b/lib/upipe-av/upipe_avcodec_decode.c @@ -1023,13 +1023,6 @@ static int set_output_pic_properties(struct upipe *upipe, struct uref **uref_inp return UBASE_ERR_ALLOC; } - UBASE_RETURN(uref_pic_set_tf(uref)) - UBASE_RETURN(uref_pic_set_bf(uref)) - if (!frame->interlaced_frame) - UBASE_RETURN(uref_pic_set_progressive(uref)) - else if (frame->top_field_first) - UBASE_RETURN(uref_pic_set_tff(uref)) - if (context->time_base.den) UBASE_RETURN(uref_clock_set_duration(uref, (uint64_t)(2 + frame->repeat_pict) * context->ticks_per_frame * @@ -1098,6 +1091,14 @@ static void draw_horiz_band(AVCodecContext *avctx, const AVFrame *frame, uref_clock_add_date_orig(uref, y * duration / frame->height); } + if (type == 3) + uref_pic_set_progressive(uref); + else if (type == 1) + uref_pic_set_tf(uref); + else if (type == 2) + uref_pic_set_bf(uref); + /* Other values are undocumented/invalid. */ + struct upipe_avcdec *upipe_avcdec = upipe_avcdec_from_upipe(upipe); upipe_avcdec_output(upipe, uref, NULL); } From e26ecb39186efc5fa1d6d5311b36f2aaf0c28a29 Mon Sep 17 00:00:00 2001 From: James Darnley Date: Wed, 1 Aug 2018 17:38:56 +0100 Subject: [PATCH 09/11] upipe-av: don't output mixed chunks and frames --- lib/upipe-av/upipe_avcodec_decode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/upipe-av/upipe_avcodec_decode.c b/lib/upipe-av/upipe_avcodec_decode.c index a6e0a7a44..354bcef81 100644 --- a/lib/upipe-av/upipe_avcodec_decode.c +++ b/lib/upipe-av/upipe_avcodec_decode.c @@ -1356,7 +1356,7 @@ static bool upipe_avcdec_decode_avpkt(struct upipe *upipe, AVPacket *avpkt, } /* output frame if any has been decoded */ - if (gotframe) { + if (gotframe && !upipe_avcdec->context->draw_horiz_band) { upipe_avcdec_output_pic(upipe, upump_p); } break; From 224e0a22cf8c5a9e201e87c8b1cd36136d4c203f Mon Sep 17 00:00:00 2001 From: James Darnley Date: Thu, 2 Aug 2018 17:20:25 +0100 Subject: [PATCH 10/11] avcdec: improve setting progressive, tf, and bf attributes in draw_horiz_band --- lib/upipe-av/upipe_avcodec_decode.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/upipe-av/upipe_avcodec_decode.c b/lib/upipe-av/upipe_avcodec_decode.c index 354bcef81..2cbbd3827 100644 --- a/lib/upipe-av/upipe_avcodec_decode.c +++ b/lib/upipe-av/upipe_avcodec_decode.c @@ -1091,12 +1091,19 @@ static void draw_horiz_band(AVCodecContext *avctx, const AVFrame *frame, uref_clock_add_date_orig(uref, y * duration / frame->height); } - if (type == 3) + if (type == 3) { uref_pic_set_progressive(uref); - else if (type == 1) uref_pic_set_tf(uref); - else if (type == 2) uref_pic_set_bf(uref); + } + else if (type == 1) { + uref_pic_delete_progressive(uref); + uref_pic_set_tf(uref); + } + else if (type == 2) { + uref_pic_delete_progressive(uref); + uref_pic_set_bf(uref); + } /* Other values are undocumented/invalid. */ struct upipe_avcdec *upipe_avcdec = upipe_avcdec_from_upipe(upipe); From 2a59092ea99301dcefec2464f5e17f2e8d0e049e Mon Sep 17 00:00:00 2001 From: James Darnley Date: Mon, 6 Aug 2018 13:18:21 +0100 Subject: [PATCH 11/11] avcdec: signal separated fields in flow_def draw_horiz_band urefs --- lib/upipe-av/upipe_avcodec_decode.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/upipe-av/upipe_avcodec_decode.c b/lib/upipe-av/upipe_avcodec_decode.c index 2cbbd3827..0971bc843 100644 --- a/lib/upipe-av/upipe_avcodec_decode.c +++ b/lib/upipe-av/upipe_avcodec_decode.c @@ -297,6 +297,9 @@ static int upipe_avcdec_get_buffer_pic(struct AVCodecContext *context, return -1; } + if (frame->interlaced_frame) + UBASE_FATAL(upipe, uref_attr_set_void(flow_def_attr, NULL, UDICT_TYPE_VOID, "sepfields")); + UBASE_FATAL(upipe, uref_pic_flow_set_align(flow_def_attr, align)) UBASE_FATAL(upipe, uref_pic_flow_set_hsize(flow_def_attr, context->width)) UBASE_FATAL(upipe, uref_pic_flow_set_vsize(flow_def_attr, context->height)) @@ -1099,10 +1102,12 @@ static void draw_horiz_band(AVCodecContext *avctx, const AVFrame *frame, else if (type == 1) { uref_pic_delete_progressive(uref); uref_pic_set_tf(uref); + uref_attr_set_void(uref, NULL, UDICT_TYPE_VOID, "sepfields"); } else if (type == 2) { uref_pic_delete_progressive(uref); uref_pic_set_bf(uref); + uref_attr_set_void(uref, NULL, UDICT_TYPE_VOID, "sepfields"); } /* Other values are undocumented/invalid. */