Function downmix_required

libhb/decavcodec.c:159–190

#define HB_AV_CH_BOTH_MASK (HB_AV_CH_SIDE_MASK|HB_AV_CH_BACK_MASK)

static int downmix_required(const AVChannelLayout *target_layout_mask, const AVChannelLayout *input_layout_mask)
{
    /*
     * Side channels can easily be remapped to back channels and vice-versa.
     * Provided the other channels are the same, downmixing is not required.
     */
    if (av_channel_layout_subset(input_layout_mask, HB_AV_CH_SIDE_MASK) == 0 &&
        av_channel_layout_subset(input_layout_mask, HB_AV_CH_BACK_MASK) == HB_AV_CH_BACK_MASK)
    {
        if (av_channel_layout_subset(target_layout_mask, HB_AV_CH_BACK_MASK) == 0 &&
            av_channel_layout_subset(target_layout_mask, HB_AV_CH_SIDE_MASK) == HB_AV_CH_SIDE_MASK)
        {
            // input has back channels but not side channels
            // target has the opposite (sides but not backs)
            return (av_channel_layout_subset(input_layout_mask, ~HB_AV_CH_BOTH_MASK) !=
                    av_channel_layout_subset(target_layout_mask, ~HB_AV_CH_BOTH_MASK));
        }
    }
    if (av_channel_layout_subset(input_layout_mask, HB_AV_CH_BACK_MASK) == 0 &&
        av_channel_layout_subset(input_layout_mask, HB_AV_CH_SIDE_MASK) == HB_AV_CH_SIDE_MASK)
    {
        if (av_channel_layout_subset(target_layout_mask, HB_AV_CH_SIDE_MASK) == 0 &&
            av_channel_layout_subset(target_layout_mask, HB_AV_CH_BACK_MASK) == HB_AV_CH_BACK_MASK)
        {
            // input has side channels but not back channels
            // target has the opposite (backs but not sides)
            return (av_channel_layout_subset(input_layout_mask, ~HB_AV_CH_BOTH_MASK) !=
                    av_channel_layout_subset(target_layout_mask, ~HB_AV_CH_BOTH_MASK));
        }
    }
    return av_channel_layout_compare(input_layout_mask, target_layout_mask);
}

static uint64_t ac3_downmix_mask(int hb_mixdown, int normalized, const AVChannelLayout *input_layout, const char **dmix_mode)
{