Add monochrome/gray pixel format to libaom-av1

[Ewout]

Monochrome encoding is supported in AV1 using 4:0:0 chroma subsampling. aomenc used the --monochrome tag, but instead of an extra option pixel formats gray, gray10le and gray12le could also be used.

Encoding with the gray pixel formats already delivers usable output, but the CLI states yuv444p is used.

[Carl Eugen Hoyos]

Please test this patch:

diff --git a/libavcodec/libaomenc.c b/libavcodec/libaomenc.c
index 1756501..e8dde36 100644
--- a/libavcodec/libaomenc.c
+++ b/libavcodec/libaomenc.c
@@ -231,6 +231,8 @@ static int set_pix_fmt(AVCodecContext *avctx, aom_codec_caps_t codec_caps,
     AOMContext av_unused *ctx = avctx->priv_data;
     enccfg->g_bit_depth = enccfg->g_input_bit_depth = 8;
     switch (avctx->pix_fmt) {
+    case AV_PIX_FMT_GRAY8:
+        ctx->rawimg.monochrome = 1;
     case AV_PIX_FMT_YUV420P:
         enccfg->g_profile = FF_PROFILE_AV1_MAIN;
         *img_fmt = AOM_IMG_FMT_I420;
@@ -243,11 +245,14 @@ static int set_pix_fmt(AVCodecContext *avctx, aom_codec_caps_t codec_caps,
         enccfg->g_profile = FF_PROFILE_AV1_HIGH;
         *img_fmt = AOM_IMG_FMT_I444;
         return 0;
+    case AV_PIX_FMT_GRAY10:
+    case AV_PIX_FMT_GRAY12:
+        ctx->rawimg.monochrome = 1;
     case AV_PIX_FMT_YUV420P10:
     case AV_PIX_FMT_YUV420P12:
         if (codec_caps & AOM_CODEC_CAP_HIGHBITDEPTH) {
             enccfg->g_bit_depth = enccfg->g_input_bit_depth =
-                avctx->pix_fmt == AV_PIX_FMT_YUV420P10 ? 10 : 12;
+                avctx->pix_fmt == AV_PIX_FMT_YUV420P10 || avctx->pix_fmt == AV_PIX_FMT_GRAY10 ? 10 : 12;
             enccfg->g_profile =
                 enccfg->g_bit_depth == 10 ? FF_PROFILE_AV1_MAIN : FF_PROFILE_AV1_PROFESSIONAL;
             *img_fmt = AOM_IMG_FMT_I42016;
@@ -937,6 +942,7 @@ static const enum AVPixelFormat av1_pix_fmts[] = {
     AV_PIX_FMT_YUV420P,
     AV_PIX_FMT_YUV422P,
     AV_PIX_FMT_YUV444P,
+    AV_PIX_FMT_GRAY8,
     AV_PIX_FMT_NONE
 };

@@ -944,12 +950,15 @@ static const enum AVPixelFormat av1_pix_fmts_highbd[] = {
     AV_PIX_FMT_YUV420P,
     AV_PIX_FMT_YUV422P,
     AV_PIX_FMT_YUV444P,
+    AV_PIX_FMT_GRAY8,
     AV_PIX_FMT_YUV420P10,
     AV_PIX_FMT_YUV422P10,
     AV_PIX_FMT_YUV444P10,
+    AV_PIX_FMT_GRAY10,
     AV_PIX_FMT_YUV420P12,
     AV_PIX_FMT_YUV422P12,
     AV_PIX_FMT_YUV444P12,
+    AV_PIX_FMT_GRAY12,
     AV_PIX_FMT_NONE
 };

[Ewout]

With the following commands FFmpeg doesn't start encoding:

Downloads\ffmpeg-tickets\ffmpeg-7599 -y -i "E:\Morocco8K-8b-1920.y4m" -t 0.25 -strict -2 -c:v libaom-av1 -pix_fmt gray -cpu-used 4 "E:\Morocco8K-8b-gray.mp4"
Downloads\ffmpeg-tickets\ffmpeg-7599 -y -i "E:\Morocco8K-10b-1920.y4m" -t 0.25 -strict -2 -c:v libaom-av1 -pix_fmt gray10le -cpu-used 4 "E:\Morocco8K-10b-gray.mp4"
Downloads\ffmpeg-tickets\ffmpeg-7599 -y -i "E:\Morocco8K-10b-1920.y4m" -t 0.25 -strict -2 -c:v libaom-av1 -pix_fmt gray12le -cpu-used 4 "E:\Morocco8K-12b-gray.mp4"

Changing -pix_fmt to -vf format= doesn't help. When -pix_fmt gray is removed or replaced with -pix_fmt yuv420p or -pix_fmt yuv444p it encodes fine.

This build is used: ​https://mega.nz/#!Y55ACKyA!cUqpJg7CNmjmwgbJNdhETkZWVqTBx_U5_KA2RWXzgpU
Debug logs attached.

[SmilingWolf]

I digged a bit into this.

It looks like libaom doesn't truly handle single planes and invariably expects 3 to be present, so when receiving a Cmono Y4M file in input it artificially creates the U and V planes filling them with 0x80: ​https://aomedia.googlesource.com/aom/+/2395ed028ca3a5a597cda74054440fd2d21330a3/common/y4minput.c#764

If I understood correctly, FFmpeg with the gray pix_fmt hands a single Y plane to libaom, which subsequently chokes in ​https://aomedia.googlesource.com/aom/+/2395ed028ca3a5a597cda74054440fd2d21330a3/av1/encoder/extend.c#144 because it tries to read invalid data

[Carl Eugen Hoyos]

Does this work any better?

diff --git a/libavcodec/libaomenc.c b/libavcodec/libaomenc.c
index faec61c..e8fb507 100644
--- a/libavcodec/libaomenc.c
+++ b/libavcodec/libaomenc.c
@@ -248,6 +248,8 @@ static int set_pix_fmt(AVCodecContext *avctx, aom_codec_caps_t codec_caps,
         enccfg->g_profile = FF_PROFILE_AV1_PROFESSIONAL;
         *img_fmt = AOM_IMG_FMT_I422;
         return 0;
+    case AV_PIX_FMT_GRAY8:
+        ctx->rawimg.monochrome = 1;
     case AV_PIX_FMT_YUV444P:
         enccfg->g_profile = FF_PROFILE_AV1_HIGH;
         *img_fmt = AOM_IMG_FMT_I444;
@@ -275,11 +277,14 @@ static int set_pix_fmt(AVCodecContext *avctx, aom_codec_caps_t codec_caps,
             return 0;
         }
         break;
+    case AV_PIX_FMT_GRAY10:
+    case AV_PIX_FMT_GRAY12:
+        ctx->rawimg.monochrome = 1;
     case AV_PIX_FMT_YUV444P10:
     case AV_PIX_FMT_YUV444P12:
         if (codec_caps & AOM_CODEC_CAP_HIGHBITDEPTH) {
             enccfg->g_bit_depth = enccfg->g_input_bit_depth =
-                avctx->pix_fmt == AV_PIX_FMT_YUV444P10 ? 10 : 12;
+                avctx->pix_fmt == AV_PIX_FMT_YUV444P10 || avctx->pix_fmt == AV_PIX_FMT_GRAY10 ? 10 : 12;
             enccfg->g_profile =
                 enccfg->g_bit_depth == 10 ? FF_PROFILE_AV1_HIGH : FF_PROFILE_AV1_PROFESSIONAL;
             *img_fmt = AOM_IMG_FMT_I44416;
@@ -902,11 +907,18 @@ static int aom_encode(AVCodecContext *avctx, AVPacket *pkt,
     if (frame) {
         rawimg                      = &ctx->rawimg;
         rawimg->planes[AOM_PLANE_Y] = frame->data[0];
-        rawimg->planes[AOM_PLANE_U] = frame->data[1];
-        rawimg->planes[AOM_PLANE_V] = frame->data[2];
         rawimg->stride[AOM_PLANE_Y] = frame->linesize[0];
-        rawimg->stride[AOM_PLANE_U] = frame->linesize[1];
-        rawimg->stride[AOM_PLANE_V] = frame->linesize[2];
+        if (rawimg->monochrome) {
+            rawimg->planes[AOM_PLANE_U] = frame->data[0];
+            rawimg->planes[AOM_PLANE_V] = frame->data[0];
+            rawimg->stride[AOM_PLANE_U] = frame->linesize[0];
+            rawimg->stride[AOM_PLANE_V] = frame->linesize[0];
+        } else {
+            rawimg->planes[AOM_PLANE_U] = frame->data[1];
+            rawimg->planes[AOM_PLANE_V] = frame->data[2];
+            rawimg->stride[AOM_PLANE_U] = frame->linesize[1];
+            rawimg->stride[AOM_PLANE_V] = frame->linesize[2];
+        }
         timestamp                   = frame->pts;
         switch (frame->color_range) {
         case AVCOL_RANGE_MPEG:
@@ -950,6 +962,7 @@ static const enum AVPixelFormat av1_pix_fmts[] = {
     AV_PIX_FMT_YUV420P,
     AV_PIX_FMT_YUV422P,
     AV_PIX_FMT_YUV444P,
+    AV_PIX_FMT_GRAY8,
     AV_PIX_FMT_NONE
 };

@@ -957,12 +970,15 @@ static const enum AVPixelFormat av1_pix_fmts_highbd[] = {
     AV_PIX_FMT_YUV420P,
     AV_PIX_FMT_YUV422P,
     AV_PIX_FMT_YUV444P,
+    AV_PIX_FMT_GRAY8,
     AV_PIX_FMT_YUV420P10,
     AV_PIX_FMT_YUV422P10,
     AV_PIX_FMT_YUV444P10,
+    AV_PIX_FMT_GRAY10,
     AV_PIX_FMT_YUV420P12,
     AV_PIX_FMT_YUV422P12,
     AV_PIX_FMT_YUV444P12,
+    AV_PIX_FMT_GRAY12,
     AV_PIX_FMT_NONE
 };

[Ewout]

cehoyos quick update: I had a conversation with Smilingwolf yesterday, he was unable to build it but hadn't had time yet to properly debug it. I asked him to post the details here when he knows more (I don't understand this issue at a level he does), I hope he does soon.

In the meantime, #7600 is good to go. Could you submit the patch?

[Philip Langdale]

Fix in commit 40135829b613f875ce71c2cc2265e74ccc6b4c71