garbled output with transparent GIF

[bjorn]

Summary of the bug:
How to reproduce:

$ ./ffmpeg -i source_sticker.gif -filter_complex '[0:v]scale=100:-1,split[a][b];[a]palettegen[p];[b][p]paletteuse' out.gif
On OS X produces various problems:
- in FFMPeg 4.1.3 it enters an infinite loop.
- in FFMPEG master, it produces garbled output.

Here's the patch:

diff --git a/libavcodec/gif.c b/libavcodec/gif.c
index 94c8b1af49..d7768b4426 100644
--- a/libavcodec/gif.c
+++ b/libavcodec/gif.c
@@ -136,7 +136,7 @@ static void gif_crop_translucent(AVCodecContext *avctx,
         while (*y_start < y_end) {
             int is_trans = 1;
             for (int i = 0; i < w; i++) {
-                if (buf[w * *y_start + i] != trans) {
+                if (buf[linesize * *y_start + i] != trans) {
                     is_trans = 0;
                     break;
                 }
@@ -148,10 +148,10 @@ static void gif_crop_translucent(AVCodecContext *avctx,
         }
 
         // crop bottom
-        while (y_end < h) {
+        while (y_end > *y_start) {
             int is_trans = 1;
             for (int i = 0; i < w; i++) {
-                if (buf[w * y_end + i] != trans) {
+                if (buf[linesize * y_end + i] != trans) {
                     is_trans = 0;
                     break;
                 }
@@ -165,7 +165,7 @@ static void gif_crop_translucent(AVCodecContext *avctx,
         while (*x_start < x_end) {
             int is_trans = 1;
             for (int i = *y_start; i < y_end; i++) {
-                if (buf[w * i + *x_start] != trans) {
+                if (buf[linesize * i + *x_start] != trans) {
                     is_trans = 0;
                     break;
                 }
@@ -176,10 +176,10 @@ static void gif_crop_translucent(AVCodecContext *avctx,
         }
 
         // crop right
-        while (x_end < w) {
+        while (x_end > *x_start) {
             int is_trans = 1;
             for (int i = *y_start; i < y_end; i++) {
-                if (buf[w * i + x_end] != trans) {
+                if (buf[linesize * i + x_end] != trans) {
                     is_trans = 0;
                     break;
                 }

[bjorn]

@cehoyos: Which ticket does this duplicate?

[Elon Musk]

#7890