Ticket #587: ffmpeg-resample2-rate-overflow.patch

ffmpeg-0.8.5/libavcodec/resample2.c

@@ -207 +208 @@
     memcpy(&c->filter_bank[c->filter_length*phase_count+1], c->filter_bank, (c->filter_length-1)*sizeof(FELEM));
     c->filter_bank[c->filter_length*phase_count]= c->filter_bank[c->filter_length - 1];
 
-    c->src_incr= out_rate;
-    c->ideal_dst_incr= c->dst_incr= in_rate * phase_count;
+    int64_t dst_incr = (int64_t)in_rate * phase_count, src_incr = out_rate;
+    int64_t gcd = av_gcd(dst_incr, src_incr);
+    if (dst_incr / gcd > INT_MAX || src_incr / gcd > INT_MAX)
+        goto error;
+
+    c->src_incr= src_incr / gcd;
+    c->ideal_dst_incr= c->dst_incr= dst_incr / gcd;
     c->index= -phase_count*((c->filter_length-1)/2);
 
     return c;