RTMP handshake fails with some encoders

[rubensanchez]

Summary of the bug:
How to reproduce:

Run a RTMP server using libavformat (I run a custom one) and try to stream to that server with Wirecast, Amazon's Elemental LIVE or iOS app wich uses ​VideoCore library.

Then you will see how the encoder do not start the stream or even crashes (Wirecast does). Wireshark shows a RST TCP packet when server sends S1 and waits for C2.

The root cause of this is the implementation of the RTMP protocol. These encoders send C0 and C1 packets together and they expect S0 and S1 sent together also. So, sending S0 and S1 on different packets results on the encoder closing the connection with the server.

Patches should be submitted to the ffmpeg-devel mailing list and not this bug tracker.

[rubensanchez]

I implemented the following fix. Basically it sends S0 and S1 using the same buffer and receives C0 and C1 on another resized buffer. I noticed the C2 packet has a 0x00 byte at the beginning so the array for C2 needs to be resized also.

static int rtmp_server_handshake(URLContext *s, RTMPContext *rt)
{
    uint8_t hs_s0s1[RTMP_HANDSHAKE_PACKET_SIZE + 1];
    uint8_t hs_c0c1[RTMP_HANDSHAKE_PACKET_SIZE + 1];
    uint8_t hs_c2[RTMP_HANDSHAKE_PACKET_SIZE + 1];
    uint8_t hs_s2[RTMP_HANDSHAKE_PACKET_SIZE];
    uint8_t dummy_uint;
    uint32_t hs_my_epoch;
    uint32_t temp       = 0;
    int randomidx       = 0;
    int inoutsize       = 0;
    int ret;

    /****************
     * Receive C0+C1
     ***************/
    ret = rtmp_receive_hs_packet(rt, &dummy_uint, &dummy_uint, hs_c0c1,
                                 RTMP_HANDSHAKE_PACKET_SIZE + 1);
    if (ret) {
        av_log(s, AV_LOG_ERROR, "RTMP Handshake C1 Error %d\n", ret);
        return ret;
    }
    // Check Version
    if (hs_c0c1[0] != 3) {
        av_log(s, AV_LOG_ERROR, "RTMP protocol version mismatch. Expected 0x03 received %02x\n", hs_c0c1[0]);
        return AVERROR(EIO);
    }
    // Get client epoch and set our with the same value
    hs_my_epoch = AV_RB32(hs_c0c1 + 1);

    /*************
     * Send S0+S1
     ************/
    // Generate random data to send it on S0+S1
    for (randomidx = 9; randomidx < (RTMP_HANDSHAKE_PACKET_SIZE + 1);
         randomidx += 4)
        AV_WB32(hs_s0s1 + randomidx, av_get_random_seed());
    // Set the RTMP protocol code on S0+S1 (First byte)
    hs_s0s1[0] = 0x03;
    // Copy the random data from C1 to S1
    memcpy(hs_s0s1 + 1, hs_c0c1 + 1, RTMP_HANDSHAKE_PACKET_SIZE);
    AV_WB32(hs_s0s1 + 1, hs_my_epoch);
    AV_WB32(hs_s0s1 + 5, 0);
    inoutsize = ffurl_write(rt->stream, hs_s0s1,
                            RTMP_HANDSHAKE_PACKET_SIZE + 1);
    if (inoutsize != RTMP_HANDSHAKE_PACKET_SIZE + 1) {
        av_log(s, AV_LOG_ERROR, "RTMP Handshake S1 Error %d\n", ret);
        return AVERROR(EIO);
    }

    /***********
     * Send S2
     **********/
    // Get the S2 random data from C0+C1
    memcpy(hs_s2, hs_c0c1, RTMP_HANDSHAKE_PACKET_SIZE);
    ret = rtmp_send_hs_packet(rt, hs_my_epoch, 0, hs_s2,
                              RTMP_HANDSHAKE_PACKET_SIZE);
    if (ret) {
        av_log(s, AV_LOG_ERROR, "RTMP Handshake S2 Error\n");
        return ret;
    }

    /*************
     * Receive C2
     ************/

    ret = ffurl_read_complete(rt->stream, hs_c2,
                                    RTMP_HANDSHAKE_PACKET_SIZE + 1);
    if (ret <= 0)
        return AVERROR(EIO);
    if (ret != RTMP_HANDSHAKE_PACKET_SIZE + 1) {
        av_log(rt, AV_LOG_ERROR, "Erroneous Message size %d"
               " not following standard\n", (int)inoutsize);
        return AVERROR(EINVAL);
    }

    // Check timestamp and random data from C2
    temp  = AV_RB32(hs_c2 + 1);
    if (temp != hs_my_epoch)
        av_log(s, AV_LOG_WARNING,
               "Erroneous C2 Message epoch does not match up with C1 epoch");
    if (memcmp(hs_c2 + 9, hs_c0c1 + 9,
               RTMP_HANDSHAKE_PACKET_SIZE - 8))
        av_log(s, AV_LOG_WARNING,
               "Erroneous C2 Message random does not match up\n");

    return 0;
}