Ticket #1163: 0001-lavu-avstring-add-av_utf8_decode-function.patch

doc/APIchanges

@@ -15 +15 @@
 
 API changes, most recent first:
 
+2013-11-12 - xxxxxxx - lavu 52.53.100 - avstring.h
+  Add av_utf8_decode() function.
+
 2013-11-xx - xxxxxxx - lavc 55.41.100 / 55.25.0 - avcodec.h
                        lavu 52.51.100 - frame.h
   Add ITU-R BT.2020 and other not yet included values to color primaries,

libavutil/Makefile

@@ -155 +155 @@
             sha                                                         \
             sha512                                                      \
             tree                                                        \
+            utf8                                                        \
             xtea                                                        \
 
 TESTPROGS-$(HAVE_LZO1X_999_COMPRESS) += lzo

libavutil/avstring.c

@@ -307 +307 @@
     return av_isdigit(c) || (c >= 'a' && c <= 'f');
 }
 
+int av_utf8_decode(int32_t *code, const uint8_t **buf, size_t left, int flags)
+{
+    const uint8_t *p = *buf;
+    uint32_t top;
+    int ret = 0;
+
+    if (!left)
+        return 0;
+
+    *code = *p++;
+
+    /* first sequence byte starts with 10, or is 1111-1110 or 1111-1111,
+       which is not admitted */
+    if ((*code & 0xc0) == 0x80 || *code >= 0xFE) {
+        ret = AVERROR(EILSEQ);
+        goto end;
+    }
+    top = (*code & 128) >> 1;
+
+    while (*code & top) {
+        int tmp;
+        if (!--left) {
+            ret = AVERROR(EILSEQ); /* incomplete sequence */
+            goto end;
+        }
+
+        /* we assume the byte to be in the form 10xx-xxxx */
+        tmp = *p++ - 128;   /* strip leading 1 */
+        if (tmp>>6) {
+            ret = AVERROR(EILSEQ);
+            goto end;
+        }
+        *code = (*code<<6) + tmp;
+        top <<= 5;
+    }
+    *code &= (top << 1) - 1;
+
+    if (flags & AV_UTF8_DECODE_FLAG_CHECK_RANGE) {
+        /* only accepts valid Unicode points:
+           #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]
+           that is any Unicode character, excluding the surrogate blocks, FFFE, and FFFF */
+        if (*code != 0x9 && *code != 0xA && *code != 0xD &&
+            (*code <    0x20 || *code >   0xD7FF) &&
+            (*code <  0xE000 || *code >   0xFFFD) &&
+            (*code < 0x10000 || *code > 0x10FFFF))
+            ret = AVERROR(EILSEQ);
+    }
+
+end:
+    *buf = p;
+    return ret;
+}
+
 #ifdef TEST
 
 int main(void)

libavutil/avstring.h

@@ -22 +22 @@
 #define AVUTIL_AVSTRING_H
 
 #include <stddef.h>
+#include <stdint.h>
 #include "attributes.h"
 
 /**
@@ -296 +297 @@
               enum AVEscapeMode mode, int flags);
 
 /**
+ * Check UNICODE range validity of the decoded code.
+ */
+#define AV_UTF8_DECODE_FLAG_CHECK_RANGE 1
+
+/**
+ * Read and decode a single UTF-8 character sequence from buffer in
+ * *buf, and update *buf to point to the next byte after the parsed
+ * sequence.
+ *
+ * In case of invalid sequence, the pointer will be updated to the
+ * next byte after the invalid sequence.
+ *
+ * @param code pointer whose pointed value is updated to keep the
+ * parsed code in case of success
+ * @param left bytes left to read in the buffer. By default it won't
+ * read more than 6 bytes (maximum number of bytes in an UTF-8
+ * sequence).
+ * @param flags a collection of AV_UTF8_DECODE_FLAG_* flags
+ * @return >= 0 in case a sequence was successfully read, a negative
+ * value in case of invalid sequence
+ * @see GET_UTF8()
+ */
+int av_utf8_decode(int32_t *code, const uint8_t **buf, size_t left, int flags);
+
+/**
  * @}
  */
 

new file libavutil/utf8.c

@@ +1 @@
+/*
+ * Copyright (c) 2013 Stefano Sabatini
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <stdio.h>
+
+#include "libavutil/avstring.h"
+#include "libavutil/file.h"
+
+static void print_sequence(const char *p, int l, int indent)
+{
+    int i;
+    for (i = 0; i < l; i++)
+        printf("%02X", (uint8_t)p[i]);
+    for (i = l*2; i < indent; i++)
+        printf("%c", ' ');
+}
+
+int main(int argc, char **argv)
+{
+    int ret;
+    char *filename = argv[1];
+    uint8_t *file_buf;
+    size_t file_buf_size;
+    uint32_t code;
+    const uint8_t *p;
+    int i = 0;
+
+    ret = av_file_map(filename, &file_buf, &file_buf_size, 0, NULL);
+    if (ret < 0)
+        goto end;
+
+    p = file_buf;
+    while (i < file_buf_size) {
+        int l, r;
+        const uint8_t *p0 = p;
+        r = av_utf8_decode(&code, &p, file_buf_size - i, 0);
+        l = (int)(p-p0);
+        print_sequence(p0, l, 20);
+        if (r >= 0) {
+            printf("%-10d 0x%-10X %-5d ", code, code, l);
+            if (*p0 == '\n') { printf("\\n\n"); }
+            else             { printf ("%.*s\n", l, p0); }
+        } else {
+            printf("-> illegal sequence\n");
+        }
+        i += l;
+    }
+
+end:
+    av_file_unmap(file_buf, file_buf_size);
+    return ret < 0;
+}

libavutil/version.h

@@ -75 +75 @@
  */
 
 #define LIBAVUTIL_VERSION_MAJOR  52
-#define LIBAVUTIL_VERSION_MINOR  52
+#define LIBAVUTIL_VERSION_MINOR  53
 #define LIBAVUTIL_VERSION_MICRO 100
 
 #define LIBAVUTIL_VERSION_INT   AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \