Ticket #1163: 0002-lavu-avstring-add-av_get_utf8-function.patch

libavutil/Makefile

@@ -155 +155 @@
             sha                                                         \
             sha512                                                      \
             tree                                                        \
+            utf8                                                        \
             xtea                                                        \
 
 TESTPROGS-$(HAVE_LZO1X_999_COMPRESS) += lzo

libavutil/avstring.c

@@ -307 +307 @@
     return av_isdigit(c) || (c >= 'a' && c <= 'f');
 }
 
+int av_get_utf8(int32_t *code, const uint8_t **buf)
+{
+    const uint8_t *p = *buf;
+    uint32_t top;
+
+    *code = *p++;
+
+    /* first sequence byte starts with 10, or is 1111-1110 or 1111-1111,
+       which is not admitted */
+    if ((*code & 0xc0) == 0x80 || *code >= 0xFE) {
+        *buf = p;
+        return AVERROR(EINVAL);
+    }
+    top = (*code & 128) >> 1;
+
+    while (*code & top) {
+        /* we assume the byte to be in the form 10xx-xxxx */
+        int tmp = *p++ - 128;   /* strip leading 1 */
+        if (tmp>>6) {
+            *buf = p;
+            return AVERROR(EINVAL);
+        }
+        *code = (*code<<6) + tmp;
+        top <<= 5;
+    }
+    *code &= (top << 1) - 1;
+
+    *buf = p;
+    return 0;
+}
+
 #ifdef TEST
 
 int main(void)

libavutil/avstring.h

@@ -22 +22 @@
 #define AVUTIL_AVSTRING_H
 
 #include <stddef.h>
+#include <stdint.h>
 #include "attributes.h"
 
 /**
@@ -226 +227 @@
 int av_isxdigit(int c);
 
 /**
+ * Read an UTF-8 character from buffer in *buf, and update *buf to point to the
+ * next sequence after the parsed sequence.
+ * In case of invalid sequence, the point will be updated to the next
+ * character after the invalid sequence.
+ *
+ * @return >= 0 in case a sequence was successfully read, a negative
+ * value in case of invalid sequence
+ */
+int av_get_utf8(int32_t *code, const uint8_t **buf);
+
+/**
  * Locale-independent case-insensitive compare.
  * @note This means only ASCII-range characters are case-insensitive
  */

new file libavutil/utf8.c

@@ +1 @@
+/*
+ * Copyright (c) 2013 Stefano Sabatini
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <stdio.h>
+
+#include "libavutil/avstring.h"
+#include "libavutil/file.h"
+
+static void print_sequence(const char *p, int l, int indent)
+{
+    int i;
+    for (i = 0; i < l; i++)
+        printf("%02X", (uint8_t)p[i]);
+    for (i=l*2; i < indent; i++)
+        printf("%c", ' ');
+}
+
+int main(int argc, char **argv)
+{
+    int ret;
+    char *filename = argv[1];
+    uint8_t *file_buf;
+    size_t file_buf_size;
+    uint32_t code;
+    const uint8_t *p;
+    int i = 0;
+
+    ret = av_file_map(filename, &file_buf, &file_buf_size, 0, NULL);
+    if (ret < 0)
+        goto end;
+
+    p = file_buf;
+    while (i < file_buf_size) {
+        int l, r;
+        const uint8_t *p0 = p;
+        r = av_get_utf8(&code, &p, 0);
+        l = (int)(p-p0);
+        print_sequence(p0, l, 20);
+        if (r >= 0) {
+            printf("%-10d 0x%-10X %-5d ", code, code, l);
+            if (*p0 == '\n') { printf("\\n\n"); }
+            else { printf ("%.*s\n", l, p0); }
+        } else {
+            printf("-> illegal sequence\n");
+        }
+        i += l;
+    }
+
+end:
+    av_file_unmap(file_buf, file_buf_size);
+    return ret < 0;
+}