Ticket #1163: 0003-ffprobe-implement-string-validation-policy-setting.2.patch

doc/ffprobe.texi

@@ -317 +317 @@
 equivalent of setting both @option{-show_program_version} and
 @option{-show_library_versions} options.
 
+@item -string_validation_policy @var{policy}
+Set string validation policy.
+
+@var{policy} must be "fail", "ignore", or in the form
+"replace[=@var{replacement}]".
+
+If it is set to "fail", the program will fail immediately in case an
+invalid string (UTF-8) sequence is found in the input. This is
+especially useful to validate input metadata.
+
+If it is set to "ignore", any invalidation error will be ignored. This
+will result in possibly broken output, especially with the json or xml
+writer.
+
+In case the "replace" form is choosen, the program will substitute the
+invalid UTF-8 sequences with the string specified in
+@var{replacement}, which is typically a simple character such as "?",
+but any valid UTF-8 sequence is accepted.
+
+In case the replacement string is not specified, the program will
+assume the empty string, that is it will remove the invalid sequences
+from the input strings.
+
+This is especially useful to create validate metadata output from
+invalid sources.
+
+By default the program will apply the replace policy with an empty
+replacement.
+
 @item -bitexact
 Force bitexact output, useful to produce output which is not dependent
 on the specific build.

ffprobe.c

@@ -75 +75 @@
 static char *print_format;
 static char *stream_specifier;
 
+typedef enum {
+    STRING_VALIDATION_POLICY_FAIL,
+    STRING_VALIDATION_POLICY_REPLACE,
+    STRING_VALIDATION_POLICY_IGNORE,
+} StringValidationPolicy;
+
+StringValidationPolicy string_validation_policy = STRING_VALIDATION_POLICY_REPLACE;
+static char *string_validation_replace;
+
 typedef struct {
     int id;             ///< identifier
     int64_t start, end; ///< start, end in second/AV_TIME_BASE units
@@ -428 +437 @@
     }
 }
 
+static char *get_utf8_sequence_string(char *buf, size_t buf_size,
+                                      const uint8_t *ubuf, size_t ubuf_size)
+{
+    AVBPrint bp;
+    int i;
+
+    av_bprint_init_for_buffer(&bp, buf, buf_size);
+    av_bprintf(&bp, "0X");
+    for (i = 0; i < ubuf_size; i++)
+        av_bprintf(&bp, "%02X", ubuf[i]);
+    return buf;
+}
+
+static inline int validate_string(char **dstp, const char *src, void *log_ctx)
+{
+    const uint8_t *p;
+    AVBPrint dstbuf;
+    int invalid_chars_nb = 0, ret = 0;
+    size_t len = strlen(src);
+
+    av_bprint_init(&dstbuf, 0, AV_BPRINT_SIZE_UNLIMITED);
+
+    for (p = (uint8_t *)src; *p;) {
+        uint32_t code;
+        uint8_t tmp;
+        int invalid = 0;
+        const uint8_t *p0 = p;
+
+        if (av_utf8_decode(&code, &p, len, AV_UTF8_DECODE_FLAG_CHECK_RANGE) < 0) {
+            char buf[32];
+            av_log(log_ctx, AV_LOG_WARNING,
+                   "Invalid UTF-8 sequence %s found in string '%s'\n",
+                   get_utf8_sequence_string(buf, sizeof(buf), p0, p-p0), src);
+            invalid = 1;
+        }
+        len -= p-p0;
+
+        if (invalid) {
+            invalid_chars_nb++;
+
+            switch (string_validation_policy) {
+            case STRING_VALIDATION_POLICY_FAIL:
+                av_log(log_ctx, AV_LOG_ERROR,
+                       "Invalid UTF-8 sequence found in string '%s'\n", src);
+                ret = AVERROR_INVALIDDATA;
+                goto end;
+                break;
+
+            case STRING_VALIDATION_POLICY_REPLACE:
+                if (string_validation_replace)
+                    av_bprintf(&dstbuf, "%s", string_validation_replace);
+                break;
+            }
+        }
+
+        if (!invalid || string_validation_policy == STRING_VALIDATION_POLICY_IGNORE) {
+            PUT_UTF8(code, tmp, av_bprint_chars(&dstbuf, tmp, 1););
+        }
+    }
+
+    if (invalid_chars_nb && string_validation_policy == STRING_VALIDATION_POLICY_REPLACE) {
+        av_log(log_ctx, AV_LOG_WARNING,
+               "%d invalid UTF-8 sequence(s) found in string '%s', replaced with '%s'\n",
+               invalid_chars_nb, src, (char *)av_x_if_null(string_validation_replace, ""));
+    }
+
+end:
+    av_bprint_finalize(&dstbuf, dstp);
+    return ret;
+}
+
+#define PRINT_STRING_OPT      1
+#define PRINT_STRING_VALIDATE 2
+
 static inline int writer_print_string(WriterContext *wctx,
-                                      const char *key, const char *val, int opt)
+                                      const char *key, const char *val, int flags)
 {
     const struct section *section = wctx->section[wctx->level];
     int ret = 0;
 
-    if (opt && !(wctx->writer->flags & WRITER_FLAG_DISPLAY_OPTIONAL_FIELDS))
+    if ((flags & PRINT_STRING_OPT)
+        && !(wctx->writer->flags & WRITER_FLAG_DISPLAY_OPTIONAL_FIELDS))
         return 0;
 
     if (section->show_all_entries || av_dict_get(section->entries_to_show, key, NULL, 0)) {
-        wctx->writer->print_string(wctx, key, val);
+        if (flags & PRINT_STRING_VALIDATE) {
+            char *key1 = NULL, *val1 = NULL;
+            ret = validate_string(&key1, key, wctx);
+            if (ret < 0) goto end;
+            ret = validate_string(&val1, val, wctx);
+            if (ret < 0) goto end;
+            wctx->writer->print_string(wctx, key1, val1);
+        end:
+            if (ret < 0) {
+                av_log(wctx, AV_LOG_ERROR,
+                       "Invalid key=value string combination %s=%s in section %s\n",
+                       key, val, section->unique_name);
+            }
+            av_free(key1);
+            av_free(val1);
+        } else {
+            wctx->writer->print_string(wctx, key, val);
+        }
+
         wctx->nb_item[wctx->level]++;
     }
 
@@ -460 +562 @@
     char buf[128];
 
     if ((!is_duration && ts == AV_NOPTS_VALUE) || (is_duration && ts == 0)) {
-        writer_print_string(wctx, key, "N/A", 1);
+        writer_print_string(wctx, key, "N/A", PRINT_STRING_OPT);
     } else {
         double d = ts * av_q2d(*time_base);
         struct unit_value uv;
@@ -474 +576 @@
 static void writer_print_ts(WriterContext *wctx, const char *key, int64_t ts, int is_duration)
 {
     if ((!is_duration && ts == AV_NOPTS_VALUE) || (is_duration && ts == 0)) {
-        writer_print_string(wctx, key, "N/A", 1);
+        writer_print_string(wctx, key, "N/A", PRINT_STRING_OPT);
     } else {
         writer_print_integer(wctx, key, ts);
     }
@@ -1443 +1545 @@
 #define print_int(k, v)         writer_print_integer(w, k, v)
 #define print_q(k, v, s)        writer_print_rational(w, k, v, s)
 #define print_str(k, v)         writer_print_string(w, k, v, 0)
-#define print_str_opt(k, v)     writer_print_string(w, k, v, 1)
+#define print_str_opt(k, v)     writer_print_string(w, k, v, PRINT_STRING_OPT)
+#define print_str_validate(k, v) writer_print_string(w, k, v, PRINT_STRING_VALIDATE)
 #define print_time(k, v, tb)    writer_print_time(w, k, v, tb, 0)
 #define print_ts(k, v)          writer_print_ts(w, k, v, 0)
 #define print_duration_time(k, v, tb) writer_print_time(w, k, v, tb, 1)
@@ -1458 +1561 @@
 #define print_section_header(s) writer_print_section_header(w, s)
 #define print_section_footer(s) writer_print_section_footer(w, s)
 
-static inline int show_tags(WriterContext *wctx, AVDictionary *tags, int section_id)
+static inline int show_tags(WriterContext *w, AVDictionary *tags, int section_id)
 {
     AVDictionaryEntry *tag = NULL;
     int ret = 0;
 
     if (!tags)
         return 0;
-    writer_print_section_header(wctx, section_id);
+    writer_print_section_header(w, section_id);
 
     while ((tag = av_dict_get(tags, "", tag, AV_DICT_IGNORE_SUFFIX))) {
-        ret = writer_print_string(wctx, tag->key, tag->value, 0);
-        if (ret < 0)
+        if ((ret = print_str_validate(tag->key, tag->value)) < 0)
             break;
     }
-    writer_print_section_footer(wctx);
+    writer_print_section_footer(w);
 
     return ret;
 }
@@ -2021 +2123 @@
     int ret = 0;
 
     writer_print_section_header(w, SECTION_ID_FORMAT);
-    print_str("filename",         fmt_ctx->filename);
+    print_str_validate("filename", fmt_ctx->filename);
     print_int("nb_streams",       fmt_ctx->nb_streams);
     print_int("nb_programs",      fmt_ctx->nb_programs);
     print_str("format_name",      fmt_ctx->iformat->name);
@@ -2537 +2639 @@
     return parse_read_intervals(arg);
 }
 
+static int opt_string_validation_policy(void *optctx, const char *opt, const char *arg)
+{
+    char *mode = av_strdup(arg);
+    char *next;
+    int ret = 0;
+
+    if (!mode) return AVERROR(ENOMEM);
+
+    next = strchr(mode, '=');
+    if (next)
+        *next++ = 0;
+
+    if (!strcmp(mode, "fail")) {
+        string_validation_policy = STRING_VALIDATION_POLICY_FAIL;
+    } else if (!strcmp(mode, "ignore")) {
+        string_validation_policy = STRING_VALIDATION_POLICY_IGNORE;
+    } else if (!strcmp(mode, "replace")) {
+        string_validation_policy = STRING_VALIDATION_POLICY_REPLACE;
+        string_validation_replace = av_strdup(next);
+
+        if (next && !string_validation_replace) {
+            ret = AVERROR(ENOMEM);
+            goto end;
+        }
+
+        {
+            /* validate replace string */
+            const uint8_t *p = string_validation_replace;
+            while (*p) {
+                char buf[32];
+                const uint8_t *p0 = p;
+                int32_t code;
+                ret = av_utf8_decode(&code, &p, strlen(p), AV_UTF8_DECODE_FLAG_CHECK_RANGE);
+                if (ret < 0) {
+                    av_log(NULL, AV_LOG_ERROR,
+                           "Invalid UTF8 sequence %s found in string validation replace '%s'\n",
+                           get_utf8_sequence_string(buf, sizeof(buf), p0, p-p0),
+                           string_validation_replace);
+                    goto end;
+                }
+            }
+        }
+    } else {
+        av_log(NULL, AV_LOG_ERROR,
+               "Invalid argument '%s' for option '%s', "
+               "choose between fail, ignore, or replace[=REPLACEMENT]\n", arg, opt);
+        ret = AVERROR(EINVAL);
+        goto end;
+    }
+
+    if (next && string_validation_policy != STRING_VALIDATION_POLICY_REPLACE) {
+        av_log(NULL, AV_LOG_ERROR,
+               "No argument must be specified for option '%s' with fail or ignore policy\n", opt);
+        ret = AVERROR(EINVAL);
+        goto end;
+    }
+
+end:
+    av_free(mode);
+    return ret;
+}
+
 static int opt_pretty(void *optctx, const char *opt, const char *arg)
 {
     show_value_unit              = 1;
@@ -2636 +2800 @@
     { "private",           OPT_BOOL, {(void*)&show_private_data}, "same as show_private_data" },
     { "bitexact", OPT_BOOL, {&do_bitexact}, "force bitexact output" },
     { "read_intervals", HAS_ARG, {.func_arg = opt_read_intervals}, "set read intervals", "read_intervals" },
+    { "string_validation_policy",  HAS_ARG, {.func_arg = opt_string_validation_policy}, "select the string validation policy", "policy_specification" },
     { "default", HAS_ARG | OPT_AUDIO | OPT_VIDEO | OPT_EXPERT, {.func_arg = opt_default}, "generic catch all option", "" },
     { "i", HAS_ARG, {.func_arg = opt_input_file_i}, "read specified file", "input_file"},
     { NULL, },
@@ -2748 +2913 @@
 
 end:
     av_freep(&print_format);
+    av_freep(&string_validation_replace);
     av_freep(&read_intervals);
 
     uninit_opts();