From: Jonathan Nieder Date: Tue, 11 Jan 2011 21:48:50 +0000 (-0600) Subject: userdiff: simplify word-diff safeguard X-Git-Tag: ko-pu~1^2~2 X-Git-Url: https://git.tokkee.org/?a=commitdiff_plain;h=664d44ee7fb18bdfdd66a1be760c7ee1bbe911c6;p=git.git userdiff: simplify word-diff safeguard git's diff-words support has a detail that can be a little dangerous: any text not matched by a given language's tokenization pattern is treated as whitespace and changes in such text would go unnoticed. Therefore each of the built-in regexes allows a special token type consisting of a single non-whitespace character [^[:space:]]. To make sure UTF-8 sequences remain human readable, the builtin regexes also have a special token type for runs of bytes with the high bit set. In English, non-ASCII characters are usually isolated so this is analogous to the [^[:space:]] pattern, except it matches a single _multibyte_ character despite use of the C locale. Unfortunately it is easy to make typos or forget entirely to include these catch-all token types when adding support for new languages (see v1.7.3.5~16, userdiff: fix typo in ruby and python word regexes, 2010-12-18). Avoid this by including them automatically within the PATTERNS and IPATTERN macros. While at it, change the UTF-8 sequence token type to match exactly one non-ASCII multi-byte character, rather than an arbitrary run of them. Suggested-by: Thomas Rast Signed-off-by: Jonathan Nieder Signed-off-by: Junio C Hamano --- diff --git a/userdiff.c b/userdiff.c index c384b39e4..3a1c392bb 100644 --- a/userdiff.c +++ b/userdiff.c @@ -8,9 +8,11 @@ static int ndrivers; static int drivers_alloc; #define PATTERNS(name, pattern, word_regex) \ - { name, NULL, -1, { pattern, REG_EXTENDED }, word_regex } + { name, NULL, -1, { pattern, REG_EXTENDED }, \ + word_regex "|[^[:space:]]|[\xc0-\xff][\x80-\xbf]+" } #define IPATTERN(name, pattern, word_regex) \ - { name, NULL, -1, { pattern, REG_EXTENDED | REG_ICASE }, word_regex } + { name, NULL, -1, { pattern, REG_EXTENDED | REG_ICASE }, \ + word_regex "|[^[:space:]]|[\xc0-\xff][\x80-\xbf]+" } static struct userdiff_driver builtin_drivers[] = { IPATTERN("fortran", "!^([C*]|[ \t]*!)\n" @@ -24,10 +26,9 @@ IPATTERN("fortran", * Don't worry about format statements without leading digits since * they would have been matched above as a variable anyway. */ "|[-+]?[0-9.]+([AaIiDdEeFfLlTtXx][Ss]?[-+]?[0-9.]*)?(_[a-zA-Z0-9][a-zA-Z0-9_]*)?" - "|//|\\*\\*|::|[/<>=]=" - "|[^[:space:]]|[\x80-\xff]+"), + "|//|\\*\\*|::|[/<>=]="), PATTERNS("html", "^[ \t]*(<[Hh][1-6][ \t].*>.*)$", - "[^<>= \t]+|[^[:space:]]|[\x80-\xff]+"), + "[^<>= \t]+"), PATTERNS("java", "!^[ \t]*(catch|do|for|if|instanceof|new|return|switch|throw|while)\n" "^[ \t]*(([A-Za-z_][A-Za-z_0-9]*[ \t]+)+[A-Za-z_][A-Za-z_0-9]*[ \t]*\\([^;]*)$", @@ -35,8 +36,7 @@ PATTERNS("java", "[a-zA-Z_][a-zA-Z0-9_]*" "|[-+0-9.e]+[fFlL]?|0[xXbB]?[0-9a-fA-F]+[lL]?" "|[-+*/<>%&^|=!]=" - "|--|\\+\\+|<<=?|>>>?=?|&&|\\|\\|" - "|[^[:space:]]|[\x80-\xff]+"), + "|--|\\+\\+|<<=?|>>>?=?|&&|\\|\\|"), PATTERNS("objc", /* Negate C statements that can look like functions */ "!^[ \t]*(do|for|if|else|return|switch|while)\n" @@ -49,8 +49,7 @@ PATTERNS("objc", /* -- */ "[a-zA-Z_][a-zA-Z0-9_]*" "|[-+0-9.e]+[fFlL]?|0[xXbB]?[0-9a-fA-F]+[lL]?" - "|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->" - "|[^[:space:]]|[\x80-\xff]+"), + "|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->"), PATTERNS("pascal", "^((procedure|function|constructor|destructor|interface|" "implementation|initialization|finalization)[ \t]*.*)$" @@ -59,8 +58,7 @@ PATTERNS("pascal", /* -- */ "[a-zA-Z_][a-zA-Z0-9_]*" "|[-+0-9.e]+|0[xXbB]?[0-9a-fA-F]+" - "|<>|<=|>=|:=|\\.\\." - "|[^[:space:]]|[\x80-\xff]+"), + "|<>|<=|>=|:=|\\.\\."), PATTERNS("perl", "^[ \t]*package .*;\n" "^[ \t]*sub .* \\{\n" @@ -76,33 +74,29 @@ PATTERNS("perl", "|&&|\\|\\||//|\\+\\+|--|\\*\\*|\\.\\.\\.?" "|[-+*/%.^&<>=!|]=" "|=~|!~" - "|<<|<>|<=>|>>" - "|[^[:space:]]"), + "|<<|<>|<=>|>>"), PATTERNS("php", "^[\t ]*(((public|protected|private|static)[\t ]+)*function.*)$\n" "^[\t ]*(class.*)$", /* -- */ "[a-zA-Z_][a-zA-Z0-9_]*" "|[-+0-9.e]+|0[xXbB]?[0-9a-fA-F]+" - "|[-+*/<>%&^|=!.]=|--|\\+\\+|<<=?|>>=?|===|&&|\\|\\||::|->" - "|[^[:space:]]|[\x80-\xff]+"), + "|[-+*/<>%&^|=!.]=|--|\\+\\+|<<=?|>>=?|===|&&|\\|\\||::|->"), PATTERNS("python", "^[ \t]*((class|def)[ \t].*)$", /* -- */ "[a-zA-Z_][a-zA-Z0-9_]*" "|[-+0-9.e]+[jJlL]?|0[xX]?[0-9a-fA-F]+[lL]?" - "|[-+*/<>%&^|=!]=|//=?|<<=?|>>=?|\\*\\*=?" - "|[^[:space:]]|[\x80-\xff]+"), + "|[-+*/<>%&^|=!]=|//=?|<<=?|>>=?|\\*\\*=?"), /* -- */ PATTERNS("ruby", "^[ \t]*((class|module|def)[ \t].*)$", /* -- */ "(@|@@|\\$)?[a-zA-Z_][a-zA-Z0-9_]*" "|[-+0-9.e]+|0[xXbB]?[0-9a-fA-F]+|\\?(\\\\C-)?(\\\\M-)?." - "|//=?|[-+*/<>%&^|=!]=|<<=?|>>=?|===|\\.{1,3}|::|[!=]~" - "|[^[:space:]]|[\x80-\xff]+"), + "|//=?|[-+*/<>%&^|=!]=|<<=?|>>=?|===|\\.{1,3}|::|[!=]~"), PATTERNS("bibtex", "(@[a-zA-Z]{1,}[ \t]*\\{{0,1}[ \t]*[^ \t\"@',\\#}{~%]*).*$", "[={}\"]|[^={}\" \t]+"), PATTERNS("tex", "^(\\\\((sub)*section|chapter|part)\\*{0,1}\\{.*)$", - "\\\\[a-zA-Z@]+|\\\\.|[a-zA-Z0-9\x80-\xff]+|[^[:space:]]"), + "\\\\[a-zA-Z@]+|\\\\.|[a-zA-Z0-9\x80-\xff]+"), PATTERNS("cpp", /* Jump targets or access declarations */ "!^[ \t]*[A-Za-z_][A-Za-z_0-9]*:.*$\n" @@ -113,8 +107,7 @@ PATTERNS("cpp", /* -- */ "[a-zA-Z_][a-zA-Z0-9_]*" "|[-+0-9.e]+[fFlL]?|0[xXbB]?[0-9a-fA-F]+[lL]?" - "|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->" - "|[^[:space:]]|[\x80-\xff]+"), + "|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->"), PATTERNS("csharp", /* Keywords */ "!^[ \t]*(do|while|for|if|else|instanceof|new|return|switch|case|throw|catch|using)\n" @@ -129,8 +122,7 @@ PATTERNS("csharp", /* -- */ "[a-zA-Z_][a-zA-Z0-9_]*" "|[-+0-9.e]+[fFlL]?|0[xXbB]?[0-9a-fA-F]+[lL]?" - "|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->" - "|[^[:space:]]|[\x80-\xff]+"), + "|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->"), { "default", NULL, -1, { NULL, 0 } }, }; #undef PATTERNS