diff --git a/chibicc.h b/chibicc.h index 75e5fdc25d0f5eaaac0263d07ff7cee843a6d0a7..d9dc41bbb3093f8d8665cafd1af73526507d25d4 100644 --- a/chibicc.h +++ b/chibicc.h @@ -46,6 +46,7 @@ typedef enum { TK_IDENT, // Identifiers TK_STR, // String literals TK_NUM, // Numeric literals + TK_PP_NUM, // Preprocessing numbers TK_EOF, // End-of-file markers } TokenKind; @@ -82,7 +83,7 @@ void warn_tok(Token *tok, char *fmt, ...); bool equal(Token *tok, char *op); Token *skip(Token *tok, char *op); bool consume(Token **rest, Token *tok, char *str); -void convert_keywords(Token *tok); +void convert_pp_tokens(Token *tok); File **get_input_files(void); File *new_file(char *name, int file_no, char *contents); Token *tokenize(File *file); diff --git a/preprocess.c b/preprocess.c index 9e508d01ced5d57142aa4717b5ae509898efd764..b0401c903fcd04e2b10743650df6d1c0c2db5f81 100644 --- a/preprocess.c +++ b/preprocess.c @@ -309,6 +309,9 @@ static long eval_const_expr(Token **rest, Token *tok) { } } + // Convert pp-numbers to regular numbers + convert_pp_tokens(expr); + Token *rest2; long val = const_expr(&rest2, expr); if (rest2->kind != TK_EOF) @@ -952,7 +955,7 @@ Token *preprocess(Token *tok) { tok = preprocess2(tok); if (cond_incl) error_tok(cond_incl->tok, "unterminated conditional directive"); - convert_keywords(tok); + convert_pp_tokens(tok); join_adjacent_string_literals(tok); return tok; } diff --git a/test/macro.c b/test/macro.c index 73aab66c9f762bfd0f8ae0c4225b0a72232bd094..9bb20d2083f504b0c655a5222f22a60f0827b7e9 100644 --- a/test/macro.c +++ b/test/macro.c @@ -362,6 +362,10 @@ int main() { #define M14(x, ...) x ASSERT(5, M14(5)); +#define CONCAT(x,y) x##y + ASSERT(5, ({ int f0zz=5; CONCAT(f,0zz); })); + ASSERT(5, ({ CONCAT(4,.57) + 0.5; })); + printf("OK\n"); return 0; } diff --git a/tokenize.c b/tokenize.c index 760235b569375265f43c931f3f8ad57ee6fd8259..71dc14698411b58e660ecaeb809aa149eb1addc4 100644 --- a/tokenize.c +++ b/tokenize.c @@ -242,8 +242,8 @@ static Token *read_char_literal(Token *cur, char *start) { return tok; } -static Token *read_int_literal(Token *cur, char *start) { - char *p = start; +static bool convert_pp_int(Token *tok) { + char *p = tok->loc; // Read a binary, octal, decimal or hexadecimal number. int base = 10; @@ -283,6 +283,9 @@ static Token *read_int_literal(Token *cur, char *start) { u = true; } + if (p != tok->loc + tok->len) + return false; + // Infer a type. Type *ty; if (base == 10) { @@ -311,21 +314,27 @@ static Token *read_int_literal(Token *cur, char *start) { ty = ty_int; } - Token *tok = new_token(TK_NUM, cur, start, p - start); + tok->kind = TK_NUM; tok->val = val; tok->ty = ty; - return tok; + return true; } -static Token *read_number(Token *cur, char *start) { +// The definition of the numeric literal at the preprocessing stage +// is more relaxed than the definition of that at the later stages. +// In order to handle that, a numeric literal is tokenized as a +// "pp-number" token first and then converted to a regular number +// token after preprocessing. +// +// This function converts a pp-number token to a regular number token. +static void convert_pp_number(Token *tok) { // Try to parse as an integer constant. - Token *tok = read_int_literal(cur, start); - if (!strchr(".eEfF", start[tok->len])) - return tok; + if (convert_pp_int(tok)) + return; // If it's not an integer, it must be a floating point constant. char *end; - double val = strtod(start, &end); + double val = strtod(tok->loc, &end); Type *ty; if (*end == 'f' || *end == 'F') { @@ -338,16 +347,21 @@ static Token *read_number(Token *cur, char *start) { ty = ty_double; } - tok = new_token(TK_NUM, cur, start, end - start); + if (tok->loc + tok->len != end) + error_tok(tok, "invalid numeric constant"); + + tok->kind = TK_NUM; tok->fval = val; tok->ty = ty; - return tok; } -void convert_keywords(Token *tok) { - for (Token *t = tok; t->kind != TK_EOF; t = t->next) +void convert_pp_tokens(Token *tok) { + for (Token *t = tok; t->kind != TK_EOF; t = t->next) { if (is_keyword(t)) t->kind = TK_RESERVED; + else if (t->kind == TK_PP_NUM) + convert_pp_number(t); + } } // Initialize line info for all tokens. @@ -413,8 +427,16 @@ Token *tokenize(File *file) { // Numeric literal if (isdigit(*p) || (*p == '.' && isdigit(p[1]))) { - cur = read_number(cur, p); - p += cur->len; + char *q = p++; + for (;;) { + if (p[0] && p[1] && strchr("eEpP", p[0]) && strchr("+-", p[1])) + p += 2; + else if (isalnum(*p) || *p == '.') + p++; + else + break; + } + cur = new_token(TK_PP_NUM, cur, q, p - q); continue; }