src/dom/js/jsscan.c

   1 /* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
   2  * vim: set sw=4 ts=8 et tw=78:
   3  *
   4  * ***** BEGIN LICENSE BLOCK *****
   5  * Version: MPL 1.1/GPL 2.0/LGPL 2.1
   6  *
   7  * The contents of this file are subject to the Mozilla Public License Version
   8  * 1.1 (the "License"); you may not use this file except in compliance with
   9  * the License. You may obtain a copy of the License at
  10  * http://www.mozilla.org/MPL/
  11  *
  12  * Software distributed under the License is distributed on an "AS IS" basis,
  13  * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  14  * for the specific language governing rights and limitations under the
  15  * License.
  16  *
  17  * The Original Code is Mozilla Communicator client code, released
  18  * March 31, 1998.
  19  *
  20  * The Initial Developer of the Original Code is
  21  * Netscape Communications Corporation.
  22  * Portions created by the Initial Developer are Copyright (C) 1998
  23  * the Initial Developer. All Rights Reserved.
  24  *
  25  * Contributor(s):
  26  *
  27  * Alternatively, the contents of this file may be used under the terms of
  28  * either of the GNU General Public License Version 2 or later (the "GPL"),
  29  * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  30  * in which case the provisions of the GPL or the LGPL are applicable instead
  31  * of those above. If you wish to allow use of your version of this file only
  32  * under the terms of either the GPL or the LGPL, and not to allow others to
  33  * use your version of this file under the terms of the MPL, indicate your
  34  * decision by deleting the provisions above and replace them with the notice
  35  * and other provisions required by the GPL or the LGPL. If you do not delete
  36  * the provisions above, a recipient may use your version of this file under
  37  * the terms of any one of the MPL, the GPL or the LGPL.
  38  *
  39  * ***** END LICENSE BLOCK ***** */
  40
  41 /*
  42  * JS lexical scanner.
  43  */
  44 #include "jsstddef.h"
  45 #include <stdio.h>      /* first to avoid trouble on some systems */
  46 #include <errno.h>
  47 #include <limits.h>
  48 #include <math.h>
  49 #ifdef HAVE_MEMORY_H
  50 #include <memory.h>
  51 #endif
  52 #include <stdarg.h>
  53 #include <stdlib.h>
  54 #include <string.h>
  55 #include "jstypes.h"
  56 #include "jsarena.h" /* Added by JSIFY */
  57 #include "jsutil.h" /* Added by JSIFY */
  58 #include "jsdtoa.h"
  59 #include "jsprf.h"
  60 #include "jsapi.h"
  61 #include "jsatom.h"
  62 #include "jscntxt.h"
  63 #include "jsconfig.h"
  64 #include "jsemit.h"
  65 #include "jsexn.h"
  66 #include "jsnum.h"
  67 #include "jsopcode.h"
  68 #include "jsregexp.h"
  69 #include "jsscan.h"
  70 #include "jsscript.h"
  71
  72 #if JS_HAS_XML_SUPPORT
  73 #include "jsparse.h"
  74 #include "jsxml.h"
  75 #endif
  76
  77 #define RESERVE_JAVA_KEYWORDS
  78 #define RESERVE_ECMA_KEYWORDS
  79
  80 #define MAX_KEYWORD_LENGTH      12
  81
  82 static struct keyword {
  83     const char  *name;
  84     JSTokenType tokentype;      /* JSTokenType */
  85     JSOp        op;             /* JSOp */
  86     JSVersion   version;        /* JSVersion */
  87 } keywords[] = {
  88     {"break",           TOK_BREAK,              JSOP_NOP,   JSVERSION_DEFAULT},
  89     {"case",            TOK_CASE,               JSOP_NOP,   JSVERSION_DEFAULT},
  90     {"continue",        TOK_CONTINUE,           JSOP_NOP,   JSVERSION_DEFAULT},
  91     {js_default_str,    TOK_DEFAULT,            JSOP_NOP,   JSVERSION_DEFAULT},
  92     {js_delete_str,     TOK_DELETE,             JSOP_NOP,   JSVERSION_DEFAULT},
  93     {"do",              TOK_DO,                 JSOP_NOP,   JSVERSION_DEFAULT},
  94     {"else",            TOK_ELSE,               JSOP_NOP,   JSVERSION_DEFAULT},
  95     {"export",          TOK_EXPORT,             JSOP_NOP,   JSVERSION_1_2},
  96     {js_false_str,      TOK_PRIMARY,            JSOP_FALSE, JSVERSION_DEFAULT},
  97     {"for",             TOK_FOR,                JSOP_NOP,   JSVERSION_DEFAULT},
  98     {js_function_str,   TOK_FUNCTION,           JSOP_NOP,   JSVERSION_DEFAULT},
  99     {"if",              TOK_IF,                 JSOP_NOP,   JSVERSION_DEFAULT},
 100     {js_in_str,         TOK_IN,                 JSOP_IN,    JSVERSION_DEFAULT},
 101     {js_new_str,        TOK_NEW,                JSOP_NEW,   JSVERSION_DEFAULT},
 102     {js_null_str,       TOK_PRIMARY,            JSOP_NULL,  JSVERSION_DEFAULT},
 103     {"return",          TOK_RETURN,             JSOP_NOP,   JSVERSION_DEFAULT},
 104     {"switch",          TOK_SWITCH,             JSOP_NOP,   JSVERSION_DEFAULT},
 105     {js_this_str,       TOK_PRIMARY,            JSOP_THIS,  JSVERSION_DEFAULT},
 106     {js_true_str,       TOK_PRIMARY,            JSOP_TRUE,  JSVERSION_DEFAULT},
 107     {js_typeof_str,     TOK_UNARYOP,            JSOP_TYPEOF,JSVERSION_DEFAULT},
 108     {js_var_str,        TOK_VAR,                JSOP_DEFVAR,JSVERSION_DEFAULT},
 109     {js_void_str,       TOK_UNARYOP,            JSOP_VOID,  JSVERSION_DEFAULT},
 110     {"while",           TOK_WHILE,              JSOP_NOP,   JSVERSION_DEFAULT},
 111     {"with",            TOK_WITH,               JSOP_NOP,   JSVERSION_DEFAULT},
 112
 113 #if JS_HAS_CONST
 114     {js_const_str,      TOK_VAR,                JSOP_DEFCONST,JSVERSION_DEFAULT},
 115 #else
 116     {js_const_str,      TOK_RESERVED,           JSOP_NOP,   JSVERSION_DEFAULT},
 117 #endif
 118
 119 #if JS_HAS_EXCEPTIONS
 120     {"try",             TOK_TRY,                JSOP_NOP,   JSVERSION_DEFAULT},
 121     {"catch",           TOK_CATCH,              JSOP_NOP,   JSVERSION_DEFAULT},
 122     {"finally",         TOK_FINALLY,            JSOP_NOP,   JSVERSION_DEFAULT},
 123     {"throw",           TOK_THROW,              JSOP_NOP,   JSVERSION_DEFAULT},
 124 #else
 125     {"try",             TOK_RESERVED,           JSOP_NOP,   JSVERSION_DEFAULT},
 126     {"catch",           TOK_RESERVED,           JSOP_NOP,   JSVERSION_DEFAULT},
 127     {"finally",         TOK_RESERVED,           JSOP_NOP,   JSVERSION_DEFAULT},
 128     {"throw",           TOK_RESERVED,           JSOP_NOP,   JSVERSION_DEFAULT},
 129 #endif
 130
 131 #if JS_HAS_INSTANCEOF
 132     {js_instanceof_str, TOK_INSTANCEOF,         JSOP_INSTANCEOF,JSVERSION_1_4},
 133 #else
 134     {js_instanceof_str, TOK_RESERVED,           JSOP_NOP,   JSVERSION_1_4},
 135 #endif
 136
 137 #ifdef RESERVE_JAVA_KEYWORDS
 138     {"abstract",        TOK_RESERVED,           JSOP_NOP,   JSVERSION_DEFAULT},
 139     {"boolean",         TOK_RESERVED,           JSOP_NOP,   JSVERSION_DEFAULT},
 140     {"byte",            TOK_RESERVED,           JSOP_NOP,   JSVERSION_DEFAULT},
 141     {"char",            TOK_RESERVED,           JSOP_NOP,   JSVERSION_DEFAULT},
 142     {"class",           TOK_RESERVED,           JSOP_NOP,   JSVERSION_DEFAULT},
 143     {"double",          TOK_RESERVED,           JSOP_NOP,   JSVERSION_DEFAULT},
 144     {"extends",         TOK_RESERVED,           JSOP_NOP,   JSVERSION_DEFAULT},
 145     {"final",           TOK_RESERVED,           JSOP_NOP,   JSVERSION_DEFAULT},
 146     {"float",           TOK_RESERVED,           JSOP_NOP,   JSVERSION_DEFAULT},
 147     {"goto",            TOK_RESERVED,           JSOP_NOP,   JSVERSION_DEFAULT},
 148     {"implements",      TOK_RESERVED,           JSOP_NOP,   JSVERSION_DEFAULT},
 149     {"import",          TOK_IMPORT,             JSOP_NOP,   JSVERSION_DEFAULT},
 150     {"int",             TOK_RESERVED,           JSOP_NOP,   JSVERSION_DEFAULT},
 151     {"interface",       TOK_RESERVED,           JSOP_NOP,   JSVERSION_DEFAULT},
 152     {"long",            TOK_RESERVED,           JSOP_NOP,   JSVERSION_DEFAULT},
 153     {"native",          TOK_RESERVED,           JSOP_NOP,   JSVERSION_DEFAULT},
 154     {"package",         TOK_RESERVED,           JSOP_NOP,   JSVERSION_DEFAULT},
 155     {"private",         TOK_RESERVED,           JSOP_NOP,   JSVERSION_DEFAULT},
 156     {"protected",       TOK_RESERVED,           JSOP_NOP,   JSVERSION_DEFAULT},
 157     {"public",          TOK_RESERVED,           JSOP_NOP,   JSVERSION_DEFAULT},
 158     {"short",           TOK_RESERVED,           JSOP_NOP,   JSVERSION_DEFAULT},
 159     {"static",          TOK_RESERVED,           JSOP_NOP,   JSVERSION_DEFAULT},
 160     {"super",           TOK_RESERVED,           JSOP_NOP,   JSVERSION_DEFAULT},
 161     {"synchronized",    TOK_RESERVED,           JSOP_NOP,   JSVERSION_DEFAULT},
 162     {"throws",          TOK_RESERVED,           JSOP_NOP,   JSVERSION_DEFAULT},
 163     {"transient",       TOK_RESERVED,           JSOP_NOP,   JSVERSION_DEFAULT},
 164     {"volatile",        TOK_RESERVED,           JSOP_NOP,   JSVERSION_DEFAULT},
 165 #endif
 166
 167 #ifdef RESERVE_ECMA_KEYWORDS
 168     {"enum",           TOK_RESERVED,            JSOP_NOP,   JSVERSION_1_3},
 169 #endif
 170
 171 #if JS_HAS_DEBUGGER_KEYWORD
 172     {"debugger",       TOK_DEBUGGER,            JSOP_NOP,   JSVERSION_1_3},
 173 #elif defined(RESERVE_ECMA_KEYWORDS)
 174     {"debugger",       TOK_RESERVED,            JSOP_NOP,   JSVERSION_1_3},
 175 #endif
 176     {0,                TOK_EOF,                 JSOP_NOP,   JSVERSION_DEFAULT}
 177 };
 178
 179 JSBool
 180 js_InitScanner(JSContext *cx)
 181 {
 182     struct keyword *kw;
 183     size_t length;
 184     JSAtom *atom;
 185
 186     for (kw = keywords; kw->name; kw++) {
 187         length = strlen(kw->name);
 188         JS_ASSERT(length <= MAX_KEYWORD_LENGTH);
 189         atom = js_Atomize(cx, kw->name, length, ATOM_PINNED);
 190         if (!atom)
 191             return JS_FALSE;
 192         ATOM_SET_KEYWORD(atom, kw);
 193     }
 194     return JS_TRUE;
 195 }
 196
 197 JS_FRIEND_API(void)
 198 js_MapKeywords(void (*mapfun)(const char *))
 199 {
 200     struct keyword *kw;
 201
 202     for (kw = keywords; kw->name; kw++)
 203         mapfun(kw->name);
 204 }
 205
 206 JSTokenStream *
 207 js_NewTokenStream(JSContext *cx, const jschar *base, size_t length,
 208                   const char *filename, uintN lineno,
 209                   JSPrincipals *principals)
 210 {
 211     JSTokenStream *ts;
 212
 213     ts = js_NewBufferTokenStream(cx, base, length);
 214     if (!ts)
 215         return NULL;
 216     ts->filename = filename;
 217     ts->lineno = lineno;
 218     if (principals)
 219         JSPRINCIPALS_HOLD(cx, principals);
 220     ts->principals = principals;
 221     return ts;
 222 }
 223
 224 #define TBMIN   64
 225
 226 static JSBool
 227 GrowTokenBuf(JSStringBuffer *sb, size_t newlength)
 228 {
 229     JSContext *cx;
 230     jschar *base;
 231     ptrdiff_t offset, length;
 232     size_t tbsize;
 233     JSArenaPool *pool;
 234
 235     cx = sb->data;
 236     base = sb->base;
 237     offset = PTRDIFF(sb->ptr, base, jschar);
 238     pool = &cx->tempPool;
 239     if (!base) {
 240         tbsize = TBMIN * sizeof(jschar);
 241         length = TBMIN - 1;
 242         JS_ARENA_ALLOCATE_CAST(base, jschar *, pool, tbsize);
 243     } else {
 244         length = PTRDIFF(sb->limit, base, jschar);
 245         tbsize = (length + 1) * sizeof(jschar);
 246         length += length + 1;
 247         JS_ARENA_GROW_CAST(base, jschar *, pool, tbsize, tbsize);
 248     }
 249     if (!base) {
 250         JS_ReportOutOfMemory(cx);
 251         sb->base = STRING_BUFFER_ERROR_BASE;
 252         return JS_FALSE;
 253     }
 254     sb->base = base;
 255     sb->limit = base + length;
 256     sb->ptr = base + offset;
 257     return JS_TRUE;
 258 }
 259
 260 JS_FRIEND_API(JSTokenStream *)
 261 js_NewBufferTokenStream(JSContext *cx, const jschar *base, size_t length)
 262 {
 263     size_t nb;
 264     JSTokenStream *ts;
 265
 266     nb = sizeof(JSTokenStream) + JS_LINE_LIMIT * sizeof(jschar);
 267     JS_ARENA_ALLOCATE_CAST(ts, JSTokenStream *, &cx->tempPool, nb);
 268     if (!ts) {
 269         JS_ReportOutOfMemory(cx);
 270         return NULL;
 271     }
 272     memset(ts, 0, nb);
 273     ts->lineno = 1;
 274     ts->linebuf.base = ts->linebuf.limit = ts->linebuf.ptr = (jschar *)(ts + 1);
 275     ts->userbuf.base = (jschar *)base;
 276     ts->userbuf.limit = (jschar *)base + length;
 277     ts->userbuf.ptr = (jschar *)base;
 278     ts->tokenbuf.grow = GrowTokenBuf;
 279     ts->tokenbuf.data = cx;
 280     ts->listener = cx->runtime->sourceHandler;
 281     ts->listenerData = cx->runtime->sourceHandlerData;
 282     return ts;
 283 }
 284
 285 JS_FRIEND_API(JSTokenStream *)
 286 js_NewFileTokenStream(JSContext *cx, const char *filename, FILE *defaultfp)
 287 {
 288     jschar *base;
 289     JSTokenStream *ts;
 290     FILE *file;
 291
 292     JS_ARENA_ALLOCATE_CAST(base, jschar *, &cx->tempPool,
 293                            JS_LINE_LIMIT * sizeof(jschar));
 294     if (!base)
 295         return NULL;
 296     ts = js_NewBufferTokenStream(cx, base, JS_LINE_LIMIT);
 297     if (!ts)
 298         return NULL;
 299     if (!filename || strcmp(filename, "-") == 0) {
 300         file = defaultfp;
 301     } else {
 302         file = fopen(filename, "r");
 303         if (!file) {
 304             JS_ReportErrorNumber(cx, js_GetErrorMessage, NULL, JSMSG_CANT_OPEN,
 305                                  filename, "No such file or directory");
 306             return NULL;
 307         }
 308     }
 309     ts->userbuf.ptr = ts->userbuf.limit;
 310     ts->file = file;
 311     ts->filename = filename;
 312     return ts;
 313 }
 314
 315 JS_FRIEND_API(JSBool)
 316 js_CloseTokenStream(JSContext *cx, JSTokenStream *ts)
 317 {
 318     if (ts->flags & TSF_OWNFILENAME)
 319         JS_free(cx, (void *) ts->filename);
 320     if (ts->principals)
 321         JSPRINCIPALS_DROP(cx, ts->principals);
 322     return !ts->file || fclose(ts->file) == 0;
 323 }
 324
 325 JS_FRIEND_API(int)
 326 js_fgets(char *buf, int size, FILE *file)
 327 {
 328     int n, i, c;
 329     JSBool crflag;
 330
 331     n = size - 1;
 332     if (n < 0)
 333         return -1;
 334
 335     crflag = JS_FALSE;
 336     for (i = 0; i < n && (c = getc(file)) != EOF; i++) {
 337         buf[i] = c;
 338         if (c == '\n') {        /* any \n ends a line */
 339             i++;                /* keep the \n; we know there is room for \0 */
 340             break;
 341         }
 342         if (crflag) {           /* \r not followed by \n ends line at the \r */
 343             ungetc(c, file);
 344             break;              /* and overwrite c in buf with \0 */
 345         }
 346         crflag = (c == '\r');
 347     }
 348
 349     buf[i] = '\0';
 350     return i;
 351 }
 352
 353 static int32
 354 GetChar(JSTokenStream *ts)
 355 {
 356     int32 c;
 357     ptrdiff_t i, j, len, olen;
 358     JSBool crflag;
 359     char cbuf[JS_LINE_LIMIT];
 360     jschar *ubuf, *nl;
 361
 362     if (ts->ungetpos != 0) {
 363         c = ts->ungetbuf[--ts->ungetpos];
 364     } else {
 365         do {
 366             if (ts->linebuf.ptr == ts->linebuf.limit) {
 367                 len = PTRDIFF(ts->userbuf.limit, ts->userbuf.ptr, jschar);
 368                 if (len <= 0) {
 369                     if (!ts->file) {
 370                         ts->flags |= TSF_EOF;
 371                         return EOF;
 372                     }
 373
 374                     /* Fill ts->userbuf so that \r and \r\n convert to \n. */
 375                     crflag = (ts->flags & TSF_CRFLAG) != 0;
 376                     len = js_fgets(cbuf, JS_LINE_LIMIT - crflag, ts->file);
 377                     if (len <= 0) {
 378                         ts->flags |= TSF_EOF;
 379                         return EOF;
 380                     }
 381                     olen = len;
 382                     ubuf = ts->userbuf.base;
 383                     i = 0;
 384                     if (crflag) {
 385                         ts->flags &= ~TSF_CRFLAG;
 386                         if (cbuf[0] != '\n') {
 387                             ubuf[i++] = '\n';
 388                             len++;
 389                             ts->linepos--;
 390                         }
 391                     }
 392                     for (j = 0; i < len; i++, j++)
 393                         ubuf[i] = (jschar) (unsigned char) cbuf[j];
 394                     ts->userbuf.limit = ubuf + len;
 395                     ts->userbuf.ptr = ubuf;
 396                 }
 397                 if (ts->listener) {
 398                     ts->listener(ts->filename, ts->lineno, ts->userbuf.ptr, len,
 399                                  &ts->listenerTSData, ts->listenerData);
 400                 }
 401
 402                 nl = ts->saveEOL;
 403                 if (!nl) {
 404                     /*
 405                      * Any one of \n, \r, or \r\n ends a line (the longest
 406                      * match wins).  Also allow the Unicode line and paragraph
 407                      * separators.
 408                      */
 409                     for (nl = ts->userbuf.ptr; nl < ts->userbuf.limit; nl++) {
 410                         /*
 411                          * Try to prevent value-testing on most characters by
 412                          * filtering out characters that aren't 000x or 202x.
 413                          */
 414                         if ((*nl & 0xDFD0) == 0) {
 415                             if (*nl == '\n')
 416                                 break;
 417                             if (*nl == '\r') {
 418                                 if (nl + 1 < ts->userbuf.limit && nl[1] == '\n')
 419                                     nl++;
 420                                 break;
 421                             }
 422                             if (*nl == LINE_SEPARATOR || *nl == PARA_SEPARATOR)
 423                                 break;
 424                         }
 425                     }
 426                 }
 427
 428                 /*
 429                  * If there was a line terminator, copy thru it into linebuf.
 430                  * Else copy JS_LINE_LIMIT-1 bytes into linebuf.
 431                  */
 432                 if (nl < ts->userbuf.limit)
 433                     len = PTRDIFF(nl, ts->userbuf.ptr, jschar) + 1;
 434                 if (len >= JS_LINE_LIMIT) {
 435                     len = JS_LINE_LIMIT - 1;
 436                     ts->saveEOL = nl;
 437                 } else {
 438                     ts->saveEOL = NULL;
 439                 }
 440                 js_strncpy(ts->linebuf.base, ts->userbuf.ptr, len);
 441                 ts->userbuf.ptr += len;
 442                 olen = len;
 443
 444                 /*
 445                  * Make sure linebuf contains \n for EOL (don't do this in
 446                  * userbuf because the user's string might be readonly).
 447                  */
 448                 if (nl < ts->userbuf.limit) {
 449                     if (*nl == '\r') {
 450                         if (ts->linebuf.base[len-1] == '\r') {
 451                             /*
 452                              * Does the line segment end in \r?  We must check
 453                              * for a \n at the front of the next segment before
 454                              * storing a \n into linebuf.  This case matters
 455                              * only when we're reading from a file.
 456                              */
 457                             if (nl + 1 == ts->userbuf.limit && ts->file) {
 458                                 len--;
 459                                 ts->flags |= TSF_CRFLAG; /* clear NLFLAG? */
 460                                 if (len == 0) {
 461                                     /*
 462                                      * This can happen when a segment ends in
 463                                      * \r\r.  Start over.  ptr == limit in this
 464                                      * case, so we'll fall into buffer-filling
 465                                      * code.
 466                                      */
 467                                     return GetChar(ts);
 468                                 }
 469                             } else {
 470                                 ts->linebuf.base[len-1] = '\n';
 471                             }
 472                         }
 473                     } else if (*nl == '\n') {
 474                         if (nl > ts->userbuf.base &&
 475                             nl[-1] == '\r' &&
 476                             ts->linebuf.base[len-2] == '\r') {
 477                             len--;
 478                             JS_ASSERT(ts->linebuf.base[len] == '\n');
 479                             ts->linebuf.base[len-1] = '\n';
 480                         }
 481                     } else if (*nl == LINE_SEPARATOR || *nl == PARA_SEPARATOR) {
 482                         ts->linebuf.base[len-1] = '\n';
 483                     }
 484                 }
 485
 486                 /* Reset linebuf based on adjusted segment length. */
 487                 ts->linebuf.limit = ts->linebuf.base + len;
 488                 ts->linebuf.ptr = ts->linebuf.base;
 489
 490                 /* Update position of linebuf within physical userbuf line. */
 491                 if (!(ts->flags & TSF_NLFLAG))
 492                     ts->linepos += ts->linelen;
 493                 else
 494                     ts->linepos = 0;
 495                 if (ts->linebuf.limit[-1] == '\n')
 496                     ts->flags |= TSF_NLFLAG;
 497                 else
 498                     ts->flags &= ~TSF_NLFLAG;
 499
 500                 /* Update linelen from original segment length. */
 501                 ts->linelen = olen;
 502             }
 503             c = *ts->linebuf.ptr++;
 504         } while (JS_ISFORMAT(c));
 505     }
 506     if (c == '\n')
 507         ts->lineno++;
 508     return c;
 509 }
 510
 511 static void
 512 UngetChar(JSTokenStream *ts, int32 c)
 513 {
 514     if (c == EOF)
 515         return;
 516     JS_ASSERT(ts->ungetpos < sizeof ts->ungetbuf / sizeof ts->ungetbuf[0]);
 517     if (c == '\n')
 518         ts->lineno--;
 519     ts->ungetbuf[ts->ungetpos++] = (jschar)c;
 520 }
 521
 522 static int32
 523 PeekChar(JSTokenStream *ts)
 524 {
 525     int32 c;
 526
 527     c = GetChar(ts);
 528     UngetChar(ts, c);
 529     return c;
 530 }
 531
 532 /*
 533  * Peek n chars ahead into ts.  Return true if n chars were read, false if
 534  * there weren't enough characters in the input stream.  This function cannot
 535  * be used to peek into or past a newline.
 536  */
 537 static JSBool
 538 PeekChars(JSTokenStream *ts, intN n, jschar *cp)
 539 {
 540     intN i, j;
 541     int32 c;
 542
 543     for (i = 0; i < n; i++) {
 544         c = GetChar(ts);
 545         if (c == EOF)
 546             break;
 547         if (c == '\n') {
 548             UngetChar(ts, c);
 549             break;
 550         }
 551         cp[i] = (jschar)c;
 552     }
 553     for (j = i - 1; j >= 0; j--)
 554         UngetChar(ts, cp[j]);
 555     return i == n;
 556 }
 557
 558 static void
 559 SkipChars(JSTokenStream *ts, intN n)
 560 {
 561     while (--n >= 0)
 562         GetChar(ts);
 563 }
 564
 565 static JSBool
 566 MatchChar(JSTokenStream *ts, int32 expect)
 567 {
 568     int32 c;
 569
 570     c = GetChar(ts);
 571     if (c == expect)
 572         return JS_TRUE;
 573     UngetChar(ts, c);
 574     return JS_FALSE;
 575 }
 576
 577 static JSBool
 578 ReportCompileErrorNumber(JSContext *cx, void *handle, uintN flags,
 579                          uintN errorNumber, JSErrorReport *report,
 580                          JSBool charArgs, va_list ap)
 581 {
 582     JSString *linestr = NULL;
 583     JSTokenStream *ts = NULL;
 584     JSCodeGenerator *cg = NULL;
 585 #if JS_HAS_XML_SUPPORT
 586     JSParseNode *pn = NULL;
 587 #endif
 588     JSErrorReporter onError;
 589     JSTokenPos *tp;
 590     JSStackFrame *fp;
 591     uintN index;
 592     char *message;
 593     JSBool warning;
 594
 595     memset(report, 0, sizeof (struct JSErrorReport));
 596     report->flags = flags;
 597     report->errorNumber = errorNumber;
 598     message = NULL;
 599
 600     if (!js_ExpandErrorArguments(cx, js_GetErrorMessage, NULL,
 601                                  errorNumber, &message, report, &warning,
 602                                  charArgs, ap)) {
 603         return JS_FALSE;
 604     }
 605
 606     js_AddRoot(cx, &linestr, "error line buffer");
 607
 608     switch (flags & JSREPORT_HANDLE) {
 609       case JSREPORT_TS:
 610         ts = handle;
 611         break;
 612       case JSREPORT_CG:
 613         cg = handle;
 614         break;
 615 #if JS_HAS_XML_SUPPORT
 616       case JSREPORT_PN:
 617         pn = handle;
 618         ts = pn->pn_ts;
 619         break;
 620 #endif
 621     }
 622
 623     JS_ASSERT(!ts || ts->linebuf.limit < ts->linebuf.base + JS_LINE_LIMIT);
 624     onError = cx->errorReporter;
 625     if (onError) {
 626         /*
 627          * We are typically called with non-null ts and null cg from jsparse.c.
 628          * We can be called with null ts from the regexp compilation functions.
 629          * The code generator (jsemit.c) may pass null ts and non-null cg.
 630          */
 631         do {
 632             if (ts) {
 633                 report->filename = ts->filename;
 634 #if JS_HAS_XML_SUPPORT
 635                 if (pn) {
 636                     report->lineno = pn->pn_pos.begin.lineno;
 637                     if (report->lineno != ts->lineno)
 638                         break;
 639                 }
 640 #endif
 641                 report->lineno = ts->lineno;
 642                 linestr = js_NewStringCopyN(cx, ts->linebuf.base,
 643                                             PTRDIFF(ts->linebuf.limit,
 644                                                     ts->linebuf.base,
 645                                                     jschar),
 646                                             0);
 647                 report->linebuf = linestr
 648                                  ? JS_GetStringBytes(linestr)
 649                                  : NULL;
 650                 tp = &ts->tokens[(ts->cursor+ts->lookahead) & NTOKENS_MASK].pos;
 651 #if JS_HAS_XML_SUPPORT
 652                 if (pn)
 653                     tp = &pn->pn_pos;
 654 #endif
 655                 /*
 656                  * FIXME: What should instead happen here is that we should
 657                  * find error-tokens in userbuf, if !ts->file.  That will
 658                  * allow us to deliver a more helpful error message, which
 659                  * includes all or part of the bad string or bad token.  The
 660                  * code here yields something that looks truncated.
 661                  * See https://bugzilla.mozilla.org/show_bug.cgi?id=352970
 662                  */
 663                 index = 0;
 664                 if (tp->begin.lineno == tp->end.lineno) {
 665                     if (tp->begin.index < ts->linepos)
 666                         break;
 667
 668                     index = tp->begin.index - ts->linepos;
 669                 }
 670
 671                 report->tokenptr = linestr ? report->linebuf + index : NULL;
 672                 report->uclinebuf = linestr ? JS_GetStringChars(linestr) : NULL;
 673                 report->uctokenptr = linestr ? report->uclinebuf + index : NULL;
 674                 break;
 675             }
 676
 677             if (cg) {
 678                 report->filename = cg->filename;
 679                 report->lineno = CG_CURRENT_LINE(cg);
 680                 break;
 681             }
 682
 683             /*
 684              * If we can't find out where the error was based on the current frame,
 685              * see if the next frame has a script/pc combo we can use.
 686              */
 687             for (fp = cx->fp; fp; fp = fp->down) {
 688                 if (fp->script && fp->pc) {
 689                     report->filename = fp->script->filename;
 690                     report->lineno = js_PCToLineNumber(cx, fp->script, fp->pc);
 691                     break;
 692                 }
 693             }
 694         } while (0);
 695
 696 #if JS_HAS_ERROR_EXCEPTIONS
 697         /*
 698          * If there's a runtime exception type associated with this error
 699          * number, set that as the pending exception.  For errors occuring at
 700          * compile time, this is very likely to be a JSEXN_SYNTAXERR.
 701          *
 702          * If an exception is thrown but not caught, the JSREPORT_EXCEPTION
 703          * flag will be set in report.flags.  Proper behavior for an error
 704          * reporter is to ignore a report with this flag for all but top-level
 705          * compilation errors.  The exception will remain pending, and so long
 706          * as the non-top-level "load", "eval", or "compile" native function
 707          * returns false, the top-level reporter will eventually receive the
 708          * uncaught exception report.
 709          *
 710          * XXX it'd probably be best if there was only one call to this
 711          * function, but there seem to be two error reporter call points.
 712          */
 713
 714         /*
 715          * Try to raise an exception only if there isn't one already set --
 716          * otherwise the exception will describe the last compile-time error,
 717          * which is likely spurious.
 718          */
 719         if (!ts || !(ts->flags & TSF_ERROR)) {
 720             if (js_ErrorToException(cx, message, report))
 721                 onError = NULL;
 722         }
 723
 724         /*
 725          * Suppress any compile-time errors that don't occur at the top level.
 726          * This may still fail, as interplevel may be zero in contexts where we
 727          * don't really want to call the error reporter, as when js is called
 728          * by other code which could catch the error.
 729          */
 730         if (cx->interpLevel != 0 && !JSREPORT_IS_WARNING(flags))
 731             onError = NULL;
 732 #endif
 733         if (onError) {
 734             JSDebugErrorHook hook = cx->runtime->debugErrorHook;
 735
 736             /*
 737              * If debugErrorHook is present then we give it a chance to veto
 738              * sending the error on to the regular error reporter.
 739              */
 740             if (hook && !hook(cx, message, report,
 741                               cx->runtime->debugErrorHookData)) {
 742                 onError = NULL;
 743             }
 744         }
 745         if (onError)
 746             (*onError)(cx, message, report);
 747     }
 748
 749     if (message)
 750         JS_free(cx, message);
 751     if (report->ucmessage)
 752         JS_free(cx, (void *)report->ucmessage);
 753
 754     js_RemoveRoot(cx->runtime, &linestr);
 755
 756     if (ts && !JSREPORT_IS_WARNING(flags)) {
 757         /* Set the error flag to suppress spurious reports. */
 758         ts->flags |= TSF_ERROR;
 759     }
 760
 761     return warning;
 762 }
 763
 764 JSBool
 765 js_ReportCompileErrorNumber(JSContext *cx, void *handle, uintN flags,
 766                             uintN errorNumber, ...)
 767 {
 768     va_list ap;
 769     JSErrorReport report;
 770     JSBool warning;
 771
 772     if ((flags & JSREPORT_STRICT) && !JS_HAS_STRICT_OPTION(cx))
 773         return JS_TRUE;
 774
 775     va_start(ap, errorNumber);
 776     warning = ReportCompileErrorNumber(cx, handle, flags, errorNumber,
 777                                        &report, JS_TRUE, ap);
 778     va_end(ap);
 779
 780     /*
 781      * We have to do this here because js_ReportCompileErrorNumberUC doesn't
 782      * need to do this.
 783      */
 784     if (report.messageArgs) {
 785         int i = 0;
 786         while (report.messageArgs[i])
 787             JS_free(cx, (void *)report.messageArgs[i++]);
 788         JS_free(cx, (void *)report.messageArgs);
 789     }
 790
 791     return warning;
 792 }
 793
 794 JSBool
 795 js_ReportCompileErrorNumberUC(JSContext *cx, void *handle, uintN flags,
 796                               uintN errorNumber, ...)
 797 {
 798     va_list ap;
 799     JSErrorReport report;
 800     JSBool warning;
 801
 802     if ((flags & JSREPORT_STRICT) && !JS_HAS_STRICT_OPTION(cx))
 803         return JS_TRUE;
 804
 805     va_start(ap, errorNumber);
 806     warning = ReportCompileErrorNumber(cx, handle, flags, errorNumber,
 807                                        &report, JS_FALSE, ap);
 808     va_end(ap);
 809
 810     if (report.messageArgs)
 811         JS_free(cx, (void *)report.messageArgs);
 812
 813     return warning;
 814 }
 815
 816 static JSBool
 817 GrowStringBuffer(JSStringBuffer *sb, size_t newlength)
 818 {
 819     ptrdiff_t offset;
 820     jschar *bp;
 821
 822     offset = PTRDIFF(sb->ptr, sb->base, jschar);
 823     JS_ASSERT(offset >= 0);
 824     newlength += offset + 1;
 825     if ((size_t)offset < newlength && newlength < ~(size_t)0 / sizeof(jschar))
 826         bp = realloc(sb->base, newlength * sizeof(jschar));
 827     else
 828         bp = NULL;
 829     if (!bp) {
 830         free(sb->base);
 831         sb->base = STRING_BUFFER_ERROR_BASE;
 832         return JS_FALSE;
 833     }
 834     sb->base = bp;
 835     sb->ptr = bp + offset;
 836     sb->limit = bp + newlength - 1;
 837     return JS_TRUE;
 838 }
 839
 840 static void
 841 FreeStringBuffer(JSStringBuffer *sb)
 842 {
 843     JS_ASSERT(STRING_BUFFER_OK(sb));
 844     if (sb->base)
 845         free(sb->base);
 846 }
 847
 848 void
 849 js_InitStringBuffer(JSStringBuffer *sb)
 850 {
 851     sb->base = sb->limit = sb->ptr = NULL;
 852     sb->data = NULL;
 853     sb->grow = GrowStringBuffer;
 854     sb->free = FreeStringBuffer;
 855 }
 856
 857 void
 858 js_FinishStringBuffer(JSStringBuffer *sb)
 859 {
 860     sb->free(sb);
 861 }
 862
 863 #define ENSURE_STRING_BUFFER(sb,n) \
 864     ((sb)->ptr + (n) <= (sb)->limit || sb->grow(sb, n))
 865
 866 static void
 867 FastAppendChar(JSStringBuffer *sb, jschar c)
 868 {
 869     if (!STRING_BUFFER_OK(sb))
 870         return;
 871     if (!ENSURE_STRING_BUFFER(sb, 1))
 872         return;
 873     *sb->ptr++ = c;
 874 }
 875
 876 void
 877 js_AppendChar(JSStringBuffer *sb, jschar c)
 878 {
 879     jschar *bp;
 880
 881     if (!STRING_BUFFER_OK(sb))
 882         return;
 883     if (!ENSURE_STRING_BUFFER(sb, 1))
 884         return;
 885     bp = sb->ptr;
 886     *bp++ = c;
 887     *bp = 0;
 888     sb->ptr = bp;
 889 }
 890
 891 #if JS_HAS_XML_SUPPORT
 892
 893 void
 894 js_RepeatChar(JSStringBuffer *sb, jschar c, uintN count)
 895 {
 896     jschar *bp;
 897
 898     if (!STRING_BUFFER_OK(sb) || count == 0)
 899         return;
 900     if (!ENSURE_STRING_BUFFER(sb, count))
 901         return;
 902     for (bp = sb->ptr; count; --count)
 903         *bp++ = c;
 904     *bp = 0;
 905     sb->ptr = bp;
 906 }
 907
 908 void
 909 js_AppendCString(JSStringBuffer *sb, const char *asciiz)
 910 {
 911     size_t length;
 912     jschar *bp;
 913
 914     if (!STRING_BUFFER_OK(sb) || *asciiz == '\0')
 915         return;
 916     length = strlen(asciiz);
 917     if (!ENSURE_STRING_BUFFER(sb, length))
 918         return;
 919     for (bp = sb->ptr; length; --length)
 920         *bp++ = (jschar) *asciiz++;
 921     *bp = 0;
 922     sb->ptr = bp;
 923 }
 924
 925 void
 926 js_AppendJSString(JSStringBuffer *sb, JSString *str)
 927 {
 928     size_t length;
 929     jschar *bp;
 930
 931     if (!STRING_BUFFER_OK(sb))
 932         return;
 933     length = JSSTRING_LENGTH(str);
 934     if (length == 0 || !ENSURE_STRING_BUFFER(sb, length))
 935         return;
 936     bp = sb->ptr;
 937     js_strncpy(bp, JSSTRING_CHARS(str), length);
 938     bp += length;
 939     *bp = 0;
 940     sb->ptr = bp;
 941 }
 942
 943 static JSBool
 944 GetXMLEntity(JSContext *cx, JSTokenStream *ts)
 945 {
 946     ptrdiff_t offset, length, i;
 947     int32 c, d;
 948     JSBool ispair;
 949     jschar *bp, digit;
 950     char *bytes;
 951     JSErrNum msg;
 952
 953     /* Put the entity, including the '&' already scanned, in ts->tokenbuf. */
 954     offset = PTRDIFF(ts->tokenbuf.ptr, ts->tokenbuf.base, jschar);
 955     FastAppendChar(&ts->tokenbuf, '&');
 956     while ((c = GetChar(ts)) != ';') {
 957         if (c == EOF || c == '\n') {
 958             js_ReportCompileErrorNumber(cx, ts,
 959                                         JSREPORT_TS | JSREPORT_ERROR,
 960                                         JSMSG_END_OF_XML_ENTITY);
 961             return JS_FALSE;
 962         }
 963         FastAppendChar(&ts->tokenbuf, (jschar) c);
 964     }
 965
 966     /* Let length be the number of jschars after the '&', including the ';'. */
 967     length = PTRDIFF(ts->tokenbuf.ptr, ts->tokenbuf.base, jschar) - offset;
 968     bp = ts->tokenbuf.base + offset;
 969     c = d = 0;
 970     ispair = JS_FALSE;
 971     if (length > 2 && bp[1] == '#') {
 972         /* Match a well-formed XML Character Reference. */
 973         i = 2;
 974         if (length > 3 && JS_TOLOWER(bp[i]) == 'x') {
 975             if (length > 9)     /* at most 6 hex digits allowed */
 976                 goto badncr;
 977             while (++i < length) {
 978                 digit = bp[i];
 979                 if (!JS7_ISHEX(digit))
 980                     goto badncr;
 981                 c = (c << 4) + JS7_UNHEX(digit);
 982             }
 983         } else {
 984             while (i < length) {
 985                 digit = bp[i++];
 986                 if (!JS7_ISDEC(digit))
 987                     goto badncr;
 988                 c = (c * 10) + JS7_UNDEC(digit);
 989                 if (c < 0)
 990                     goto badncr;
 991             }
 992         }
 993
 994         if (0x10000 <= c && c <= 0x10FFFF) {
 995             /* Form a surrogate pair (c, d) -- c is the high surrogate. */
 996             d = 0xDC00 + (c & 0x3FF);
 997             c = 0xD7C0 + (c >> 10);
 998             ispair = JS_TRUE;
 999         } else {
1000             /* Enforce the http://www.w3.org/TR/REC-xml/#wf-Legalchar WFC. */
1001             if (c != 0x9 && c != 0xA && c != 0xD &&
1002                 !(0x20 <= c && c <= 0xD7FF) &&
1003                 !(0xE000 <= c && c <= 0xFFFD)) {
1004                 goto badncr;
1005             }
1006         }
1007     } else {
1008         /* Try to match one of the five XML 1.0 predefined entities. */
1009         switch (length) {
1010           case 3:
1011             if (bp[2] == 't') {
1012                 if (bp[1] == 'l')
1013                     c = '<';
1014                 else if (bp[1] == 'g')
1015                     c = '>';
1016             }
1017             break;
1018           case 4:
1019             if (bp[1] == 'a' && bp[2] == 'm' && bp[3] == 'p')
1020                 c = '&';
1021             break;
1022           case 5:
1023             if (bp[3] == 'o') {
1024                 if (bp[1] == 'a' && bp[2] == 'p' && bp[4] == 's')
1025                     c = '\'';
1026                 else if (bp[1] == 'q' && bp[2] == 'u' && bp[4] == 't')
1027                     c = '"';
1028             }
1029             break;
1030         }
1031         if (c == 0) {
1032             msg = JSMSG_UNKNOWN_XML_ENTITY;
1033             goto bad;
1034         }
1035     }
1036
1037     /* If we matched, retract ts->tokenbuf and store the entity's value. */
1038     *bp++ = (jschar) c;
1039     if (ispair)
1040         *bp++ = (jschar) d;
1041     *bp = 0;
1042     ts->tokenbuf.ptr = bp;
1043     return JS_TRUE;
1044
1045 badncr:
1046     msg = JSMSG_BAD_XML_NCR;
1047 bad:
1048     /* No match: throw a TypeError per ECMA-357 10.3.2.1 step 8(a). */
1049     bytes = js_DeflateString(cx, bp + 1,
1050                              PTRDIFF(ts->tokenbuf.ptr, bp, jschar) - 1);
1051     if (bytes) {
1052         js_ReportCompileErrorNumber(cx, ts, JSREPORT_TS | JSREPORT_ERROR,
1053                                     msg, bytes);
1054         JS_free(cx, bytes);
1055     }
1056     return JS_FALSE;
1057 }
1058
1059 #endif /* JS_HAS_XML_SUPPORT */
1060
1061 JSTokenType
1062 js_PeekToken(JSContext *cx, JSTokenStream *ts)
1063 {
1064     JSTokenType tt;
1065
1066     if (ts->lookahead != 0) {
1067         tt = ts->tokens[(ts->cursor + ts->lookahead) & NTOKENS_MASK].type;
1068     } else {
1069         tt = js_GetToken(cx, ts);
1070         js_UngetToken(ts);
1071     }
1072     return tt;
1073 }
1074
1075 JSTokenType
1076 js_PeekTokenSameLine(JSContext *cx, JSTokenStream *ts)
1077 {
1078     JSTokenType tt;
1079
1080     JS_ASSERT(ts->lookahead == 0 ||
1081               ON_CURRENT_LINE(ts, CURRENT_TOKEN(ts).pos) ||
1082               ts->tokens[(ts->cursor + ts->lookahead) & NTOKENS_MASK].type
1083                   == TOK_EOL);
1084     ts->flags |= TSF_NEWLINES;
1085     tt = js_PeekToken(cx, ts);
1086     ts->flags &= ~TSF_NEWLINES;
1087     return tt;
1088 }
1089
1090 /*
1091  * We have encountered a '\': check for a Unicode escape sequence after it,
1092  * returning the character code value if we found a Unicode escape sequence.
1093  * Otherwise, non-destructively return the original '\'.
1094  */
1095 static int32
1096 GetUnicodeEscape(JSTokenStream *ts)
1097 {
1098     jschar cp[5];
1099     int32 c;
1100
1101     if (PeekChars(ts, 5, cp) && cp[0] == 'u' &&
1102         JS7_ISHEX(cp[1]) && JS7_ISHEX(cp[2]) &&
1103         JS7_ISHEX(cp[3]) && JS7_ISHEX(cp[4]))
1104     {
1105         c = (((((JS7_UNHEX(cp[1]) << 4)
1106                 + JS7_UNHEX(cp[2])) << 4)
1107               + JS7_UNHEX(cp[3])) << 4)
1108             + JS7_UNHEX(cp[4]);
1109         SkipChars(ts, 5);
1110         return c;
1111     }
1112     return '\\';
1113 }
1114
1115 static JSToken *
1116 NewToken(JSTokenStream *ts, ptrdiff_t adjust)
1117 {
1118     JSToken *tp;
1119
1120     ts->cursor = (ts->cursor + 1) & NTOKENS_MASK;
1121     tp = &CURRENT_TOKEN(ts);
1122     tp->ptr = ts->linebuf.ptr + adjust;
1123     tp->pos.begin.index = ts->linepos +
1124                           PTRDIFF(tp->ptr, ts->linebuf.base, jschar) -
1125                           ts->ungetpos;
1126     tp->pos.begin.lineno = tp->pos.end.lineno = (uint16)ts->lineno;
1127     return tp;
1128 }
1129
1130 JSTokenType
1131 js_GetToken(JSContext *cx, JSTokenStream *ts)
1132 {
1133     JSTokenType tt;
1134     int32 c, qc;
1135     JSToken *tp;
1136     JSAtom *atom;
1137     JSBool hadUnicodeEscape;
1138
1139 #define INIT_TOKENBUF()     (ts->tokenbuf.ptr = ts->tokenbuf.base)
1140 #define TOKENBUF_LENGTH()   PTRDIFF(ts->tokenbuf.ptr, ts->tokenbuf.base, jschar)
1141 #define TOKENBUF_OK()       STRING_BUFFER_OK(&ts->tokenbuf)
1142 #define TOKENBUF_TO_ATOM()  (TOKENBUF_OK()                                    \
1143                              ? js_AtomizeChars(cx,                            \
1144                                                TOKENBUF_BASE(),               \
1145                                                TOKENBUF_LENGTH(),             \
1146                                                0)                             \
1147                              : NULL)
1148 #define ADD_TO_TOKENBUF(c)  FastAppendChar(&ts->tokenbuf, (jschar) (c))
1149
1150 /* The following 4 macros should only be used when TOKENBUF_OK() is true. */
1151 #define TOKENBUF_BASE()     (ts->tokenbuf.base)
1152 #define TOKENBUF_CHAR(i)    (ts->tokenbuf.base[i])
1153 #define TRIM_TOKENBUF(i)    (ts->tokenbuf.ptr = ts->tokenbuf.base + i)
1154 #define NUL_TERM_TOKENBUF() (*ts->tokenbuf.ptr = 0)
1155
1156     /* If there was a fatal error, keep returning TOK_ERROR. */
1157     if (ts->flags & TSF_ERROR)
1158         return TOK_ERROR;
1159
1160     /* Check for a pushed-back token resulting from mismatching lookahead. */
1161     while (ts->lookahead != 0) {
1162         JS_ASSERT(!(ts->flags & TSF_XMLTEXTMODE));
1163         ts->lookahead--;
1164         ts->cursor = (ts->cursor + 1) & NTOKENS_MASK;
1165         tt = CURRENT_TOKEN(ts).type;
1166         if (tt != TOK_EOL || (ts->flags & TSF_NEWLINES))
1167             return tt;
1168     }
1169
1170 #if JS_HAS_XML_SUPPORT
1171     if (ts->flags & TSF_XMLTEXTMODE) {
1172         tt = TOK_XMLSPACE;      /* veto if non-space, return TOK_XMLTEXT */
1173         tp = NewToken(ts, 0);
1174         INIT_TOKENBUF();
1175         qc = (ts->flags & TSF_XMLONLYMODE) ? '<' : '{';
1176
1177         while ((c = GetChar(ts)) != qc && c != '<' && c != EOF) {
1178             if (c == '&' && qc == '<') {
1179                 if (!GetXMLEntity(cx, ts))
1180                     goto error;
1181                 tt = TOK_XMLTEXT;
1182                 continue;
1183             }
1184
1185             if (!JS_ISXMLSPACE(c))
1186                 tt = TOK_XMLTEXT;
1187             ADD_TO_TOKENBUF(c);
1188         }
1189         UngetChar(ts, c);
1190
1191         if (TOKENBUF_LENGTH() == 0) {
1192             atom = NULL;
1193         } else {
1194             atom = TOKENBUF_TO_ATOM();
1195             if (!atom)
1196                 goto error;
1197         }
1198         tp->pos.end.lineno = (uint16)ts->lineno;
1199         tp->t_op = JSOP_STRING;
1200         tp->t_atom = atom;
1201         goto out;
1202     }
1203
1204     if (ts->flags & TSF_XMLTAGMODE) {
1205         tp = NewToken(ts, 0);
1206         c = GetChar(ts);
1207         if (JS_ISXMLSPACE(c)) {
1208             do {
1209                 c = GetChar(ts);
1210             } while (JS_ISXMLSPACE(c));
1211             UngetChar(ts, c);
1212             tt = TOK_XMLSPACE;
1213             goto out;
1214         }
1215
1216         if (c == EOF) {
1217             tt = TOK_EOF;
1218             goto out;
1219         }
1220
1221         INIT_TOKENBUF();
1222         if (JS_ISXMLNSSTART(c)) {
1223             JSBool sawColon = JS_FALSE;
1224
1225             ADD_TO_TOKENBUF(c);
1226             while ((c = GetChar(ts)) != EOF && JS_ISXMLNAME(c)) {
1227                 if (c == ':') {
1228                     int nextc;
1229
1230                     if (sawColon ||
1231                         (nextc = PeekChar(ts),
1232                          ((ts->flags & TSF_XMLONLYMODE) || nextc != '{') &&
1233                          !JS_ISXMLNAME(nextc))) {
1234                         js_ReportCompileErrorNumber(cx, ts,
1235                                                     JSREPORT_TS |
1236                                                     JSREPORT_ERROR,
1237                                                     JSMSG_BAD_XML_QNAME);
1238                         goto error;
1239                     }
1240                     sawColon = JS_TRUE;
1241                 }
1242
1243                 ADD_TO_TOKENBUF(c);
1244             }
1245
1246             UngetChar(ts, c);
1247             atom = TOKENBUF_TO_ATOM();
1248             if (!atom)
1249                 goto error;
1250             tp->t_op = JSOP_STRING;
1251             tp->t_atom = atom;
1252             tt = TOK_XMLNAME;
1253             goto out;
1254         }
1255
1256         switch (c) {
1257           case '{':
1258             if (ts->flags & TSF_XMLONLYMODE)
1259                 goto bad_xml_char;
1260             tt = TOK_LC;
1261             goto out;
1262
1263           case '=':
1264             tt = TOK_ASSIGN;
1265             goto out;
1266
1267           case '"':
1268           case '\'':
1269             qc = c;
1270             while ((c = GetChar(ts)) != qc) {
1271                 if (c == EOF) {
1272                     js_ReportCompileErrorNumber(cx, ts,
1273                                                 JSREPORT_TS | JSREPORT_ERROR,
1274                                                 JSMSG_UNTERMINATED_STRING);
1275                     goto error;
1276                 }
1277
1278                 /*
1279                  * XML attribute values are double-quoted when pretty-printed,
1280                  * so escape " if it is expressed directly in a single-quoted
1281                  * attribute value.
1282                  */
1283                 if (c == '"' && !(ts->flags & TSF_XMLONLYMODE)) {
1284                     JS_ASSERT(qc == '\'');
1285                     js_AppendCString(&ts->tokenbuf, js_quot_entity_str);
1286                     continue;
1287                 }
1288
1289                 if (c == '&' && (ts->flags & TSF_XMLONLYMODE)) {
1290                     if (!GetXMLEntity(cx, ts))
1291                         goto error;
1292                     continue;
1293                 }
1294
1295                 ADD_TO_TOKENBUF(c);
1296             }
1297             atom = TOKENBUF_TO_ATOM();
1298             if (!atom)
1299                 goto error;
1300             tp->pos.end.lineno = (uint16)ts->lineno;
1301             tp->t_op = JSOP_STRING;
1302             tp->t_atom = atom;
1303             tt = TOK_XMLATTR;
1304             goto out;
1305
1306           case '>':
1307             tt = TOK_XMLTAGC;
1308             goto out;
1309
1310           case '/':
1311             if (MatchChar(ts, '>')) {
1312                 tt = TOK_XMLPTAGC;
1313                 goto out;
1314             }
1315             /* FALL THROUGH */
1316
1317           bad_xml_char:
1318           default:
1319             js_ReportCompileErrorNumber(cx, ts, JSREPORT_TS | JSREPORT_ERROR,
1320                                         JSMSG_BAD_XML_CHARACTER);
1321             goto error;
1322         }
1323         /* NOTREACHED */
1324     }
1325 #endif /* JS_HAS_XML_SUPPORT */
1326
1327 retry:
1328     do {
1329         c = GetChar(ts);
1330         if (c == '\n') {
1331             ts->flags &= ~TSF_DIRTYLINE;
1332             if (ts->flags & TSF_NEWLINES)
1333                 break;
1334         }
1335     } while (JS_ISSPACE(c));
1336
1337     tp = NewToken(ts, -1);
1338     if (c == EOF) {
1339         tt = TOK_EOF;
1340         goto out;
1341     }
1342
1343     hadUnicodeEscape = JS_FALSE;
1344     if (JS_ISIDSTART(c) ||
1345         (c == '\\' &&
1346          (c = GetUnicodeEscape(ts),
1347           hadUnicodeEscape = JS_ISIDSTART(c)))) {
1348         INIT_TOKENBUF();
1349         for (;;) {
1350             ADD_TO_TOKENBUF(c);
1351             c = GetChar(ts);
1352             if (c == '\\') {
1353                 c = GetUnicodeEscape(ts);
1354                 if (!JS_ISIDENT(c))
1355                     break;
1356                 hadUnicodeEscape = JS_TRUE;
1357             } else {
1358                 if (!JS_ISIDENT(c))
1359                     break;
1360             }
1361         }
1362         UngetChar(ts, c);
1363
1364         atom = TOKENBUF_TO_ATOM();
1365         if (!atom)
1366             goto error;
1367         if (!hadUnicodeEscape && ATOM_KEYWORD(atom)) {
1368             struct keyword *kw;
1369
1370             JS_ASSERT(!(atom->flags & ATOM_HIDDEN));
1371             kw = ATOM_KEYWORD(atom);
1372             if (kw->tokentype == TOK_RESERVED) {
1373                 char buf[MAX_KEYWORD_LENGTH + 1];
1374                 size_t buflen = sizeof(buf) - 1;
1375                 if (!js_DeflateStringToBuffer(cx, TOKENBUF_BASE(), TOKENBUF_LENGTH(),
1376                                                   buf, &buflen))
1377                     goto error;
1378                 buf [buflen] = 0;
1379                 if (!js_ReportCompileErrorNumber(cx, ts,
1380                                                  JSREPORT_TS |
1381                                                  JSREPORT_WARNING |
1382                                                  JSREPORT_STRICT,
1383                                                  JSMSG_RESERVED_ID, buf)) {
1384                     goto error;
1385                 }
1386             } else if (JS_VERSION_IS_ECMA(cx) ||
1387                        kw->version <= (cx->version & JSVERSION_MASK)) {
1388                 tt = kw->tokentype;
1389                 tp->t_op = (JSOp) kw->op;
1390                 goto out;
1391             }
1392         }
1393         tp->t_op = JSOP_NAME;
1394         tp->t_atom = atom;
1395         tt = TOK_NAME;
1396         goto out;
1397     }
1398
1399     if (JS7_ISDEC(c) || (c == '.' && JS7_ISDEC(PeekChar(ts)))) {
1400         jsint radix;
1401         const jschar *endptr;
1402         jsdouble dval;
1403
1404         radix = 10;
1405         INIT_TOKENBUF();
1406
1407         if (c == '0') {
1408             ADD_TO_TOKENBUF(c);
1409             c = GetChar(ts);
1410             if (JS_TOLOWER(c) == 'x') {
1411                 ADD_TO_TOKENBUF(c);
1412                 c = GetChar(ts);
1413                 radix = 16;
1414             } else if (JS7_ISDEC(c)) {
1415                 radix = 8;
1416             }
1417         }
1418
1419         while (JS7_ISHEX(c)) {
1420             if (radix < 16) {
1421                 if (JS7_ISLET(c))
1422                     break;
1423
1424                 /*
1425                  * We permit 08 and 09 as decimal numbers, which makes our
1426                  * behaviour a superset of the ECMA numeric grammar.  We might
1427                  * not always be so permissive, so we warn about it.
1428                  */
1429                 if (radix == 8 && c >= '8') {
1430                     if (!js_ReportCompileErrorNumber(cx, ts,
1431                                                      JSREPORT_TS |
1432                                                      JSREPORT_WARNING,
1433                                                      JSMSG_BAD_OCTAL,
1434                                                      c == '8' ? "08" : "09")) {
1435                         goto error;
1436                     }
1437                     radix = 10;
1438                 }
1439             }
1440             ADD_TO_TOKENBUF(c);
1441             c = GetChar(ts);
1442         }
1443
1444         if (radix == 10 && (c == '.' || JS_TOLOWER(c) == 'e')) {
1445             if (c == '.') {
1446                 do {
1447                     ADD_TO_TOKENBUF(c);
1448                     c = GetChar(ts);
1449                 } while (JS7_ISDEC(c));
1450             }
1451             if (JS_TOLOWER(c) == 'e') {
1452                 ADD_TO_TOKENBUF(c);
1453                 c = GetChar(ts);
1454                 if (c == '+' || c == '-') {
1455                     ADD_TO_TOKENBUF(c);
1456                     c = GetChar(ts);
1457                 }
1458                 if (!JS7_ISDEC(c)) {
1459                     js_ReportCompileErrorNumber(cx, ts,
1460                                                 JSREPORT_TS | JSREPORT_ERROR,
1461                                                 JSMSG_MISSING_EXPONENT);
1462                     goto error;
1463                 }
1464                 do {
1465                     ADD_TO_TOKENBUF(c);
1466                     c = GetChar(ts);
1467                 } while (JS7_ISDEC(c));
1468             }
1469         }
1470
1471         /* Put back the next char and NUL-terminate tokenbuf for js_strto*. */
1472         UngetChar(ts, c);
1473         ADD_TO_TOKENBUF(0);
1474
1475         if (!TOKENBUF_OK())
1476             goto error;
1477         if (radix == 10) {
1478             if (!js_strtod(cx, TOKENBUF_BASE(), &endptr, &dval)) {
1479                 js_ReportCompileErrorNumber(cx, ts,
1480                                             JSREPORT_TS | JSREPORT_ERROR,
1481                                             JSMSG_OUT_OF_MEMORY);
1482                 goto error;
1483             }
1484         } else {
1485             if (!js_strtointeger(cx, TOKENBUF_BASE(), &endptr, radix, &dval)) {
1486                 js_ReportCompileErrorNumber(cx, ts,
1487                                             JSREPORT_TS | JSREPORT_ERROR,
1488                                             JSMSG_OUT_OF_MEMORY);
1489                 goto error;
1490             }
1491         }
1492         tp->t_dval = dval;
1493         tt = TOK_NUMBER;
1494         goto out;
1495     }
1496
1497     if (c == '"' || c == '\'') {
1498         qc = c;
1499         INIT_TOKENBUF();
1500         while ((c = GetChar(ts)) != qc) {
1501             if (c == '\n' || c == EOF) {
1502                 UngetChar(ts, c);
1503                 js_ReportCompileErrorNumber(cx, ts,
1504                                             JSREPORT_TS | JSREPORT_ERROR,
1505                                             JSMSG_UNTERMINATED_STRING);
1506                 goto error;
1507             }
1508             if (c == '\\') {
1509                 switch (c = GetChar(ts)) {
1510                   case 'b': c = '\b'; break;
1511                   case 'f': c = '\f'; break;
1512                   case 'n': c = '\n'; break;
1513                   case 'r': c = '\r'; break;
1514                   case 't': c = '\t'; break;
1515                   case 'v': c = '\v'; break;
1516
1517                   default:
1518                     if ('0' <= c && c < '8') {
1519                         int32 val = JS7_UNDEC(c);
1520
1521                         c = PeekChar(ts);
1522                         if ('0' <= c && c < '8') {
1523                             val = 8 * val + JS7_UNDEC(c);
1524                             GetChar(ts);
1525                             c = PeekChar(ts);
1526                             if ('0' <= c && c < '8') {
1527                                 int32 save = val;
1528                                 val = 8 * val + JS7_UNDEC(c);
1529                                 if (val <= 0377)
1530                                     GetChar(ts);
1531                                 else
1532                                     val = save;
1533                             }
1534                         }
1535
1536                         c = (jschar)val;
1537                     } else if (c == 'u') {
1538                         jschar cp[4];
1539                         if (PeekChars(ts, 4, cp) &&
1540                             JS7_ISHEX(cp[0]) && JS7_ISHEX(cp[1]) &&
1541                             JS7_ISHEX(cp[2]) && JS7_ISHEX(cp[3])) {
1542                             c = (((((JS7_UNHEX(cp[0]) << 4)
1543                                     + JS7_UNHEX(cp[1])) << 4)
1544                                   + JS7_UNHEX(cp[2])) << 4)
1545                                 + JS7_UNHEX(cp[3]);
1546                             SkipChars(ts, 4);
1547                         }
1548                     } else if (c == 'x') {
1549                         jschar cp[2];
1550                         if (PeekChars(ts, 2, cp) &&
1551                             JS7_ISHEX(cp[0]) && JS7_ISHEX(cp[1])) {
1552                             c = (JS7_UNHEX(cp[0]) << 4) + JS7_UNHEX(cp[1]);
1553                             SkipChars(ts, 2);
1554                         }
1555                     } else if (c == '\n' && JS_VERSION_IS_ECMA(cx)) {
1556                         /* ECMA follows C by removing escaped newlines. */
1557                         continue;
1558                     }
1559                     break;
1560                 }
1561             }
1562             ADD_TO_TOKENBUF(c);
1563         }
1564         atom = TOKENBUF_TO_ATOM();
1565         if (!atom)
1566             goto error;
1567         tp->pos.end.lineno = (uint16)ts->lineno;
1568         tp->t_op = JSOP_STRING;
1569         tp->t_atom = atom;
1570         tt = TOK_STRING;
1571         goto out;
1572     }
1573
1574     switch (c) {
1575       case '\n': tt = TOK_EOL; goto eol_out;
1576       case ';':  tt = TOK_SEMI; break;
1577       case '[':  tt = TOK_LB; break;
1578       case ']':  tt = TOK_RB; break;
1579       case '{':  tt = TOK_LC; break;
1580       case '}':  tt = TOK_RC; break;
1581       case '(':  tt = TOK_LP; break;
1582       case ')':  tt = TOK_RP; break;
1583       case ',':  tt = TOK_COMMA; break;
1584       case '?':  tt = TOK_HOOK; break;
1585
1586       case '.':
1587 #if JS_HAS_XML_SUPPORT
1588         if (MatchChar(ts, c))
1589             tt = TOK_DBLDOT;
1590         else
1591 #endif
1592             tt = TOK_DOT;
1593         break;
1594
1595       case ':':
1596 #if JS_HAS_XML_SUPPORT
1597         if (MatchChar(ts, c)) {
1598             tt = TOK_DBLCOLON;
1599             break;
1600         }
1601 #endif
1602         /*
1603          * Default so compiler can modify to JSOP_GETTER if 'p getter: v' in an
1604          * object initializer, likewise for setter.
1605          */
1606         tp->t_op = JSOP_NOP;
1607         tt = TOK_COLON;
1608         break;
1609
1610       case '|':
1611         if (MatchChar(ts, c)) {
1612             tt = TOK_OR;
1613         } else if (MatchChar(ts, '=')) {
1614             tp->t_op = JSOP_BITOR;
1615             tt = TOK_ASSIGN;
1616         } else {
1617             tt = TOK_BITOR;
1618         }
1619         break;
1620
1621       case '^':
1622         if (MatchChar(ts, '=')) {
1623             tp->t_op = JSOP_BITXOR;
1624             tt = TOK_ASSIGN;
1625         } else {
1626             tt = TOK_BITXOR;
1627         }
1628         break;
1629
1630       case '&':
1631         if (MatchChar(ts, c)) {
1632             tt = TOK_AND;
1633         } else if (MatchChar(ts, '=')) {
1634             tp->t_op = JSOP_BITAND;
1635             tt = TOK_ASSIGN;
1636         } else {
1637             tt = TOK_BITAND;
1638         }
1639         break;
1640
1641       case '=':
1642         if (MatchChar(ts, c)) {
1643 #if JS_HAS_TRIPLE_EQOPS
1644             tp->t_op = MatchChar(ts, c) ? JSOP_NEW_EQ : (JSOp)cx->jsop_eq;
1645 #else
1646             tp->t_op = cx->jsop_eq;
1647 #endif
1648             tt = TOK_EQOP;
1649         } else {
1650             tp->t_op = JSOP_NOP;
1651             tt = TOK_ASSIGN;
1652         }
1653         break;
1654
1655       case '!':
1656         if (MatchChar(ts, '=')) {
1657 #if JS_HAS_TRIPLE_EQOPS
1658             tp->t_op = MatchChar(ts, '=') ? JSOP_NEW_NE : (JSOp)cx->jsop_ne;
1659 #else
1660             tp->t_op = cx->jsop_ne;
1661 #endif
1662             tt = TOK_EQOP;
1663         } else {
1664             tp->t_op = JSOP_NOT;
1665             tt = TOK_UNARYOP;
1666         }
1667         break;
1668
1669 #if JS_HAS_XML_SUPPORT
1670       case '@':
1671         tt = TOK_AT;
1672         break;
1673 #endif
1674
1675       case '<':
1676 #if JS_HAS_XML_SUPPORT
1677         /*
1678          * After much testing, it's clear that Postel's advice to protocol
1679          * designers ("be liberal in what you accept, and conservative in what
1680          * you send") invites a natural-law repercussion for JS as "protocol":
1681          *
1682          * "If you are liberal in what you accept, others will utterly fail to
1683          *  be conservative in what they send."
1684          *
1685          * Which means you will get <!-- comments to end of line in the middle
1686          * of .js files, and after if conditions whose then statements are on
1687          * the next line, and other wonders.  See at least the following bugs:
1688          * https://bugzilla.mozilla.org/show_bug.cgi?id=309242
1689          * https://bugzilla.mozilla.org/show_bug.cgi?id=309712
1690          * https://bugzilla.mozilla.org/show_bug.cgi?id=310993
1691          *
1692          * So without JSOPTION_XML, we never scan an XML comment or CDATA
1693          * literal.  We always scan <! as the start of an HTML comment hack
1694          * to end of line, used since Netscape 2 to hide script tag content
1695          * from script-unaware browsers.
1696          */
1697         if ((ts->flags & TSF_OPERAND) &&
1698             (JS_HAS_XML_OPTION(cx) || PeekChar(ts) != '!')) {
1699             /* Check for XML comment or CDATA section. */
1700             if (MatchChar(ts, '!')) {
1701                 INIT_TOKENBUF();
1702
1703                 /* Scan XML comment. */
1704                 if (MatchChar(ts, '-')) {
1705                     if (!MatchChar(ts, '-'))
1706                         goto bad_xml_markup;
1707                     while ((c = GetChar(ts)) != '-' || !MatchChar(ts, '-')) {
1708                         if (c == EOF)
1709                             goto bad_xml_markup;
1710                         ADD_TO_TOKENBUF(c);
1711                     }
1712                     tt = TOK_XMLCOMMENT;
1713                     tp->t_op = JSOP_XMLCOMMENT;
1714                     goto finish_xml_markup;
1715                 }
1716
1717                 /* Scan CDATA section. */
1718                 if (MatchChar(ts, '[')) {
1719                     jschar cp[6];
1720                     if (PeekChars(ts, 6, cp) &&
1721                         cp[0] == 'C' &&
1722                         cp[1] == 'D' &&
1723                         cp[2] == 'A' &&
1724                         cp[3] == 'T' &&
1725                         cp[4] == 'A' &&
1726                         cp[5] == '[') {
1727                         SkipChars(ts, 6);
1728                         while ((c = GetChar(ts)) != ']' ||
1729                                !PeekChars(ts, 2, cp) ||
1730                                cp[0] != ']' ||
1731                                cp[1] != '>') {
1732                             if (c == EOF)
1733                                 goto bad_xml_markup;
1734                             ADD_TO_TOKENBUF(c);
1735                         }
1736                         GetChar(ts);            /* discard ] but not > */
1737                         tt = TOK_XMLCDATA;
1738                         tp->t_op = JSOP_XMLCDATA;
1739                         goto finish_xml_markup;
1740                     }
1741                     goto bad_xml_markup;
1742                 }
1743             }
1744
1745             /* Check for processing instruction. */
1746             if (MatchChar(ts, '?')) {
1747                 JSBool inTarget = JS_TRUE;
1748                 size_t targetLength = 0;
1749                 ptrdiff_t contentIndex = -1;
1750
1751                 INIT_TOKENBUF();
1752                 while ((c = GetChar(ts)) != '?' || PeekChar(ts) != '>') {
1753                     if (c == EOF)
1754                         goto bad_xml_markup;
1755                     if (inTarget) {
1756                         if (JS_ISXMLSPACE(c)) {
1757                             if (TOKENBUF_LENGTH() == 0)
1758                                 goto bad_xml_markup;
1759                             inTarget = JS_FALSE;
1760                         } else {
1761                             if (!((TOKENBUF_LENGTH() == 0)
1762                                   ? JS_ISXMLNSSTART(c)
1763                                   : JS_ISXMLNS(c))) {
1764                                 goto bad_xml_markup;
1765                             }
1766                             ++targetLength;
1767                         }
1768                     } else {
1769                         if (contentIndex < 0 && !JS_ISXMLSPACE(c))
1770                             contentIndex = TOKENBUF_LENGTH();
1771                     }
1772                     ADD_TO_TOKENBUF(c);
1773                 }
1774                 if (contentIndex < 0) {
1775                     atom = cx->runtime->atomState.emptyAtom;
1776                 } else {
1777                     if (!TOKENBUF_OK())
1778                         goto error;
1779                     atom = js_AtomizeChars(cx,
1780                                            &TOKENBUF_CHAR(contentIndex),
1781                                            TOKENBUF_LENGTH() - contentIndex,
1782                                            0);
1783                     if (!atom)
1784                         goto error;
1785                     TRIM_TOKENBUF(targetLength);
1786                 }
1787                 tp->t_atom2 = atom;
1788                 tt = TOK_XMLPI;
1789
1790         finish_xml_markup:
1791                 if (!MatchChar(ts, '>'))
1792                     goto bad_xml_markup;
1793                 atom = TOKENBUF_TO_ATOM();
1794                 if (!atom)
1795                     goto error;
1796                 tp->t_atom = atom;
1797                 tp->pos.end.lineno = (uint16)ts->lineno;
1798                 goto out;
1799             }
1800
1801             /* An XML start-of-tag character. */
1802             tt = MatchChar(ts, '/') ? TOK_XMLETAGO : TOK_XMLSTAGO;
1803             goto out;
1804
1805         bad_xml_markup:
1806             js_ReportCompileErrorNumber(cx, ts, JSREPORT_TS | JSREPORT_ERROR,
1807                                         JSMSG_BAD_XML_MARKUP);
1808             goto error;
1809         }
1810 #endif /* JS_HAS_XML_SUPPORT */
1811
1812         /* NB: treat HTML begin-comment as comment-till-end-of-line */
1813         if (MatchChar(ts, '!')) {
1814             if (MatchChar(ts, '-')) {
1815                 if (MatchChar(ts, '-')) {
1816                     ts->flags |= TSF_IN_HTML_COMMENT;
1817                     goto skipline;
1818                 }
1819                 UngetChar(ts, '-');
1820             }
1821             UngetChar(ts, '!');
1822         }
1823         if (MatchChar(ts, c)) {
1824             tp->t_op = JSOP_LSH;
1825             tt = MatchChar(ts, '=') ? TOK_ASSIGN : TOK_SHOP;
1826         } else {
1827             tp->t_op = MatchChar(ts, '=') ? JSOP_LE : JSOP_LT;
1828             tt = TOK_RELOP;
1829         }
1830         break;
1831
1832       case '>':
1833         if (MatchChar(ts, c)) {
1834             tp->t_op = MatchChar(ts, c) ? JSOP_URSH : JSOP_RSH;
1835             tt = MatchChar(ts, '=') ? TOK_ASSIGN : TOK_SHOP;
1836         } else {
1837             tp->t_op = MatchChar(ts, '=') ? JSOP_GE : JSOP_GT;
1838             tt = TOK_RELOP;
1839         }
1840         break;
1841
1842       case '*':
1843         tp->t_op = JSOP_MUL;
1844         tt = MatchChar(ts, '=') ? TOK_ASSIGN : TOK_STAR;
1845         break;
1846
1847       case '/':
1848         if (MatchChar(ts, '/')) {
1849             /*
1850              * Hack for source filters such as the Mozilla XUL preprocessor:
1851              * "//@line 123\n" sets the number of the *next* line after the
1852              * comment to 123.
1853              */
1854             if (JS_HAS_ATLINE_OPTION(cx)) {
1855                 jschar cp[5];
1856                 uintN i, line, temp;
1857                 char filename[1024];
1858
1859                 if (PeekChars(ts, 5, cp) &&
1860                     cp[0] == '@' &&
1861                     cp[1] == 'l' &&
1862                     cp[2] == 'i' &&
1863                     cp[3] == 'n' &&
1864                     cp[4] == 'e') {
1865                     SkipChars(ts, 5);
1866                     while ((c = GetChar(ts)) != '\n' && JS_ISSPACE(c))
1867                         continue;
1868                     if (JS7_ISDEC(c)) {
1869                         line = JS7_UNDEC(c);
1870                         while ((c = GetChar(ts)) != EOF && JS7_ISDEC(c)) {
1871                             temp = 10 * line + JS7_UNDEC(c);
1872                             if (temp < line) {
1873                                 /* Ignore overlarge line numbers. */
1874                                 goto skipline;
1875                             }
1876                             line = temp;
1877                         }
1878                         while (c != '\n' && JS_ISSPACE(c))
1879                             c = GetChar(ts);
1880                         i = 0;
1881                         if (c == '"') {
1882                             while ((c = GetChar(ts)) != EOF && c != '"') {
1883                                 if (c == '\n') {
1884                                     UngetChar(ts, c);
1885                                     goto skipline;
1886                                 }
1887                                 if ((c >> 8) != 0 || i >= sizeof filename - 1)
1888                                     goto skipline;
1889                                 filename[i++] = (char) c;
1890                             }
1891                             if (c == '"') {
1892                                 while ((c = GetChar(ts)) != '\n' &&
1893                                        JS_ISSPACE(c)) {
1894                                     continue;
1895                                 }
1896                             }
1897                         }
1898                         filename[i] = '\0';
1899                         if (c == '\n') {
1900                             if (i > 0) {
1901                                 if (ts->flags & TSF_OWNFILENAME)
1902                                     JS_free(cx, (void *) ts->filename);
1903                                 ts->filename = JS_strdup(cx, filename);
1904                                 if (!ts->filename)
1905                                     goto error;
1906                                 ts->flags |= TSF_OWNFILENAME;
1907                             }
1908                             ts->lineno = line;
1909                         }
1910                     }
1911                     UngetChar(ts, c);
1912                 }
1913             }
1914
1915 skipline:
1916             /* Optimize line skipping if we are not in an HTML comment. */
1917             if (ts->flags & TSF_IN_HTML_COMMENT) {
1918                 while ((c = GetChar(ts)) != EOF && c != '\n') {
1919                     if (c == '-' && MatchChar(ts, '-') && MatchChar(ts, '>'))
1920                         ts->flags &= ~TSF_IN_HTML_COMMENT;
1921                 }
1922             } else {
1923                 while ((c = GetChar(ts)) != EOF && c != '\n')
1924                     continue;
1925             }
1926             UngetChar(ts, c);
1927             ts->cursor = (ts->cursor - 1) & NTOKENS_MASK;
1928             goto retry;
1929         }
1930
1931         if (MatchChar(ts, '*')) {
1932             while ((c = GetChar(ts)) != EOF &&
1933                    !(c == '*' && MatchChar(ts, '/'))) {
1934                 /* Ignore all characters until comment close. */
1935             }
1936             if (c == EOF) {
1937                 js_ReportCompileErrorNumber(cx, ts,
1938                                             JSREPORT_TS | JSREPORT_ERROR,
1939                                             JSMSG_UNTERMINATED_COMMENT);
1940                 goto error;
1941             }
1942             ts->cursor = (ts->cursor - 1) & NTOKENS_MASK;
1943             goto retry;
1944         }
1945
1946 #if JS_HAS_REGEXPS
1947         if (ts->flags & TSF_OPERAND) {
1948             JSObject *obj;
1949             uintN flags;
1950             JSBool inCharClass = JS_FALSE;
1951
1952             INIT_TOKENBUF();
1953             for (;;) {
1954                 c = GetChar(ts);
1955                 if (c == '\n' || c == EOF) {
1956                     UngetChar(ts, c);
1957                     js_ReportCompileErrorNumber(cx, ts,
1958                                                 JSREPORT_TS | JSREPORT_ERROR,
1959                                                 JSMSG_UNTERMINATED_REGEXP);
1960                     goto error;
1961                 }
1962                 if (c == '\\') {
1963                     ADD_TO_TOKENBUF(c);
1964                     c = GetChar(ts);
1965                 } else if (c == '[') {
1966                     inCharClass = JS_TRUE;
1967                 } else if (c == ']') {
1968                     inCharClass = JS_FALSE;
1969                 } else if (c == '/' && !inCharClass) {
1970                     /* For compat with IE, allow unescaped / in char classes. */
1971                     break;
1972                 }
1973                 ADD_TO_TOKENBUF(c);
1974             }
1975             for (flags = 0; ; ) {
1976                 if (MatchChar(ts, 'g'))
1977                     flags |= JSREG_GLOB;
1978                 else if (MatchChar(ts, 'i'))
1979                     flags |= JSREG_FOLD;
1980                 else if (MatchChar(ts, 'm'))
1981                     flags |= JSREG_MULTILINE;
1982                 else
1983                     break;
1984             }
1985             c = PeekChar(ts);
1986             if (JS7_ISLET(c)) {
1987                 tp->ptr = ts->linebuf.ptr - 1;
1988                 js_ReportCompileErrorNumber(cx, ts,
1989                                             JSREPORT_TS | JSREPORT_ERROR,
1990                                             JSMSG_BAD_REGEXP_FLAG);
1991                 (void) GetChar(ts);
1992                 goto error;
1993             }
1994             /* XXXbe fix jsregexp.c so it doesn't depend on NUL termination */
1995             if (!TOKENBUF_OK())
1996                 goto error;
1997             NUL_TERM_TOKENBUF();
1998             obj = js_NewRegExpObject(cx, ts,
1999                                      TOKENBUF_BASE(),
2000                                      TOKENBUF_LENGTH(),
2001                                      flags);
2002             if (!obj)
2003                 goto error;
2004             atom = js_AtomizeObject(cx, obj, 0);
2005             if (!atom)
2006                 goto error;
2007
2008             /*
2009              * If the regexp's script is one-shot, we can avoid the extra
2010              * fork-on-exec costs of JSOP_REGEXP by selecting JSOP_OBJECT.
2011              * Otherwise, to avoid incorrect proto, parent, and lastIndex
2012              * sharing among threads and sequentially across re-execution,
2013              * select JSOP_REGEXP.
2014              */
2015             tp->t_op = (cx->fp->flags & (JSFRAME_EVAL | JSFRAME_COMPILE_N_GO))
2016                        ? JSOP_OBJECT
2017                        : JSOP_REGEXP;
2018             tp->t_atom = atom;
2019             tt = TOK_OBJECT;
2020             break;
2021         }
2022 #endif /* JS_HAS_REGEXPS */
2023
2024         tp->t_op = JSOP_DIV;
2025         tt = MatchChar(ts, '=') ? TOK_ASSIGN : TOK_DIVOP;
2026         break;
2027
2028       case '%':
2029         tp->t_op = JSOP_MOD;
2030         tt = MatchChar(ts, '=') ? TOK_ASSIGN : TOK_DIVOP;
2031         break;
2032
2033       case '~':
2034         tp->t_op = JSOP_BITNOT;
2035         tt = TOK_UNARYOP;
2036         break;
2037
2038       case '+':
2039         if (MatchChar(ts, '=')) {
2040             tp->t_op = JSOP_ADD;
2041             tt = TOK_ASSIGN;
2042         } else if (MatchChar(ts, c)) {
2043             tt = TOK_INC;
2044         } else {
2045             tp->t_op = JSOP_POS;
2046             tt = TOK_PLUS;
2047         }
2048         break;
2049
2050       case '-':
2051         if (MatchChar(ts, '=')) {
2052             tp->t_op = JSOP_SUB;
2053             tt = TOK_ASSIGN;
2054         } else if (MatchChar(ts, c)) {
2055             if (PeekChar(ts) == '>' && !(ts->flags & TSF_DIRTYLINE)) {
2056                 ts->flags &= ~TSF_IN_HTML_COMMENT;
2057                 goto skipline;
2058             }
2059             tt = TOK_DEC;
2060         } else {
2061             tp->t_op = JSOP_NEG;
2062             tt = TOK_MINUS;
2063         }
2064         break;
2065
2066 #if JS_HAS_SHARP_VARS
2067       case '#':
2068       {
2069         uint32 n;
2070
2071         c = GetChar(ts);
2072         if (!JS7_ISDEC(c)) {
2073             UngetChar(ts, c);
2074             goto badchar;
2075         }
2076         n = (uint32)JS7_UNDEC(c);
2077         for (;;) {
2078             c = GetChar(ts);
2079             if (!JS7_ISDEC(c))
2080                 break;
2081             n = 10 * n + JS7_UNDEC(c);
2082             if (n >= ATOM_INDEX_LIMIT) {
2083                 js_ReportCompileErrorNumber(cx, ts,
2084                                             JSREPORT_TS | JSREPORT_ERROR,
2085                                             JSMSG_SHARPVAR_TOO_BIG);
2086                 goto error;
2087             }
2088         }
2089         tp->t_dval = (jsdouble) n;
2090         if (JS_HAS_STRICT_OPTION(cx) &&
2091             (c == '=' || c == '#')) {
2092             char buf[20];
2093             JS_snprintf(buf, sizeof buf, "#%u%c", n, c);
2094             if (!js_ReportCompileErrorNumber(cx, ts,
2095                                              JSREPORT_TS |
2096                                              JSREPORT_WARNING |
2097                                              JSREPORT_STRICT,
2098                                              JSMSG_DEPRECATED_USAGE,
2099                                              buf)) {
2100                 goto error;
2101             }
2102         }
2103         if (c == '=')
2104             tt = TOK_DEFSHARP;
2105         else if (c == '#')
2106             tt = TOK_USESHARP;
2107         else
2108             goto badchar;
2109         break;
2110       }
2111 #endif /* JS_HAS_SHARP_VARS */
2112
2113 #if JS_HAS_SHARP_VARS || JS_HAS_XML_SUPPORT
2114       badchar:
2115 #endif
2116
2117       default:
2118         js_ReportCompileErrorNumber(cx, ts, JSREPORT_TS | JSREPORT_ERROR,
2119                                     JSMSG_ILLEGAL_CHARACTER);
2120         goto error;
2121     }
2122
2123 out:
2124     JS_ASSERT(tt != TOK_EOL);
2125     ts->flags |= TSF_DIRTYLINE;
2126
2127 eol_out:
2128     if (!STRING_BUFFER_OK(&ts->tokenbuf))
2129         tt = TOK_ERROR;
2130     JS_ASSERT(tt < TOK_LIMIT);
2131     tp->pos.end.index = ts->linepos +
2132                         PTRDIFF(ts->linebuf.ptr, ts->linebuf.base, jschar) -
2133                         ts->ungetpos;
2134     tp->type = tt;
2135     return tt;
2136
2137 error:
2138     tt = TOK_ERROR;
2139     ts->flags |= TSF_ERROR;
2140     goto out;
2141
2142 #undef INIT_TOKENBUF
2143 #undef TOKENBUF_LENGTH
2144 #undef TOKENBUF_OK
2145 #undef TOKENBUF_TO_ATOM
2146 #undef ADD_TO_TOKENBUF
2147 #undef TOKENBUF_BASE
2148 #undef TOKENBUF_CHAR
2149 #undef TRIM_TOKENBUF
2150 #undef NUL_TERM_TOKENBUF
2151 }
2152
2153 void
2154 js_UngetToken(JSTokenStream *ts)
2155 {
2156     JS_ASSERT(ts->lookahead < NTOKENS_MASK);
2157     if (ts->flags & TSF_ERROR)
2158         return;
2159     ts->lookahead++;
2160     ts->cursor = (ts->cursor - 1) & NTOKENS_MASK;
2161 }
2162
2163 JSBool
2164 js_MatchToken(JSContext *cx, JSTokenStream *ts, JSTokenType tt)
2165 {
2166     if (js_GetToken(cx, ts) == tt)
2167         return JS_TRUE;
2168     js_UngetToken(ts);
2169     return JS_FALSE;
2170 }