media/webrtc/trunk/testing/gtest/scripts/pump.py

Wed, 31 Dec 2014 06:09:35 +0100

author
Michael Schloh von Bennewitz <michael@schloh.com>
date
Wed, 31 Dec 2014 06:09:35 +0100
changeset 0
6474c204b198
permissions
-rwxr-xr-x

Cloned upstream origin tor-browser at tor-browser-31.3.0esr-4.5-1-build1
revision ID fc1c9ff7c1b2defdbc039f12214767608f46423f for hacking purpose.

michael@0 1 #!/usr/bin/env python
michael@0 2 #
michael@0 3 # Copyright 2008, Google Inc.
michael@0 4 # All rights reserved.
michael@0 5 #
michael@0 6 # Redistribution and use in source and binary forms, with or without
michael@0 7 # modification, are permitted provided that the following conditions are
michael@0 8 # met:
michael@0 9 #
michael@0 10 # * Redistributions of source code must retain the above copyright
michael@0 11 # notice, this list of conditions and the following disclaimer.
michael@0 12 # * Redistributions in binary form must reproduce the above
michael@0 13 # copyright notice, this list of conditions and the following disclaimer
michael@0 14 # in the documentation and/or other materials provided with the
michael@0 15 # distribution.
michael@0 16 # * Neither the name of Google Inc. nor the names of its
michael@0 17 # contributors may be used to endorse or promote products derived from
michael@0 18 # this software without specific prior written permission.
michael@0 19 #
michael@0 20 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
michael@0 21 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
michael@0 22 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
michael@0 23 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
michael@0 24 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
michael@0 25 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
michael@0 26 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
michael@0 27 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
michael@0 28 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
michael@0 29 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
michael@0 30 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
michael@0 31
michael@0 32 """pump v0.2.0 - Pretty Useful for Meta Programming.
michael@0 33
michael@0 34 A tool for preprocessor meta programming. Useful for generating
michael@0 35 repetitive boilerplate code. Especially useful for writing C++
michael@0 36 classes, functions, macros, and templates that need to work with
michael@0 37 various number of arguments.
michael@0 38
michael@0 39 USAGE:
michael@0 40 pump.py SOURCE_FILE
michael@0 41
michael@0 42 EXAMPLES:
michael@0 43 pump.py foo.cc.pump
michael@0 44 Converts foo.cc.pump to foo.cc.
michael@0 45
michael@0 46 GRAMMAR:
michael@0 47 CODE ::= ATOMIC_CODE*
michael@0 48 ATOMIC_CODE ::= $var ID = EXPRESSION
michael@0 49 | $var ID = [[ CODE ]]
michael@0 50 | $range ID EXPRESSION..EXPRESSION
michael@0 51 | $for ID SEPARATOR [[ CODE ]]
michael@0 52 | $($)
michael@0 53 | $ID
michael@0 54 | $(EXPRESSION)
michael@0 55 | $if EXPRESSION [[ CODE ]] ELSE_BRANCH
michael@0 56 | [[ CODE ]]
michael@0 57 | RAW_CODE
michael@0 58 SEPARATOR ::= RAW_CODE | EMPTY
michael@0 59 ELSE_BRANCH ::= $else [[ CODE ]]
michael@0 60 | $elif EXPRESSION [[ CODE ]] ELSE_BRANCH
michael@0 61 | EMPTY
michael@0 62 EXPRESSION has Python syntax.
michael@0 63 """
michael@0 64
michael@0 65 __author__ = 'wan@google.com (Zhanyong Wan)'
michael@0 66
michael@0 67 import os
michael@0 68 import re
michael@0 69 import sys
michael@0 70
michael@0 71
michael@0 72 TOKEN_TABLE = [
michael@0 73 (re.compile(r'\$var\s+'), '$var'),
michael@0 74 (re.compile(r'\$elif\s+'), '$elif'),
michael@0 75 (re.compile(r'\$else\s+'), '$else'),
michael@0 76 (re.compile(r'\$for\s+'), '$for'),
michael@0 77 (re.compile(r'\$if\s+'), '$if'),
michael@0 78 (re.compile(r'\$range\s+'), '$range'),
michael@0 79 (re.compile(r'\$[_A-Za-z]\w*'), '$id'),
michael@0 80 (re.compile(r'\$\(\$\)'), '$($)'),
michael@0 81 (re.compile(r'\$'), '$'),
michael@0 82 (re.compile(r'\[\[\n?'), '[['),
michael@0 83 (re.compile(r'\]\]\n?'), ']]'),
michael@0 84 ]
michael@0 85
michael@0 86
michael@0 87 class Cursor:
michael@0 88 """Represents a position (line and column) in a text file."""
michael@0 89
michael@0 90 def __init__(self, line=-1, column=-1):
michael@0 91 self.line = line
michael@0 92 self.column = column
michael@0 93
michael@0 94 def __eq__(self, rhs):
michael@0 95 return self.line == rhs.line and self.column == rhs.column
michael@0 96
michael@0 97 def __ne__(self, rhs):
michael@0 98 return not self == rhs
michael@0 99
michael@0 100 def __lt__(self, rhs):
michael@0 101 return self.line < rhs.line or (
michael@0 102 self.line == rhs.line and self.column < rhs.column)
michael@0 103
michael@0 104 def __le__(self, rhs):
michael@0 105 return self < rhs or self == rhs
michael@0 106
michael@0 107 def __gt__(self, rhs):
michael@0 108 return rhs < self
michael@0 109
michael@0 110 def __ge__(self, rhs):
michael@0 111 return rhs <= self
michael@0 112
michael@0 113 def __str__(self):
michael@0 114 if self == Eof():
michael@0 115 return 'EOF'
michael@0 116 else:
michael@0 117 return '%s(%s)' % (self.line + 1, self.column)
michael@0 118
michael@0 119 def __add__(self, offset):
michael@0 120 return Cursor(self.line, self.column + offset)
michael@0 121
michael@0 122 def __sub__(self, offset):
michael@0 123 return Cursor(self.line, self.column - offset)
michael@0 124
michael@0 125 def Clone(self):
michael@0 126 """Returns a copy of self."""
michael@0 127
michael@0 128 return Cursor(self.line, self.column)
michael@0 129
michael@0 130
michael@0 131 # Special cursor to indicate the end-of-file.
michael@0 132 def Eof():
michael@0 133 """Returns the special cursor to denote the end-of-file."""
michael@0 134 return Cursor(-1, -1)
michael@0 135
michael@0 136
michael@0 137 class Token:
michael@0 138 """Represents a token in a Pump source file."""
michael@0 139
michael@0 140 def __init__(self, start=None, end=None, value=None, token_type=None):
michael@0 141 if start is None:
michael@0 142 self.start = Eof()
michael@0 143 else:
michael@0 144 self.start = start
michael@0 145 if end is None:
michael@0 146 self.end = Eof()
michael@0 147 else:
michael@0 148 self.end = end
michael@0 149 self.value = value
michael@0 150 self.token_type = token_type
michael@0 151
michael@0 152 def __str__(self):
michael@0 153 return 'Token @%s: \'%s\' type=%s' % (
michael@0 154 self.start, self.value, self.token_type)
michael@0 155
michael@0 156 def Clone(self):
michael@0 157 """Returns a copy of self."""
michael@0 158
michael@0 159 return Token(self.start.Clone(), self.end.Clone(), self.value,
michael@0 160 self.token_type)
michael@0 161
michael@0 162
michael@0 163 def StartsWith(lines, pos, string):
michael@0 164 """Returns True iff the given position in lines starts with 'string'."""
michael@0 165
michael@0 166 return lines[pos.line][pos.column:].startswith(string)
michael@0 167
michael@0 168
michael@0 169 def FindFirstInLine(line, token_table):
michael@0 170 best_match_start = -1
michael@0 171 for (regex, token_type) in token_table:
michael@0 172 m = regex.search(line)
michael@0 173 if m:
michael@0 174 # We found regex in lines
michael@0 175 if best_match_start < 0 or m.start() < best_match_start:
michael@0 176 best_match_start = m.start()
michael@0 177 best_match_length = m.end() - m.start()
michael@0 178 best_match_token_type = token_type
michael@0 179
michael@0 180 if best_match_start < 0:
michael@0 181 return None
michael@0 182
michael@0 183 return (best_match_start, best_match_length, best_match_token_type)
michael@0 184
michael@0 185
michael@0 186 def FindFirst(lines, token_table, cursor):
michael@0 187 """Finds the first occurrence of any string in strings in lines."""
michael@0 188
michael@0 189 start = cursor.Clone()
michael@0 190 cur_line_number = cursor.line
michael@0 191 for line in lines[start.line:]:
michael@0 192 if cur_line_number == start.line:
michael@0 193 line = line[start.column:]
michael@0 194 m = FindFirstInLine(line, token_table)
michael@0 195 if m:
michael@0 196 # We found a regex in line.
michael@0 197 (start_column, length, token_type) = m
michael@0 198 if cur_line_number == start.line:
michael@0 199 start_column += start.column
michael@0 200 found_start = Cursor(cur_line_number, start_column)
michael@0 201 found_end = found_start + length
michael@0 202 return MakeToken(lines, found_start, found_end, token_type)
michael@0 203 cur_line_number += 1
michael@0 204 # We failed to find str in lines
michael@0 205 return None
michael@0 206
michael@0 207
michael@0 208 def SubString(lines, start, end):
michael@0 209 """Returns a substring in lines."""
michael@0 210
michael@0 211 if end == Eof():
michael@0 212 end = Cursor(len(lines) - 1, len(lines[-1]))
michael@0 213
michael@0 214 if start >= end:
michael@0 215 return ''
michael@0 216
michael@0 217 if start.line == end.line:
michael@0 218 return lines[start.line][start.column:end.column]
michael@0 219
michael@0 220 result_lines = ([lines[start.line][start.column:]] +
michael@0 221 lines[start.line + 1:end.line] +
michael@0 222 [lines[end.line][:end.column]])
michael@0 223 return ''.join(result_lines)
michael@0 224
michael@0 225
michael@0 226 def StripMetaComments(str):
michael@0 227 """Strip meta comments from each line in the given string."""
michael@0 228
michael@0 229 # First, completely remove lines containing nothing but a meta
michael@0 230 # comment, including the trailing \n.
michael@0 231 str = re.sub(r'^\s*\$\$.*\n', '', str)
michael@0 232
michael@0 233 # Then, remove meta comments from contentful lines.
michael@0 234 return re.sub(r'\s*\$\$.*', '', str)
michael@0 235
michael@0 236
michael@0 237 def MakeToken(lines, start, end, token_type):
michael@0 238 """Creates a new instance of Token."""
michael@0 239
michael@0 240 return Token(start, end, SubString(lines, start, end), token_type)
michael@0 241
michael@0 242
michael@0 243 def ParseToken(lines, pos, regex, token_type):
michael@0 244 line = lines[pos.line][pos.column:]
michael@0 245 m = regex.search(line)
michael@0 246 if m and not m.start():
michael@0 247 return MakeToken(lines, pos, pos + m.end(), token_type)
michael@0 248 else:
michael@0 249 print 'ERROR: %s expected at %s.' % (token_type, pos)
michael@0 250 sys.exit(1)
michael@0 251
michael@0 252
michael@0 253 ID_REGEX = re.compile(r'[_A-Za-z]\w*')
michael@0 254 EQ_REGEX = re.compile(r'=')
michael@0 255 REST_OF_LINE_REGEX = re.compile(r'.*?(?=$|\$\$)')
michael@0 256 OPTIONAL_WHITE_SPACES_REGEX = re.compile(r'\s*')
michael@0 257 WHITE_SPACE_REGEX = re.compile(r'\s')
michael@0 258 DOT_DOT_REGEX = re.compile(r'\.\.')
michael@0 259
michael@0 260
michael@0 261 def Skip(lines, pos, regex):
michael@0 262 line = lines[pos.line][pos.column:]
michael@0 263 m = re.search(regex, line)
michael@0 264 if m and not m.start():
michael@0 265 return pos + m.end()
michael@0 266 else:
michael@0 267 return pos
michael@0 268
michael@0 269
michael@0 270 def SkipUntil(lines, pos, regex, token_type):
michael@0 271 line = lines[pos.line][pos.column:]
michael@0 272 m = re.search(regex, line)
michael@0 273 if m:
michael@0 274 return pos + m.start()
michael@0 275 else:
michael@0 276 print ('ERROR: %s expected on line %s after column %s.' %
michael@0 277 (token_type, pos.line + 1, pos.column))
michael@0 278 sys.exit(1)
michael@0 279
michael@0 280
michael@0 281 def ParseExpTokenInParens(lines, pos):
michael@0 282 def ParseInParens(pos):
michael@0 283 pos = Skip(lines, pos, OPTIONAL_WHITE_SPACES_REGEX)
michael@0 284 pos = Skip(lines, pos, r'\(')
michael@0 285 pos = Parse(pos)
michael@0 286 pos = Skip(lines, pos, r'\)')
michael@0 287 return pos
michael@0 288
michael@0 289 def Parse(pos):
michael@0 290 pos = SkipUntil(lines, pos, r'\(|\)', ')')
michael@0 291 if SubString(lines, pos, pos + 1) == '(':
michael@0 292 pos = Parse(pos + 1)
michael@0 293 pos = Skip(lines, pos, r'\)')
michael@0 294 return Parse(pos)
michael@0 295 else:
michael@0 296 return pos
michael@0 297
michael@0 298 start = pos.Clone()
michael@0 299 pos = ParseInParens(pos)
michael@0 300 return MakeToken(lines, start, pos, 'exp')
michael@0 301
michael@0 302
michael@0 303 def RStripNewLineFromToken(token):
michael@0 304 if token.value.endswith('\n'):
michael@0 305 return Token(token.start, token.end, token.value[:-1], token.token_type)
michael@0 306 else:
michael@0 307 return token
michael@0 308
michael@0 309
michael@0 310 def TokenizeLines(lines, pos):
michael@0 311 while True:
michael@0 312 found = FindFirst(lines, TOKEN_TABLE, pos)
michael@0 313 if not found:
michael@0 314 yield MakeToken(lines, pos, Eof(), 'code')
michael@0 315 return
michael@0 316
michael@0 317 if found.start == pos:
michael@0 318 prev_token = None
michael@0 319 prev_token_rstripped = None
michael@0 320 else:
michael@0 321 prev_token = MakeToken(lines, pos, found.start, 'code')
michael@0 322 prev_token_rstripped = RStripNewLineFromToken(prev_token)
michael@0 323
michael@0 324 if found.token_type == '$var':
michael@0 325 if prev_token_rstripped:
michael@0 326 yield prev_token_rstripped
michael@0 327 yield found
michael@0 328 id_token = ParseToken(lines, found.end, ID_REGEX, 'id')
michael@0 329 yield id_token
michael@0 330 pos = Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX)
michael@0 331
michael@0 332 eq_token = ParseToken(lines, pos, EQ_REGEX, '=')
michael@0 333 yield eq_token
michael@0 334 pos = Skip(lines, eq_token.end, r'\s*')
michael@0 335
michael@0 336 if SubString(lines, pos, pos + 2) != '[[':
michael@0 337 exp_token = ParseToken(lines, pos, REST_OF_LINE_REGEX, 'exp')
michael@0 338 yield exp_token
michael@0 339 pos = Cursor(exp_token.end.line + 1, 0)
michael@0 340 elif found.token_type == '$for':
michael@0 341 if prev_token_rstripped:
michael@0 342 yield prev_token_rstripped
michael@0 343 yield found
michael@0 344 id_token = ParseToken(lines, found.end, ID_REGEX, 'id')
michael@0 345 yield id_token
michael@0 346 pos = Skip(lines, id_token.end, WHITE_SPACE_REGEX)
michael@0 347 elif found.token_type == '$range':
michael@0 348 if prev_token_rstripped:
michael@0 349 yield prev_token_rstripped
michael@0 350 yield found
michael@0 351 id_token = ParseToken(lines, found.end, ID_REGEX, 'id')
michael@0 352 yield id_token
michael@0 353 pos = Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX)
michael@0 354
michael@0 355 dots_pos = SkipUntil(lines, pos, DOT_DOT_REGEX, '..')
michael@0 356 yield MakeToken(lines, pos, dots_pos, 'exp')
michael@0 357 yield MakeToken(lines, dots_pos, dots_pos + 2, '..')
michael@0 358 pos = dots_pos + 2
michael@0 359 new_pos = Cursor(pos.line + 1, 0)
michael@0 360 yield MakeToken(lines, pos, new_pos, 'exp')
michael@0 361 pos = new_pos
michael@0 362 elif found.token_type == '$':
michael@0 363 if prev_token:
michael@0 364 yield prev_token
michael@0 365 yield found
michael@0 366 exp_token = ParseExpTokenInParens(lines, found.end)
michael@0 367 yield exp_token
michael@0 368 pos = exp_token.end
michael@0 369 elif (found.token_type == ']]' or found.token_type == '$if' or
michael@0 370 found.token_type == '$elif' or found.token_type == '$else'):
michael@0 371 if prev_token_rstripped:
michael@0 372 yield prev_token_rstripped
michael@0 373 yield found
michael@0 374 pos = found.end
michael@0 375 else:
michael@0 376 if prev_token:
michael@0 377 yield prev_token
michael@0 378 yield found
michael@0 379 pos = found.end
michael@0 380
michael@0 381
michael@0 382 def Tokenize(s):
michael@0 383 """A generator that yields the tokens in the given string."""
michael@0 384 if s != '':
michael@0 385 lines = s.splitlines(True)
michael@0 386 for token in TokenizeLines(lines, Cursor(0, 0)):
michael@0 387 yield token
michael@0 388
michael@0 389
michael@0 390 class CodeNode:
michael@0 391 def __init__(self, atomic_code_list=None):
michael@0 392 self.atomic_code = atomic_code_list
michael@0 393
michael@0 394
michael@0 395 class VarNode:
michael@0 396 def __init__(self, identifier=None, atomic_code=None):
michael@0 397 self.identifier = identifier
michael@0 398 self.atomic_code = atomic_code
michael@0 399
michael@0 400
michael@0 401 class RangeNode:
michael@0 402 def __init__(self, identifier=None, exp1=None, exp2=None):
michael@0 403 self.identifier = identifier
michael@0 404 self.exp1 = exp1
michael@0 405 self.exp2 = exp2
michael@0 406
michael@0 407
michael@0 408 class ForNode:
michael@0 409 def __init__(self, identifier=None, sep=None, code=None):
michael@0 410 self.identifier = identifier
michael@0 411 self.sep = sep
michael@0 412 self.code = code
michael@0 413
michael@0 414
michael@0 415 class ElseNode:
michael@0 416 def __init__(self, else_branch=None):
michael@0 417 self.else_branch = else_branch
michael@0 418
michael@0 419
michael@0 420 class IfNode:
michael@0 421 def __init__(self, exp=None, then_branch=None, else_branch=None):
michael@0 422 self.exp = exp
michael@0 423 self.then_branch = then_branch
michael@0 424 self.else_branch = else_branch
michael@0 425
michael@0 426
michael@0 427 class RawCodeNode:
michael@0 428 def __init__(self, token=None):
michael@0 429 self.raw_code = token
michael@0 430
michael@0 431
michael@0 432 class LiteralDollarNode:
michael@0 433 def __init__(self, token):
michael@0 434 self.token = token
michael@0 435
michael@0 436
michael@0 437 class ExpNode:
michael@0 438 def __init__(self, token, python_exp):
michael@0 439 self.token = token
michael@0 440 self.python_exp = python_exp
michael@0 441
michael@0 442
michael@0 443 def PopFront(a_list):
michael@0 444 head = a_list[0]
michael@0 445 a_list[:1] = []
michael@0 446 return head
michael@0 447
michael@0 448
michael@0 449 def PushFront(a_list, elem):
michael@0 450 a_list[:0] = [elem]
michael@0 451
michael@0 452
michael@0 453 def PopToken(a_list, token_type=None):
michael@0 454 token = PopFront(a_list)
michael@0 455 if token_type is not None and token.token_type != token_type:
michael@0 456 print 'ERROR: %s expected at %s' % (token_type, token.start)
michael@0 457 print 'ERROR: %s found instead' % (token,)
michael@0 458 sys.exit(1)
michael@0 459
michael@0 460 return token
michael@0 461
michael@0 462
michael@0 463 def PeekToken(a_list):
michael@0 464 if not a_list:
michael@0 465 return None
michael@0 466
michael@0 467 return a_list[0]
michael@0 468
michael@0 469
michael@0 470 def ParseExpNode(token):
michael@0 471 python_exp = re.sub(r'([_A-Za-z]\w*)', r'self.GetValue("\1")', token.value)
michael@0 472 return ExpNode(token, python_exp)
michael@0 473
michael@0 474
michael@0 475 def ParseElseNode(tokens):
michael@0 476 def Pop(token_type=None):
michael@0 477 return PopToken(tokens, token_type)
michael@0 478
michael@0 479 next = PeekToken(tokens)
michael@0 480 if not next:
michael@0 481 return None
michael@0 482 if next.token_type == '$else':
michael@0 483 Pop('$else')
michael@0 484 Pop('[[')
michael@0 485 code_node = ParseCodeNode(tokens)
michael@0 486 Pop(']]')
michael@0 487 return code_node
michael@0 488 elif next.token_type == '$elif':
michael@0 489 Pop('$elif')
michael@0 490 exp = Pop('code')
michael@0 491 Pop('[[')
michael@0 492 code_node = ParseCodeNode(tokens)
michael@0 493 Pop(']]')
michael@0 494 inner_else_node = ParseElseNode(tokens)
michael@0 495 return CodeNode([IfNode(ParseExpNode(exp), code_node, inner_else_node)])
michael@0 496 elif not next.value.strip():
michael@0 497 Pop('code')
michael@0 498 return ParseElseNode(tokens)
michael@0 499 else:
michael@0 500 return None
michael@0 501
michael@0 502
michael@0 503 def ParseAtomicCodeNode(tokens):
michael@0 504 def Pop(token_type=None):
michael@0 505 return PopToken(tokens, token_type)
michael@0 506
michael@0 507 head = PopFront(tokens)
michael@0 508 t = head.token_type
michael@0 509 if t == 'code':
michael@0 510 return RawCodeNode(head)
michael@0 511 elif t == '$var':
michael@0 512 id_token = Pop('id')
michael@0 513 Pop('=')
michael@0 514 next = PeekToken(tokens)
michael@0 515 if next.token_type == 'exp':
michael@0 516 exp_token = Pop()
michael@0 517 return VarNode(id_token, ParseExpNode(exp_token))
michael@0 518 Pop('[[')
michael@0 519 code_node = ParseCodeNode(tokens)
michael@0 520 Pop(']]')
michael@0 521 return VarNode(id_token, code_node)
michael@0 522 elif t == '$for':
michael@0 523 id_token = Pop('id')
michael@0 524 next_token = PeekToken(tokens)
michael@0 525 if next_token.token_type == 'code':
michael@0 526 sep_token = next_token
michael@0 527 Pop('code')
michael@0 528 else:
michael@0 529 sep_token = None
michael@0 530 Pop('[[')
michael@0 531 code_node = ParseCodeNode(tokens)
michael@0 532 Pop(']]')
michael@0 533 return ForNode(id_token, sep_token, code_node)
michael@0 534 elif t == '$if':
michael@0 535 exp_token = Pop('code')
michael@0 536 Pop('[[')
michael@0 537 code_node = ParseCodeNode(tokens)
michael@0 538 Pop(']]')
michael@0 539 else_node = ParseElseNode(tokens)
michael@0 540 return IfNode(ParseExpNode(exp_token), code_node, else_node)
michael@0 541 elif t == '$range':
michael@0 542 id_token = Pop('id')
michael@0 543 exp1_token = Pop('exp')
michael@0 544 Pop('..')
michael@0 545 exp2_token = Pop('exp')
michael@0 546 return RangeNode(id_token, ParseExpNode(exp1_token),
michael@0 547 ParseExpNode(exp2_token))
michael@0 548 elif t == '$id':
michael@0 549 return ParseExpNode(Token(head.start + 1, head.end, head.value[1:], 'id'))
michael@0 550 elif t == '$($)':
michael@0 551 return LiteralDollarNode(head)
michael@0 552 elif t == '$':
michael@0 553 exp_token = Pop('exp')
michael@0 554 return ParseExpNode(exp_token)
michael@0 555 elif t == '[[':
michael@0 556 code_node = ParseCodeNode(tokens)
michael@0 557 Pop(']]')
michael@0 558 return code_node
michael@0 559 else:
michael@0 560 PushFront(tokens, head)
michael@0 561 return None
michael@0 562
michael@0 563
michael@0 564 def ParseCodeNode(tokens):
michael@0 565 atomic_code_list = []
michael@0 566 while True:
michael@0 567 if not tokens:
michael@0 568 break
michael@0 569 atomic_code_node = ParseAtomicCodeNode(tokens)
michael@0 570 if atomic_code_node:
michael@0 571 atomic_code_list.append(atomic_code_node)
michael@0 572 else:
michael@0 573 break
michael@0 574 return CodeNode(atomic_code_list)
michael@0 575
michael@0 576
michael@0 577 def ParseToAST(pump_src_text):
michael@0 578 """Convert the given Pump source text into an AST."""
michael@0 579 tokens = list(Tokenize(pump_src_text))
michael@0 580 code_node = ParseCodeNode(tokens)
michael@0 581 return code_node
michael@0 582
michael@0 583
michael@0 584 class Env:
michael@0 585 def __init__(self):
michael@0 586 self.variables = []
michael@0 587 self.ranges = []
michael@0 588
michael@0 589 def Clone(self):
michael@0 590 clone = Env()
michael@0 591 clone.variables = self.variables[:]
michael@0 592 clone.ranges = self.ranges[:]
michael@0 593 return clone
michael@0 594
michael@0 595 def PushVariable(self, var, value):
michael@0 596 # If value looks like an int, store it as an int.
michael@0 597 try:
michael@0 598 int_value = int(value)
michael@0 599 if ('%s' % int_value) == value:
michael@0 600 value = int_value
michael@0 601 except Exception:
michael@0 602 pass
michael@0 603 self.variables[:0] = [(var, value)]
michael@0 604
michael@0 605 def PopVariable(self):
michael@0 606 self.variables[:1] = []
michael@0 607
michael@0 608 def PushRange(self, var, lower, upper):
michael@0 609 self.ranges[:0] = [(var, lower, upper)]
michael@0 610
michael@0 611 def PopRange(self):
michael@0 612 self.ranges[:1] = []
michael@0 613
michael@0 614 def GetValue(self, identifier):
michael@0 615 for (var, value) in self.variables:
michael@0 616 if identifier == var:
michael@0 617 return value
michael@0 618
michael@0 619 print 'ERROR: meta variable %s is undefined.' % (identifier,)
michael@0 620 sys.exit(1)
michael@0 621
michael@0 622 def EvalExp(self, exp):
michael@0 623 try:
michael@0 624 result = eval(exp.python_exp)
michael@0 625 except Exception, e:
michael@0 626 print 'ERROR: caught exception %s: %s' % (e.__class__.__name__, e)
michael@0 627 print ('ERROR: failed to evaluate meta expression %s at %s' %
michael@0 628 (exp.python_exp, exp.token.start))
michael@0 629 sys.exit(1)
michael@0 630 return result
michael@0 631
michael@0 632 def GetRange(self, identifier):
michael@0 633 for (var, lower, upper) in self.ranges:
michael@0 634 if identifier == var:
michael@0 635 return (lower, upper)
michael@0 636
michael@0 637 print 'ERROR: range %s is undefined.' % (identifier,)
michael@0 638 sys.exit(1)
michael@0 639
michael@0 640
michael@0 641 class Output:
michael@0 642 def __init__(self):
michael@0 643 self.string = ''
michael@0 644
michael@0 645 def GetLastLine(self):
michael@0 646 index = self.string.rfind('\n')
michael@0 647 if index < 0:
michael@0 648 return ''
michael@0 649
michael@0 650 return self.string[index + 1:]
michael@0 651
michael@0 652 def Append(self, s):
michael@0 653 self.string += s
michael@0 654
michael@0 655
michael@0 656 def RunAtomicCode(env, node, output):
michael@0 657 if isinstance(node, VarNode):
michael@0 658 identifier = node.identifier.value.strip()
michael@0 659 result = Output()
michael@0 660 RunAtomicCode(env.Clone(), node.atomic_code, result)
michael@0 661 value = result.string
michael@0 662 env.PushVariable(identifier, value)
michael@0 663 elif isinstance(node, RangeNode):
michael@0 664 identifier = node.identifier.value.strip()
michael@0 665 lower = int(env.EvalExp(node.exp1))
michael@0 666 upper = int(env.EvalExp(node.exp2))
michael@0 667 env.PushRange(identifier, lower, upper)
michael@0 668 elif isinstance(node, ForNode):
michael@0 669 identifier = node.identifier.value.strip()
michael@0 670 if node.sep is None:
michael@0 671 sep = ''
michael@0 672 else:
michael@0 673 sep = node.sep.value
michael@0 674 (lower, upper) = env.GetRange(identifier)
michael@0 675 for i in range(lower, upper + 1):
michael@0 676 new_env = env.Clone()
michael@0 677 new_env.PushVariable(identifier, i)
michael@0 678 RunCode(new_env, node.code, output)
michael@0 679 if i != upper:
michael@0 680 output.Append(sep)
michael@0 681 elif isinstance(node, RawCodeNode):
michael@0 682 output.Append(node.raw_code.value)
michael@0 683 elif isinstance(node, IfNode):
michael@0 684 cond = env.EvalExp(node.exp)
michael@0 685 if cond:
michael@0 686 RunCode(env.Clone(), node.then_branch, output)
michael@0 687 elif node.else_branch is not None:
michael@0 688 RunCode(env.Clone(), node.else_branch, output)
michael@0 689 elif isinstance(node, ExpNode):
michael@0 690 value = env.EvalExp(node)
michael@0 691 output.Append('%s' % (value,))
michael@0 692 elif isinstance(node, LiteralDollarNode):
michael@0 693 output.Append('$')
michael@0 694 elif isinstance(node, CodeNode):
michael@0 695 RunCode(env.Clone(), node, output)
michael@0 696 else:
michael@0 697 print 'BAD'
michael@0 698 print node
michael@0 699 sys.exit(1)
michael@0 700
michael@0 701
michael@0 702 def RunCode(env, code_node, output):
michael@0 703 for atomic_code in code_node.atomic_code:
michael@0 704 RunAtomicCode(env, atomic_code, output)
michael@0 705
michael@0 706
michael@0 707 def IsSingleLineComment(cur_line):
michael@0 708 return '//' in cur_line
michael@0 709
michael@0 710
michael@0 711 def IsInPreprocessorDirective(prev_lines, cur_line):
michael@0 712 if cur_line.lstrip().startswith('#'):
michael@0 713 return True
michael@0 714 return prev_lines and prev_lines[-1].endswith('\\')
michael@0 715
michael@0 716
michael@0 717 def WrapComment(line, output):
michael@0 718 loc = line.find('//')
michael@0 719 before_comment = line[:loc].rstrip()
michael@0 720 if before_comment == '':
michael@0 721 indent = loc
michael@0 722 else:
michael@0 723 output.append(before_comment)
michael@0 724 indent = len(before_comment) - len(before_comment.lstrip())
michael@0 725 prefix = indent*' ' + '// '
michael@0 726 max_len = 80 - len(prefix)
michael@0 727 comment = line[loc + 2:].strip()
michael@0 728 segs = [seg for seg in re.split(r'(\w+\W*)', comment) if seg != '']
michael@0 729 cur_line = ''
michael@0 730 for seg in segs:
michael@0 731 if len((cur_line + seg).rstrip()) < max_len:
michael@0 732 cur_line += seg
michael@0 733 else:
michael@0 734 if cur_line.strip() != '':
michael@0 735 output.append(prefix + cur_line.rstrip())
michael@0 736 cur_line = seg.lstrip()
michael@0 737 if cur_line.strip() != '':
michael@0 738 output.append(prefix + cur_line.strip())
michael@0 739
michael@0 740
michael@0 741 def WrapCode(line, line_concat, output):
michael@0 742 indent = len(line) - len(line.lstrip())
michael@0 743 prefix = indent*' ' # Prefix of the current line
michael@0 744 max_len = 80 - indent - len(line_concat) # Maximum length of the current line
michael@0 745 new_prefix = prefix + 4*' ' # Prefix of a continuation line
michael@0 746 new_max_len = max_len - 4 # Maximum length of a continuation line
michael@0 747 # Prefers to wrap a line after a ',' or ';'.
michael@0 748 segs = [seg for seg in re.split(r'([^,;]+[,;]?)', line.strip()) if seg != '']
michael@0 749 cur_line = '' # The current line without leading spaces.
michael@0 750 for seg in segs:
michael@0 751 # If the line is still too long, wrap at a space.
michael@0 752 while cur_line == '' and len(seg.strip()) > max_len:
michael@0 753 seg = seg.lstrip()
michael@0 754 split_at = seg.rfind(' ', 0, max_len)
michael@0 755 output.append(prefix + seg[:split_at].strip() + line_concat)
michael@0 756 seg = seg[split_at + 1:]
michael@0 757 prefix = new_prefix
michael@0 758 max_len = new_max_len
michael@0 759
michael@0 760 if len((cur_line + seg).rstrip()) < max_len:
michael@0 761 cur_line = (cur_line + seg).lstrip()
michael@0 762 else:
michael@0 763 output.append(prefix + cur_line.rstrip() + line_concat)
michael@0 764 prefix = new_prefix
michael@0 765 max_len = new_max_len
michael@0 766 cur_line = seg.lstrip()
michael@0 767 if cur_line.strip() != '':
michael@0 768 output.append(prefix + cur_line.strip())
michael@0 769
michael@0 770
michael@0 771 def WrapPreprocessorDirective(line, output):
michael@0 772 WrapCode(line, ' \\', output)
michael@0 773
michael@0 774
michael@0 775 def WrapPlainCode(line, output):
michael@0 776 WrapCode(line, '', output)
michael@0 777
michael@0 778
michael@0 779 def IsMultiLineIWYUPragma(line):
michael@0 780 return re.search(r'/\* IWYU pragma: ', line)
michael@0 781
michael@0 782
michael@0 783 def IsHeaderGuardIncludeOrOneLineIWYUPragma(line):
michael@0 784 return (re.match(r'^#(ifndef|define|endif\s*//)\s*[\w_]+\s*$', line) or
michael@0 785 re.match(r'^#include\s', line) or
michael@0 786 # Don't break IWYU pragmas, either; that causes iwyu.py problems.
michael@0 787 re.search(r'// IWYU pragma: ', line))
michael@0 788
michael@0 789
michael@0 790 def WrapLongLine(line, output):
michael@0 791 line = line.rstrip()
michael@0 792 if len(line) <= 80:
michael@0 793 output.append(line)
michael@0 794 elif IsSingleLineComment(line):
michael@0 795 if IsHeaderGuardIncludeOrOneLineIWYUPragma(line):
michael@0 796 # The style guide made an exception to allow long header guard lines,
michael@0 797 # includes and IWYU pragmas.
michael@0 798 output.append(line)
michael@0 799 else:
michael@0 800 WrapComment(line, output)
michael@0 801 elif IsInPreprocessorDirective(output, line):
michael@0 802 if IsHeaderGuardIncludeOrOneLineIWYUPragma(line):
michael@0 803 # The style guide made an exception to allow long header guard lines,
michael@0 804 # includes and IWYU pragmas.
michael@0 805 output.append(line)
michael@0 806 else:
michael@0 807 WrapPreprocessorDirective(line, output)
michael@0 808 elif IsMultiLineIWYUPragma(line):
michael@0 809 output.append(line)
michael@0 810 else:
michael@0 811 WrapPlainCode(line, output)
michael@0 812
michael@0 813
michael@0 814 def BeautifyCode(string):
michael@0 815 lines = string.splitlines()
michael@0 816 output = []
michael@0 817 for line in lines:
michael@0 818 WrapLongLine(line, output)
michael@0 819 output2 = [line.rstrip() for line in output]
michael@0 820 return '\n'.join(output2) + '\n'
michael@0 821
michael@0 822
michael@0 823 def ConvertFromPumpSource(src_text):
michael@0 824 """Return the text generated from the given Pump source text."""
michael@0 825 ast = ParseToAST(StripMetaComments(src_text))
michael@0 826 output = Output()
michael@0 827 RunCode(Env(), ast, output)
michael@0 828 return BeautifyCode(output.string)
michael@0 829
michael@0 830
michael@0 831 def main(argv):
michael@0 832 if len(argv) == 1:
michael@0 833 print __doc__
michael@0 834 sys.exit(1)
michael@0 835
michael@0 836 file_path = argv[-1]
michael@0 837 output_str = ConvertFromPumpSource(file(file_path, 'r').read())
michael@0 838 if file_path.endswith('.pump'):
michael@0 839 output_file_path = file_path[:-5]
michael@0 840 else:
michael@0 841 output_file_path = '-'
michael@0 842 if output_file_path == '-':
michael@0 843 print output_str,
michael@0 844 else:
michael@0 845 output_file = file(output_file_path, 'w')
michael@0 846 output_file.write('// This file was GENERATED by command:\n')
michael@0 847 output_file.write('// %s %s\n' %
michael@0 848 (os.path.basename(__file__), os.path.basename(file_path)))
michael@0 849 output_file.write('// DO NOT EDIT BY HAND!!!\n\n')
michael@0 850 output_file.write(output_str)
michael@0 851 output_file.close()
michael@0 852
michael@0 853
michael@0 854 if __name__ == '__main__':
michael@0 855 main(sys.argv)

mercurial