Lines Matching refs:tokens
218 self.tokens = None
256 tokens = []
264 tokens.append(token)
266 return tokens
274 self.tokens = self._getTokensWithCursors()
280 self.tokens = self._getTokensWithCursors()
284 if self._index < len(self.tokens):
285 t = self.tokens[self._index]
314 tokens = CppStringTokenizer(token_string)
317 token = tokens.nextToken()
390 def __init__(self, tokens): argument
392 self.tokens = tokens
393 self._num_tokens = len(tokens)
397 print "CppExpr: trying to parse %s" % repr(tokens)
403 % (self._index, self._num_tokens, repr(tokens)))
407 tok = self.tokens[self._index]
415 token = self.tokens[self._index]
423 token = self.tokens[self._index].id
434 token = self.tokens[self._index].id
447 token = self.tokens[self._index].id
460 if self.tokens[self._index].kind != TokenKind.LITERAL:
478 t = self.tokens[self._index]
494 t = self.tokens[self._index]
502 self.tokens[self._index].id == tokLPAREN):
511 t = self.tokens[self._index]
526 t = self.tokens[self._index]
534 self.tokens[self._index].id != tokLPAREN):
542 id = self.tokens[self._index].id
547 param = self.tokens[j:k]
608 self.tokens[self._index].id))
623 return self.tokens[self._index]
629 return self.tokens[self._index]
809 return repr(CppExpr(CppStringTokenizer(expr).tokens))
841 e = CppExpr(CppStringTokenizer(expr).tokens)
918 return str(CppExpr(CppStringTokenizer(expr).tokens))
959 def __init__(self, tokens, directive=None, lineno=0, identifier=None): argument
969 tokens = [tok] + tokens
977 tokens = [tok1, tok2] + tokens
980 self.tokens = tokens
986 self.lineno = self.tokens[0].location.line
989 self.expr = CppExpr(self.tokens)
1023 return ''.join([str(x) for x in self.tokens])
1026 def format_blocks(tokens, indent=0): argument
1032 while i < len(tokens):
1033 t = tokens[i]
1038 if i < 2 or tokens[i-2].id != 'extern' or tokens[i-1].id != '"C"':
1048 if (i + 1 < len(tokens) and
1049 (tokens[i+1].id == ';' or
1050 tokens[i+1].id in ['else', '__attribute__',
1052 tokens[i+1].kind == TokenKind.IDENTIFIER)):
1088 if self.directive is None and not self.tokens:
1094 lines, indent = self.format_blocks(self.tokens, indent)
1107 for tok in self.tokens:
1111 for tok in self.tokens:
1134 if self.tokens:
1136 expr = strip_space(' '.join([tok.id for tok in self.tokens]))
1144 result = "#error %s" % ' '.join([tok.id for tok in self.tokens])
1148 if self.tokens:
1150 result += ''.join([tok.id for tok in self.tokens])
1152 lines, _ = self.format_blocks(self.tokens)
1204 num_tokens = len(b.tokens)
1219 if (b.tokens[i].kind != TokenKind.KEYWORD or
1220 b.tokens[i].id != "struct"):
1223 if (b.tokens[i + 1].kind == TokenKind.IDENTIFIER and
1224 b.tokens[i + 2].kind == TokenKind.PUNCTUATION and
1225 b.tokens[i + 2].id == "{" and b.tokens[i + 1].id in structs):
1231 if (b.tokens[j].kind == TokenKind.PUNCTUATION and
1232 b.tokens[j].id == "}" and
1233 b.tokens[j + 1].kind == TokenKind.PUNCTUATION and
1234 b.tokens[j + 1].id == ";"):
1235 b.tokens = b.tokens[0:i] + b.tokens[j + 2:num_tokens]
1236 num_tokens = len(b.tokens)
1312 numTokens = len(block.tokens)
1316 token_id = block.tokens[i].id
1320 if (i >= 2 and block.tokens[i-2].id == 'extern' and
1321 block.tokens[i-1].id == '"C"'):
1332 if block.tokens[i].id == ';':
1348 block.tokens[i-1].kind == TokenKind.IDENTIFIER and
1349 block.tokens[i-1].id in macros):
1354 blocksToKeep.append(Block(block.tokens[lastTerminatorIndex:i-1]))
1362 if block.tokens[i].id == ')':
1366 elif block.tokens[i].id == '(':
1381 elif block.tokens[i].kind == TokenKind.IDENTIFIER:
1387 blocksInProgress.append(Block(block.tokens[lastTerminatorIndex:i+1]))
1400 blocksInProgress.append(Block(block.tokens[lastTerminatorIndex:numTokens]))
1412 while i < len(b.tokens):
1413 tok = b.tokens[i]
1415 and (i + 2) < len(b.tokens) and b.tokens[i + 2].id == '{'):
1416 struct_name = b.tokens[i + 1].id
1420 while end < len(b.tokens) and b.tokens[end].id != '}':
1423 while end < len(b.tokens) and b.tokens[end].id != ';':
1427 b.tokens[i:end] = []
1443 b.expr = CppExpr(b.tokens)
1447 self.blocks.insert(2, Block(replacement.tokens, directive='include'))
1498 def consume_extent(i, tokens, extent=None, detect_change=False): argument
1508 extent = tokens[i].cursor.extent
1510 while i < len(tokens) and tokens[i].location in extent:
1511 t = tokens[i]
1521 def consume_line(i, tokens): argument
1524 line = tokens[i].location.line
1525 while i < len(tokens) and tokens[i].location.line == line:
1526 if tokens[i].cursor.kind == CursorKind.PREPROCESSING_DIRECTIVE:
1528 result.append(tokens[i])
1534 tokens = tokzer.tokens
1540 while i < len(tokens):
1541 t = tokens[i]
1556 if j + 1 >= len(tokens):
1558 directive = tokens[j+1].id
1561 if i+2 >= len(tokens):
1563 (tokens[i].location))
1566 extent = tokens[i].cursor.extent
1571 if (i + 1 < len(tokens) and tokens[i+1].id == '(' and
1572 (tokens[i].location.column + len(tokens[i].spelling) ==
1573 tokens[i+1].location.column)):
1574 while i < len(tokens):
1575 id += tokens[i].id
1576 if tokens[i].spelling == ')':
1581 id += tokens[i].id
1585 (i, ret) = consume_extent(i, tokens, extent=extent)
1590 (i, ret) = consume_extent(i, tokens)
1598 directive = tokens[i+1].id
1599 (i, ret) = consume_extent(i, tokens)
1609 (i, ret) = consume_extent(i, tokens, detect_change=True)
1617 (i, ret) = consume_extent(i, tokens, detect_change=True)
1621 (i, ret) = consume_line(i, tokens)
1739 blocks[j].expr = CppExpr(CppStringTokenizer("1").tokens)
1766 blocks[j].tokens = CppStringTokenizer("1").tokens
1779 blocks[j].tokens = CppStringTokenizer("1").tokens
1786 blocks[k].expr = CppExpr(CppStringTokenizer("0").tokens)
1793 blocks[k].expr = CppExpr(CppStringTokenizer("0").tokens)