pydoctor.tokenutils.TokenProcessor

Processes tokens from source code lines.

Example usage: >>> from token import NAME, OP, NUMBER, STRING >>> src = ["foo = 123 # comment\n"] >>> tp = TokenProcessor(src) >>> isinstance(tp.buffers, list) True

# get_line returns the line at given lineno (1-based) >>> tp.get_line(1) 'foo = 123 # comment\n'

# fetch_token yields tokens one by one >>> t1 = tp.fetch_token() >>> t1.kind == NAME True >>> t2 = tp.fetch_token() >>> t2.kind == OP True >>> t3 = tp.fetch_token() >>> t3.kind == NUMBER True

# previous and current attributes >>> tp.previous == t2 True >>> tp.current == t3 True

# fetch_until collects tokens until matching condition >>> tp2 = TokenProcessor(["foo = (1 + 2)\n"]) >>> tokens = tp2.fetch_until([OP, ')']) >>> any(t.value == ')' for t in tokens) True