Lines Matching refs:tokens
43 TokenStream(const std::vector<std::string>& tokens, const FileCtx& fc) in TokenStream() argument
44 : fc_(fc), ix_(0u), tokens_(tokens) {} in TokenStream()
70 const std::vector<std::string>& tokens, in process_line() argument
75 auto& first = acc.empty() ? tokens[0] : acc[0]; in process_line()
76 auto& last = tokens.back(); in process_line()
85 TokenStream ts(tokens, fc); in process_line()
96 t += tokens; in process_line()
102 acc += tokens; in process_line()
109 fc.print_error("missing terminator", tokens[0]); in process_line()
111 fc.print_error("unknown token", tokens[0]); in process_line()
135 auto tokens = tokenize_string(line); in run_parser() local
136 if (tokens.empty()) in run_parser()
139 if (!process_line(parser, table, tokens, fc)) { in run_parser()