From 16d94a6e50eb81de9d9d438e1cce0746928597f3 Mon Sep 17 00:00:00 2001 From: Owen Jacobson Date: Sat, 11 Nov 2017 01:51:06 -0500 Subject: Introduce input ports. Ports are the lisp abstraction of files and streams. Actinide ports additionally guarantee a peek operation. This makes ``tokenize`` (now ``read_token``) callable as a lisp function, as it takes a port and reads one token from it. This is a substantial refactoring. As most of the state is now captured by closures, it's no longer practical to test individual states as readily. However, the top-level tokenizer tests exercise the full state space. --- tests/tokens.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'tests/tokens.py') diff --git a/tests/tokens.py b/tests/tokens.py index 0027fb2..3eb58b8 100644 --- a/tests/tokens.py +++ b/tests/tokens.py @@ -48,6 +48,10 @@ def whitespace_characters(): def tokens(): return one_of(symbols(), strings(), open_parens(), close_parens()) +# Generates a string which may not be empty, but which does not contain a token. +def nontokens(): + return one_of(whitespace(), comments(), just('')) + # Generates at least one character of whitespace. def whitespace(): return text(whitespace_characters(), min_size=1) -- cgit v1.2.3