The purplex from mtomwing

purplex's People

Contributors

Stargazers

Watchers

purplex's Issues

Using OrderedDict for token list is slow

Switching to a regular list improves performance.

diff --git a/purplex/lex.py b/purplex/lex.py
index e70af02..93a0e94 100644
--- a/purplex/lex.py
+++ b/purplex/lex.py
@@ -15,11 +15,10 @@ class LexerBase(type):
         for name, attr in dct.items():
             if isinstance(attr, TokenDef):
                 tokens[name] = attr
-                del dct[name]
         ret = super(LexerBase, cls).__new__(cls, name, bases, dct)
         if hasattr(ret, 'tokens'):
             tokens.update(ret.tokens)
-        ret.tokens = tokens
+        ret.tokens = list(tokens.items())
         return ret


@@ -49,7 +48,7 @@ class Lexer(metaclass=LexerBase):
     def next_token(self):
         matches = []

-        for name, token_defn in self.tokens.items():
+        for name, token_defn in self.tokens:
             match = token_defn.regexp.match(self.input_text, self.input_pos)
             if match:
                 token = Token(name, match.group(0), token_defn,
diff --git a/purplex/parse.py b/purplex/parse.py
index b42dd99..bdf4bc6 100644
--- a/purplex/parse.py
+++ b/purplex/parse.py
@@ -94,7 +94,7 @@ class Parser(metaclass=ParserBase):

     def _build(self, start, debug):
         magic = MagicParser()
-        magic.tokens = [name for name, tokendef in self.LEXER.tokens.items()
+        magic.tokens = [name for name, tokendef in self.LEXER.tokens
                         if not tokendef.ignore]
         if self.PRECEDENCE:
             magic.precedence = self.PRECEDENCE
diff --git a/purplex/test/test_lexer.py b/purplex/test/test_lexer.py
index 5665db9..a0b7c51 100644
--- a/purplex/test/test_lexer.py
+++ b/purplex/test/test_lexer.py
@@ -26,13 +26,13 @@ def test_lex_foobar():

     assert tokens[0].name == 'FOO'
     assert tokens[0].value == 'foo'
-    assert tokens[0].defn == FooBarLexer.tokens['FOO']
+    assert tokens[0].defn == FooBarLexer.FOO
     assert tokens[0].line_num == 1
     assert tokens[0].line_pos == 1

     assert tokens[1].name == 'BAR'
     assert tokens[1].value == 'bar'
-    assert tokens[1].defn == FooBarLexer.tokens['BAR']
+    assert tokens[1].defn == FooBarLexer.BAR
     assert tokens[1].line_num == 1
     assert tokens[1].line_pos == 4