Lexer examples

Example #1 Tokenize comma separated integer list

<?php

use Parle\Token;
use 
Parle\Lexer;
use 
Parle\LexerException;

/* name => id */
$token = array(
        
"COMMA" => 1,
        
"CRLF" => 2,
        
"DECIMAL" => 3,
);
/* id => name */
$tokenIdToName array_flip($token);

$lex = new Lexer;
$lex->push("[\x2c]"$token["COMMA"]);
$lex->push("[\r][\n]"$token["CRLF"]);
$lex->push("[\d]+"$token["DECIMAL"]);
$lex->build();

$in "0,1,2\r\n3,42,5\r\n6,77,8\r\n";

$lex->consume($in);

do {
        
$lex->advance();
        
$tok $lex->getToken();

        if (
Token::UNKNOWN == $tok->id) {
                throw new 
LexerException("Unknown token '{$tok->value}' at offset {$lex->marker}.");
        }

        echo 
"TOKEN: "$tokenIdToName[$tok->id], PHP_EOL;
} while (
Token::EOI != $tok->id);

Example #2 Tokenize assign statement

<?php

use Parle\{TokenLexer};

$lex = new Lexer;

$lex->push("\$[a-zA-Z_][a-zA-Z0-9_]*"1);
$lex->push("="2);
$lex->push("\d+"3);
$lex->push(";"4);

$lex->build();

$lex->consume('$x = 42; $y = 24;');


do {
    
$lex->advance();
    
$tok $lex->getToken();
    
var_dump($tok);
} while (
Token::EOI != $tok->id);

User Contributed Notes

There are no user contributed notes for this page.