Contenu connexe Similaire à Climbing the Abstract Syntax Tree (phpDay 2017) (20) Plus de James Titcumb (20) Climbing the Abstract Syntax Tree (phpDay 2017)2. Who is this guy?
James Titcumb
www.jamestitcumb.com
www.roave.com
www.phphants.co.uk
www.phpsouthcoast.co.uk
@asgrim
22. @asgrim
zend_language_parser.y
if_stmt:
if_stmt_without_else %prec T_NOELSE { $$ = $1; }
| if_stmt_without_else T_ELSE statement
{ $$ = zend_ast_list_add($1, zend_ast_create(ZEND_AST_IF_ELEM, NULL, $3)); }
;
if_stmt_without_else:
T_IF '(' expr ')' statement
{ $$ = zend_ast_create_list(1, ZEND_AST_IF,
zend_ast_create(ZEND_AST_IF_ELEM, $3, $5)); }
| if_stmt_without_else T_ELSEIF '(' expr ')' statement
{ $$ = zend_ast_list_add($1,
zend_ast_create(ZEND_AST_IF_ELEM, $4, $6)); }
;
23. @asgrim
if_stmt:
if_stmt_without_else %prec T_NOELSE { $$ = $1; }
| if_stmt_without_else T_ELSE statement
{ $$ = zend_ast_list_add($1, zend_ast_create(ZEND_AST_IF_ELEM, NULL, $3)); }
;
if_stmt_without_else:
T_IF '(' expr ')' statement
{ $$ = zend_ast_create_list(1, ZEND_AST_IF,
zend_ast_create(ZEND_AST_IF_ELEM, $3, $5)); }
| if_stmt_without_else T_ELSEIF '(' expr ')' statement
{ $$ = zend_ast_list_add($1,
zend_ast_create(ZEND_AST_IF_ELEM, $4, $6)); }
;
zend_language_parser.y
24. @asgrim
if_stmt:
if_stmt_without_else %prec T_NOELSE { $$ = $1; }
| if_stmt_without_else T_ELSE statement
{ $$ = zend_ast_list_add($1, zend_ast_create(ZEND_AST_IF_ELEM, NULL, $3)); }
;
if_stmt_without_else:
T_IF '(' expr ')' statement
{ $$ = zend_ast_create_list(1, ZEND_AST_IF,
zend_ast_create(ZEND_AST_IF_ELEM, $3, $5)); }
| if_stmt_without_else T_ELSEIF '(' expr ')' statement
{ $$ = zend_ast_list_add($1,
zend_ast_create(ZEND_AST_IF_ELEM, $4, $6)); }
;
zend_language_parser.y
25. @asgrim
if_stmt:
if_stmt_without_else %prec T_NOELSE { $$ = $1; }
| if_stmt_without_else T_ELSE statement
{ $$ = zend_ast_list_add($1, zend_ast_create(ZEND_AST_IF_ELEM, NULL, $3)); }
;
if_stmt_without_else:
T_IF '(' expr ')' statement
{ $$ = zend_ast_create_list(1, ZEND_AST_IF,
zend_ast_create(ZEND_AST_IF_ELEM, $3, $5)); }
| if_stmt_without_else T_ELSEIF '(' expr ')' statement
{ $$ = zend_ast_list_add($1,
zend_ast_create(ZEND_AST_IF_ELEM, $4, $6)); }
;
zend_language_parser.y
26. @asgrim
if_stmt:
if_stmt_without_else %prec T_NOELSE { $$ = $1; }
| if_stmt_without_else T_ELSE statement
{ $$ = zend_ast_list_add($1, zend_ast_create(ZEND_AST_IF_ELEM, NULL, $3)); }
;
if_stmt_without_else:
T_IF '(' expr ')' statement
{ $$ = zend_ast_create_list(1, ZEND_AST_IF,
zend_ast_create(ZEND_AST_IF_ELEM, $3, $5)); }
| if_stmt_without_else T_ELSEIF '(' expr ')' statement
{ $$ = zend_ast_list_add($1,
zend_ast_create(ZEND_AST_IF_ELEM, $4, $6)); }
;
zend_language_parser.y
27. @asgrim
if_stmt:
if_stmt_without_else %prec T_NOELSE { $$ = $1; }
| if_stmt_without_else T_ELSE statement
{ $$ = zend_ast_list_add($1, zend_ast_create(ZEND_AST_IF_ELEM, NULL, $3)); }
;
if_stmt_without_else:
T_IF '(' expr ')' statement
{ $$ = zend_ast_create_list(1, ZEND_AST_IF,
zend_ast_create(ZEND_AST_IF_ELEM, $3, $5)); }
| if_stmt_without_else T_ELSEIF '(' expr ')' statement
{ $$ = zend_ast_list_add($1,
zend_ast_create(ZEND_AST_IF_ELEM, $4, $6)); }
;
zend_language_parser.y
28. @asgrim
if_stmt:
if_stmt_without_else %prec T_NOELSE { $$ = $1; }
| if_stmt_without_else T_ELSE statement
{ $$ = zend_ast_list_add($1, zend_ast_create(ZEND_AST_IF_ELEM, NULL, $3)); }
;
if_stmt_without_else:
T_IF '(' expr ')' statement
{ $$ = zend_ast_create_list(1, ZEND_AST_IF,
zend_ast_create(ZEND_AST_IF_ELEM, $3, $5)); }
| if_stmt_without_else T_ELSEIF '(' expr ')' statement
{ $$ = zend_ast_list_add($1,
zend_ast_create(ZEND_AST_IF_ELEM, $4, $6)); }
;
zend_language_parser.y
29. @asgrim
if_stmt:
if_stmt_without_else %prec T_NOELSE { $$ = $1; }
| if_stmt_without_else T_ELSE statement
{ $$ = zend_ast_list_add($1, zend_ast_create(ZEND_AST_IF_ELEM, NULL, $3)); }
;
if_stmt_without_else:
T_IF '(' expr ')' statement
{ $$ = zend_ast_create_list(1, ZEND_AST_IF,
zend_ast_create(ZEND_AST_IF_ELEM, $3, $5)); }
| if_stmt_without_else T_ELSEIF '(' expr ')' statement
{ $$ = zend_ast_list_add($1,
zend_ast_create(ZEND_AST_IF_ELEM, $4, $6)); }
;
zend_language_parser.y
30. @asgrim
if ($a == 1)
{
a();
}
else if ($b == 1)
{
b();
}
else
{
c();
}
Using the rules to parse
31. @asgrim
if ($a == 1)
{
a();
}
else if ($b == 1)
{
b();
}
else
{
c();
}
Using the rules to parse
if_stmt_without_else (A)
32. @asgrim
if ($a == 1)
{
a();
}
else if ($b == 1)
{
b();
}
else
{
c();
}
Using the rules to parse
if_stmt_without_else (A)
if_stmt_without_else (B)
33. @asgrim
if ($a == 1)
{
a();
}
else if ($b == 1)
{
b();
}
else
{
c();
}
Using the rules to parse
if_stmt_without_else (A)
if_stmt_without_else (B)
if_stmt
34. @asgrim
Zend_language_parser.y (PHP 7.0.10)
if_stmt:
if_stmt_without_else %prec T_NOELSE { $$ = $1; }
| if_stmt_without_else T_ELSE statement
{ $$ = zend_ast_list_add($1, zend_ast_create(ZEND_AST_IF_ELEM, NULL, $3)); }
;
if_stmt_without_else:
T_IF '(' expr ')' statement
{ $$ = zend_ast_create_list(1, ZEND_AST_IF,
zend_ast_create(ZEND_AST_IF_ELEM, $3, $5)); }
| if_stmt_without_else T_ELSEIF '(' expr ')' statement
{ $$ = zend_ast_list_add($1,
zend_ast_create(ZEND_AST_IF_ELEM, $4, $6)); }
;
35. @asgrim
zend_language_parser.y (PHP 5.6.26)
T_IF parenthesis_expr { zend_do_if_cond(&$2, &$1 TSRMLS_CC); }
statement { zend_do_if_after_statement(&$1, 1 TSRMLS_CC); }
void zend_do_if_cond(const znode *cond, znode *closing_bracket_token TSRMLS_DC)
{
int if_cond_op_number = get_next_op_number(CG(active_op_array));
zend_op *opline = get_next_op(CG(active_op_array) TSRMLS_CC);
opline->opcode = ZEND_JMPZ;
SET_NODE(opline->op1, cond);
closing_bracket_token->u.op.opline_num = if_cond_op_number;
SET_UNUSED(opline->op2);
INC_BPC(CG(active_op_array));
}
46. @asgrim
An AST representation
Assign statement
|-- Variable $a
`-- Integer, value 5
Assign statement
|-- Variable $b
`-- Integer, value 3
Echo statement
`-- Add operation
|-- Left
| `-- Variable $a
`-- Right
`-- Multiply operation
|-- Left
| `-- Variable $b
`-- Right
`-- Integer, value 2
49. @asgrim
AST compilation: pre-order traversal
Statements
EchoAssign
Scalar
value: (int)5
Variable
name: $a
Assign
Scalar
value: (int)3
Variable
name: $b
Add op
Right operandLeft operand
Variable
name: $a
Multiply op
Right operandLeft operand
Variable
name: $b
Scalar
value: (int)2
54. @asgrim
Order of precedence
1 + 2 * 3
= 1 + (2 * 3) = 7?
= (1 + 2) * 3 = 9?
+ 1 * 2 3
Operator Left operand Right operand
Operator Left operand
Right operand
70. @asgrim
Using regular expressions
private static $matches = [
'/^(+)/' => Token::T_ADD,
'/^(-)/' => Token::T_SUBTRACT,
'/^(*)/' => Token::T_MULTIPLY,
'/^(/)/' => Token::T_DIVIDE,
'/^(d+)/' => Token::T_INTEGER,
'/^(s+)/' => Token::T_WHITESPACE,
];
71. @asgrim
Step through the input string
public function __invoke(string $input) : array
{
$tokens = [];
$offset = 0;
while ($offset < strlen($input)) {
$focus = substr($input, $offset);
$result = $this->match($focus);
$tokens[] = $result;
$offset += strlen($result->getLexeme());
}
return $tokens;
}
72. @asgrim
The matching method
private function match(string $input) : Token
{
foreach (self::$matches as $pattern => $token) {
if (preg_match($pattern, $input, $matches)) {
return new Token($token, $matches[1]);
}
}
throw new RuntimeException(sprintf(
'Unmatched token, next 15 chars were: %s', substr($input, 0, 15)
));
}
74. @asgrim
Order tokens by operator precedence
/**
* Higher number is higher precedence.
* @var int[]
*/
private static $operatorPrecedence = [
Token::T_SUBTRACT => 0,
Token::T_ADD => 1,
Token::T_DIVIDE => 2,
Token::T_MULTIPLY => 3,
];
75. @asgrim
Order tokens by operator precedence
/** @var Token[] $stack */
$stack = [];
/** @var Token[] $operators */
$operators = [];
while (false !== ($token = current($tokens))) {
if ($token->isOperator()) {
// ...
}
$stack[] = $token;
next($tokens);
}
76. @asgrim
Order tokens by operator precedence
/** @var Token[] $stack */
$stack = [];
/** @var Token[] $operators */
$operators = [];
while (false !== ($token = current($tokens))) {
if ($token->isOperator()) {
// ...
}
$stack[] = $token;
next($tokens);
}
77. @asgrim
Order tokens by operator precedence
/** @var Token[] $stack */
$stack = [];
/** @var Token[] $operators */
$operators = [];
while (false !== ($token = current($tokens))) {
if ($token->isOperator()) {
// ...
}
$stack[] = $token;
next($tokens);
}
78. @asgrim
Order tokens by operator precedence
/** @var Token[] $stack */
$stack = [];
/** @var Token[] $operators */
$operators = [];
while (false !== ($token = current($tokens))) {
if ($token->isOperator()) {
// ...
}
$stack[] = $token;
next($tokens);
}
79. @asgrim
Order tokens by operator precedence
if ($token->isOperator()) {
$tokenPrecedence = self::$operatorPrecedence[$token->getToken()];
while (
count($operators)
&& self::$operatorPrecedence[$operators[count($operators) - 1]->getToken()]
> $tokenPrecedence
) {
$higherOp = array_pop($operators);
$stack[] = $higherOp;
}
$operators[] = $token;
next($tokens);
continue;
}
80. @asgrim
Order tokens by operator precedence
if ($token->isOperator()) {
$tokenPrecedence = self::$operatorPrecedence[$token->getToken()];
while (
count($operators)
&& self::$operatorPrecedence[$operators[count($operators) - 1]->getToken()]
> $tokenPrecedence
) {
$higherOp = array_pop($operators);
$stack[] = $higherOp;
}
$operators[] = $token;
next($tokens);
continue;
}
81. @asgrim
Order tokens by operator precedence
if ($token->isOperator()) {
$tokenPrecedence = self::$operatorPrecedence[$token->getToken()];
while (
count($operators)
&& self::$operatorPrecedence[$operators[count($operators) - 1]->getToken()]
> $tokenPrecedence
) {
$higherOp = array_pop($operators);
$stack[] = $higherOp;
}
$operators[] = $token;
next($tokens);
continue;
}
82. @asgrim
Order tokens by operator precedence
if ($token->isOperator()) {
$tokenPrecedence = self::$operatorPrecedence[$token->getToken()];
while (
count($operators)
&& self::$operatorPrecedence[$operators[count($operators) - 1]->getToken()]
> $tokenPrecedence
) {
$higherOp = array_pop($operators);
$stack[] = $higherOp;
}
$operators[] = $token;
next($tokens);
continue;
}
83. @asgrim
Order tokens by operator precedence
// Clean up by moving any remaining operators onto the token stack
while (count($operators)) {
$stack[] = array_pop($operators);
}
return $stack;
92. @asgrim
Create AST
while ($ip < count($tokenStack)) {
$token = $tokenStack[$ip++];
if ($token->isOperator()) {
// (figure out $nodeType)
$right = array_pop($astStack);
$left = array_pop($astStack);
$astStack[] = new $nodeType($left, $right);
continue;
}
$astStack[] = new NodeScalarIntegerValue((int)$token->getLexeme());
}
93. @asgrim
Create AST
while ($ip < count($tokenStack)) {
$token = $tokenStack[$ip++];
if ($token->isOperator()) {
// (figure out $nodeType)
$right = array_pop($astStack);
$left = array_pop($astStack);
$astStack[] = new $nodeType($left, $right);
continue;
}
$astStack[] = new NodeScalarIntegerValue((int)$token->getLexeme());
}
94. @asgrim
Create AST
while ($ip < count($tokenStack)) {
$token = $tokenStack[$ip++];
if ($token->isOperator()) {
// (figure out $nodeType)
$right = array_pop($astStack);
$left = array_pop($astStack);
$astStack[] = new $nodeType($left, $right);
continue;
}
$astStack[] = new NodeScalarIntegerValue((int)$token->getLexeme());
}
95. @asgrim
Create AST
while ($ip < count($tokenStack)) {
$token = $tokenStack[$ip++];
if ($token->isOperator()) {
// (figure out $nodeType)
$right = array_pop($astStack);
$left = array_pop($astStack);
$astStack[] = new $nodeType($left, $right);
continue;
}
$astStack[] = new NodeScalarIntegerValue((int)$token->getLexeme());
}
98. @asgrim
Compile & execute AST
private function compileNode(NodeInterface $node)
{
if ($node instanceof NodeBinaryOpAbstractBinaryOp) {
return $this->compileBinaryOp($node);
}
if ($node instanceof NodeScalarIntegerValue) {
return $node->getValue();
}
}
99. @asgrim
Compile & execute AST
private function compileBinaryOp(NodeBinaryOpAbstractBinaryOp $node)
{
$left = $this->compileNode($node->getLeft());
$right = $this->compileNode($node->getRight());
switch (get_class($node)) {
case NodeBinaryOpAdd::class:
return $left + $right;
case NodeBinaryOpSubtract::class:
return $left - $right;
case NodeBinaryOpMultiply::class:
return $left * $right;
case NodeBinaryOpDivide::class:
return $left / $right;
}
}
103. @asgrim
php-ast example usage
<?php
require 'path/to/util.php';
$code = <<<'EOC'
<?php
$var = 42;
EOC;
echo ast_dump(astparse_code($code, $version=35)), "n";
// Output:
AST_STMT_LIST
0: AST_ASSIGN
var: AST_VAR
name: "var"
expr: 42
105. @asgrim
astkit example usage
$if = AstKit::parseString(<<<EOD
if (true) {
echo "This is a triumph.n";
} else {
echo "The cake is a lie.n";
}
EOD
);
$if->execute(); // First run, program is as-seen above
$const = $if->getChild(0)->getChild(0);
// Replace the "true" constant in the condition with false
$const->graft(0, false);
// Can also graft other AstKit nodes, instead of constants
$if->execute(); // Second run now takes the else path
115. @asgrim
… we get the AST!
Class, name Foo
|-- Statements
| |-- Property, name bar
| | |-- Type [private]
| | `-- Attributes [start line: 7, end line: 9]
| `-- Method, name thing
| |-- Type [public]
| |-- Parameters [...]
| |-- Statements [...]
| `-- Attributes [start line: 7, end line: 9]
`-- Attributes [start line: 3, end line: 10]