PK €UU6ýqo o README.mdnu €žÙ˜ # Doctrine Lexer
[![Build Status](https://github.com/doctrine/lexer/workflows/Continuous%20Integration/badge.svg)](https://github.com/doctrine/lexer/actions)
Base library for a lexer that can be used in Top-Down, Recursive Descent Parsers.
This lexer is used in Doctrine Annotations and in Doctrine ORM (DQL).
https://www.doctrine-project.org/projects/lexer.html
PK €UUk©4RG
G
psalm.xmlnu €žÙ˜
PK €UUo½”Kw w
UPGRADE.mdnu €žÙ˜ Note about upgrading: Doctrine uses static and runtime mechanisms to raise
awareness about deprecated code.
- Use of `@deprecated` docblock that is detected by IDEs (like PHPStorm) or
Static Analysis tools (like Psalm, phpstan)
- Use of our low-overhead runtime deprecation API, details:
https://github.com/doctrine/deprecations/
# Upgrade to 2.0.0
`AbstractLexer::glimpse()` and `AbstractLexer::peek()` now return
instances of `Doctrine\Common\Lexer\Token`, which is an array-like class
Using it as an array is deprecated in favor of using properties of that class.
Using `count()` on it is deprecated with no replacement.
PK €UU*nØT T
.gitignorenu €žÙ˜ /vendor
/composer.lock
/phpunit.xml
/.phpunit.result.cache
/phpcs.xml
/.phpcs-cache
PK €UU˜¥¥vI I src/AbstractLexer.phpnu €žÙ˜ >
*/
private $tokens = [];
/**
* Current lexer position in input string.
*
* @var int
*/
private $position = 0;
/**
* Current peek of current lexer position.
*
* @var int
*/
private $peek = 0;
/**
* The next token in the input.
*
* @var Token|null
*/
public $lookahead;
/**
* The last matched/seen token.
*
* @var Token|null
*/
public $token;
/**
* Composed regex for input parsing.
*
* @var string|null
*/
private $regex;
/**
* Sets the input data to be tokenized.
*
* The Lexer is immediately reset and the new input tokenized.
* Any unprocessed tokens from any previous input are lost.
*
* @param string $input The input to be tokenized.
*
* @return void
*/
public function setInput($input)
{
$this->input = $input;
$this->tokens = [];
$this->reset();
$this->scan($input);
}
/**
* Resets the lexer.
*
* @return void
*/
public function reset()
{
$this->lookahead = null;
$this->token = null;
$this->peek = 0;
$this->position = 0;
}
/**
* Resets the peek pointer to 0.
*
* @return void
*/
public function resetPeek()
{
$this->peek = 0;
}
/**
* Resets the lexer position on the input to the given position.
*
* @param int $position Position to place the lexical scanner.
*
* @return void
*/
public function resetPosition($position = 0)
{
$this->position = $position;
}
/**
* Retrieve the original lexer's input until a given position.
*
* @param int $position
*
* @return string
*/
public function getInputUntilPosition($position)
{
return substr($this->input, 0, $position);
}
/**
* Checks whether a given token matches the current lookahead.
*
* @param T $type
*
* @return bool
*
* @psalm-assert-if-true !=null $this->lookahead
*/
public function isNextToken($type)
{
return $this->lookahead !== null && $this->lookahead->isA($type);
}
/**
* Checks whether any of the given tokens matches the current lookahead.
*
* @param list $types
*
* @return bool
*
* @psalm-assert-if-true !=null $this->lookahead
*/
public function isNextTokenAny(array $types)
{
return $this->lookahead !== null && $this->lookahead->isA(...$types);
}
/**
* Moves to the next token in the input string.
*
* @return bool
*
* @psalm-assert-if-true !null $this->lookahead
*/
public function moveNext()
{
$this->peek = 0;
$this->token = $this->lookahead;
$this->lookahead = isset($this->tokens[$this->position])
? $this->tokens[$this->position++] : null;
return $this->lookahead !== null;
}
/**
* Tells the lexer to skip input tokens until it sees a token with the given value.
*
* @param T $type The token type to skip until.
*
* @return void
*/
public function skipUntil($type)
{
while ($this->lookahead !== null && ! $this->lookahead->isA($type)) {
$this->moveNext();
}
}
/**
* Checks if given value is identical to the given token.
*
* @param string $value
* @param int|string $token
*
* @return bool
*/
public function isA($value, $token)
{
return $this->getType($value) === $token;
}
/**
* Moves the lookahead token forward.
*
* @return Token|null The next token or NULL if there are no more tokens ahead.
*/
public function peek()
{
if (isset($this->tokens[$this->position + $this->peek])) {
return $this->tokens[$this->position + $this->peek++];
}
return null;
}
/**
* Peeks at the next token, returns it and immediately resets the peek.
*
* @return Token|null The next token or NULL if there are no more tokens ahead.
*/
public function glimpse()
{
$peek = $this->peek();
$this->peek = 0;
return $peek;
}
/**
* Scans the input string for tokens.
*
* @param string $input A query string.
*
* @return void
*/
protected function scan($input)
{
if (! isset($this->regex)) {
$this->regex = sprintf(
'/(%s)|%s/%s',
implode(')|(', $this->getCatchablePatterns()),
implode('|', $this->getNonCatchablePatterns()),
$this->getModifiers()
);
}
$flags = PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_OFFSET_CAPTURE;
$matches = preg_split($this->regex, $input, -1, $flags);
if ($matches === false) {
// Work around https://bugs.php.net/78122
$matches = [[$input, 0]];
}
foreach ($matches as $match) {
// Must remain before 'value' assignment since it can change content
$firstMatch = $match[0];
$type = $this->getType($firstMatch);
$this->tokens[] = new Token(
$firstMatch,
$type,
$match[1]
);
}
}
/**
* Gets the literal for a given token.
*
* @param T $token
*
* @return int|string
*/
public function getLiteral($token)
{
if ($token instanceof UnitEnum) {
return get_class($token) . '::' . $token->name;
}
$className = static::class;
$reflClass = new ReflectionClass($className);
$constants = $reflClass->getConstants();
foreach ($constants as $name => $value) {
if ($value === $token) {
return $className . '::' . $name;
}
}
return $token;
}
/**
* Regex modifiers
*
* @return string
*/
protected function getModifiers()
{
return 'iu';
}
/**
* Lexical catchable patterns.
*
* @return string[]
*/
abstract protected function getCatchablePatterns();
/**
* Lexical non-catchable patterns.
*
* @return string[]
*/
abstract protected function getNonCatchablePatterns();
/**
* Retrieve token type. Also processes the token value if necessary.
*
* @param string $value
*
* @return T|null
*
* @param-out V $value
*/
abstract protected function getType(&$value);
}
PK €UU…+{Úñ ñ
src/Token.phpnu €žÙ˜
*/
final class Token implements ArrayAccess
{
/**
* The string value of the token in the input string
*
* @readonly
* @var V
*/
public $value;
/**
* The type of the token (identifier, numeric, string, input parameter, none)
*
* @readonly
* @var T|null
*/
public $type;
/**
* The position of the token in the input string
*
* @readonly
* @var int
*/
public $position;
/**
* @param V $value
* @param T|null $type
*/
public function __construct($value, $type, int $position)
{
$this->value = $value;
$this->type = $type;
$this->position = $position;
}
/** @param T ...$types */
public function isA(...$types): bool
{
return in_array($this->type, $types, true);
}
/**
* @deprecated Use the value, type or position property instead
* {@inheritDoc}
*/
public function offsetExists($offset): bool
{
Deprecation::trigger(
'doctrine/lexer',
'https://github.com/doctrine/lexer/pull/79',
'Accessing %s properties via ArrayAccess is deprecated, use the value, type or position property instead',
self::class
);
return in_array($offset, ['value', 'type', 'position'], true);
}
/**
* @deprecated Use the value, type or position property instead
* {@inheritDoc}
*
* @param O $offset
*
* @return mixed
* @psalm-return (
* O is 'value'
* ? V
* : (
* O is 'type'
* ? T|null
* : (
* O is 'position'
* ? int
* : mixed
* )
* )
* )
*
* @template O of array-key
*/
#[ReturnTypeWillChange]
public function offsetGet($offset)
{
Deprecation::trigger(
'doctrine/lexer',
'https://github.com/doctrine/lexer/pull/79',
'Accessing %s properties via ArrayAccess is deprecated, use the value, type or position property instead',
self::class
);
return $this->$offset;
}
/**
* @deprecated no replacement planned
* {@inheritDoc}
*/
public function offsetSet($offset, $value): void
{
Deprecation::trigger(
'doctrine/lexer',
'https://github.com/doctrine/lexer/pull/79',
'Setting %s properties via ArrayAccess is deprecated',
self::class
);
$this->$offset = $value;
}
/**
* @deprecated no replacement planned
* {@inheritDoc}
*/
public function offsetUnset($offset): void
{
Deprecation::trigger(
'doctrine/lexer',
'https://github.com/doctrine/lexer/pull/79',
'Setting %s properties via ArrayAccess is deprecated',
self::class
);
$this->$offset = null;
}
}
PK €UUµó‰| phpcs.xml.distnu €žÙ˜
src
tests
tests/*
PK €UUiYÀŸ % .github/workflows/static-analysis.ymlnu €žÙ˜
name: "Static Analysis"
on:
pull_request:
branches:
- "*.x"
paths:
- .github/workflows/static-analysis.yml
- composer.*
- src/**
- phpstan*
- psalm*
- tests/**
push:
branches:
- "*.x"
paths:
- .github/workflows/static-analysis.yml
- composer.*
- src/**
- phpstan*
- psalm*
- tests/**
jobs:
static-analysis:
name: "Static Analysis"
uses: "doctrine/.github/.github/workflows/static-analysis.yml@3.0.0"
PK €UU%YP“ù ù &