PHP Syntax highlighting using PhpToken
Some checks failed
timw4mail/php-kilo/pipeline/head There was a failure building this commit

This commit is contained in:
Timothy Warren 2021-03-03 11:50:29 -05:00
parent 7cd7d1baa5
commit 03fed4d667
6 changed files with 130 additions and 42 deletions

View File

@ -24,6 +24,7 @@
"test-update": "vendor/bin/phpunit -c phpunit.xml --no-coverage -d --update-snapshots tests"
},
"require": {
"ext-ffi": "*"
"ext-ffi": "*",
"php": ">= 8.0.0"
}
}

View File

@ -3,7 +3,7 @@
namespace Aviat\Kilo;
use Aviat\Kilo\Enum\{Color, KeyCode, KeyType, Highlight};
use Aviat\Kilo\Tokens\PHP;
use Aviat\Kilo\Tokens\PHP8;
/**
* // Don't highlight this!
@ -88,7 +88,8 @@ class Editor {
{
$c = read_stdin();
$map = [
return match($c)
{
// Unambiguous mappings
KeyCode::ARROW_DOWN => KeyType::ARROW_DOWN,
KeyCode::ARROW_LEFT => KeyType::ARROW_LEFT,
@ -100,29 +101,19 @@ class Editor {
KeyCode::PAGE_UP => KeyType::PAGE_UP,
// Backspace
KeyCode::CTRL('h') => KeyType::BACKSPACE,
KeyCode::BACKSPACE => KeyType::BACKSPACE,
KeyCode::CTRL('h'), KeyCode::BACKSPACE => KeyType::BACKSPACE,
// Escape
KeyCode::CTRL('l') => KeyType::ESCAPE,
KeyCode::ESCAPE => KeyType::ESCAPE,
KeyCode::CTRL('l'), KeyCode::ESCAPE => KeyType::ESCAPE,
// Home Key
"\eOH" => KeyType::HOME_KEY,
"\e[1~" => KeyType::HOME_KEY,
"\e[7~" => KeyType::HOME_KEY,
ANSI::RESET_CURSOR => KeyType::HOME_KEY,
"\eOH", "\e[7~", "\e[1~", ANSI::RESET_CURSOR => KeyType::HOME_KEY,
// End Key
"\eOF" => KeyType::END_KEY,
"\e[4~" => KeyType::END_KEY,
"\e[8~" => KeyType::END_KEY,
"\e[F" => KeyType::END_KEY,
];
"\eOF", "\e[4~", "\e[8~", "\e[F" => KeyType::END_KEY,
return (array_key_exists($c, $map))
? $map[$c]
: $c;
default => $c,
};
}
protected function selectSyntaxHighlight(): void
@ -147,7 +138,7 @@ class Editor {
// Pre-tokenize the file
if ($this->syntax->filetype === 'PHP')
{
$this->tokens = PHP::getFileTokens($this->filename);
$this->tokens = PHP8::getFileTokens($this->filename);
}
$this->refreshSyntax();
@ -696,7 +687,7 @@ class Editor {
echo $this->ab;
}
public function setStatusMessage(string $fmt, ...$args): void
public function setStatusMessage(string $fmt, mixed ...$args): void
{
$this->statusMsg = (count($args) > 0)
? sprintf($fmt, ...$args)
@ -923,7 +914,7 @@ class Editor {
return;
}
$this->tokens = PHP::getTokens($this->rowsToString());
$this->tokens = PHP8::getTokens($this->rowsToString());
$this->refreshSyntax();
}
}

View File

@ -74,6 +74,7 @@ class Row {
T_MOD_EQUAL => Highlight::OPERATOR,
T_MUL_EQUAL => Highlight::OPERATOR,
T_NS_SEPARATOR => Highlight::OPERATOR,
T_NULLSAFE_OBJECT_OPERATOR => Highlight::OPERATOR,
T_OBJECT_OPERATOR => Highlight::OPERATOR,
T_OR_EQUAL => Highlight::OPERATOR,
T_PLUS_EQUAL => Highlight::OPERATOR,
@ -198,19 +199,16 @@ class Row {
public function __get(string $name)
{
switch ($name)
return match ($name)
{
case 'size': return strlen($this->chars);
case 'rsize': return strlen($this->render);
case 'chars': return $this->chars;
default: return NULL;
}
'size' => strlen($this->chars),
'rsize' => strlen($this->render),
'chars' => $this->chars,
default => NULL,
};
}
public function __set(string $key, $value): void
public function __set(string $key, mixed $value): void
{
if ($key === 'chars')
{
@ -514,7 +512,7 @@ class Row {
if (in_array($token['type'], [T_DOC_COMMENT, T_COMMENT], TRUE))
{
// Single line comments
if (strpos($char, '//') !== FALSE || strpos($char, '#') !== FALSE)
if (str_contains($char, '//') || str_contains($char, '#'))
{
array_replace_range($this->hl, $charStart, $charLen, Highlight::COMMENT);
break;

View File

@ -2,6 +2,8 @@
namespace Aviat\Kilo\Tokens;
use PhpToken;
use function Aviat\Kilo\str_contains;
use function Aviat\Kilo\tabs_to_spaces;
@ -35,11 +37,6 @@ class PHP {
*/
public static function getTokens(string $code): array
{
if (class_exists('PhpToken'))
{
return \PhpToken::tokenize($code);
}
return (new self($code))->organizeTokens();
}
@ -64,10 +61,16 @@ class PHP {
protected function organizeTokens(): array
{
$rawTokens = token_get_all($this->code);
$rawTokens = (class_exists('PhpToken'))
? \PhpToken::tokenize($this->code)
: token_get_all($this->code);
foreach ($rawTokens as $t)
{
if (is_array($t))
if ($t instanceof \PhpToken)
{
$this->processObjectToken($t);
}
else if (is_array($t))
{
$this->processArrayToken($t);
}
@ -82,9 +85,19 @@ class PHP {
return $this->tokens;
}
protected function processObjectToken(\PhpToken $token)
protected function processObjectToken(\PhpToken $token): void
{
$currentLine = $token->line;
$char = tabs_to_spaces($token->text);
$current = [
'type' => $token->id,
'typeName' => $token->getTokenName(),
'char' => $char,
'line' => $currentLine,
];
$this->tokens[$currentLine][] = $current;
}
protected function processArrayToken(array $token): void
@ -102,7 +115,7 @@ class PHP {
];
// Single new line, or starts with a new line with other whitespace
if (strpos($char, "\n") === 0 && trim($char) === '')
if (str_starts_with($char, "\n") && trim($char) === '')
{
$this->tokens[$this->lineNum][] = $current;
$this->lineNum++;

83
src/Tokens/PHP8.php Normal file
View File

@ -0,0 +1,83 @@
<?php declare(strict_types=1);
namespace Aviat\Kilo\Tokens;
use PhpToken;
use function Aviat\Kilo\str_contains;
use function Aviat\Kilo\tabs_to_spaces;
class PHP8 extends PhpToken {
private array $rawLines = [];
private array $tokens = [];
/**
* Use 'token_get_all' to get the tokens for a file,
* organized by row number
*
* @param string $code
* @return array
*/
public static function getTokens(string $code): array
{
$lines = explode("\n", $code);
array_unshift($lines, '');
unset($lines[0]);
$self = (new self(0, $code));
// $->code = $code;
$self->rawLines = $lines;
$self->tokens = array_fill(1, count($lines), []);
return $self->organizeTokens($code);
}
/**
* Return tokens for the current $filename, organized
* by row number
*
* @param string $filename
* @return array
*/
public static function getFileTokens(string $filename): array
{
$code = @file_get_contents($filename);
if ($code === FALSE)
{
return [];
}
return self::getTokens($code);
}
protected function organizeTokens(string $code): array
{
$rawTokens = self::tokenize($code);
foreach ($rawTokens as $t)
{
$this->processObjectToken($t);
}
ksort($this->tokens);
return $this->tokens;
}
protected function processObjectToken(\PhpToken $token): void
{
$currentLine = $token->line;
$char = tabs_to_spaces($token->text);
$current = [
'type' => $token->id,
'typeName' => $token->getTokenName(),
'char' => $char,
'line' => $currentLine,
];
$this->tokens[$currentLine][] = $current;
}
}

View File

@ -30,6 +30,8 @@ class FooBar extends Foo implements Ifoo {
private function operations(int $a, int $b): int
{
$this?->x?->bar();
$this->doNothing();
$c = $a + $b;