PHP Syntax highlighting using PhpToken
Some checks failed
timw4mail/php-kilo/pipeline/head There was a failure building this commit

This commit is contained in:
Timothy Warren 2021-03-03 11:50:29 -05:00
parent 7cd7d1baa5
commit 03fed4d667
6 changed files with 130 additions and 42 deletions

View File

@ -24,6 +24,7 @@
"test-update": "vendor/bin/phpunit -c phpunit.xml --no-coverage -d --update-snapshots tests" "test-update": "vendor/bin/phpunit -c phpunit.xml --no-coverage -d --update-snapshots tests"
}, },
"require": { "require": {
"ext-ffi": "*" "ext-ffi": "*",
"php": ">= 8.0.0"
} }
} }

View File

@ -3,7 +3,7 @@
namespace Aviat\Kilo; namespace Aviat\Kilo;
use Aviat\Kilo\Enum\{Color, KeyCode, KeyType, Highlight}; use Aviat\Kilo\Enum\{Color, KeyCode, KeyType, Highlight};
use Aviat\Kilo\Tokens\PHP; use Aviat\Kilo\Tokens\PHP8;
/** /**
* // Don't highlight this! * // Don't highlight this!
@ -88,7 +88,8 @@ class Editor {
{ {
$c = read_stdin(); $c = read_stdin();
$map = [ return match($c)
{
// Unambiguous mappings // Unambiguous mappings
KeyCode::ARROW_DOWN => KeyType::ARROW_DOWN, KeyCode::ARROW_DOWN => KeyType::ARROW_DOWN,
KeyCode::ARROW_LEFT => KeyType::ARROW_LEFT, KeyCode::ARROW_LEFT => KeyType::ARROW_LEFT,
@ -100,29 +101,19 @@ class Editor {
KeyCode::PAGE_UP => KeyType::PAGE_UP, KeyCode::PAGE_UP => KeyType::PAGE_UP,
// Backspace // Backspace
KeyCode::CTRL('h') => KeyType::BACKSPACE, KeyCode::CTRL('h'), KeyCode::BACKSPACE => KeyType::BACKSPACE,
KeyCode::BACKSPACE => KeyType::BACKSPACE,
// Escape // Escape
KeyCode::CTRL('l') => KeyType::ESCAPE, KeyCode::CTRL('l'), KeyCode::ESCAPE => KeyType::ESCAPE,
KeyCode::ESCAPE => KeyType::ESCAPE,
// Home Key // Home Key
"\eOH" => KeyType::HOME_KEY, "\eOH", "\e[7~", "\e[1~", ANSI::RESET_CURSOR => KeyType::HOME_KEY,
"\e[1~" => KeyType::HOME_KEY,
"\e[7~" => KeyType::HOME_KEY,
ANSI::RESET_CURSOR => KeyType::HOME_KEY,
// End Key // End Key
"\eOF" => KeyType::END_KEY, "\eOF", "\e[4~", "\e[8~", "\e[F" => KeyType::END_KEY,
"\e[4~" => KeyType::END_KEY,
"\e[8~" => KeyType::END_KEY,
"\e[F" => KeyType::END_KEY,
];
return (array_key_exists($c, $map)) default => $c,
? $map[$c] };
: $c;
} }
protected function selectSyntaxHighlight(): void protected function selectSyntaxHighlight(): void
@ -147,7 +138,7 @@ class Editor {
// Pre-tokenize the file // Pre-tokenize the file
if ($this->syntax->filetype === 'PHP') if ($this->syntax->filetype === 'PHP')
{ {
$this->tokens = PHP::getFileTokens($this->filename); $this->tokens = PHP8::getFileTokens($this->filename);
} }
$this->refreshSyntax(); $this->refreshSyntax();
@ -696,7 +687,7 @@ class Editor {
echo $this->ab; echo $this->ab;
} }
public function setStatusMessage(string $fmt, ...$args): void public function setStatusMessage(string $fmt, mixed ...$args): void
{ {
$this->statusMsg = (count($args) > 0) $this->statusMsg = (count($args) > 0)
? sprintf($fmt, ...$args) ? sprintf($fmt, ...$args)
@ -923,7 +914,7 @@ class Editor {
return; return;
} }
$this->tokens = PHP::getTokens($this->rowsToString()); $this->tokens = PHP8::getTokens($this->rowsToString());
$this->refreshSyntax(); $this->refreshSyntax();
} }
} }

View File

@ -74,6 +74,7 @@ class Row {
T_MOD_EQUAL => Highlight::OPERATOR, T_MOD_EQUAL => Highlight::OPERATOR,
T_MUL_EQUAL => Highlight::OPERATOR, T_MUL_EQUAL => Highlight::OPERATOR,
T_NS_SEPARATOR => Highlight::OPERATOR, T_NS_SEPARATOR => Highlight::OPERATOR,
T_NULLSAFE_OBJECT_OPERATOR => Highlight::OPERATOR,
T_OBJECT_OPERATOR => Highlight::OPERATOR, T_OBJECT_OPERATOR => Highlight::OPERATOR,
T_OR_EQUAL => Highlight::OPERATOR, T_OR_EQUAL => Highlight::OPERATOR,
T_PLUS_EQUAL => Highlight::OPERATOR, T_PLUS_EQUAL => Highlight::OPERATOR,
@ -198,19 +199,16 @@ class Row {
public function __get(string $name) public function __get(string $name)
{ {
switch ($name) return match ($name)
{ {
case 'size': return strlen($this->chars); 'size' => strlen($this->chars),
'rsize' => strlen($this->render),
case 'rsize': return strlen($this->render); 'chars' => $this->chars,
default => NULL,
case 'chars': return $this->chars; };
default: return NULL;
}
} }
public function __set(string $key, $value): void public function __set(string $key, mixed $value): void
{ {
if ($key === 'chars') if ($key === 'chars')
{ {
@ -514,7 +512,7 @@ class Row {
if (in_array($token['type'], [T_DOC_COMMENT, T_COMMENT], TRUE)) if (in_array($token['type'], [T_DOC_COMMENT, T_COMMENT], TRUE))
{ {
// Single line comments // Single line comments
if (strpos($char, '//') !== FALSE || strpos($char, '#') !== FALSE) if (str_contains($char, '//') || str_contains($char, '#'))
{ {
array_replace_range($this->hl, $charStart, $charLen, Highlight::COMMENT); array_replace_range($this->hl, $charStart, $charLen, Highlight::COMMENT);
break; break;

View File

@ -2,6 +2,8 @@
namespace Aviat\Kilo\Tokens; namespace Aviat\Kilo\Tokens;
use PhpToken;
use function Aviat\Kilo\str_contains; use function Aviat\Kilo\str_contains;
use function Aviat\Kilo\tabs_to_spaces; use function Aviat\Kilo\tabs_to_spaces;
@ -35,11 +37,6 @@ class PHP {
*/ */
public static function getTokens(string $code): array public static function getTokens(string $code): array
{ {
if (class_exists('PhpToken'))
{
return \PhpToken::tokenize($code);
}
return (new self($code))->organizeTokens(); return (new self($code))->organizeTokens();
} }
@ -64,10 +61,16 @@ class PHP {
protected function organizeTokens(): array protected function organizeTokens(): array
{ {
$rawTokens = token_get_all($this->code); $rawTokens = (class_exists('PhpToken'))
? \PhpToken::tokenize($this->code)
: token_get_all($this->code);
foreach ($rawTokens as $t) foreach ($rawTokens as $t)
{ {
if (is_array($t)) if ($t instanceof \PhpToken)
{
$this->processObjectToken($t);
}
else if (is_array($t))
{ {
$this->processArrayToken($t); $this->processArrayToken($t);
} }
@ -82,9 +85,19 @@ class PHP {
return $this->tokens; return $this->tokens;
} }
protected function processObjectToken(\PhpToken $token) protected function processObjectToken(\PhpToken $token): void
{ {
$currentLine = $token->line;
$char = tabs_to_spaces($token->text);
$current = [
'type' => $token->id,
'typeName' => $token->getTokenName(),
'char' => $char,
'line' => $currentLine,
];
$this->tokens[$currentLine][] = $current;
} }
protected function processArrayToken(array $token): void protected function processArrayToken(array $token): void
@ -102,7 +115,7 @@ class PHP {
]; ];
// Single new line, or starts with a new line with other whitespace // Single new line, or starts with a new line with other whitespace
if (strpos($char, "\n") === 0 && trim($char) === '') if (str_starts_with($char, "\n") && trim($char) === '')
{ {
$this->tokens[$this->lineNum][] = $current; $this->tokens[$this->lineNum][] = $current;
$this->lineNum++; $this->lineNum++;

83
src/Tokens/PHP8.php Normal file
View File

@ -0,0 +1,83 @@
<?php declare(strict_types=1);
namespace Aviat\Kilo\Tokens;
use PhpToken;
use function Aviat\Kilo\str_contains;
use function Aviat\Kilo\tabs_to_spaces;
class PHP8 extends PhpToken {
private array $rawLines = [];
private array $tokens = [];
/**
* Use 'token_get_all' to get the tokens for a file,
* organized by row number
*
* @param string $code
* @return array
*/
public static function getTokens(string $code): array
{
$lines = explode("\n", $code);
array_unshift($lines, '');
unset($lines[0]);
$self = (new self(0, $code));
// $->code = $code;
$self->rawLines = $lines;
$self->tokens = array_fill(1, count($lines), []);
return $self->organizeTokens($code);
}
/**
* Return tokens for the current $filename, organized
* by row number
*
* @param string $filename
* @return array
*/
public static function getFileTokens(string $filename): array
{
$code = @file_get_contents($filename);
if ($code === FALSE)
{
return [];
}
return self::getTokens($code);
}
protected function organizeTokens(string $code): array
{
$rawTokens = self::tokenize($code);
foreach ($rawTokens as $t)
{
$this->processObjectToken($t);
}
ksort($this->tokens);
return $this->tokens;
}
protected function processObjectToken(\PhpToken $token): void
{
$currentLine = $token->line;
$char = tabs_to_spaces($token->text);
$current = [
'type' => $token->id,
'typeName' => $token->getTokenName(),
'char' => $char,
'line' => $currentLine,
];
$this->tokens[$currentLine][] = $current;
}
}

View File

@ -30,6 +30,8 @@ class FooBar extends Foo implements Ifoo {
private function operations(int $a, int $b): int private function operations(int $a, int $b): int
{ {
$this?->x?->bar();
$this->doNothing(); $this->doNothing();
$c = $a + $b; $c = $a + $b;