PHP Syntax highlighting using PhpToken

composer.json

@@ -24,6 +24,7 @@
 		"test-update": "vendor/bin/phpunit -c phpunit.xml --no-coverage -d --update-snapshots tests"
 	},
 	"require": {
-		"ext-ffi": "*"
+		"ext-ffi": "*",
+	  	"php": ">= 8.0.0"
 	}
 }

src/Editor.php

@@ -3,7 +3,7 @@
 namespace Aviat\Kilo;
 
 use Aviat\Kilo\Enum\{Color, KeyCode, KeyType, Highlight};
-use Aviat\Kilo\Tokens\PHP;
+use Aviat\Kilo\Tokens\PHP8;
 
 /**
  * // Don't highlight this!
@@ -88,7 +88,8 @@ class Editor {
 	{
 		$c = read_stdin();
 
-		$map = [
+		return match($c)
+		{
 			// Unambiguous mappings
 			KeyCode::ARROW_DOWN => KeyType::ARROW_DOWN,
 			KeyCode::ARROW_LEFT => KeyType::ARROW_LEFT,
@@ -100,29 +101,19 @@ class Editor {
 			KeyCode::PAGE_UP => KeyType::PAGE_UP,
 
 			// Backspace
-			KeyCode::CTRL('h') => KeyType::BACKSPACE,
-			KeyCode::BACKSPACE => KeyType::BACKSPACE,
+			KeyCode::CTRL('h'), KeyCode::BACKSPACE => KeyType::BACKSPACE,
 
 			// Escape
-			KeyCode::CTRL('l') => KeyType::ESCAPE,
-			KeyCode::ESCAPE => KeyType::ESCAPE,
+			KeyCode::CTRL('l'), KeyCode::ESCAPE => KeyType::ESCAPE,
 
 			// Home Key
-			"\eOH" => KeyType::HOME_KEY,
-			"\e[1~" => KeyType::HOME_KEY,
-			"\e[7~" => KeyType::HOME_KEY,
-			ANSI::RESET_CURSOR => KeyType::HOME_KEY,
+			"\eOH", "\e[7~", "\e[1~", ANSI::RESET_CURSOR => KeyType::HOME_KEY,
 
 			// End Key
-			"\eOF" => KeyType::END_KEY,
-			"\e[4~" => KeyType::END_KEY,
-			"\e[8~" => KeyType::END_KEY,
-			"\e[F" => KeyType::END_KEY,
-		];
+			"\eOF", "\e[4~", "\e[8~", "\e[F" => KeyType::END_KEY,
 
-		return (array_key_exists($c, $map))
-			? $map[$c]
-			: $c;
+			default => $c,
+		};
 	}
 
 	protected function selectSyntaxHighlight(): void
@@ -147,7 +138,7 @@ class Editor {
 				// Pre-tokenize the file
 				if ($this->syntax->filetype === 'PHP')
 				{
-					$this->tokens = PHP::getFileTokens($this->filename);
+					$this->tokens = PHP8::getFileTokens($this->filename);
 				}
 
 				$this->refreshSyntax();
@@ -696,7 +687,7 @@ class Editor {
 		echo $this->ab;
 	}
 
-	public function setStatusMessage(string $fmt, ...$args): void
+	public function setStatusMessage(string $fmt, mixed ...$args): void
 	{
 		$this->statusMsg = (count($args) > 0)
 			? sprintf($fmt, ...$args)
@@ -923,7 +914,7 @@ class Editor {
 			return;
 		}
 
-		$this->tokens = PHP::getTokens($this->rowsToString());
+		$this->tokens = PHP8::getTokens($this->rowsToString());
 		$this->refreshSyntax();
 	}
 }

src/Row.php

@@ -74,6 +74,7 @@ class Row {
 		T_MOD_EQUAL => Highlight::OPERATOR,
 		T_MUL_EQUAL => Highlight::OPERATOR,
 		T_NS_SEPARATOR => Highlight::OPERATOR,
+		T_NULLSAFE_OBJECT_OPERATOR => Highlight::OPERATOR,
 		T_OBJECT_OPERATOR => Highlight::OPERATOR,
 		T_OR_EQUAL => Highlight::OPERATOR,
 		T_PLUS_EQUAL => Highlight::OPERATOR,
@@ -198,19 +199,16 @@ class Row {
 
 	public function __get(string $name)
 	{
-		switch ($name)
+		return match ($name)
 		{
-			case 'size': return strlen($this->chars);
-
-			case 'rsize': return strlen($this->render);
-
-			case 'chars': return $this->chars;
-
-			default: return NULL;
-		}
+			'size' => strlen($this->chars),
+			'rsize' => strlen($this->render),
+			'chars' => $this->chars,
+			default => NULL,
+		};
 	}
 
-	public function __set(string $key, $value): void
+	public function __set(string $key, mixed $value): void
 	{
 		if ($key === 'chars')
 		{
@@ -514,7 +512,7 @@ class Row {
 			if (in_array($token['type'], [T_DOC_COMMENT, T_COMMENT], TRUE))
 			{
 				// Single line comments
-				if (strpos($char, '//') !== FALSE || strpos($char, '#') !== FALSE)
+				if (str_contains($char, '//') || str_contains($char, '#'))
 				{
 					array_replace_range($this->hl, $charStart, $charLen, Highlight::COMMENT);
 					break;

src/Tokens/PHP.php

@@ -2,6 +2,8 @@
 
 namespace Aviat\Kilo\Tokens;
 
+use PhpToken;
+
 use function Aviat\Kilo\str_contains;
 use function Aviat\Kilo\tabs_to_spaces;
 
@@ -35,11 +37,6 @@ class PHP {
 		 */
 		public static function getTokens(string $code): array
 		{
-			if (class_exists('PhpToken'))
-			{
-				return \PhpToken::tokenize($code);
-			}
-
 			return (new self($code))->organizeTokens();
 		}
 
@@ -64,10 +61,16 @@ class PHP {
 
 		protected function organizeTokens(): array
 		{
-			$rawTokens = token_get_all($this->code);
+			$rawTokens = (class_exists('PhpToken'))
+				? \PhpToken::tokenize($this->code)
+				: token_get_all($this->code);
 			foreach ($rawTokens as $t)
 			{
-				if (is_array($t))
+				if ($t instanceof \PhpToken)
+				{
+					$this->processObjectToken($t);
+				}
+				else if (is_array($t))
 				{
 					$this->processArrayToken($t);
 				}
@@ -82,9 +85,19 @@ class PHP {
 			return $this->tokens;
 		}
 
-		protected function processObjectToken(\PhpToken $token)
+		protected function processObjectToken(\PhpToken $token): void
 		{
+			$currentLine = $token->line;
+			$char = tabs_to_spaces($token->text);
 
+			$current = [
+				'type' => $token->id,
+				'typeName' => $token->getTokenName(),
+				'char' => $char,
+				'line' => $currentLine,
+			];
+
+			$this->tokens[$currentLine][] = $current;
 		}
 
 		protected function processArrayToken(array $token): void
@@ -102,7 +115,7 @@ class PHP {
 			];
 
 			// Single new line, or starts with a new line with other whitespace
-			if (strpos($char, "\n") === 0 && trim($char) === '')
+			if (str_starts_with($char, "\n") && trim($char) === '')
 			{
 				$this->tokens[$this->lineNum][] = $current;
 				$this->lineNum++;

src/Tokens/PHP8.php

@@ -0,0 +1,83 @@
+<?php declare(strict_types=1);
+
+namespace Aviat\Kilo\Tokens;
+
+use PhpToken;
+
+use function Aviat\Kilo\str_contains;
+use function Aviat\Kilo\tabs_to_spaces;
+
+class PHP8 extends PhpToken {
+	private array $rawLines = [];
+
+	private array $tokens = [];
+
+	/**
+	 * Use 'token_get_all' to get the tokens for a file,
+	 * organized by row number
+	 *
+	 * @param string $code
+	 * @return array
+	 */
+	public static function getTokens(string $code): array
+	{
+		$lines = explode("\n", $code);
+		array_unshift($lines, '');
+		unset($lines[0]);
+
+		$self = (new self(0, $code));
+
+		// $->code = $code;
+		$self->rawLines = $lines;
+		$self->tokens = array_fill(1, count($lines), []);
+
+		return $self->organizeTokens($code);
+	}
+
+	/**
+	 * Return tokens for the current $filename, organized
+	 * by row number
+	 *
+	 * @param string $filename
+	 * @return array
+	 */
+	public static function getFileTokens(string $filename): array
+	{
+		$code = @file_get_contents($filename);
+
+		if ($code === FALSE)
+		{
+			return [];
+		}
+
+		return self::getTokens($code);
+	}
+
+	protected function organizeTokens(string $code): array
+	{
+		$rawTokens = self::tokenize($code);
+		foreach ($rawTokens as $t)
+		{
+			$this->processObjectToken($t);
+		}
+
+		ksort($this->tokens);
+
+		return $this->tokens;
+	}
+
+	protected function processObjectToken(\PhpToken $token): void
+	{
+		$currentLine = $token->line;
+		$char = tabs_to_spaces($token->text);
+
+		$current = [
+			'type' => $token->id,
+			'typeName' => $token->getTokenName(),
+			'char' => $char,
+			'line' => $currentLine,
+		];
+
+		$this->tokens[$currentLine][] = $current;
+	}
+}

test.php

@@ -30,6 +30,8 @@ class FooBar extends Foo implements Ifoo {
 
 	private function operations(int $a, int $b): int
 	{
+		$this?->x?->bar();
+
 		$this->doNothing();
 
 		$c = $a + $b;