Work in progress on PHP highlighting using token_get_all

src/Editor.php

@@ -25,7 +25,7 @@ class Editor {
 	public array $rows = [];
 
 	public int $dirty = 0;
-	protected string $filename = '';
+	public string $filename = '';
 	protected string $statusMsg = '';
 	protected int $statusMsgTime;
 
@@ -122,6 +122,12 @@ class Editor {
 				) {
 					$this->syntax = $syntax;
 
+					// Pre-tokenize the file
+					if ($this->syntax->filetype === 'PHP')
+					{
+						$this->syntax->tokens = get_php_tokens($this->filename);
+					}
+
 					// Update the syntax highlighting for all the rows of the file
 					for ($i = 0; $i < $this->numRows; $i++)
 					{
@@ -180,6 +186,12 @@ class Editor {
 			return;
 		}
 
+		// Re-tokenize the file
+		if ($this->syntax->filetype === 'PHP')
+		{
+			$this->syntax->tokens = get_php_tokens($this->filename);
+		}
+
 		$row = Row::new($this, $s, $at);
 		// Update other row indices
 		for ($i = $at + 1; $i <= $this->numRows; $i++)
@@ -213,6 +225,12 @@ class Editor {
 			return;
 		}
 
+		// Re-tokenize the file
+		if ($this->syntax->filetype === 'PHP')
+		{
+			$this->syntax->tokens = get_php_tokens($this->filename);
+		}
+
 		// Remove the row
 		unset($this->rows[$at]);
 
@@ -232,6 +250,12 @@ class Editor {
 
 	protected function insertChar(string $c): void
 	{
+		// Re-tokenize the file
+		if ($this->syntax->filetype === 'PHP')
+		{
+			$this->syntax->tokens = get_php_tokens($this->filename);
+		}
+
 		if ($this->cursorY === $this->numRows)
 		{
 			$this->insertRow($this->numRows, '');
@@ -242,6 +266,12 @@ class Editor {
 
 	protected function insertNewline(): void
 	{
+		// Re-tokenize the file
+		if ($this->syntax->filetype === 'PHP')
+		{
+			$this->syntax->tokens = get_php_tokens($this->filename);
+		}
+
 		if ($this->cursorX === 0)
 		{
 			$this->insertRow($this->cursorY, '');
@@ -269,6 +299,12 @@ class Editor {
 			return;
 		}
 
+		// Re-tokenize the file
+		if ($this->syntax->filetype === 'PHP')
+		{
+			$this->syntax->tokens = get_php_tokens($this->filename);
+		}
+
 		$row = $this->rows[$this->cursorY];
 		if ($this->cursorX > 0)
 		{

src/Highlight.php

@@ -10,5 +10,8 @@ class Highlight {
 	public const KEYWORD2 = 4;
 	public const STRING = 5;
 	public const NUMBER = 6;
-	public const MATCH = 7;
+	public const OPERATOR = 7;
+	public const VARIABLE = 8;
+	public const INVALID = 9;
+	public const MATCH = 10;
 }

src/Row.php

@@ -118,6 +118,12 @@ class Row {
 	{
 		$this->hl = array_fill(0, $this->rsize, Highlight::NORMAL);
 
+		if ($this->parent->syntax->filetype === 'PHP')
+		{
+			$this->updateSyntaxPHP();
+			return;
+		}
+
 		$keywords1 = $this->parent->syntax->keywords1;
 		$keywords2 = $this->parent->syntax->keywords2;
 
@@ -274,4 +280,116 @@ class Row {
 			$this->parent->rows[$this->idx + 1]->update();
 		}
 	}
+
+	protected function updateSyntaxPHP():void
+	{
+		$tokens = $this->parent->syntax->tokens[$this->idx + 1];
+		$inComment = ($this->idx > 0 && $this->parent->rows[$this->idx - 1]->hlOpenComment);
+
+		// The line is probably just empty
+		if ($tokens === NULL)
+		{
+			return;
+		}
+
+		// Keep track of where you are in the line, so that
+		// multiples of the same tokens can be effectively matched
+		$offset = 0;
+
+		foreach ($tokens as $token)
+		{
+			$char = $token['char'];
+			$charLen = strlen($char);
+			$charStart = strpos($this->render, $char, $offset);
+			$charEnd = $charStart + $charLen;
+
+			switch ($token['type'])
+			{
+				case T_LNUMBER:
+					array_replace_range($this->hl, $charStart, $charLen, Highlight::NUMBER);
+					$offset = $charEnd;
+					continue 2;
+
+				case T_CONSTANT_ENCAPSED_STRING:
+					array_replace_range($this->hl, $charStart, $charLen, Highlight::STRING);
+					$offset = $charEnd;
+					continue 2;
+
+				// Operators
+				case T_AND_EQUAL:
+				case T_BOOLEAN_AND:
+				case T_BOOLEAN_OR:
+				case T_COALESCE:
+				case T_CONCAT_EQUAL:
+				case T_DIV_EQUAL:
+				case T_DOUBLE_ARROW:
+				case T_DOUBLE_COLON:
+				case T_ELLIPSIS:
+				case T_INC:
+				case T_IS_EQUAL:
+				case T_IS_GREATER_OR_EQUAL:
+				case T_IS_IDENTICAL:
+				case T_IS_NOT_EQUAL:
+				case T_IS_NOT_IDENTICAL:
+				case T_IS_SMALLER_OR_EQUAL:
+				case T_SPACESHIP:
+				case T_LOGICAL_AND:
+				case T_LOGICAL_OR:
+				case T_LOGICAL_XOR:
+				case T_MINUS_EQUAL:
+				case T_MOD_EQUAL:
+				case T_MUL_EQUAL:
+				case T_OBJECT_OPERATOR:
+				case T_OR_EQUAL:
+				case T_PAAMAYIM_NEKUDOTAYIM:
+				case T_PLUS_EQUAL:
+				case T_POW:
+				case T_POW_EQUAL:
+				case T_SL:
+				case T_SL_EQUAL:
+				case T_SR:
+				case T_SR_EQUAL:
+				case T_XOR_EQUAL:
+					array_replace_range($this->hl, $charStart, $charLen, Highlight::OPERATOR);
+					$offset = $charEnd;
+					continue 2;
+
+				case T_VARIABLE:
+					array_replace_range($this->hl, $charStart, $charLen, Highlight::VARIABLE);
+					$offset = $charEnd;
+					continue 2;
+
+				case T_DOC_COMMENT:
+					// TODO
+				break;
+
+				// Keywords1
+				case T_ABSTRACT:
+				case T_AS:
+				case T_BREAK:
+				case T_CASE:
+				case T_DO:
+					array_replace_range($this->hl, $charStart, $charLen, Highlight::KEYWORD1);
+					// $keyword = $this->getKeywordFromToken($token['type']);
+					$offset = $charEnd;
+					continue 2;
+				break;
+
+				// Keywords 2
+			}
+		}
+	}
+
+	private function getKeywordFromToken(int $token): ?string
+	{
+		$map = [
+			T_ABSTRACT => 'abstract',
+			T_AS => 'as',
+			T_BREAK => 'break',
+			T_CASE => 'case',
+			T_DO => 'do',
+		];
+
+		return $map[$token] ?? NULL;
+	}
 }

src/Syntax.php

@@ -16,6 +16,9 @@ class Syntax {
 	public array $keywords1 = [];
 	public array $keywords2 = [];
 
+	// Tokens for PHP files
+	public array $tokens = [];
+
 	public int $flags = 0;
 
 	public static function new(string $name, array $extList, array $keywords1, array $keywords2, string $slcs, string $mcs, string $mce, int $flags): self

src/functions.php

@@ -279,6 +279,12 @@ function array_replace_range(array &$array, int $offset, int $length, $value):vo
 	} */
 }
 
+/**
+ * Get the ASCII color escape number for the specified syntax type
+ *
+ * @param int $hl
+ * @return int
+ */
 function syntax_to_color(int $hl): int
 {
 	$map = [
@@ -288,6 +294,9 @@ function syntax_to_color(int $hl): int
 		Highlight::KEYWORD2 => 32, // Foreground Green
 		Highlight::STRING => 35, // Foreground Magenta
 		Highlight::NUMBER => 31, // Foreground Red
+		Highlight::OPERATOR => 92, // Foreground Bright Green
+		Highlight::VARIABLE => 96, // Foreground Bright Cyan
+		Highlight::INVALID => 101, // Background Bright Red
 		Highlight::MATCH => 7, // Reverse!
 	];
 
@@ -295,3 +304,53 @@ function syntax_to_color(int $hl): int
 		? $map[$hl]
 		: 37; // Foreground White
 }
+
+/**
+ * Use 'token_get_all' to get the tokens for a file,
+ * organized by row number
+ *
+ * @param string $filename
+ * @return array
+ */
+function get_php_tokens(string $filename): array
+{
+	$raw_tokens = token_get_all(file_get_contents($filename), TOKEN_PARSE);
+	$tokens = [];
+	$lineNum = 1;
+	$line = [];
+	foreach($raw_tokens as $token)
+	{
+		// Simple characters, usually delimiters
+		if ( ! is_array($token))
+		{
+			$line[] = [
+				'typeName' => 'RAW',
+				'char' => $token,
+			];
+			continue;
+		}
+
+		[$type, $char, $currentLine] = $token;
+
+		$current = [
+			'type' => $type,
+			'typeName' => token_name($type),
+			'char' => $char,
+			'line' => $currentLine,
+		];
+
+		if ($current['line'] !== $lineNum)
+		{
+			$tokens[$lineNum] = $line;
+			$lineNum = $current['line'];
+			$line = [];
+			$line[] = $current;
+		}
+		else
+		{
+			$line[] = $current;
+		}
+	}
+
+	return $tokens;
+}

src/hldb.php

@@ -85,7 +85,7 @@ function get_hldb(): array
 			),
 			Syntax::new(
 				'PHP',
-				['.php'],
+				['.php', 'kilo'],
 				[
 					'?php', '$this', '__halt_compiler', 'abstract', 'and', 'array', 'as', 'break',
 					'callable', 'case', 'catch', 'class', 'clone', 'const', 'continue', 'declare',