Work in progress on PHP highlighting using token_get_all

This commit is contained in:
Timothy Warren 2019-10-29 17:02:03 -04:00
parent ca9ec17d49
commit 346ef67a08
6 changed files with 222 additions and 3 deletions

View File

@ -25,7 +25,7 @@ class Editor {
public array $rows = []; public array $rows = [];
public int $dirty = 0; public int $dirty = 0;
protected string $filename = ''; public string $filename = '';
protected string $statusMsg = ''; protected string $statusMsg = '';
protected int $statusMsgTime; protected int $statusMsgTime;
@ -122,6 +122,12 @@ class Editor {
) { ) {
$this->syntax = $syntax; $this->syntax = $syntax;
// Pre-tokenize the file
if ($this->syntax->filetype === 'PHP')
{
$this->syntax->tokens = get_php_tokens($this->filename);
}
// Update the syntax highlighting for all the rows of the file // Update the syntax highlighting for all the rows of the file
for ($i = 0; $i < $this->numRows; $i++) for ($i = 0; $i < $this->numRows; $i++)
{ {
@ -180,6 +186,12 @@ class Editor {
return; return;
} }
// Re-tokenize the file
if ($this->syntax->filetype === 'PHP')
{
$this->syntax->tokens = get_php_tokens($this->filename);
}
$row = Row::new($this, $s, $at); $row = Row::new($this, $s, $at);
// Update other row indices // Update other row indices
for ($i = $at + 1; $i <= $this->numRows; $i++) for ($i = $at + 1; $i <= $this->numRows; $i++)
@ -213,6 +225,12 @@ class Editor {
return; return;
} }
// Re-tokenize the file
if ($this->syntax->filetype === 'PHP')
{
$this->syntax->tokens = get_php_tokens($this->filename);
}
// Remove the row // Remove the row
unset($this->rows[$at]); unset($this->rows[$at]);
@ -232,6 +250,12 @@ class Editor {
protected function insertChar(string $c): void protected function insertChar(string $c): void
{ {
// Re-tokenize the file
if ($this->syntax->filetype === 'PHP')
{
$this->syntax->tokens = get_php_tokens($this->filename);
}
if ($this->cursorY === $this->numRows) if ($this->cursorY === $this->numRows)
{ {
$this->insertRow($this->numRows, ''); $this->insertRow($this->numRows, '');
@ -242,6 +266,12 @@ class Editor {
protected function insertNewline(): void protected function insertNewline(): void
{ {
// Re-tokenize the file
if ($this->syntax->filetype === 'PHP')
{
$this->syntax->tokens = get_php_tokens($this->filename);
}
if ($this->cursorX === 0) if ($this->cursorX === 0)
{ {
$this->insertRow($this->cursorY, ''); $this->insertRow($this->cursorY, '');
@ -269,6 +299,12 @@ class Editor {
return; return;
} }
// Re-tokenize the file
if ($this->syntax->filetype === 'PHP')
{
$this->syntax->tokens = get_php_tokens($this->filename);
}
$row = $this->rows[$this->cursorY]; $row = $this->rows[$this->cursorY];
if ($this->cursorX > 0) if ($this->cursorX > 0)
{ {

View File

@ -10,5 +10,8 @@ class Highlight {
public const KEYWORD2 = 4; public const KEYWORD2 = 4;
public const STRING = 5; public const STRING = 5;
public const NUMBER = 6; public const NUMBER = 6;
public const MATCH = 7; public const OPERATOR = 7;
public const VARIABLE = 8;
public const INVALID = 9;
public const MATCH = 10;
} }

View File

@ -118,6 +118,12 @@ class Row {
{ {
$this->hl = array_fill(0, $this->rsize, Highlight::NORMAL); $this->hl = array_fill(0, $this->rsize, Highlight::NORMAL);
if ($this->parent->syntax->filetype === 'PHP')
{
$this->updateSyntaxPHP();
return;
}
$keywords1 = $this->parent->syntax->keywords1; $keywords1 = $this->parent->syntax->keywords1;
$keywords2 = $this->parent->syntax->keywords2; $keywords2 = $this->parent->syntax->keywords2;
@ -274,4 +280,116 @@ class Row {
$this->parent->rows[$this->idx + 1]->update(); $this->parent->rows[$this->idx + 1]->update();
} }
} }
protected function updateSyntaxPHP():void
{
$tokens = $this->parent->syntax->tokens[$this->idx + 1];
$inComment = ($this->idx > 0 && $this->parent->rows[$this->idx - 1]->hlOpenComment);
// The line is probably just empty
if ($tokens === NULL)
{
return;
}
// Keep track of where you are in the line, so that
// multiples of the same tokens can be effectively matched
$offset = 0;
foreach ($tokens as $token)
{
$char = $token['char'];
$charLen = strlen($char);
$charStart = strpos($this->render, $char, $offset);
$charEnd = $charStart + $charLen;
switch ($token['type'])
{
case T_LNUMBER:
array_replace_range($this->hl, $charStart, $charLen, Highlight::NUMBER);
$offset = $charEnd;
continue 2;
case T_CONSTANT_ENCAPSED_STRING:
array_replace_range($this->hl, $charStart, $charLen, Highlight::STRING);
$offset = $charEnd;
continue 2;
// Operators
case T_AND_EQUAL:
case T_BOOLEAN_AND:
case T_BOOLEAN_OR:
case T_COALESCE:
case T_CONCAT_EQUAL:
case T_DIV_EQUAL:
case T_DOUBLE_ARROW:
case T_DOUBLE_COLON:
case T_ELLIPSIS:
case T_INC:
case T_IS_EQUAL:
case T_IS_GREATER_OR_EQUAL:
case T_IS_IDENTICAL:
case T_IS_NOT_EQUAL:
case T_IS_NOT_IDENTICAL:
case T_IS_SMALLER_OR_EQUAL:
case T_SPACESHIP:
case T_LOGICAL_AND:
case T_LOGICAL_OR:
case T_LOGICAL_XOR:
case T_MINUS_EQUAL:
case T_MOD_EQUAL:
case T_MUL_EQUAL:
case T_OBJECT_OPERATOR:
case T_OR_EQUAL:
case T_PAAMAYIM_NEKUDOTAYIM:
case T_PLUS_EQUAL:
case T_POW:
case T_POW_EQUAL:
case T_SL:
case T_SL_EQUAL:
case T_SR:
case T_SR_EQUAL:
case T_XOR_EQUAL:
array_replace_range($this->hl, $charStart, $charLen, Highlight::OPERATOR);
$offset = $charEnd;
continue 2;
case T_VARIABLE:
array_replace_range($this->hl, $charStart, $charLen, Highlight::VARIABLE);
$offset = $charEnd;
continue 2;
case T_DOC_COMMENT:
// TODO
break;
// Keywords1
case T_ABSTRACT:
case T_AS:
case T_BREAK:
case T_CASE:
case T_DO:
array_replace_range($this->hl, $charStart, $charLen, Highlight::KEYWORD1);
// $keyword = $this->getKeywordFromToken($token['type']);
$offset = $charEnd;
continue 2;
break;
// Keywords 2
}
}
}
private function getKeywordFromToken(int $token): ?string
{
$map = [
T_ABSTRACT => 'abstract',
T_AS => 'as',
T_BREAK => 'break',
T_CASE => 'case',
T_DO => 'do',
];
return $map[$token] ?? NULL;
}
} }

View File

@ -16,6 +16,9 @@ class Syntax {
public array $keywords1 = []; public array $keywords1 = [];
public array $keywords2 = []; public array $keywords2 = [];
// Tokens for PHP files
public array $tokens = [];
public int $flags = 0; public int $flags = 0;
public static function new(string $name, array $extList, array $keywords1, array $keywords2, string $slcs, string $mcs, string $mce, int $flags): self public static function new(string $name, array $extList, array $keywords1, array $keywords2, string $slcs, string $mcs, string $mce, int $flags): self

View File

@ -279,6 +279,12 @@ function array_replace_range(array &$array, int $offset, int $length, $value):vo
} */ } */
} }
/**
* Get the ASCII color escape number for the specified syntax type
*
* @param int $hl
* @return int
*/
function syntax_to_color(int $hl): int function syntax_to_color(int $hl): int
{ {
$map = [ $map = [
@ -288,6 +294,9 @@ function syntax_to_color(int $hl): int
Highlight::KEYWORD2 => 32, // Foreground Green Highlight::KEYWORD2 => 32, // Foreground Green
Highlight::STRING => 35, // Foreground Magenta Highlight::STRING => 35, // Foreground Magenta
Highlight::NUMBER => 31, // Foreground Red Highlight::NUMBER => 31, // Foreground Red
Highlight::OPERATOR => 92, // Foreground Bright Green
Highlight::VARIABLE => 96, // Foreground Bright Cyan
Highlight::INVALID => 101, // Background Bright Red
Highlight::MATCH => 7, // Reverse! Highlight::MATCH => 7, // Reverse!
]; ];
@ -295,3 +304,53 @@ function syntax_to_color(int $hl): int
? $map[$hl] ? $map[$hl]
: 37; // Foreground White : 37; // Foreground White
} }
/**
* Use 'token_get_all' to get the tokens for a file,
* organized by row number
*
* @param string $filename
* @return array
*/
function get_php_tokens(string $filename): array
{
$raw_tokens = token_get_all(file_get_contents($filename), TOKEN_PARSE);
$tokens = [];
$lineNum = 1;
$line = [];
foreach($raw_tokens as $token)
{
// Simple characters, usually delimiters
if ( ! is_array($token))
{
$line[] = [
'typeName' => 'RAW',
'char' => $token,
];
continue;
}
[$type, $char, $currentLine] = $token;
$current = [
'type' => $type,
'typeName' => token_name($type),
'char' => $char,
'line' => $currentLine,
];
if ($current['line'] !== $lineNum)
{
$tokens[$lineNum] = $line;
$lineNum = $current['line'];
$line = [];
$line[] = $current;
}
else
{
$line[] = $current;
}
}
return $tokens;
}

View File

@ -85,7 +85,7 @@ function get_hldb(): array
), ),
Syntax::new( Syntax::new(
'PHP', 'PHP',
['.php'], ['.php', 'kilo'],
[ [
'?php', '$this', '__halt_compiler', 'abstract', 'and', 'array', 'as', 'break', '?php', '$this', '__halt_compiler', 'abstract', 'and', 'array', 'as', 'break',
'callable', 'case', 'catch', 'class', 'clone', 'const', 'continue', 'declare', 'callable', 'case', 'catch', 'class', 'clone', 'const', 'continue', 'declare',