rawLines = $lines; $self->tokens = array_fill(1, count($lines), []); return $self->organizeTokens($code); } /** * Return tokens for the current $filename, organized * by row number * * @param string $filename * @return array */ public static function getFileTokens(string $filename): array { $code = @file_get_contents($filename); if ($code === FALSE) { return []; } return self::getTokens($code); } protected function organizeTokens(string $code): array { $rawTokens = self::tokenize($code); foreach ($rawTokens as $t) { $this->processObjectToken($t); } ksort($this->tokens); return $this->tokens; } protected function processObjectToken(\PhpToken $token): void { $currentLine = $token->line; $char = tabs_to_spaces($token->text); $current = [ 'type' => $token->id, 'typeName' => $token->getTokenName(), 'char' => $char, 'line' => $currentLine, ]; // Single new line, or starts with a new line with other whitespace if (str_starts_with($char, "\n") && trim($char) === '') { $this->tokens[$currentLine][] = $current; return; } // Only return the first line of a multi-line token for this line array if (str_contains($char, "\n")) { $chars = explode("\n", $char); $current['original'] = [ 'string' => $char, 'lines' => $chars, ]; $current['char'] = array_shift($chars); // Add new lines for additional newline characters $nextLine = $currentLine; foreach ($chars as $char) { $nextLine++; if ( ! empty($char)) { $this->processStringToken($char, $nextLine); } } } $this->tokens[$currentLine][] = $current; } protected function processStringToken(string $token, int $startLine): void { $char = tabs_to_spaces($token); $lineNumber = $this->findCorrectLine($char, $startLine) ?? $startLine; // Simple characters, usually delimiters or single character operators $this->tokens[$lineNumber][] = [ 'type' => -1, 'typeName' => 'RAW', 'char' => tabs_to_spaces($token), 'line' => $lineNumber, ]; } private function findCorrectLine(string $search, int $rowOffset, int $searchLength = 5): ?int { $end = $rowOffset + $searchLength; if ($end > count($this->rawLines)) { $end = count($this->rawLines); } for ($i = $rowOffset; $i < $end; $i++) { if (str_contains($this->rawLines[$i], $search)) { return $i; } } return NULL; } }