code = $code; $this->rawLines = $lines; $this->tokens = array_fill(1, count($lines), []); } /** * Use 'token_get_all' to get the tokens for a file, * organized by row number * * @param string $code * @return array */ public static function getTokens(string $code): array { return (new self($code))->organizeTokens(); } /** * Return tokens for the current $filename, organized * by row number * * @param string $filename * @return array */ public static function getFileTokens(string $filename): array { $code = file_get_contents($filename); if ($code === FALSE) { return []; } return self::getTokens($code); } protected function organizeTokens(): array { $rawTokens = token_get_all($this->code); foreach ($rawTokens as $t) { if (is_array($t)) { $this->processArrayToken($t); } else if (is_string($t)) { $this->processStringToken($t); } } ksort($this->tokens); return $this->tokens; } protected function processArrayToken(array $token): void { [$type, $rawChar, $currentLine] = $token; $char = tabs_to_spaces($rawChar); if ($currentLine !== $this->lineNum) { $this->lineNum = $currentLine; } $current = [ 'type' => $type, 'typeName' => token_name($type), 'char' => $char, 'line' => $currentLine, ]; // Single new line, or starts with a new line with other whitespace if (strpos($char, "\n") === 0 && trim($char) === '') { $this->tokens[$this->lineNum][] = $current; $this->lineNum++; if ( ! array_key_exists($this->lineNum, $this->tokens)) { $this->tokens[$this->lineNum] = []; } return; } // Only return the first line of a multi-line token for this line array if (str_contains($char, "\n")) { $chars = explode("\n", $char); $current['original'] = [ 'string' => $char, 'lines' => $chars, ]; $current['char'] = array_shift($chars); // Add new lines for additional newline characters $nextLine = $currentLine; foreach ($chars as $char) { $nextLine++; if ( ! array_key_exists($nextLine, $this->tokens)) { $tokens[$nextLine] = []; } if ( ! empty($char)) { $this->processStringToken($char, $nextLine); } } } $this->tokens[$this->lineNum][] = $current; } protected function processStringToken(string $token, ?int $startLine = NULL): void { $char = tabs_to_spaces($token); $startLine = $startLine ?? $this->lineNum; $lineNumber = $this->findCorrectLine($char, $startLine) ?? $startLine; // Simple characters, usually delimiters or single character operators $this->tokens[$lineNumber][] = [ 'type' => -1, 'typeName' => 'RAW', 'char' => tabs_to_spaces($token), ]; } private function findCorrectLine(string $search, int $rowOffset, int $searchLength = 5): ?int { $end = $rowOffset + $searchLength; if ($end > count($this->rawLines)) { $end = count($this->rawLines); } for ($i = $rowOffset; $i < $end; $i++) { if (str_contains($this->rawLines[$i], $search)) { return $i; } } return NULL; } }