From 470d25f26976639e0e8f52d770dc5066d6cd1904 Mon Sep 17 00:00:00 2001 From: "Timothy J. Warren" Date: Fri, 16 Oct 2020 16:18:56 -0400 Subject: [PATCH] Sync Kitsu and Anilist both via GraphQL, see #33 --- src/AnimeClient/API/Kitsu/MangaTrait.php | 1 - src/AnimeClient/API/Kitsu/Model.php | 146 +++++++++--------- .../API/Kitsu/Queries/GetSyncLibrary.graphql | 42 +++++ src/AnimeClient/Command/SyncLists.php | 124 ++++++--------- src/AnimeClient/Enum/APISource.php | 27 ---- 5 files changed, 167 insertions(+), 173 deletions(-) create mode 100644 src/AnimeClient/API/Kitsu/Queries/GetSyncLibrary.graphql delete mode 100644 src/AnimeClient/Enum/APISource.php diff --git a/src/AnimeClient/API/Kitsu/MangaTrait.php b/src/AnimeClient/API/Kitsu/MangaTrait.php index 94168d0d..740d2633 100644 --- a/src/AnimeClient/API/Kitsu/MangaTrait.php +++ b/src/AnimeClient/API/Kitsu/MangaTrait.php @@ -19,7 +19,6 @@ namespace Aviat\AnimeClient\API\Kitsu; use Amp\Http\Client\Request; use Aviat\AnimeClient\Kitsu as K; use Aviat\AnimeClient\API\Enum\MangaReadingStatus\Kitsu as KitsuReadingStatus; -use Aviat\AnimeClient\API\JsonAPI; use Aviat\AnimeClient\API\Kitsu\Transformer\MangaHistoryTransformer; use Aviat\AnimeClient\API\Kitsu\Transformer\MangaListTransformer; use Aviat\AnimeClient\API\Kitsu\Transformer\OldMangaListTransformer; diff --git a/src/AnimeClient/API/Kitsu/Model.php b/src/AnimeClient/API/Kitsu/Model.php index 25d4083d..830d5afe 100644 --- a/src/AnimeClient/API/Kitsu/Model.php +++ b/src/AnimeClient/API/Kitsu/Model.php @@ -329,16 +329,27 @@ final class Model { */ public function getSyncList(string $type): array { - $options = [ - 'filter' => [ - 'user_id' => $this->getUserId(), - 'kind' => $type, - ], - 'include' => "{$type},{$type}.mappings", - // 'sort' => '-updated_at' + $statuses = [ + 'CURRENT', + 'PLANNED', + 'ON_HOLD', + 'DROPPED', + 'COMPLETED', ]; - return $this->getRawSyncList($type, $options); + $pages = []; + + // Although I can fetch the whole list without segregating by status, + // this way is much faster... + foreach ($statuses as $status) + { + foreach ($this->getRawSyncListPages(strtoupper($type), $status) as $page) + { + $pages[] = $page; + } + } + + return array_merge(...$pages); } /** @@ -412,6 +423,9 @@ final class Model { $page = $data['pageInfo']; if (empty($data)) { + dump($rawData); + die(); + // @TODO Error logging break; } @@ -428,6 +442,58 @@ final class Model { }); } + protected function getRawSyncListPages(string $type, string $status): ?\Generator + { + $items = $this->getRawSyncPages($type, $status); + + while (wait($items->advance())) + { + yield $items->getCurrent(); + } + } + + protected function getRawSyncPages(string $type, $status): Amp\Iterator { + $cursor = ''; + $username = $this->getUsername(); + + return new Amp\Producer(function (callable $emit) use ($type, $status, $cursor, $username) { + while (TRUE) + { + $vars = [ + 'type' => $type, + 'slug' => $username, + 'status' => $status, + ]; + if ($cursor !== '') + { + $vars['after'] = $cursor; + } + + $request = $this->requestBuilder->queryRequest('GetSyncLibrary', $vars); + $response = yield getApiClient()->request($request); + $json = yield $response->getBody()->buffer(); + + $rawData = Json::decode($json); + $data = $rawData['data']['findProfileBySlug']['library']['all'] ?? []; + $page = $data['pageInfo']; + if (empty($data)) + { + dump($rawData); + die(); + } + + $cursor = $page['endCursor']; + + yield $emit($data['nodes']); + + if ($page['hasNextPage'] === FALSE) + { + break; + } + } + }); + } + private function getUserId(): string { static $userId = NULL; @@ -467,68 +533,4 @@ final class Model { return $res['data']['findProfileBySlug']['library']['all']['totalCount']; } - - /** - * Get the full anime list - * - * @param string $type - * @param array $options - * @return array - * @throws InvalidArgumentException - * @throws Throwable - */ - private function getRawSyncList(string $type, array $options): array - { - $count = $this->getListCount($type); - $size = static::LIST_PAGE_SIZE; - $pages = ceil($count / $size); - - $requester = new ParallelAPIRequest(); - - // Set up requests - for ($i = 0; $i < $pages; $i++) - { - $offset = $i * $size; - $requester->addRequest($this->getRawSyncListPage($type, $size, $offset, $options)); - } - - $responses = $requester->makeRequests(); - $output = []; - - foreach($responses as $response) - { - $data = Json::decode($response); - $output[] = $data; - } - - return array_merge_recursive(...$output); - } - - /** - * Get the full anime list in paginated form - * - * @param string $type - * @param int $limit - * @param int $offset - * @param array $options - * @return Request - * @throws InvalidArgumentException - */ - private function getRawSyncListPage(string $type, int $limit, int $offset = 0, array $options = []): Request - { - $defaultOptions = [ - 'filter' => [ - 'user_id' => $this->getUserId(), - 'kind' => $type, - ], - 'page' => [ - 'offset' => $offset, - 'limit' => $limit - ], - 'sort' => '-updated_at' - ]; - $options = array_merge($defaultOptions, $options); - - return $this->requestBuilder->setUpRequest('GET', 'library-entries', ['query' => $options]); - } } \ No newline at end of file diff --git a/src/AnimeClient/API/Kitsu/Queries/GetSyncLibrary.graphql b/src/AnimeClient/API/Kitsu/Queries/GetSyncLibrary.graphql new file mode 100644 index 00000000..436f84c5 --- /dev/null +++ b/src/AnimeClient/API/Kitsu/Queries/GetSyncLibrary.graphql @@ -0,0 +1,42 @@ +query ( + $slug: String!, + $type: MediaTypeEnum!, + $status: [LibraryEntryStatusEnum!], + $after: String +) { + findProfileBySlug(slug: $slug) { + library { + all(first: 100, after: $after, mediaType: $type, status: $status) { + pageInfo { + endCursor + hasNextPage + hasPreviousPage + startCursor + } + totalCount + nodes { + id + notes + nsfw + private + progress + progressedAt + rating + reconsumeCount + reconsuming + status + media { + id + slug + mappings(first: 10) { + nodes { + externalId + externalSite + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/src/AnimeClient/Command/SyncLists.php b/src/AnimeClient/Command/SyncLists.php index d16fc9c5..aa3c0cb0 100644 --- a/src/AnimeClient/Command/SyncLists.php +++ b/src/AnimeClient/Command/SyncLists.php @@ -20,14 +20,12 @@ use ConsoleKit\Widgets; use Aviat\AnimeClient\API\{ Anilist\MissingIdException, - FailedResponseException, - JsonAPI, ParallelAPIRequest }; use Aviat\AnimeClient\API; use Aviat\AnimeClient\API\Anilist; use Aviat\AnimeClient\API\Mapping\{AnimeWatchingStatus, MangaReadingStatus}; -use Aviat\AnimeClient\Enum\{APISource, ListType, SyncAction}; +use Aviat\AnimeClient\Enum\{ListType, SyncAction}; use Aviat\AnimeClient\Types\FormItem; use Aviat\Ion\Di\Exception\ContainerException; use Aviat\Ion\Di\Exception\NotFoundException; @@ -40,6 +38,9 @@ use function in_array; * Syncs list data between Anilist and Kitsu */ final class SyncLists extends BaseCommand { + protected const KITSU_GREATER = 1; + protected const ANILIST_GREATER = -1; + protected const SAME = 0; /** * Model for making requests to Anilist API @@ -315,37 +316,24 @@ final class SyncLists extends BaseCommand { return []; } - if ( ! array_key_exists('included', $data)) - { - dump([ - 'problem' => 'Missing included data in method ' . __METHOD__, - 'data' => $data - ]); - return []; - } - - $includes = JsonAPI::organizeIncludes($data['included']); - $includes['mappings'] = $this->filterMappings($includes['mappings'], $type); - $output = []; - foreach($data['data'] as $listItem) + foreach($data as $listItem) { - $id = $listItem['relationships'][$type]['data']['id']; - - $potentialMappings = $includes[$type][$id]['relationships']['mappings']; + // If there's no mapping, we can't sync, so continue + if ( ! is_array($listItem['media']['mappings']['nodes'])) + { + continue; + } $malId = NULL; - foreach ($potentialMappings as $mappingId) + foreach ($listItem['media']['mappings']['nodes'] as $mapping) { - if (is_array($mappingId)) + $uType = strtoupper($type); + if ($mapping['externalSite'] === "MYANIMELIST_{$uType}") { - continue; - } - - if (array_key_exists($mappingId, $includes['mappings'])) - { - $malId = $includes['mappings'][$mappingId]['externalId']; + $malId = $mapping['externalId']; + break; } } @@ -355,10 +343,22 @@ final class SyncLists extends BaseCommand { continue; } - $output[$listItem['id']] = [ - 'id' => $listItem['id'], + $output[$listItem['media']['id']] = [ + 'id' => $listItem['media']['id'], + 'slug' => $listItem['media']['slug'], 'malId' => $malId, - 'data' => $listItem['attributes'], + 'data' => [ + 'notes' => $listItem['notes'], + 'private' => $listItem['private'], + 'progress' => $listItem['progress'], + // Comparision is done on 1-10 scale, + // Kitsu returns 1-20 scale. + 'rating' => $listItem['rating'] / 2, + 'reconsumeCount' => $listItem['reconsumeCount'], + 'reconsuming' => $listItem['reconsuming'], + 'status' => strtolower($listItem['status']), + 'updatedAt' => $listItem['progressedAt'], + ] ]; } @@ -460,7 +460,7 @@ final class SyncLists extends BaseCommand { 'private' => $kItem['private'], 'progress' => $kItem['progress'], 'repeat' => $kItem['reconsumeCount'], - 'score' => $kItem['ratingTwenty'] * 5, // 100 point score on Anilist + 'score' => $kItem['rating'] * 10, // 100 point score on Anilist 'status' => $newItemStatus, ], ]; @@ -492,10 +492,9 @@ final class SyncLists extends BaseCommand { 'status', ]; $diff = []; - $dateDiff = new DateTime($kitsuItem['data']['updatedAt']) <=> new DateTime((string)$anilistItem['data']['updatedAt']); - - // Correct differences in notation - $kitsuItem['data']['rating'] = $kitsuItem['data']['ratingTwenty'] / 2; + $dateDiff = ($kitsuItem['data']['updatedAt'] !== NULL) + ? new DateTime($kitsuItem['data']['updatedAt']) <=> new DateTime((string)$anilistItem['data']['updatedAt']) + : 0; foreach($compareKeys as $key) { @@ -535,12 +534,12 @@ final class SyncLists extends BaseCommand { // If status is the same, and progress count is different, use greater progress if ($sameStatus && ( ! $sameProgress)) { - if ($diff['progress'] === 1) + if ($diff['progress'] === self::KITSU_GREATER) { $update['data']['progress'] = $kitsuItem['data']['progress']; $return['updateType'][] = API::ANILIST; } - else if($diff['progress'] === -1) + else if($diff['progress'] === self::ANILIST_GREATER) { $update['data']['progress'] = $anilistItem['data']['progress']; $return['updateType'][] = API::KITSU; @@ -550,12 +549,12 @@ final class SyncLists extends BaseCommand { // If status is different, use the status of the more recently updated item if ( ! $sameStatus) { - if ($dateDiff === 1) + if ($dateDiff === self::KITSU_GREATER) { $update['data']['status'] = $kitsuItem['data']['status']; $return['updateType'][] = API::ANILIST; } - else if ($dateDiff === -1) + else if ($dateDiff === self::ANILIST_GREATER) { $update['data']['status'] = $anilistItem['data']['status']; $return['updateType'][] = API::KITSU; @@ -566,7 +565,7 @@ final class SyncLists extends BaseCommand { // But, at least for now, assume newer record is correct if ( ! ($sameStatus || $sameProgress)) { - if ($dateDiff === 1) + if ($dateDiff === self::KITSU_GREATER) { $update['data']['status'] = $kitsuItem['data']['status']; @@ -577,7 +576,7 @@ final class SyncLists extends BaseCommand { $return['updateType'][] = API::ANILIST; } - else if($dateDiff === -1) + else if($dateDiff === self::ANILIST_GREATER) { $update['data']['status'] = $anilistItem['data']['status']; @@ -594,7 +593,7 @@ final class SyncLists extends BaseCommand { if ( ! $sameRating) { if ( - $dateDiff === 1 && + $dateDiff === self::KITSU_GREATER && $kitsuItem['data']['rating'] !== 0 && $kitsuItem['data']['ratingTwenty'] !== 0 ) @@ -602,7 +601,7 @@ final class SyncLists extends BaseCommand { $update['data']['ratingTwenty'] = $kitsuItem['data']['ratingTwenty']; $return['updateType'][] = API::ANILIST; } - else if($dateDiff === -1 && $anilistItem['data']['rating'] !== 0) + else if($dateDiff === self::ANILIST_GREATER && $anilistItem['data']['rating'] !== 0) { $update['data']['ratingTwenty'] = $anilistItem['data']['rating'] * 2; $return['updateType'][] = API::KITSU; @@ -612,7 +611,7 @@ final class SyncLists extends BaseCommand { // If notes are set, use kitsu, otherwise, set kitsu from anilist if ( ! $sameNotes) { - if ($kitsuItem['data']['notes'] !== '') + if ( ! empty($kitsuItem['data']['notes'])) { $update['data']['notes'] = $kitsuItem['data']['notes']; $return['updateType'][] = API::ANILIST; @@ -627,12 +626,12 @@ final class SyncLists extends BaseCommand { // Assume the larger reconsumeCount is correct if ( ! $sameRewatchCount) { - if ($diff['reconsumeCount'] === 1) + if ($diff['reconsumeCount'] === self::KITSU_GREATER) { $update['data']['reconsumeCount'] = $kitsuItem['data']['reconsumeCount']; $return['updateType'][] = API::ANILIST; } - else if ($diff['reconsumeCount'] === -1) + else if ($diff['reconsumeCount'] === self::ANILIST_GREATER) { $update['data']['reconsumeCount'] = $anilistItem['data']['reconsumeCount']; $return['updateType'][] = API::KITSU; @@ -659,11 +658,14 @@ final class SyncLists extends BaseCommand { // to handle each combination of fields if ($return['updateType'][0] === API::ANILIST) { + // Anilist GraphQL expects a rating from 1-100 $prevData = [ 'notes' => $kitsuItem['data']['notes'], 'private' => $kitsuItem['data']['private'], 'progress' => $kitsuItem['data']['progress'], - 'rating' => $kitsuItem['data']['ratingTwenty'] * 5, + // Transformed Kitsu data returns a rating from 1-10 + // Anilist expects a rating from 1-100 + 'rating' => $kitsuItem['data']['rating'] * 10, 'reconsumeCount' => $kitsuItem['data']['reconsumeCount'], 'reconsuming' => $kitsuItem['data']['reconsuming'], 'status' => $kitsuItem['data']['status'], @@ -677,6 +679,8 @@ final class SyncLists extends BaseCommand { 'notes' => $anilistItem['data']['notes'], 'private' => $anilistItem['data']['private'], 'progress' => $anilistItem['data']['progress'] ?? 0, + // Anilist returns a rating between 1-100 + // Kitsu expects a rating from 1-20 'rating' => (((int)$anilistItem['data']['rating']) > 0) ? $anilistItem['data']['rating'] / 5 : 0, @@ -824,30 +828,4 @@ final class SyncLists extends BaseCommand { } } } - - // ------------------------------------------------------------------------ - // Other Helpers - // ------------------------------------------------------------------------ - - /** - * Filter Kitsu mappings for the specified type - * - * @param array $includes - * @param string $type - * @return array - */ - private function filterMappings(array $includes, string $type = ListType::ANIME): array - { - $output = []; - - foreach($includes as $id => $mapping) - { - if ($mapping['externalSite'] === "myanimelist/{$type}") - { - $output[$id] = $mapping; - } - } - - return $output; - } } diff --git a/src/AnimeClient/Enum/APISource.php b/src/AnimeClient/Enum/APISource.php deleted file mode 100644 index 66bccf17..00000000 --- a/src/AnimeClient/Enum/APISource.php +++ /dev/null @@ -1,27 +0,0 @@ - - * @copyright 2015 - 2020 Timothy J. Warren - * @license http://www.opensource.org/licenses/mit-license.html MIT License - * @version 5.1 - * @link https://git.timshomepage.net/timw4mail/HummingBirdAnimeClient - */ - -namespace Aviat\AnimeClient\Enum; - -use Aviat\Ion\Enum as BaseEnum; - -/** - * Types of lists - */ -final class APISource extends BaseEnum { - public const KITSU = 'kitsu'; - public const ANILIST = 'anilist'; -} \ No newline at end of file