From 652eac5be00866a390553a68318b4ad896957321 Mon Sep 17 00:00:00 2001 From: "Timothy J. Warren" Date: Tue, 14 Feb 2017 15:29:13 -0500 Subject: [PATCH] Get sync-lists command to create missing entries on MAL --- src/API/Kitsu/KitsuTrait.php | 9 +- src/API/Kitsu/Model.php | 47 +++- src/API/MAL.php | 8 + src/API/MAL/ListItem.php | 16 +- src/API/MAL/Model.php | 9 +- .../MAL/Transformer/AnimeListTransformer.php | 6 + .../MAL/Transformer/MALToKitsuTransformer.php | 33 --- src/Command/BaseCommand.php | 23 +- src/Command/SyncKitsuWithMal.php | 255 +++++++++++++----- 9 files changed, 275 insertions(+), 131 deletions(-) delete mode 100644 src/API/MAL/Transformer/MALToKitsuTransformer.php diff --git a/src/API/Kitsu/KitsuTrait.php b/src/API/Kitsu/KitsuTrait.php index 171d0817..84bcf42c 100644 --- a/src/API/Kitsu/KitsuTrait.php +++ b/src/API/Kitsu/KitsuTrait.php @@ -20,7 +20,7 @@ use const Aviat\AnimeClient\SESSION_SEGMENT; use function Amp\wait; -use Amp\Artax\Client; +use Amp\Artax\{Client, Request}; use Aviat\AnimeClient\AnimeClient; use Aviat\AnimeClient\API\Kitsu as K; use Aviat\Ion\Json; @@ -53,9 +53,9 @@ trait KitsuTrait { * @param string $type * @param string $url * @param array $options - * @return \Amp\Artax\Response + * @return \Amp\Artax\Request */ - public function setUpRequest(string $type, string $url, array $options = []) + public function setUpRequest(string $type, string $url, array $options = []): Request { $config = $this->container->get('config'); @@ -69,7 +69,6 @@ trait KitsuTrait { { $token = $sessionSegment->get('auth_token'); $request = $request->setAuth('bearer', $token); - // $defaultOptions['headers']['Authorization'] = "bearer {$token}"; } if (array_key_exists('form_params', $options)) @@ -138,7 +137,7 @@ trait KitsuTrait { { if ($logger) { - $logger->warning('Non 200 response for api call', $response->getBody()); + $logger->warning('Non 200 response for api call', (array)$response->getBody()); } } diff --git a/src/API/Kitsu/Model.php b/src/API/Kitsu/Model.php index cd87a4cf..96fd54c5 100644 --- a/src/API/Kitsu/Model.php +++ b/src/API/Kitsu/Model.php @@ -203,15 +203,56 @@ class Model { $baseData = $this->getRawMediaData('manga', $mangaId); return $this->mangaTransformer->transform($baseData); } + + /** + * Get the number of anime list items + * + * @return int + */ + public function getAnimeListCount() : int + { + $options = [ + 'query' => [ + 'filter' => [ + 'user_id' => $this->getUserIdByUsername(), + 'media_type' => 'Anime' + ], + 'page' => [ + 'limit' => 1 + ], + 'sort' => '-updated_at' + ] + ]; + + $response = $this->getRequest('library-entries', $options); + + return $response['meta']['count']; + + } /** * Get and transform the entirety of the user's anime list * - * @return array + * @return Request */ - public function getFullAnimeList(): array + public function getFullAnimeList(int $limit = 100, int $offset = 0): Request { - + $options = [ + 'query' => [ + 'filter' => [ + 'user_id' => $this->getUserIdByUsername($this->getUsername()), + 'media_type' => 'Anime' + ], + 'include' => 'anime.mappings', + 'page' => [ + 'offset' => $offset, + 'limit' => $limit + ], + 'sort' => '-updated_at' + ] + ]; + + return $this->setUpRequest('GET', 'library-entries', $options); } /** diff --git a/src/API/MAL.php b/src/API/MAL.php index a346993a..972b4534 100644 --- a/src/API/MAL.php +++ b/src/API/MAL.php @@ -36,6 +36,14 @@ class MAL { KAWS::DROPPED => AnimeWatchingStatus::DROPPED, KAWS::PLAN_TO_WATCH => AnimeWatchingStatus::PLAN_TO_WATCH ]; + + const MAL_KITSU_WATCHING_STATUS_MAP = [ + 1 => KAWS::WATCHING, + 2 => KAWS::COMPLETED, + 3 => KAWS::ON_HOLD, + 4 => KAWS::DROPPED, + 6 => KAWS::PLAN_TO_WATCH + ]; public static function getIdToWatchingStatusMap() { diff --git a/src/API/MAL/ListItem.php b/src/API/MAL/ListItem.php index ca2d9086..686c3455 100644 --- a/src/API/MAL/ListItem.php +++ b/src/API/MAL/ListItem.php @@ -46,12 +46,6 @@ class ListItem { ->setFormFields($createData) ->setBasicAuth($config->get(['mal','username']), $config->get(['mal', 'password'])) ->getFullRequest(); - - /* $response = $this->getResponse('POST', "animelist/add/{$id}.xml", [ - 'body' => $this->fixBody((new FormBody)->addFields($createData)) - ]); - - return $response->getBody() === 'Created'; */ } public function delete(string $id): Request @@ -65,11 +59,7 @@ class ListItem { ->setBasicAuth($config->get(['mal','username']), $config->get(['mal', 'password'])) ->getFullRequest(); - /*$response = $this->getResponse('DELETE', "animelist/delete/{$id}.xml", [ - 'body' => $this->fixBody((new FormBody)->addField('id', $id)) - ]); - - return $response->getBody() === 'Deleted';*/ + // return $response->getBody() === 'Deleted' } public function get(string $id): array @@ -93,9 +83,5 @@ class ListItem { ]) ->setBasicAuth($config->get(['mal','username']), $config->get(['mal', 'password'])) ->getFullRequest(); - - /* return $this->getResponse('POST', "animelist/update/{$id}.xml", [ - 'body' => $this->fixBody($body) - ]); */ } } \ No newline at end of file diff --git a/src/API/MAL/Model.php b/src/API/MAL/Model.php index e8458121..46355b27 100644 --- a/src/API/MAL/Model.php +++ b/src/API/MAL/Model.php @@ -36,13 +36,18 @@ class Model { protected $animeListTransformer; /** - * KitsuModel constructor. + * MAL Model constructor. */ public function __construct(ListItem $listItem) { $this->animeListTransformer = new AnimeListTransformer(); $this->listItem = $listItem; } + + public function createFullListItem(array $data): Request + { + return $this->listItem->create($data); + } public function createListItem(array $data): Request { @@ -70,7 +75,7 @@ class Model { ] ]); - return $list;//['anime']; + return $list['myanimelist']['anime']; } public function getListItem(string $listId): array diff --git a/src/API/MAL/Transformer/AnimeListTransformer.php b/src/API/MAL/Transformer/AnimeListTransformer.php index dc48adda..16f59d8a 100644 --- a/src/API/MAL/Transformer/AnimeListTransformer.php +++ b/src/API/MAL/Transformer/AnimeListTransformer.php @@ -32,6 +32,12 @@ class AnimeListTransformer extends AbstractTransformer { AnimeWatchingStatus::PLAN_TO_WATCH => '6' ]; + /** + * Transform MAL episode data to Kitsu episode data + * + * @param array $item + * @return array + */ public function transform($item) { $rewatching = (array_key_exists('rewatching', $item) && $item['rewatching']); diff --git a/src/API/MAL/Transformer/MALToKitsuTransformer.php b/src/API/MAL/Transformer/MALToKitsuTransformer.php deleted file mode 100644 index dc8fed17..00000000 --- a/src/API/MAL/Transformer/MALToKitsuTransformer.php +++ /dev/null @@ -1,33 +0,0 @@ - - * @copyright 2015 - 2017 Timothy J. Warren - * @license http://www.opensource.org/licenses/mit-license.html MIT License - * @version 4.0 - * @link https://github.com/timw4mail/HummingBirdAnimeClient - */ - -namespace Aviat\AnimeClient\API\MAL; - -use Aviat\Ion\Transformer\AbstractTransformer; - -class MALToKitsuTransformer extends AbstractTransformer { - - - public function transform($item) - { - - } - - public function untransform($item) - { - - } -} \ No newline at end of file diff --git a/src/Command/BaseCommand.php b/src/Command/BaseCommand.php index f3dccc1d..cff2621c 100644 --- a/src/Command/BaseCommand.php +++ b/src/Command/BaseCommand.php @@ -24,11 +24,10 @@ use Aviat\AnimeClient\{ Model, Util }; -use Aviat\AnimeClient\API\{ - CacheTrait, - Kitsu, - MAL -}; +use Aviat\AnimeClient\API\CacheTrait; +use Aviat\AnimeClient\API\{Kitsu, MAL}; +use Aviat\AnimeClient\API\Kitsu\KitsuRequestBuilder; +use Aviat\AnimeClient\API\MAL\MALRequestBuilder; use Aviat\Banker\Pool; use Aviat\Ion\Config; use Aviat\Ion\Di\{Container, ContainerAware}; @@ -109,21 +108,35 @@ class BaseCommand extends Command { // Models $container->set('kitsu-model', function($container) { + $requestBuilder = new KitsuRequestBuilder(); + $requestBuilder->setLogger($container->getLogger('kitsu-request')); + $listItem = new Kitsu\ListItem(); $listItem->setContainer($container); + $listItem->setRequestBuilder($requestBuilder); + $model = new Kitsu\Model($listItem); $model->setContainer($container); + $model->setRequestBuilder($requestBuilder); + $cache = $container->get('cache'); $model->setCache($cache); return $model; }); $container->set('mal-model', function($container) { + $requestBuilder = new MALRequestBuilder(); + $requestBuilder->setLogger($container->getLogger('mal-request')); + $listItem = new MAL\ListItem(); $listItem->setContainer($container); + $listItem->setRequestBuilder($requestBuilder); + $model = new MAL\Model($listItem); $model->setContainer($container); + $model->setRequestBuilder($requestBuilder); return $model; }); + $container->set('util', function($container) { return new Util($container); }); diff --git a/src/Command/SyncKitsuWithMal.php b/src/Command/SyncKitsuWithMal.php index d7f13daf..c219bfff 100644 --- a/src/Command/SyncKitsuWithMal.php +++ b/src/Command/SyncKitsuWithMal.php @@ -16,8 +16,13 @@ namespace Aviat\AnimeClient\Command; +use function Amp\{all, wait}; + use Amp\Artax; -use Aviat\AnimeClient\API\Kitsu; +use Amp\Artax\Client; +use Aviat\AnimeClient\API\{JsonAPI, Kitsu, MAL}; +use Aviat\AnimeClient\API\MAL\Transformer\AnimeListTransformer as ALT; +use Aviat\Ion\Json; /** * Clears the API Cache @@ -41,91 +46,173 @@ class SyncKitsuWithMal extends BaseCommand { $this->setCache($this->container->get('cache')); $this->kitsuModel = $this->container->get('kitsu-model'); $this->malModel = $this->container->get('mal-model'); - - //$kitsuCount = $this->getKitsuAnimeListPageCount(); - //$this->echoBox("List item count: {$kitsuCount}"); - $this->MALItemCreate(); - - //echo json_encode($this->getMALList(), \JSON_PRETTY_PRINT); + + $malCount = count($this->getMALList()); + $kitsuCount = $this->getKitsuAnimeListPageCount(); + + $this->echoBox("Number of MAL list items: {$malCount}"); + $this->echoBox("Number of Kitsu list items: {$kitsuCount}"); + + $data = $this->diffLists(); + $this->echoBox("Number of items that need to be added to MAL: " . count($data)); + + if (! empty($data['addToMAL'])) + { + $this->echoBox("Adding missing list items to MAL"); + $this->createMALListItems($data['addToMAL']); + } + } + + public function getKitsuList() + { + $count = $this->getKitsuAnimeListPageCount(); + $size = 100; + $pages = ceil($count / $size); + + $requests = []; + + // Set up requests + for ($i = 0; $i < $count; $i++) + { + $offset = $i * $size; + $requests[] = $this->kitsuModel->getFullAnimeList($size, $offset); + } + + $promiseArray = (new Client())->requestMulti($requests); + + $responses = wait(all($promiseArray)); + $output = []; + foreach($responses as $response) + { + $data = Json::decode($response->getBody()); + $output = array_merge_recursive($output, $data); + } + + return $output; + } public function getMALList() { return $this->malModel->getFullList(); } + + public function filterMappings(array $includes): array + { + $output = []; + + foreach($includes as $id => $mapping) + { + if ($mapping['externalSite'] === 'myanimelist/anime') + { + $output[$id] = $mapping; + } + } + + return $output; + } + + // 2015-05-20T23:48:47.731Z + + public function formatMALList() + { + $orig = $this->getMALList(); + $output = []; + + foreach($orig as $item) + { + $output[$item['series_animedb_id']] = [ + 'id' => $item['series_animedb_id'], + 'data' => [ + 'status' => MAL::MAL_KITSU_WATCHING_STATUS_MAP[$item['my_status']], + 'progress' => $item['my_watched_episodes'], + 'reconsuming' => (bool) $item['my_rewatching'], + 'reconsumeCount' => array_key_exists('times_rewatched', $item) + ? $item['times_rewatched'] + : 0, + // 'notes' => , + 'rating' => $item['my_score'], + 'updatedAt' => (new \DateTime()) + ->setTimestamp((int)$item['my_last_updated']) + ->format(\DateTime::W3C), + ] + ]; + } + + return $output; + } + + public function filterKitsuList() + { + $data = $this->getKitsuList(); + $includes = JsonAPI::organizeIncludes($data['included']); + $includes['mappings'] = $this->filterMappings($includes['mappings']); + + $output = []; + + foreach($data['data'] as $listItem) + { + $animeId = $listItem['relationships']['anime']['data']['id']; + $potentialMappings = $includes['anime'][$animeId]['relationships']['mappings']; + $malId = null; + + foreach ($potentialMappings as $mappingId) + { + if (array_key_exists($mappingId, $includes['mappings'])) + { + $malId = $includes['mappings'][$mappingId]['externalId']; + } + } + + // Skip to the next item if there isn't a MAL ID + if ($malId === null) + { + continue; + } + + $output[$listItem['id']] = [ + 'id' => $listItem['id'], + 'malId' => $malId, + 'data' => $listItem['attributes'], + ]; + } + + return $output; + } public function getKitsuAnimeListPageCount() { - $cacheItem = $this->cache->getItem(Kitsu::AUTH_TOKEN_CACHE_KEY); - - $query = http_build_query([ - 'filter' => [ - 'user_id' => $this->kitsuModel->getUserIdByUsername(), - 'media_type' => 'Anime' - ], - // 'include' => 'anime,anime.genres,anime.mappings,anime.streamingLinks', - 'page' => [ - 'limit' => 1 - ], - 'sort' => '-updated_at' - ]); - $request = (new Artax\Request) - ->setUri("https://kitsu.io/api/edge/library-entries?{$query}") - ->setProtocol('1.1') - ->setAllHeaders([ - 'Accept' => 'application/vnd.api+json', - 'Content-Type' => 'application/vnd.api+json', - 'User-Agent' => "Tim's Anime Client/4.0" - ]); - - if ($cacheItem->isHit()) - { - $token = $cacheItem->get(); - $request->setHeader('Authorization', "bearer {$token}"); - } - else - { - $this->echoBox("WARNING: NOT LOGGED IN\nSome data might be missing"); - } - - $response = \Amp\wait((new Artax\Client)->request($request)); - - $body = json_decode($response->getBody(), TRUE); - return $body['meta']['count']; - } - - public function MALItemCreate() - { - $input = json_decode('{ - "watching_status": "current", - "user_rating": "", - "episodes_watched": "4", - "rewatched": "0", - "notes": "", - "id": "15794526", - "mal_id": "33731", - "edit": "true" - }', TRUE); - - $response = $this->malModel->createListItem([ - 'id' => 12255, - 'status' => 'planned', - 'type' => 'anime' - ]); - - //$response = $this->malModel->updateListItem($input); - //print_r($response); - //echo $response->getBody(); - + return $this->kitsuModel->getAnimeListCount(); } public function diffLists() { // Get libraryEntries with media.mappings from Kitsu // Organize mappings, and ignore entries without mappings + $kitsuList = $this->filterKitsuList(); // Get MAL list data + $malList = $this->formatMALList(); + + $itemsToAddToMAL = []; + + foreach($kitsuList as $kitsuItem) + { + if (array_key_exists($kitsuItem['malId'], $malList)) + { + // Eventually, compare the list entries, and determine which + // needs to be updated + continue; + } + + // Looks like this item only exists on Kitsu + $itemsToAddToMAL[] = [ + 'mal_id' => $kitsuItem['malId'], + 'data' => $kitsuItem['data'] + ]; + + } // Compare each list entry // If a list item exists only on MAL, create it on Kitsu with the existing data from MAL @@ -135,7 +222,39 @@ class SyncKitsuWithMal extends BaseCommand { // Otherwise, use rewatch count, then episode progress as critera for selecting the more up // to date entry // Based on the 'newer' entry, update the other api list item + + return [ + 'addToMAL' => $itemsToAddToMAL, + ]; } + public function createMALListItems($itemsToAdd) + { + $transformer = new ALT(); + $requests = []; + + foreach($itemsToAdd as $item) + { + $data = $transformer->untransform($item); + $requests[] = $this->malModel->createFullListItem($data); + } + + $promiseArray = (new Client())->requestMulti($requests); + + $responses = wait(all($promiseArray)); + + foreach($responses as $key => $response) + { + $id = $itemsToAdd[$key]['mal_id']; + if ($response->getBody() === 'Created') + { + $this->echoBox("Successfully create list item with id: {$id}"); + } + else + { + $this->echoBox("Failed to create list item with id: {$id}"); + } + } + } } \ No newline at end of file