Sync Kitsu and Anilist both via GraphQL, see #33

This commit is contained in:
Timothy Warren 2020-10-16 16:18:56 -04:00
parent 70a33e36c0
commit 470d25f269
5 changed files with 167 additions and 173 deletions

View File

@ -19,7 +19,6 @@ namespace Aviat\AnimeClient\API\Kitsu;
use Amp\Http\Client\Request; use Amp\Http\Client\Request;
use Aviat\AnimeClient\Kitsu as K; use Aviat\AnimeClient\Kitsu as K;
use Aviat\AnimeClient\API\Enum\MangaReadingStatus\Kitsu as KitsuReadingStatus; use Aviat\AnimeClient\API\Enum\MangaReadingStatus\Kitsu as KitsuReadingStatus;
use Aviat\AnimeClient\API\JsonAPI;
use Aviat\AnimeClient\API\Kitsu\Transformer\MangaHistoryTransformer; use Aviat\AnimeClient\API\Kitsu\Transformer\MangaHistoryTransformer;
use Aviat\AnimeClient\API\Kitsu\Transformer\MangaListTransformer; use Aviat\AnimeClient\API\Kitsu\Transformer\MangaListTransformer;
use Aviat\AnimeClient\API\Kitsu\Transformer\OldMangaListTransformer; use Aviat\AnimeClient\API\Kitsu\Transformer\OldMangaListTransformer;

View File

@ -329,16 +329,27 @@ final class Model {
*/ */
public function getSyncList(string $type): array public function getSyncList(string $type): array
{ {
$options = [ $statuses = [
'filter' => [ 'CURRENT',
'user_id' => $this->getUserId(), 'PLANNED',
'kind' => $type, 'ON_HOLD',
], 'DROPPED',
'include' => "{$type},{$type}.mappings", 'COMPLETED',
// 'sort' => '-updated_at'
]; ];
return $this->getRawSyncList($type, $options); $pages = [];
// Although I can fetch the whole list without segregating by status,
// this way is much faster...
foreach ($statuses as $status)
{
foreach ($this->getRawSyncListPages(strtoupper($type), $status) as $page)
{
$pages[] = $page;
}
}
return array_merge(...$pages);
} }
/** /**
@ -412,6 +423,9 @@ final class Model {
$page = $data['pageInfo']; $page = $data['pageInfo'];
if (empty($data)) if (empty($data))
{ {
dump($rawData);
die();
// @TODO Error logging // @TODO Error logging
break; break;
} }
@ -428,6 +442,58 @@ final class Model {
}); });
} }
protected function getRawSyncListPages(string $type, string $status): ?\Generator
{
$items = $this->getRawSyncPages($type, $status);
while (wait($items->advance()))
{
yield $items->getCurrent();
}
}
protected function getRawSyncPages(string $type, $status): Amp\Iterator {
$cursor = '';
$username = $this->getUsername();
return new Amp\Producer(function (callable $emit) use ($type, $status, $cursor, $username) {
while (TRUE)
{
$vars = [
'type' => $type,
'slug' => $username,
'status' => $status,
];
if ($cursor !== '')
{
$vars['after'] = $cursor;
}
$request = $this->requestBuilder->queryRequest('GetSyncLibrary', $vars);
$response = yield getApiClient()->request($request);
$json = yield $response->getBody()->buffer();
$rawData = Json::decode($json);
$data = $rawData['data']['findProfileBySlug']['library']['all'] ?? [];
$page = $data['pageInfo'];
if (empty($data))
{
dump($rawData);
die();
}
$cursor = $page['endCursor'];
yield $emit($data['nodes']);
if ($page['hasNextPage'] === FALSE)
{
break;
}
}
});
}
private function getUserId(): string private function getUserId(): string
{ {
static $userId = NULL; static $userId = NULL;
@ -467,68 +533,4 @@ final class Model {
return $res['data']['findProfileBySlug']['library']['all']['totalCount']; return $res['data']['findProfileBySlug']['library']['all']['totalCount'];
} }
/**
* Get the full anime list
*
* @param string $type
* @param array $options
* @return array
* @throws InvalidArgumentException
* @throws Throwable
*/
private function getRawSyncList(string $type, array $options): array
{
$count = $this->getListCount($type);
$size = static::LIST_PAGE_SIZE;
$pages = ceil($count / $size);
$requester = new ParallelAPIRequest();
// Set up requests
for ($i = 0; $i < $pages; $i++)
{
$offset = $i * $size;
$requester->addRequest($this->getRawSyncListPage($type, $size, $offset, $options));
}
$responses = $requester->makeRequests();
$output = [];
foreach($responses as $response)
{
$data = Json::decode($response);
$output[] = $data;
}
return array_merge_recursive(...$output);
}
/**
* Get the full anime list in paginated form
*
* @param string $type
* @param int $limit
* @param int $offset
* @param array $options
* @return Request
* @throws InvalidArgumentException
*/
private function getRawSyncListPage(string $type, int $limit, int $offset = 0, array $options = []): Request
{
$defaultOptions = [
'filter' => [
'user_id' => $this->getUserId(),
'kind' => $type,
],
'page' => [
'offset' => $offset,
'limit' => $limit
],
'sort' => '-updated_at'
];
$options = array_merge($defaultOptions, $options);
return $this->requestBuilder->setUpRequest('GET', 'library-entries', ['query' => $options]);
}
} }

View File

@ -0,0 +1,42 @@
query (
$slug: String!,
$type: MediaTypeEnum!,
$status: [LibraryEntryStatusEnum!],
$after: String
) {
findProfileBySlug(slug: $slug) {
library {
all(first: 100, after: $after, mediaType: $type, status: $status) {
pageInfo {
endCursor
hasNextPage
hasPreviousPage
startCursor
}
totalCount
nodes {
id
notes
nsfw
private
progress
progressedAt
rating
reconsumeCount
reconsuming
status
media {
id
slug
mappings(first: 10) {
nodes {
externalId
externalSite
}
}
}
}
}
}
}
}

View File

@ -20,14 +20,12 @@ use ConsoleKit\Widgets;
use Aviat\AnimeClient\API\{ use Aviat\AnimeClient\API\{
Anilist\MissingIdException, Anilist\MissingIdException,
FailedResponseException,
JsonAPI,
ParallelAPIRequest ParallelAPIRequest
}; };
use Aviat\AnimeClient\API; use Aviat\AnimeClient\API;
use Aviat\AnimeClient\API\Anilist; use Aviat\AnimeClient\API\Anilist;
use Aviat\AnimeClient\API\Mapping\{AnimeWatchingStatus, MangaReadingStatus}; use Aviat\AnimeClient\API\Mapping\{AnimeWatchingStatus, MangaReadingStatus};
use Aviat\AnimeClient\Enum\{APISource, ListType, SyncAction}; use Aviat\AnimeClient\Enum\{ListType, SyncAction};
use Aviat\AnimeClient\Types\FormItem; use Aviat\AnimeClient\Types\FormItem;
use Aviat\Ion\Di\Exception\ContainerException; use Aviat\Ion\Di\Exception\ContainerException;
use Aviat\Ion\Di\Exception\NotFoundException; use Aviat\Ion\Di\Exception\NotFoundException;
@ -40,6 +38,9 @@ use function in_array;
* Syncs list data between Anilist and Kitsu * Syncs list data between Anilist and Kitsu
*/ */
final class SyncLists extends BaseCommand { final class SyncLists extends BaseCommand {
protected const KITSU_GREATER = 1;
protected const ANILIST_GREATER = -1;
protected const SAME = 0;
/** /**
* Model for making requests to Anilist API * Model for making requests to Anilist API
@ -315,37 +316,24 @@ final class SyncLists extends BaseCommand {
return []; return [];
} }
if ( ! array_key_exists('included', $data))
{
dump([
'problem' => 'Missing included data in method ' . __METHOD__,
'data' => $data
]);
return [];
}
$includes = JsonAPI::organizeIncludes($data['included']);
$includes['mappings'] = $this->filterMappings($includes['mappings'], $type);
$output = []; $output = [];
foreach($data['data'] as $listItem) foreach($data as $listItem)
{ {
$id = $listItem['relationships'][$type]['data']['id']; // If there's no mapping, we can't sync, so continue
if ( ! is_array($listItem['media']['mappings']['nodes']))
$potentialMappings = $includes[$type][$id]['relationships']['mappings']; {
continue;
}
$malId = NULL; $malId = NULL;
foreach ($potentialMappings as $mappingId) foreach ($listItem['media']['mappings']['nodes'] as $mapping)
{ {
if (is_array($mappingId)) $uType = strtoupper($type);
if ($mapping['externalSite'] === "MYANIMELIST_{$uType}")
{ {
continue; $malId = $mapping['externalId'];
} break;
if (array_key_exists($mappingId, $includes['mappings']))
{
$malId = $includes['mappings'][$mappingId]['externalId'];
} }
} }
@ -355,10 +343,22 @@ final class SyncLists extends BaseCommand {
continue; continue;
} }
$output[$listItem['id']] = [ $output[$listItem['media']['id']] = [
'id' => $listItem['id'], 'id' => $listItem['media']['id'],
'slug' => $listItem['media']['slug'],
'malId' => $malId, 'malId' => $malId,
'data' => $listItem['attributes'], 'data' => [
'notes' => $listItem['notes'],
'private' => $listItem['private'],
'progress' => $listItem['progress'],
// Comparision is done on 1-10 scale,
// Kitsu returns 1-20 scale.
'rating' => $listItem['rating'] / 2,
'reconsumeCount' => $listItem['reconsumeCount'],
'reconsuming' => $listItem['reconsuming'],
'status' => strtolower($listItem['status']),
'updatedAt' => $listItem['progressedAt'],
]
]; ];
} }
@ -460,7 +460,7 @@ final class SyncLists extends BaseCommand {
'private' => $kItem['private'], 'private' => $kItem['private'],
'progress' => $kItem['progress'], 'progress' => $kItem['progress'],
'repeat' => $kItem['reconsumeCount'], 'repeat' => $kItem['reconsumeCount'],
'score' => $kItem['ratingTwenty'] * 5, // 100 point score on Anilist 'score' => $kItem['rating'] * 10, // 100 point score on Anilist
'status' => $newItemStatus, 'status' => $newItemStatus,
], ],
]; ];
@ -492,10 +492,9 @@ final class SyncLists extends BaseCommand {
'status', 'status',
]; ];
$diff = []; $diff = [];
$dateDiff = new DateTime($kitsuItem['data']['updatedAt']) <=> new DateTime((string)$anilistItem['data']['updatedAt']); $dateDiff = ($kitsuItem['data']['updatedAt'] !== NULL)
? new DateTime($kitsuItem['data']['updatedAt']) <=> new DateTime((string)$anilistItem['data']['updatedAt'])
// Correct differences in notation : 0;
$kitsuItem['data']['rating'] = $kitsuItem['data']['ratingTwenty'] / 2;
foreach($compareKeys as $key) foreach($compareKeys as $key)
{ {
@ -535,12 +534,12 @@ final class SyncLists extends BaseCommand {
// If status is the same, and progress count is different, use greater progress // If status is the same, and progress count is different, use greater progress
if ($sameStatus && ( ! $sameProgress)) if ($sameStatus && ( ! $sameProgress))
{ {
if ($diff['progress'] === 1) if ($diff['progress'] === self::KITSU_GREATER)
{ {
$update['data']['progress'] = $kitsuItem['data']['progress']; $update['data']['progress'] = $kitsuItem['data']['progress'];
$return['updateType'][] = API::ANILIST; $return['updateType'][] = API::ANILIST;
} }
else if($diff['progress'] === -1) else if($diff['progress'] === self::ANILIST_GREATER)
{ {
$update['data']['progress'] = $anilistItem['data']['progress']; $update['data']['progress'] = $anilistItem['data']['progress'];
$return['updateType'][] = API::KITSU; $return['updateType'][] = API::KITSU;
@ -550,12 +549,12 @@ final class SyncLists extends BaseCommand {
// If status is different, use the status of the more recently updated item // If status is different, use the status of the more recently updated item
if ( ! $sameStatus) if ( ! $sameStatus)
{ {
if ($dateDiff === 1) if ($dateDiff === self::KITSU_GREATER)
{ {
$update['data']['status'] = $kitsuItem['data']['status']; $update['data']['status'] = $kitsuItem['data']['status'];
$return['updateType'][] = API::ANILIST; $return['updateType'][] = API::ANILIST;
} }
else if ($dateDiff === -1) else if ($dateDiff === self::ANILIST_GREATER)
{ {
$update['data']['status'] = $anilistItem['data']['status']; $update['data']['status'] = $anilistItem['data']['status'];
$return['updateType'][] = API::KITSU; $return['updateType'][] = API::KITSU;
@ -566,7 +565,7 @@ final class SyncLists extends BaseCommand {
// But, at least for now, assume newer record is correct // But, at least for now, assume newer record is correct
if ( ! ($sameStatus || $sameProgress)) if ( ! ($sameStatus || $sameProgress))
{ {
if ($dateDiff === 1) if ($dateDiff === self::KITSU_GREATER)
{ {
$update['data']['status'] = $kitsuItem['data']['status']; $update['data']['status'] = $kitsuItem['data']['status'];
@ -577,7 +576,7 @@ final class SyncLists extends BaseCommand {
$return['updateType'][] = API::ANILIST; $return['updateType'][] = API::ANILIST;
} }
else if($dateDiff === -1) else if($dateDiff === self::ANILIST_GREATER)
{ {
$update['data']['status'] = $anilistItem['data']['status']; $update['data']['status'] = $anilistItem['data']['status'];
@ -594,7 +593,7 @@ final class SyncLists extends BaseCommand {
if ( ! $sameRating) if ( ! $sameRating)
{ {
if ( if (
$dateDiff === 1 && $dateDiff === self::KITSU_GREATER &&
$kitsuItem['data']['rating'] !== 0 && $kitsuItem['data']['rating'] !== 0 &&
$kitsuItem['data']['ratingTwenty'] !== 0 $kitsuItem['data']['ratingTwenty'] !== 0
) )
@ -602,7 +601,7 @@ final class SyncLists extends BaseCommand {
$update['data']['ratingTwenty'] = $kitsuItem['data']['ratingTwenty']; $update['data']['ratingTwenty'] = $kitsuItem['data']['ratingTwenty'];
$return['updateType'][] = API::ANILIST; $return['updateType'][] = API::ANILIST;
} }
else if($dateDiff === -1 && $anilistItem['data']['rating'] !== 0) else if($dateDiff === self::ANILIST_GREATER && $anilistItem['data']['rating'] !== 0)
{ {
$update['data']['ratingTwenty'] = $anilistItem['data']['rating'] * 2; $update['data']['ratingTwenty'] = $anilistItem['data']['rating'] * 2;
$return['updateType'][] = API::KITSU; $return['updateType'][] = API::KITSU;
@ -612,7 +611,7 @@ final class SyncLists extends BaseCommand {
// If notes are set, use kitsu, otherwise, set kitsu from anilist // If notes are set, use kitsu, otherwise, set kitsu from anilist
if ( ! $sameNotes) if ( ! $sameNotes)
{ {
if ($kitsuItem['data']['notes'] !== '') if ( ! empty($kitsuItem['data']['notes']))
{ {
$update['data']['notes'] = $kitsuItem['data']['notes']; $update['data']['notes'] = $kitsuItem['data']['notes'];
$return['updateType'][] = API::ANILIST; $return['updateType'][] = API::ANILIST;
@ -627,12 +626,12 @@ final class SyncLists extends BaseCommand {
// Assume the larger reconsumeCount is correct // Assume the larger reconsumeCount is correct
if ( ! $sameRewatchCount) if ( ! $sameRewatchCount)
{ {
if ($diff['reconsumeCount'] === 1) if ($diff['reconsumeCount'] === self::KITSU_GREATER)
{ {
$update['data']['reconsumeCount'] = $kitsuItem['data']['reconsumeCount']; $update['data']['reconsumeCount'] = $kitsuItem['data']['reconsumeCount'];
$return['updateType'][] = API::ANILIST; $return['updateType'][] = API::ANILIST;
} }
else if ($diff['reconsumeCount'] === -1) else if ($diff['reconsumeCount'] === self::ANILIST_GREATER)
{ {
$update['data']['reconsumeCount'] = $anilistItem['data']['reconsumeCount']; $update['data']['reconsumeCount'] = $anilistItem['data']['reconsumeCount'];
$return['updateType'][] = API::KITSU; $return['updateType'][] = API::KITSU;
@ -659,11 +658,14 @@ final class SyncLists extends BaseCommand {
// to handle each combination of fields // to handle each combination of fields
if ($return['updateType'][0] === API::ANILIST) if ($return['updateType'][0] === API::ANILIST)
{ {
// Anilist GraphQL expects a rating from 1-100
$prevData = [ $prevData = [
'notes' => $kitsuItem['data']['notes'], 'notes' => $kitsuItem['data']['notes'],
'private' => $kitsuItem['data']['private'], 'private' => $kitsuItem['data']['private'],
'progress' => $kitsuItem['data']['progress'], 'progress' => $kitsuItem['data']['progress'],
'rating' => $kitsuItem['data']['ratingTwenty'] * 5, // Transformed Kitsu data returns a rating from 1-10
// Anilist expects a rating from 1-100
'rating' => $kitsuItem['data']['rating'] * 10,
'reconsumeCount' => $kitsuItem['data']['reconsumeCount'], 'reconsumeCount' => $kitsuItem['data']['reconsumeCount'],
'reconsuming' => $kitsuItem['data']['reconsuming'], 'reconsuming' => $kitsuItem['data']['reconsuming'],
'status' => $kitsuItem['data']['status'], 'status' => $kitsuItem['data']['status'],
@ -677,6 +679,8 @@ final class SyncLists extends BaseCommand {
'notes' => $anilistItem['data']['notes'], 'notes' => $anilistItem['data']['notes'],
'private' => $anilistItem['data']['private'], 'private' => $anilistItem['data']['private'],
'progress' => $anilistItem['data']['progress'] ?? 0, 'progress' => $anilistItem['data']['progress'] ?? 0,
// Anilist returns a rating between 1-100
// Kitsu expects a rating from 1-20
'rating' => (((int)$anilistItem['data']['rating']) > 0) 'rating' => (((int)$anilistItem['data']['rating']) > 0)
? $anilistItem['data']['rating'] / 5 ? $anilistItem['data']['rating'] / 5
: 0, : 0,
@ -824,30 +828,4 @@ final class SyncLists extends BaseCommand {
} }
} }
} }
// ------------------------------------------------------------------------
// Other Helpers
// ------------------------------------------------------------------------
/**
* Filter Kitsu mappings for the specified type
*
* @param array $includes
* @param string $type
* @return array
*/
private function filterMappings(array $includes, string $type = ListType::ANIME): array
{
$output = [];
foreach($includes as $id => $mapping)
{
if ($mapping['externalSite'] === "myanimelist/{$type}")
{
$output[$id] = $mapping;
}
}
return $output;
}
} }

View File

@ -1,27 +0,0 @@
<?php declare(strict_types=1);
/**
* Hummingbird Anime List Client
*
* An API client for Kitsu to manage anime and manga watch lists
*
* PHP version 7.4
*
* @package HummingbirdAnimeClient
* @author Timothy J. Warren <tim@timshomepage.net>
* @copyright 2015 - 2020 Timothy J. Warren
* @license http://www.opensource.org/licenses/mit-license.html MIT License
* @version 5.1
* @link https://git.timshomepage.net/timw4mail/HummingBirdAnimeClient
*/
namespace Aviat\AnimeClient\Enum;
use Aviat\Ion\Enum as BaseEnum;
/**
* Types of lists
*/
final class APISource extends BaseEnum {
public const KITSU = 'kitsu';
public const ANILIST = 'anilist';
}