timw4mail/HummingBirdAnimeClient
1
1
Fork 1
Code Issues 5 Pull Requests Projects 1 Releases Wiki Activity

Get sync-lists command to create missing entries on MAL

Browse Source
This commit is contained in:
Timothy Warren 2017-02-14 15:29:13 -05:00
parent 9c1dc50e65
commit 0d553b7dd4
9 changed files with 275 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/API/Kitsu/KitsuTrait.php
View File

@ -20,7 +20,7 @@ use const Aviat\AnimeClient\SESSION_SEGMENT;
use function Amp\wait;
use Amp\Artax\Client;
use Amp\Artax\{Client, Request};
use Aviat\AnimeClient\AnimeClient;
use Aviat\AnimeClient\API\Kitsu as K;
use Aviat\Ion\Json;
@ -53,9 +53,9 @@ trait KitsuTrait {
* @param string $type
* @param string $url
* @param array $options
* @return \Amp\Artax\Response
* @return \Amp\Artax\Request
*/
public function setUpRequest(string $type, string $url, array $options = [])
public function setUpRequest(string $type, string $url, array $options = []): Request
{
$config = $this->container->get('config');
@ -69,7 +69,6 @@ trait KitsuTrait {
{
$token = $sessionSegment->get('auth_token');
$request = $request->setAuth('bearer', $token);
// $defaultOptions['headers']['Authorization'] = "bearer {$token}";
}
if (array_key_exists('form_params', $options))
@ -138,7 +137,7 @@ trait KitsuTrait {
{
if ($logger)
{
$logger->warning('Non 200 response for api call', $response->getBody());
$logger->warning('Non 200 response for api call', (array)$response->getBody());
}
}

45
src/API/Kitsu/Model.php
View File

@ -204,14 +204,55 @@ class Model {
return $this->mangaTransformer->transform($baseData);
}
/**
* Get the number of anime list items
*
* @return int
*/
public function getAnimeListCount() : int
{
$options = [
'query' => [
'filter' => [
'user_id' => $this->getUserIdByUsername(),
'media_type' => 'Anime'
],
'page' => [
'limit' => 1
],
'sort' => '-updated_at'
]
];
$response = $this->getRequest('library-entries', $options);
return $response['meta']['count'];
}
/**
* Get and transform the entirety of the user's anime list
*
* @return array
* @return Request
*/
public function getFullAnimeList(): array
public function getFullAnimeList(int $limit = 100, int $offset = 0): Request
{
$options = [
'query' => [
'filter' => [
'user_id' => $this->getUserIdByUsername($this->getUsername()),
'media_type' => 'Anime'
],
'include' => 'anime.mappings',
'page' => [
'offset' => $offset,
'limit' => $limit
],
'sort' => '-updated_at'
]
];
return $this->setUpRequest('GET', 'library-entries', $options);
}
/**

8
src/API/MAL.php
View File

@ -37,6 +37,14 @@ class MAL {
KAWS::PLAN_TO_WATCH => AnimeWatchingStatus::PLAN_TO_WATCH
];
const MAL_KITSU_WATCHING_STATUS_MAP = [
1 => KAWS::WATCHING,
2 => KAWS::COMPLETED,
3 => KAWS::ON_HOLD,
4 => KAWS::DROPPED,
6 => KAWS::PLAN_TO_WATCH
];
public static function getIdToWatchingStatusMap()
{
return [

16
src/API/MAL/ListItem.php
View File

@ -46,12 +46,6 @@ class ListItem {
->setFormFields($createData)
->setBasicAuth($config->get(['mal','username']), $config->get(['mal', 'password']))
->getFullRequest();
/* $response = $this->getResponse('POST', "animelist/add/{$id}.xml", [
'body' => $this->fixBody((new FormBody)->addFields($createData))
]);
return $response->getBody() === 'Created'; */
}
public function delete(string $id): Request
@ -65,11 +59,7 @@ class ListItem {
->setBasicAuth($config->get(['mal','username']), $config->get(['mal', 'password']))
->getFullRequest();
/*$response = $this->getResponse('DELETE', "animelist/delete/{$id}.xml", [
'body' => $this->fixBody((new FormBody)->addField('id', $id))
]);
return $response->getBody() === 'Deleted';*/
// return $response->getBody() === 'Deleted'
}
public function get(string $id): array
@ -93,9 +83,5 @@ class ListItem {
])
->setBasicAuth($config->get(['mal','username']), $config->get(['mal', 'password']))
->getFullRequest();
/* return $this->getResponse('POST', "animelist/update/{$id}.xml", [
'body' => $this->fixBody($body)
]); */
}
}

9
src/API/MAL/Model.php
View File

@ -36,7 +36,7 @@ class Model {
protected $animeListTransformer;
/**
* KitsuModel constructor.
* MAL Model constructor.
*/
public function __construct(ListItem $listItem)
{
@ -44,6 +44,11 @@ class Model {
$this->listItem = $listItem;
}
public function createFullListItem(array $data): Request
{
return $this->listItem->create($data);
}
public function createListItem(array $data): Request
{
$createData = [
@ -70,7 +75,7 @@ class Model {
]
]);
return $list;//['anime'];
return $list['myanimelist']['anime'];
}
public function getListItem(string $listId): array

6
src/API/MAL/Transformer/AnimeListTransformer.php
View File

@ -32,6 +32,12 @@ class AnimeListTransformer extends AbstractTransformer {
AnimeWatchingStatus::PLAN_TO_WATCH => '6'
];
/**
* Transform MAL episode data to Kitsu episode data
*
* @param array $item
* @return array
*/
public function transform($item)
{
$rewatching = (array_key_exists('rewatching', $item) && $item['rewatching']);

33
src/API/MAL/Transformer/MALToKitsuTransformer.php
View File

@ -1,33 +0,0 @@
<?php declare(strict_types=1);
/**
* Anime List Client
*
* An API client for Kitsu and MyAnimeList to manage anime and manga watch lists
*
* PHP version 7
*
* @package AnimeListClient
* @author Timothy J. Warren <tim@timshomepage.net>
* @copyright 2015 - 2017 Timothy J. Warren
* @license http://www.opensource.org/licenses/mit-license.html MIT License
* @version 4.0
* @link https://github.com/timw4mail/HummingBirdAnimeClient
*/
namespace Aviat\AnimeClient\API\MAL;
use Aviat\Ion\Transformer\AbstractTransformer;
class MALToKitsuTransformer extends AbstractTransformer {
public function transform($item)
{
}
public function untransform($item)
{
}
}

23
src/Command/BaseCommand.php
View File

@ -24,11 +24,10 @@ use Aviat\AnimeClient\{
Model,
Util
};
use Aviat\AnimeClient\API\{
CacheTrait,
Kitsu,
MAL
};
use Aviat\AnimeClient\API\CacheTrait;
use Aviat\AnimeClient\API\{Kitsu, MAL};
use Aviat\AnimeClient\API\Kitsu\KitsuRequestBuilder;
use Aviat\AnimeClient\API\MAL\MALRequestBuilder;
use Aviat\Banker\Pool;
use Aviat\Ion\Config;
use Aviat\Ion\Di\{Container, ContainerAware};
@ -109,21 +108,35 @@ class BaseCommand extends Command {
// Models
$container->set('kitsu-model', function($container) {
$requestBuilder = new KitsuRequestBuilder();
$requestBuilder->setLogger($container->getLogger('kitsu-request'));
$listItem = new Kitsu\ListItem();
$listItem->setContainer($container);
$listItem->setRequestBuilder($requestBuilder);
$model = new Kitsu\Model($listItem);
$model->setContainer($container);
$model->setRequestBuilder($requestBuilder);
$cache = $container->get('cache');
$model->setCache($cache);
return $model;
});
$container->set('mal-model', function($container) {
$requestBuilder = new MALRequestBuilder();
$requestBuilder->setLogger($container->getLogger('mal-request'));
$listItem = new MAL\ListItem();
$listItem->setContainer($container);
$listItem->setRequestBuilder($requestBuilder);
$model = new MAL\Model($listItem);
$model->setContainer($container);
$model->setRequestBuilder($requestBuilder);
return $model;
});
$container->set('util', function($container) {
return new Util($container);
});

233
src/Command/SyncKitsuWithMal.php
View File

@ -16,8 +16,13 @@
namespace Aviat\AnimeClient\Command;
use function Amp\{all, wait};
use Amp\Artax;
use Aviat\AnimeClient\API\Kitsu;
use Amp\Artax\Client;
use Aviat\AnimeClient\API\{JsonAPI, Kitsu, MAL};
use Aviat\AnimeClient\API\MAL\Transformer\AnimeListTransformer as ALT;
use Aviat\Ion\Json;
/**
* Clears the API Cache
@ -42,90 +47,172 @@ class SyncKitsuWithMal extends BaseCommand {
$this->kitsuModel = $this->container->get('kitsu-model');
$this->malModel = $this->container->get('mal-model');
//$kitsuCount = $this->getKitsuAnimeListPageCount();
//$this->echoBox("List item count: {$kitsuCount}");
$this->MALItemCreate();
$malCount = count($this->getMALList());
$kitsuCount = $this->getKitsuAnimeListPageCount();
$this->echoBox("Number of MAL list items: {$malCount}");
$this->echoBox("Number of Kitsu list items: {$kitsuCount}");
$data = $this->diffLists();
$this->echoBox("Number of items that need to be added to MAL: " . count($data));
if (! empty($data['addToMAL']))
{
$this->echoBox("Adding missing list items to MAL");
$this->createMALListItems($data['addToMAL']);
}
//echo json_encode($this->getMALList(), \JSON_PRETTY_PRINT);
}
public function getKitsuList()
{
$count = $this->getKitsuAnimeListPageCount();
$size = 100;
$pages = ceil($count / $size);
$requests = [];
// Set up requests
for ($i = 0; $i < $count; $i++)
{
$offset = $i * $size;
$requests[] = $this->kitsuModel->getFullAnimeList($size, $offset);
}
$promiseArray = (new Client())->requestMulti($requests);
$responses = wait(all($promiseArray));
$output = [];
foreach($responses as $response)
{
$data = Json::decode($response->getBody());
$output = array_merge_recursive($output, $data);
}
return $output;
}
public function getMALList()
{
return $this->malModel->getFullList();
}
public function getKitsuAnimeListPageCount()
public function filterMappings(array $includes): array
{
$cacheItem = $this->cache->getItem(Kitsu::AUTH_TOKEN_CACHE_KEY);
$output = [];
$query = http_build_query([
'filter' => [
'user_id' => $this->kitsuModel->getUserIdByUsername(),
'media_type' => 'Anime'
],
// 'include' => 'anime,anime.genres,anime.mappings,anime.streamingLinks',
'page' => [
'limit' => 1
],
'sort' => '-updated_at'
]);
$request = (new Artax\Request)
->setUri("https://kitsu.io/api/edge/library-entries?{$query}")
->setProtocol('1.1')
->setAllHeaders([
'Accept' => 'application/vnd.api+json',
'Content-Type' => 'application/vnd.api+json',
'User-Agent' => "Tim's Anime Client/4.0"
]);
if ($cacheItem->isHit())
foreach($includes as $id => $mapping)
{
$token = $cacheItem->get();
$request->setHeader('Authorization', "bearer {$token}");
}
else
{
$this->echoBox("WARNING: NOT LOGGED IN\nSome data might be missing");
if ($mapping['externalSite'] === 'myanimelist/anime')
{
$output[$id] = $mapping;
}
}
$response = \Amp\wait((new Artax\Client)->request($request));
$body = json_decode($response->getBody(), TRUE);
return $body['meta']['count'];
return $output;
}
public function MALItemCreate()
// 2015-05-20T23:48:47.731Z
public function formatMALList()
{
$input = json_decode('{
"watching_status": "current",
"user_rating": "",
"episodes_watched": "4",
"rewatched": "0",
"notes": "",
"id": "15794526",
"mal_id": "33731",
"edit": "true"
}', TRUE);
$orig = $this->getMALList();
$output = [];
$response = $this->malModel->createListItem([
'id' => 12255,
'status' => 'planned',
'type' => 'anime'
]);
foreach($orig as $item)
{
$output[$item['series_animedb_id']] = [
'id' => $item['series_animedb_id'],
'data' => [
'status' => MAL::MAL_KITSU_WATCHING_STATUS_MAP[$item['my_status']],
'progress' => $item['my_watched_episodes'],
'reconsuming' => (bool) $item['my_rewatching'],
'reconsumeCount' => array_key_exists('times_rewatched', $item)
? $item['times_rewatched']
: 0,
// 'notes' => ,
'rating' => $item['my_score'],
'updatedAt' => (new \DateTime())
->setTimestamp((int)$item['my_last_updated'])
->format(\DateTime::W3C),
]
];
}
//$response = $this->malModel->updateListItem($input);
//print_r($response);
//echo $response->getBody();
return $output;
}
public function filterKitsuList()
{
$data = $this->getKitsuList();
$includes = JsonAPI::organizeIncludes($data['included']);
$includes['mappings'] = $this->filterMappings($includes['mappings']);
$output = [];
foreach($data['data'] as $listItem)
{
$animeId = $listItem['relationships']['anime']['data']['id'];
$potentialMappings = $includes['anime'][$animeId]['relationships']['mappings'];
$malId = null;
foreach ($potentialMappings as $mappingId)
{
if (array_key_exists($mappingId, $includes['mappings']))
{
$malId = $includes['mappings'][$mappingId]['externalId'];
}
}
// Skip to the next item if there isn't a MAL ID
if ($malId === null)
{
continue;
}
$output[$listItem['id']] = [
'id' => $listItem['id'],
'malId' => $malId,
'data' => $listItem['attributes'],
];
}
return $output;
}
public function getKitsuAnimeListPageCount()
{
return $this->kitsuModel->getAnimeListCount();
}
public function diffLists()
{
// Get libraryEntries with media.mappings from Kitsu
// Organize mappings, and ignore entries without mappings
$kitsuList = $this->filterKitsuList();
// Get MAL list data
$malList = $this->formatMALList();
$itemsToAddToMAL = [];
foreach($kitsuList as $kitsuItem)
{
if (array_key_exists($kitsuItem['malId'], $malList))
{
// Eventually, compare the list entries, and determine which
// needs to be updated
continue;
}
// Looks like this item only exists on Kitsu
$itemsToAddToMAL[] = [
'mal_id' => $kitsuItem['malId'],
'data' => $kitsuItem['data']
];
}
// Compare each list entry
// If a list item exists only on MAL, create it on Kitsu with the existing data from MAL
@ -135,7 +222,39 @@ class SyncKitsuWithMal extends BaseCommand {
// Otherwise, use rewatch count, then episode progress as critera for selecting the more up
// to date entry
// Based on the 'newer' entry, update the other api list item
return [
'addToMAL' => $itemsToAddToMAL,
];
}
public function createMALListItems($itemsToAdd)
{
$transformer = new ALT();
$requests = [];
foreach($itemsToAdd as $item)
{
$data = $transformer->untransform($item);
$requests[] = $this->malModel->createFullListItem($data);
}
$promiseArray = (new Client())->requestMulti($requests);
$responses = wait(all($promiseArray));
foreach($responses as $key => $response)
{
$id = $itemsToAdd[$key]['mal_id'];
if ($response->getBody() === 'Created')
{
$this->echoBox("Successfully create list item with id: {$id}");
}
else
{
$this->echoBox("Failed to create list item with id: {$id}");
}
}
}
}