jonny/jonnybarnes.uk
1
0
Fork 0
Code Issues 8 Pull requests Projects Releases 5 Packages Wiki Activity Actions

Merge pull request #638 from jonnybarnes/616-use-cloudconvert-for-webpage-screenshots

Browse source 
Use CloudConvert for webpage screenshots
This commit is contained in:
Jonny Barnes 2023-02-04 10:44:25 +00:00 • committed by GitHub
commit a1e7fe1662
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 442 additions and 1108 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -11,6 +11,7 @@ yarn-error.log
/lsp
.phpstorm.meta.php
_ide_helper.php
ray.php
# Custom paths in /public
/public/coverage
/public/hot

10
app/Jobs/ProcessBookmark.php
View file

@ -20,8 +20,7 @@ class ProcessBookmark implements ShouldQueue
use Queueable;
use SerializesModels;
/** @var Bookmark */
protected $bookmark;
protected Bookmark $bookmark;
/**
* Create a new job instance.
@ -38,14 +37,13 @@ class ProcessBookmark implements ShouldQueue
*
* @return void
*/
public function handle()
public function handle(): void
{
$uuid = (resolve(BookmarkService::class))->saveScreenshot($this->bookmark->url);
$this->bookmark->screenshot = $uuid;
SaveScreenshot::dispatch($this->bookmark);
try {
$archiveLink = (resolve(BookmarkService::class))->getArchiveLink($this->bookmark->url);
} catch (InternetArchiveException $e) {
} catch (InternetArchiveException) {
$archiveLink = null;
}
$this->bookmark->archive = $archiveLink;

109
app/Jobs/SaveScreenshot.php Executable file
View file

@ -0,0 +1,109 @@
<?php
declare(strict_types=1);
namespace App\Jobs;
use App\Models\Bookmark;
use GuzzleHttp\Client;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Storage;
use JsonException;
class SaveScreenshot implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
private Bookmark $bookmark;
/**
* Create a new job instance.
*
* @return void
*/
public function __construct(Bookmark $bookmark)
{
$this->bookmark = $bookmark;
}
/**
* Execute the job.
*
* @return void
*
* @throws JsonException
*/
public function handle(): void
{
// A normal Guzzle client
$client = resolve(Client::class);
// A Guzzle client with a custom Middleware to retry the CloudConvert API requests
$retryClient = resolve('RetryGuzzle');
// First request that CloudConvert takes a screenshot of the URL
$takeScreenshotJobResponse = $client->request('POST', 'https://api.cloudconvert.com/v2/capture-website', [
'headers' => [
'Authorization' => 'Bearer ' . config('services.cloudconvert.token'),
],
'json' => [
'url' => $this->bookmark->url,
'output_format' => 'png',
'screen_width' => 1440,
'screen_height' => 900,
'wait_until' => 'networkidle0',
'wait_time' => 100,
],
]);
$taskId = json_decode($takeScreenshotJobResponse->getBody()->getContents(), false, 512, JSON_THROW_ON_ERROR)->data->id;
// Now wait till the status job is finished
$screenshotJobStatusResponse = $retryClient->request('GET', 'https://api.cloudconvert.com/v2/tasks/' . $taskId, [
'headers' => [
'Authorization' => 'Bearer ' . config('services.cloudconvert.token'),
],
'query' => [
'include' => 'payload',
],
]);
$finishedCaptureId = json_decode($screenshotJobStatusResponse->getBody()->getContents(), false, 512, JSON_THROW_ON_ERROR)->data->id;
// Now we can create a new job to request thst the screenshot is exported to a temporary URL we can download the screenshot from
$exportImageJob = $client->request('POST', 'https://api.cloudconvert.com/v2/export/url', [
'headers' => [
'Authorization' => 'Bearer ' . config('services.cloudconvert.token'),
],
'json' => [
'input' => $finishedCaptureId,
'archive_multiple_files' => false,
],
]);
$exportImageJobId = json_decode($exportImageJob->getBody()->getContents(), false, 512, JSON_THROW_ON_ERROR)->data->id;
// Again, wait till the status of this export job is finished
$finalImageUrlResponse = $retryClient->request('GET', 'https://api.cloudconvert.com/v2/tasks/' . $exportImageJobId, [
'headers' => [
'Authorization' => 'Bearer ' . config('services.cloudconvert.token'),
],
'query' => [
'include' => 'payload',
],
]);
// Now we can download the screenshot and save it to the storage
$finalImageUrl = json_decode($finalImageUrlResponse->getBody()->getContents(), false, 512, JSON_THROW_ON_ERROR)->data->result->files[0]->url;
$finalImageUrlContent = $client->request('GET', $finalImageUrl);
Storage::disk('public')->put('/assets/img/bookmarks/' . $taskId . '.png', $finalImageUrlContent->getBody()->getContents());
$this->bookmark->screenshot = $taskId;
$this->bookmark->save();
}
}

34
app/Providers/AppServiceProvider.php
View file

@ -5,6 +5,8 @@ namespace App\Providers;
use App\Models\Note;
use App\Observers\NoteObserver;
use Codebird\Codebird;
use GuzzleHttp\Client;
use GuzzleHttp\Middleware;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Http\Request;
use Illuminate\Pagination\LengthAwarePaginator;
@ -104,6 +106,38 @@ class AppServiceProvider extends ServiceProvider
);
});
// Configure Guzzle
$this->app->bind('RetryGuzzle', function () {
$handlerStack = \GuzzleHttp\HandlerStack::create();
$handlerStack->push(Middleware::retry(
function ($retries, $request, $response, $exception) {
// Limit the number of retries to 5
if ($retries >= 5) {
return false;
}
// Retry connection exceptions
if ($exception instanceof \GuzzleHttp\Exception\ConnectException) {
return true;
}
// Retry on server errors
if ($response && $response->getStatusCode() >= 500) {
return true;
}
// Finally for CloudConvert, retry if status is not final
return json_decode($response, false, 512, JSON_THROW_ON_ERROR)->data->status !== 'finished';
},
function () {
// Retry after 1 second
return 1000;
}
));
return new Client(['handler' => $handlerStack]);
});
// Turn on Eloquent strict mode when developing
Model::shouldBeStrict(! $this->app->isProduction());
}

29
app/Services/BookmarkService.php
View file

@ -14,9 +14,6 @@ use GuzzleHttp\Client;
use GuzzleHttp\Exception\ClientException;
use Illuminate\Support\Arr;
use Illuminate\Support\Str;
use Ramsey\Uuid\Uuid;
use Spatie\Browsershot\Browsershot;
use Spatie\Browsershot\Exceptions\CouldNotTakeBrowsershot;
class BookmarkService extends Service
{
@ -24,6 +21,7 @@ class BookmarkService extends Service
* Create a new Bookmark.
*
* @param array $request Data from request()->all()
* @param string|null $client
* @return Bookmark
*/
public function create(array $request, ?string $client = null): Bookmark
@ -75,31 +73,6 @@ class BookmarkService extends Service
return $bookmark;
}
/**
* Given a URL, use `browsershot` to save an image of the page.
*
* @param string $url
* @return string The uuid for the screenshot
*
* @throws CouldNotTakeBrowsershot
*
* @codeCoverageIgnore
*/
public function saveScreenshot(string $url): string
{
$browsershot = new Browsershot();
$uuid = Uuid::uuid4();
$browsershot->url($url)
->setIncludePath('$PATH:/usr/local/bin')
->noSandbox()
->windowSize(960, 640)
->save(public_path() . '/assets/img/bookmarks/' . $uuid . '.png');
return $uuid->toString();
}
/**
* Given a URL, attempt to save it to the Internet Archive.
*

1
composer.json
View file

@ -28,7 +28,6 @@
"league/commonmark": "^2.0",
"league/flysystem-aws-s3-v3": "^3.0",
"mf2/mf2": "~0.3",
"spatie/browsershot": "~3.0",
"spatie/commonmark-highlighter": "^3.0",
"symfony/html-sanitizer": "^6.1"
},

316
composer.lock generated
View file

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "a995551674b30efcec7d8a4986cff454",
"content-hash": "8060b9de669d94b189eeeb34170e25e3",
"packages": [
{
"name": "aws/aws-crt-php",
@ -2679,71 +2679,6 @@
],
"time": "2022-10-26T18:15:09+00:00"
},
{
"name": "league/glide",
"version": "2.2.2",
"source": {
"type": "git",
"url": "https://github.com/thephpleague/glide.git",
"reference": "bff5b0fe2fd26b2fde2d6958715fde313887d79d"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/thephpleague/glide/zipball/bff5b0fe2fd26b2fde2d6958715fde313887d79d",
"reference": "bff5b0fe2fd26b2fde2d6958715fde313887d79d",
"shasum": ""
},
"require": {
"intervention/image": "^2.7",
"league/flysystem": "^2.0|^3.0",
"php": "^7.2|^8.0",
"psr/http-message": "^1.0"
},
"require-dev": {
"mockery/mockery": "^1.3.3",
"phpunit/php-token-stream": "^3.1|^4.0",
"phpunit/phpunit": "^8.5|^9.0"
},
"type": "library",
"autoload": {
"psr-4": {
"League\\Glide\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Jonathan Reinink",
"email": "jonathan@reinink.ca",
"homepage": "http://reinink.ca"
},
{
"name": "Titouan Galopin",
"email": "galopintitouan@gmail.com",
"homepage": "https://titouangalopin.com"
}
],
"description": "Wonderfully easy on-demand image manipulation library with an HTTP based API.",
"homepage": "http://glide.thephpleague.com",
"keywords": [
"ImageMagick",
"editing",
"gd",
"image",
"imagick",
"league",
"manipulation",
"processing"
],
"support": {
"issues": "https://github.com/thephpleague/glide/issues",
"source": "https://github.com/thephpleague/glide/tree/2.2.2"
},
"time": "2022-02-21T07:40:55+00:00"
},
{
"name": "league/mime-type-detection",
"version": "1.11.0",
@ -4506,72 +4441,6 @@
],
"time": "2021-12-03T06:45:28+00:00"
},
{
"name": "spatie/browsershot",
"version": "3.57.5",
"source": {
"type": "git",
"url": "https://github.com/spatie/browsershot.git",
"reference": "a4ae0f3a289cfb9384f2ee01b7f37c271f6a4159"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/spatie/browsershot/zipball/a4ae0f3a289cfb9384f2ee01b7f37c271f6a4159",
"reference": "a4ae0f3a289cfb9384f2ee01b7f37c271f6a4159",
"shasum": ""
},
"require": {
"ext-json": "*",
"php": "^7.4|^8.0",
"spatie/image": "^1.5.3|^2.0",
"spatie/temporary-directory": "^1.1|^2.0",
"symfony/process": "^4.2|^5.0|^6.0"
},
"require-dev": {
"pestphp/pest": "^1.20",
"spatie/phpunit-snapshot-assertions": "^4.2.3"
},
"type": "library",
"autoload": {
"psr-4": {
"Spatie\\Browsershot\\": "src"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Freek Van der Herten",
"email": "freek@spatie.be",
"homepage": "https://github.com/freekmurze",
"role": "Developer"
}
],
"description": "Convert a webpage to an image or pdf using headless Chrome",
"homepage": "https://github.com/spatie/browsershot",
"keywords": [
"chrome",
"convert",
"headless",
"image",
"pdf",
"puppeteer",
"screenshot",
"webpage"
],
"support": {
"source": "https://github.com/spatie/browsershot/tree/3.57.5"
},
"funding": [
{
"url": "https://github.com/spatie",
"type": "github"
}
],
"time": "2022-12-05T15:59:26+00:00"
},
{
"name": "spatie/commonmark-highlighter",
"version": "3.0.0",
@ -4626,189 +4495,6 @@
},
"time": "2021-08-04T18:03:57+00:00"
},
{
"name": "spatie/image",
"version": "2.2.4",
"source": {
"type": "git",
"url": "https://github.com/spatie/image.git",
"reference": "c2dc137c52d17bf12aff94ad051370c0f106b322"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/spatie/image/zipball/c2dc137c52d17bf12aff94ad051370c0f106b322",
"reference": "c2dc137c52d17bf12aff94ad051370c0f106b322",
"shasum": ""
},
"require": {
"ext-exif": "*",
"ext-json": "*",
"ext-mbstring": "*",
"league/glide": "^2.2.2",
"php": "^8.0",
"spatie/image-optimizer": "^1.1",
"spatie/temporary-directory": "^1.0|^2.0",
"symfony/process": "^3.0|^4.0|^5.0|^6.0"
},
"require-dev": {
"phpunit/phpunit": "^9.5",
"symfony/var-dumper": "^4.0|^5.0|^6.0",
"vimeo/psalm": "^4.6"
},
"type": "library",
"autoload": {
"psr-4": {
"Spatie\\Image\\": "src"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Freek Van der Herten",
"email": "freek@spatie.be",
"homepage": "https://spatie.be",
"role": "Developer"
}
],
"description": "Manipulate images with an expressive API",
"homepage": "https://github.com/spatie/image",
"keywords": [
"image",
"spatie"
],
"support": {
"source": "https://github.com/spatie/image/tree/2.2.4"
},
"funding": [
{
"url": "https://spatie.be/open-source/support-us",
"type": "custom"
},
{
"url": "https://github.com/spatie",
"type": "github"
}
],
"time": "2022-08-09T10:18:57+00:00"
},
{
"name": "spatie/image-optimizer",
"version": "1.6.2",
"source": {
"type": "git",
"url": "https://github.com/spatie/image-optimizer.git",
"reference": "6db75529cbf8fa84117046a9d513f277aead90a0"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/spatie/image-optimizer/zipball/6db75529cbf8fa84117046a9d513f277aead90a0",
"reference": "6db75529cbf8fa84117046a9d513f277aead90a0",
"shasum": ""
},
"require": {
"ext-fileinfo": "*",
"php": "^7.3|^8.0",
"psr/log": "^1.0 | ^2.0 | ^3.0",
"symfony/process": "^4.2|^5.0|^6.0"
},
"require-dev": {
"phpunit/phpunit": "^8.5.21|^9.4.4",
"symfony/var-dumper": "^4.2|^5.0|^6.0"
},
"type": "library",
"autoload": {
"psr-4": {
"Spatie\\ImageOptimizer\\": "src"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Freek Van der Herten",
"email": "freek@spatie.be",
"homepage": "https://spatie.be",
"role": "Developer"
}
],
"description": "Easily optimize images using PHP",
"homepage": "https://github.com/spatie/image-optimizer",
"keywords": [
"image-optimizer",
"spatie"
],
"support": {
"issues": "https://github.com/spatie/image-optimizer/issues",
"source": "https://github.com/spatie/image-optimizer/tree/1.6.2"
},
"time": "2021-12-21T10:08:05+00:00"
},
{
"name": "spatie/temporary-directory",
"version": "2.1.1",
"source": {
"type": "git",
"url": "https://github.com/spatie/temporary-directory.git",
"reference": "e2818d871783d520b319c2d38dc37c10ecdcde20"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/spatie/temporary-directory/zipball/e2818d871783d520b319c2d38dc37c10ecdcde20",
"reference": "e2818d871783d520b319c2d38dc37c10ecdcde20",
"shasum": ""
},
"require": {
"php": "^8.0"
},
"require-dev": {
"phpunit/phpunit": "^9.5"
},
"type": "library",
"autoload": {
"psr-4": {
"Spatie\\TemporaryDirectory\\": "src"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Alex Vanderbist",
"email": "alex@spatie.be",
"homepage": "https://spatie.be",
"role": "Developer"
}
],
"description": "Easily create, use and destroy temporary directories",
"homepage": "https://github.com/spatie/temporary-directory",
"keywords": [
"php",
"spatie",
"temporary-directory"
],
"support": {
"issues": "https://github.com/spatie/temporary-directory/issues",
"source": "https://github.com/spatie/temporary-directory/tree/2.1.1"
},
"funding": [
{
"url": "https://spatie.be/open-source/support-us",
"type": "custom"
},
{
"url": "https://github.com/spatie",
"type": "github"
}
],
"time": "2022-08-23T07:15:15+00:00"
},
{
"name": "stella-maris/clock",
"version": "0.1.6",

4
config/services.php
View file

@ -31,4 +31,8 @@ return [
'region' => env('AWS_DEFAULT_REGION', 'us-east-1'),
],
'cloudconvert' => [
'token' => env('CLOUDCONVERT_API_TOKEN'),
],
];

854
package-lock.json generated
View file

File diff suppressed because it is too large Load diff

3
package.json
View file

@ -5,8 +5,7 @@
"repository": "https://github.com/jonnybarnes/jonnybarnes.uk",
"license": "CC0-1.0",
"dependencies": {
"normalize.css": "^8.0.1",
"puppeteer": "^19.4.1"
"normalize.css": "^8.0.1"
},
"devDependencies": {
"@babel/core": "^7.20.7",

9
tests/Unit/ArticlesTest.php
View file

@ -85,14 +85,15 @@ class ArticlesTest extends TestCase
public function dateScopeReturnsExpectedArticlesForDecember(): void
{
Article::factory()->create([
'created_at' => Carbon::now()->setMonth(11)->toDateTimeString(),
'updated_at' => Carbon::now()->setMonth(11)->toDateTimeString(),
'created_at' => Carbon::now()->setDay(11)->setMonth(11)->toDateTimeString(),
'updated_at' => Carbon::now()->setDay(11)->setMonth(11)->toDateTimeString(),
]);
Article::factory()->create([
'created_at' => Carbon::now()->setMonth(12)->toDateTimeString(),
'updated_at' => Carbon::now()->setMonth(12)->toDateTimeString(),
'created_at' => Carbon::now()->setMonth(12)->setDay(12)->toDateTimeString(),
'updated_at' => Carbon::now()->setMonth(12)->setDay(12)->toDateTimeString(),
]);
$this->assertCount(1, Article::date(date('Y'), 12)->get());
}
}

20
tests/Unit/Jobs/ProcessBookmarkJobTest.php
View file

@ -6,10 +6,11 @@ namespace Tests\Unit\Jobs;
use App\Exceptions\InternetArchiveException;
use App\Jobs\ProcessBookmark;
use App\Jobs\SaveScreenshot;
use App\Models\Bookmark;
use App\Services\BookmarkService;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Ramsey\Uuid\Uuid;
use Illuminate\Support\Facades\Queue;
use Tests\TestCase;
class ProcessBookmarkJobTest extends TestCase
@ -17,13 +18,12 @@ class ProcessBookmarkJobTest extends TestCase
use RefreshDatabase;
/** @test */
public function screenshotAndArchiveLinkAreSavedByJob(): void
public function archiveLinkIsSavedByJobAndScreenshotJobIsQueued(): void
{
Queue::fake();
$bookmark = Bookmark::factory()->create();
$uuid = Uuid::uuid4();
$service = $this->createMock(BookmarkService::class);
$service->method('saveScreenshot')
->willReturn($uuid->toString());
$service->method('getArchiveLink')
->willReturn('https://web.archive.org/web/1234');
$this->app->instance(BookmarkService::class, $service);
@ -32,19 +32,19 @@ class ProcessBookmarkJobTest extends TestCase
$job->handle();
$this->assertDatabaseHas('bookmarks', [
'screenshot' => $uuid->toString(),
'archive' => 'https://web.archive.org/web/1234',
]);
Queue::assertPushed(SaveScreenshot::class);
}
/** @test */
public function archiveLinkSavedAsNullWhenExceptionThrown(): void
{
Queue::fake();
$bookmark = Bookmark::factory()->create();
$uuid = Uuid::uuid4();
$service = $this->createMock(BookmarkService::class);
$service->method('saveScreenshot')
->willReturn($uuid->toString());
$service->method('getArchiveLink')
->will($this->throwException(new InternetArchiveException()));
$this->app->instance(BookmarkService::class, $service);
@ -53,7 +53,7 @@ class ProcessBookmarkJobTest extends TestCase
$job->handle();
$this->assertDatabaseHas('bookmarks', [
'screenshot' => $uuid->toString(),
'id' => $bookmark->id,
'archive' => null,
]);
}

160
tests/Unit/Jobs/SaveScreenshotJobTest.php Normal file
View file

@ -0,0 +1,160 @@
<?php
declare(strict_types=1);
namespace Tests\Unit\Jobs;
use App\Jobs\SaveScreenshot;
use App\Models\Bookmark;
use GuzzleHttp\Client;
use GuzzleHttp\Handler\MockHandler;
use GuzzleHttp\HandlerStack;
use GuzzleHttp\Middleware;
use GuzzleHttp\Psr7\Response;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Storage;
use Tests\TestCase;
class SaveScreenshotJobTest extends TestCase
{
use RefreshDatabase;
/** @test */
public function screenshotIsSavedByJob(): void
{
Storage::fake('public');
$guzzleMock = new MockHandler([
new Response(201, ['Content-Type' => 'application/json'], '{"data":{"id":"68d52633-e170-465e-b13e-746c97d01ffb","job_id":null,"status":"finished","credits":null,"code":null,"message":null,"percent":100,"operation":"capture-website","engine":"chrome","engine_version":"107","result":null,"created_at":"2023-01-07T21:05:48+00:00","started_at":null,"ended_at":null,"retry_of_task_id":null,"copy_of_task_id":null,"user_id":61485254,"priority":-10,"host_name":null,"storage":"ceph-fra","depends_on_task_ids":[],"links":{"self":"https:\/\/api.cloudconvert.com\/v2\/tasks\/68d52633-e170-465e-b13e-746c97d01ffb"}}}'),
new Response(201, ['Content-Type' => 'application/json'], '{"data":{"id":"27f33137-cc03-4468-aba4-1e1aa8c096fb","job_id":null,"status":"finished","credits":null,"code":null,"message":null,"percent":100,"operation":"export\/url","result":null,"created_at":"2023-01-07T21:10:02+00:00","started_at":null,"ended_at":null,"retry_of_task_id":null,"copy_of_task_id":null,"user_id":61485254,"priority":-10,"host_name":null,"storage":"ceph-fra","depends_on_task_ids":["68d52633-e170-465e-b13e-746c97d01ffb"],"links":{"self":"https:\/\/api.cloudconvert.com\/v2\/tasks\/27f33137-cc03-4468-aba4-1e1aa8c096fb"}}}'),
new Response(200, ['Content-Type' => 'image/png'], fopen(__DIR__ . '/../../theverge.com.png', 'rb')),
]);
$guzzleHandler = HandlerStack::create($guzzleMock);
$guzzleClient = new Client(['handler' => $guzzleHandler]);
$this->app->instance(Client::class, $guzzleClient);
$retryMock = new MockHandler([
new Response(200, ['Content-Type' => 'application/json'], '{"data":{"id":"68d52633-e170-465e-b13e-746c97d01ffb","job_id":null,"status":"finished","credits":1,"code":null,"message":null,"percent":100,"operation":"capture-website","engine":"chrome","engine_version":"107","payload":{"url":"https:\/\/theverge.com","output_format":"png","screen_width":1440,"screen_height":900,"wait_until":"networkidle0","wait_time":"100"},"result":{"files":[{"filename":"theverge.com.png","size":811819}]},"created_at":"2023-01-07T21:05:48+00:00","started_at":"2023-01-07T21:05:48+00:00","ended_at":"2023-01-07T21:05:55+00:00","retry_of_task_id":null,"copy_of_task_id":null,"user_id":61485254,"priority":-10,"host_name":"virgie","storage":"ceph-fra","depends_on_task_ids":[],"links":{"self":"https:\/\/api.cloudconvert.com\/v2\/tasks\/68d52633-e170-465e-b13e-746c97d01ffb"}}}'),
new Response(200, ['Content-Type' => 'application/json'], '{"data":{"id":"27f33137-cc03-4468-aba4-1e1aa8c096fb","job_id":null,"status":"finished","credits":0,"code":null,"message":null,"percent":100,"operation":"export\/url","payload":{"input":"68d52633-e170-465e-b13e-746c97d01ffb","archive_multiple_files":false},"result":{"files":[{"filename":"theverge.com.png","size":811819,"url":"https:\/\/storage.cloudconvert.com\/tasks\/27f33137-cc03-4468-aba4-1e1aa8c096fb\/theverge.com.png?AWSAccessKeyId=cloudconvert-production&Expires=1673212203&Signature=xyz&response-content-disposition=attachment%3B%20filename%3D%22theverge.com.png%22&response-content-type=image%2Fpng"}]},"created_at":"2023-01-07T21:10:02+00:00","started_at":"2023-01-07T21:10:03+00:00","ended_at":"2023-01-07T21:10:03+00:00","retry_of_task_id":null,"copy_of_task_id":null,"user_id":61485254,"priority":-10,"host_name":"virgie","storage":"ceph-fra","depends_on_task_ids":["68d52633-e170-465e-b13e-746c97d01ffb"],"links":{"self":"https:\/\/api.cloudconvert.com\/v2\/tasks\/27f33137-cc03-4468-aba4-1e1aa8c096fb"}}}'),
]);
$retryHandler = HandlerStack::create($retryMock);
$retryHandler->push(Middleware::retry(
function ($retries, $request, $response, $exception) {
// Limit the number of retries to 5
if ($retries >= 5) {
return false;
}
// Retry connection exceptions
if ($exception instanceof \GuzzleHttp\Exception\ConnectException) {
return true;
}
// Retry on server errors
if ($response && $response->getStatusCode() >= 500) {
return true;
}
$responseBody = '';
if (is_string($response)) {
$responseBody = $response;
}
if ($response instanceof Response) {
$responseBody = $response->getBody()->getContents();
$response->getBody()->rewind();
}
// Finally for CloudConvert, retry if status is not final
return json_decode($responseBody, false, 512, JSON_THROW_ON_ERROR)?->data?->status !== 'finished';
},
function () {
// Retry after 1 second
return 1000;
}
));
$retryClient = new Client(['handler' => $retryHandler]);
$this->app->instance('RetryGuzzle', $retryClient);
$bookmark = Bookmark::factory()->create();
$job = new SaveScreenshot($bookmark);
$job->handle();
$bookmark->refresh();
$this->assertEquals('68d52633-e170-465e-b13e-746c97d01ffb', $bookmark->screenshot);
Storage::disk('public')->assertExists('/assets/img/bookmarks/' . $bookmark->screenshot . '.png');
}
/** @test */
public function screenshotJobHandlesUnfinishedTasks(): void
{
Storage::fake('public');
$guzzleMock = new MockHandler([
new Response(201, ['Content-Type' => 'application/json'], '{"id":1,"data":{"id":"68d52633-e170-465e-b13e-746c97d01ffb","job_id":null,"status":"waiting","credits":null,"code":null,"message":null,"percent":100,"operation":"capture-website","engine":"chrome","engine_version":"107","result":null,"created_at":"2023-01-07T21:05:48+00:00","started_at":null,"ended_at":null,"retry_of_task_id":null,"copy_of_task_id":null,"user_id":61485254,"priority":-10,"host_name":null,"storage":"ceph-fra","depends_on_task_ids":[],"links":{"self":"https:\/\/api.cloudconvert.com\/v2\/tasks\/68d52633-e170-465e-b13e-746c97d01ffb"}}}'),
new Response(201, ['Content-Type' => 'application/json'], '{"id":2,"data":{"id":"27f33137-cc03-4468-aba4-1e1aa8c096fb","job_id":null,"status":"waiting","credits":null,"code":null,"message":null,"percent":100,"operation":"export\/url","result":null,"created_at":"2023-01-07T21:10:02+00:00","started_at":null,"ended_at":null,"retry_of_task_id":null,"copy_of_task_id":null,"user_id":61485254,"priority":-10,"host_name":null,"storage":"ceph-fra","depends_on_task_ids":["68d52633-e170-465e-b13e-746c97d01ffb"],"links":{"self":"https:\/\/api.cloudconvert.com\/v2\/tasks\/27f33137-cc03-4468-aba4-1e1aa8c096fb"}}}'),
new Response(200, ['Content-Type' => 'image/png'], fopen(__DIR__ . '/../../theverge.com.png', 'rb')),
]);
$guzzleHandler = HandlerStack::create($guzzleMock);
$guzzleClient = new Client(['handler' => $guzzleHandler]);
$this->app->instance(Client::class, $guzzleClient);
$container = [];
$history = Middleware::history($container);
$retryMock = new MockHandler([
new Response(200, ['Content-Type' => 'application/json'], '{"id":3,"data":{"id":"68d52633-e170-465e-b13e-746c97d01ffb","job_id":null,"status":"waiting","credits":1,"code":null,"message":null,"percent":50,"operation":"capture-website","engine":"chrome","engine_version":"107","payload":{"url":"https:\/\/theverge.com","output_format":"png","screen_width":1440,"screen_height":900,"wait_until":"networkidle0","wait_time":"100"},"result":{"files":[{"filename":"theverge.com.png","size":811819}]},"created_at":"2023-01-07T21:05:48+00:00","started_at":"2023-01-07T21:05:48+00:00","ended_at":"2023-01-07T21:05:55+00:00","retry_of_task_id":null,"copy_of_task_id":null,"user_id":61485254,"priority":-10,"host_name":"virgie","storage":"ceph-fra","depends_on_task_ids":[],"links":{"self":"https:\/\/api.cloudconvert.com\/v2\/tasks\/68d52633-e170-465e-b13e-746c97d01ffb"}}}'),
new Response(200, ['Content-Type' => 'application/json'], '{"id":4,"data":{"id":"68d52633-e170-465e-b13e-746c97d01ffb","job_id":null,"status":"finished","credits":1,"code":null,"message":null,"percent":100,"operation":"capture-website","engine":"chrome","engine_version":"107","payload":{"url":"https:\/\/theverge.com","output_format":"png","screen_width":1440,"screen_height":900,"wait_until":"networkidle0","wait_time":"100"},"result":{"files":[{"filename":"theverge.com.png","size":811819}]},"created_at":"2023-01-07T21:05:48+00:00","started_at":"2023-01-07T21:05:48+00:00","ended_at":"2023-01-07T21:05:55+00:00","retry_of_task_id":null,"copy_of_task_id":null,"user_id":61485254,"priority":-10,"host_name":"virgie","storage":"ceph-fra","depends_on_task_ids":[],"links":{"self":"https:\/\/api.cloudconvert.com\/v2\/tasks\/68d52633-e170-465e-b13e-746c97d01ffb"}}}'),
new Response(200, ['Content-Type' => 'application/json'], '{"id":5,"data":{"id":"27f33137-cc03-4468-aba4-1e1aa8c096fb","job_id":null,"status":"waiting","credits":0,"code":null,"message":null,"percent":50,"operation":"export\/url","payload":{"input":"68d52633-e170-465e-b13e-746c97d01ffb","archive_multiple_files":false},"created_at":"2023-01-07T21:10:02+00:00","started_at":"2023-01-07T21:10:03+00:00","ended_at":null,"retry_of_task_id":null,"copy_of_task_id":null,"user_id":61485254,"priority":-10,"host_name":"virgie","storage":"ceph-fra","depends_on_task_ids":["68d52633-e170-465e-b13e-746c97d01ffb"],"links":{"self":"https:\/\/api.cloudconvert.com\/v2\/tasks\/27f33137-cc03-4468-aba4-1e1aa8c096fb"}}}'),
new Response(200, ['Content-Type' => 'application/json'], '{"id":6,"data":{"id":"27f33137-cc03-4468-aba4-1e1aa8c096fb","job_id":null,"status":"finished","credits":0,"code":null,"message":null,"percent":100,"operation":"export\/url","payload":{"input":"68d52633-e170-465e-b13e-746c97d01ffb","archive_multiple_files":false},"result":{"files":[{"filename":"theverge.com.png","size":811819,"url":"https:\/\/storage.cloudconvert.com\/tasks\/27f33137-cc03-4468-aba4-1e1aa8c096fb\/theverge.com.png?AWSAccessKeyId=cloudconvert-production&Expires=1673212203&Signature=xyz&response-content-disposition=attachment%3B%20filename%3D%22theverge.com.png%22&response-content-type=image%2Fpng"}]},"created_at":"2023-01-07T21:10:02+00:00","started_at":"2023-01-07T21:10:03+00:00","ended_at":"2023-01-07T21:10:03+00:00","retry_of_task_id":null,"copy_of_task_id":null,"user_id":61485254,"priority":-10,"host_name":"virgie","storage":"ceph-fra","depends_on_task_ids":["68d52633-e170-465e-b13e-746c97d01ffb"],"links":{"self":"https:\/\/api.cloudconvert.com\/v2\/tasks\/27f33137-cc03-4468-aba4-1e1aa8c096fb"}}}'),
]);
$retryHandler = HandlerStack::create($retryMock);
$retryHandler->push($history);
$retryHandler->push(Middleware::retry(
function ($retries, $request, $response, $exception) {
// Limit the number of retries to 5
if ($retries >= 5) {
return false;
}
// Retry connection exceptions
if ($exception instanceof \GuzzleHttp\Exception\ConnectException) {
return true;
}
// Retry on server errors
if ($response && $response->getStatusCode() >= 500) {
return true;
}
$responseBody = '';
if (is_string($response)) {
$responseBody = $response;
}
if ($response instanceof Response) {
$responseBody = $response->getBody()->getContents();
$response->getBody()->rewind();
}
// Finally for CloudConvert, retry if status is not final
return json_decode($responseBody, false, 512, JSON_THROW_ON_ERROR)?->data?->status !== 'finished';
},
function () {
// Retry after 1 second
return 1000;
}
));
$retryClient = new Client(['handler' => $retryHandler]);
$this->app->instance('RetryGuzzle', $retryClient);
$bookmark = Bookmark::factory()->create();
$job = new SaveScreenshot($bookmark);
$job->handle();
$bookmark->refresh();
$this->assertEquals('68d52633-e170-465e-b13e-746c97d01ffb', $bookmark->screenshot);
Storage::disk('public')->assertExists('/assets/img/bookmarks/' . $bookmark->screenshot . '.png');
// Also assert we made the correct number of requests
$this->assertCount(2, $container);
// However with retries there should be more than 4 responses for the 2 requests
$this->assertEquals(0, $retryMock->count());
}
}

BIN
tests/theverge.com.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 793 KiB