From 49e6e9a84dc6c0ed2f04b38e9944a51b9b439453 Mon Sep 17 00:00:00 2001 From: Jonny Barnes Date: Mon, 19 Sep 2016 15:32:11 +0100 Subject: [PATCH 1/8] Updated .lock --- composer.lock | 51 +++++++++++++++++++++++++-------------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/composer.lock b/composer.lock index 2de32248..74bf5039 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,6 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", "This file is @generated automatically" ], - "hash": "b65fffcf4b32d065494b01ada5391976", "content-hash": "86faec8ac49549630fc60578bcd085cc", "packages": [ { @@ -59,16 +58,16 @@ }, { "name": "aws/aws-sdk-php", - "version": "3.19.6", + "version": "3.19.8", "source": { "type": "git", "url": "https://github.com/aws/aws-sdk-php.git", - "reference": "34060bf0db260031697b17dbb37fa1bbec92f1c4" + "reference": "4e976ef6750bb177f493f3b01476136213b8d0b9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/34060bf0db260031697b17dbb37fa1bbec92f1c4", - "reference": "34060bf0db260031697b17dbb37fa1bbec92f1c4", + "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/4e976ef6750bb177f493f3b01476136213b8d0b9", + "reference": "4e976ef6750bb177f493f3b01476136213b8d0b9", "shasum": "" }, "require": { @@ -135,7 +134,7 @@ "s3", "sdk" ], - "time": "2016-09-08 20:27:15" + "time": "2016-09-15 21:43:53" }, { "name": "barnabywalters/mf-cleaner", @@ -1533,16 +1532,16 @@ }, { "name": "laravel/framework", - "version": "v5.3.8", + "version": "v5.3.9", "source": { "type": "git", "url": "https://github.com/laravel/framework.git", - "reference": "99c74afc0b99e1af1984cb55dc242ab28a0e496b" + "reference": "f6fbb481672f8dc4bc6882d5d654bbfa3588c8ec" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/framework/zipball/99c74afc0b99e1af1984cb55dc242ab28a0e496b", - "reference": "99c74afc0b99e1af1984cb55dc242ab28a0e496b", + "url": "https://api.github.com/repos/laravel/framework/zipball/f6fbb481672f8dc4bc6882d5d654bbfa3588c8ec", + "reference": "f6fbb481672f8dc4bc6882d5d654bbfa3588c8ec", "shasum": "" }, "require": { @@ -1656,7 +1655,7 @@ "framework", "laravel" ], - "time": "2016-09-09 16:33:59" + "time": "2016-09-12 14:08:29" }, { "name": "lcobucci/jwt", @@ -2297,16 +2296,16 @@ }, { "name": "nikic/php-parser", - "version": "v2.1.0", + "version": "v2.1.1", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "47b254ea51f1d6d5dc04b9b299e88346bf2369e3" + "reference": "4dd659edadffdc2143e4753df655d866dbfeedf0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/47b254ea51f1d6d5dc04b9b299e88346bf2369e3", - "reference": "47b254ea51f1d6d5dc04b9b299e88346bf2369e3", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/4dd659edadffdc2143e4753df655d866dbfeedf0", + "reference": "4dd659edadffdc2143e4753df655d866dbfeedf0", "shasum": "" }, "require": { @@ -2344,7 +2343,7 @@ "parser", "php" ], - "time": "2016-04-19 13:41:41" + "time": "2016-09-16 12:04:44" }, { "name": "paragonie/random_compat", @@ -2882,16 +2881,16 @@ }, { "name": "spatie/laravel-medialibrary", - "version": "4.8.3", + "version": "4.8.4", "source": { "type": "git", "url": "https://github.com/spatie/laravel-medialibrary.git", - "reference": "7fcb591ff0ef2d28b9b622f84f972c642ff3e8f9" + "reference": "8c862e270d49e8bbff6f0993900c8bb59ea165ea" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/spatie/laravel-medialibrary/zipball/7fcb591ff0ef2d28b9b622f84f972c642ff3e8f9", - "reference": "7fcb591ff0ef2d28b9b622f84f972c642ff3e8f9", + "url": "https://api.github.com/repos/spatie/laravel-medialibrary/zipball/8c862e270d49e8bbff6f0993900c8bb59ea165ea", + "reference": "8c862e270d49e8bbff6f0993900c8bb59ea165ea", "shasum": "" }, "require": { @@ -2945,7 +2944,7 @@ "media", "spatie" ], - "time": "2016-08-25 08:08:10" + "time": "2016-09-14 16:31:49" }, { "name": "spatie/pdf-to-image", @@ -4349,16 +4348,16 @@ }, { "name": "myclabs/deep-copy", - "version": "1.5.2", + "version": "1.5.4", "source": { "type": "git", "url": "https://github.com/myclabs/DeepCopy.git", - "reference": "da8529775f14f4fdae33f916eb0cf65f6afbddbc" + "reference": "ea74994a3dc7f8d2f65a06009348f2d63c81e61f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/da8529775f14f4fdae33f916eb0cf65f6afbddbc", - "reference": "da8529775f14f4fdae33f916eb0cf65f6afbddbc", + "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/ea74994a3dc7f8d2f65a06009348f2d63c81e61f", + "reference": "ea74994a3dc7f8d2f65a06009348f2d63c81e61f", "shasum": "" }, "require": { @@ -4387,7 +4386,7 @@ "object", "object graph" ], - "time": "2016-09-06 16:07:05" + "time": "2016-09-16 13:37:59" }, { "name": "phpdocumentor/reflection-common", From ca88bbdb915f84693dd3d22ef34d4616c2ebb453 Mon Sep 17 00:00:00 2001 From: Jonny Barnes Date: Mon, 19 Sep 2016 15:50:15 +0100 Subject: [PATCH 2/8] =?UTF-8?q?Filter=20html=20on=20output=20and=20rely=20?= =?UTF-8?q?on=20HTMLFilter=E2=80=99s=20own=20cache?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/Http/Controllers/NotesController.php | 19 ++++++- app/Observers/WebMentionObserver.php | 65 ------------------------ app/Providers/AppServiceProvider.php | 3 -- 3 files changed, 18 insertions(+), 69 deletions(-) delete mode 100644 app/Observers/WebMentionObserver.php diff --git a/app/Http/Controllers/NotesController.php b/app/Http/Controllers/NotesController.php index 590ff654..f1bddc32 100644 --- a/app/Http/Controllers/NotesController.php +++ b/app/Http/Controllers/NotesController.php @@ -6,6 +6,8 @@ use Cache; use Twitter; use App\Tag; use App\Note; +use HTMLPurifier; +use HTMLPurifier_Config; use Jonnybarnes\IndieWeb\Numbers; use Illuminate\Filesystem\Filesystem; use Jonnybarnes\WebmentionsParser\Authorship; @@ -103,7 +105,7 @@ class NotesController extends Controller case 'in-reply-to': $content['source'] = $webmention->source; $content['date'] = $carbon->parse($content['date'])->toDayDateTimeString(); - $content['reply'] = $microformats['items'][0]['properties']['content'][0]['html_purified']; + $content['reply'] = $this->filterHTML($microformats['items'][0]['properties']['content'][0]['html']); $replies[] = $content; break; @@ -260,4 +262,19 @@ class NotesController extends Controller return $oEmbed; } + + /** + * Filter the HTML in a reply webmention. + * + * @param string The reply HTML + * @return string The filtered HTML + */ + private function filterHTML($html) + { + $config = HTMLPurifier_Config::createDefault(); + $config->set('Cache.SerializerPath', storage_path() . '/HTMLPurifier'); + $purifier = new HTMLPurifier($config); + + return $purifier->purify($html); + } } diff --git a/app/Observers/WebMentionObserver.php b/app/Observers/WebMentionObserver.php deleted file mode 100644 index b15c3b3e..00000000 --- a/app/Observers/WebMentionObserver.php +++ /dev/null @@ -1,65 +0,0 @@ -addFilteredHTML($webmention); - } - - /** - * Listen for the updated event. - * - * @param WebMention $webmention - * @return void - */ - public function updated(WebMention $webmention) - { - $this->addFilteredHTML($webmention); - } - - /** - * Filter the HTML in a reply webmention. - * - * @param WebMention The WebMention model - * @return void - */ - private function addFilteredHTML(WebMention $webmention) - { - $mf2 = json_decode($webmention->mf2); - if (isset($mf2['items'][0]['properties']['content'][0]['html'])) { - $mf2['items'][0]['properties']['content'][0]['html_purified'] = $this->useHTMLPurifier( - $mf2['items'][0]['properties']['content'][0]['html'] - ); - } - $webmention->mf2 = json_encode($mf2); - $webmetion->save(); - } - - /** - * Set up and use HTMLPurifer on some HTML. - * - * @param string The HTML to be processed - * @return string The processed HTML - */ - private function useHTMLPurifier($html) - { - $config = HTMLPurifier_Config::createDefault(); - $config->set('Cache.SerializerPath', storage_path() . '/HTMLPurifier'); - $purifier = new HTMLPurifier($config); - - return $purifier->purify($html); - } -} diff --git a/app/Providers/AppServiceProvider.php b/app/Providers/AppServiceProvider.php index a83013ab..177b94cc 100644 --- a/app/Providers/AppServiceProvider.php +++ b/app/Providers/AppServiceProvider.php @@ -47,9 +47,6 @@ class AppServiceProvider extends ServiceProvider $note->tags()->attach($tagsToAdd); } }); - - //observer the webmention model - WebMention::observe(WebMentionObserver::class); } /** From 069a5e49ebf2813f5930ea0f19cc432edc74ffaf Mon Sep 17 00:00:00 2001 From: Jonny Barnes Date: Mon, 19 Sep 2016 15:51:43 +0100 Subject: [PATCH 3/8] Update changelog --- changelog.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/changelog.md b/changelog.md index 9fae8d98..c372fe20 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,8 @@ # Changelog +## Version {next} + - Simplify how we filter/cache reply html + ## Version 0.0.11.2 (2016-09-19) - Update Typekit’s js sri hash From 6804cc3776beb73dc02583736c9a80b1501415e4 Mon Sep 17 00:00:00 2001 From: Jonny Barnes Date: Mon, 19 Sep 2016 17:18:24 +0100 Subject: [PATCH 4/8] Backup a file before saving the new HTML contents --- app/Jobs/DownloadWebMention.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/Jobs/DownloadWebMention.php b/app/Jobs/DownloadWebMention.php index 592b41d3..71894270 100644 --- a/app/Jobs/DownloadWebMention.php +++ b/app/Jobs/DownloadWebMention.php @@ -41,8 +41,13 @@ class DownloadWebMention implements ShouldQueue //Laravel should catch and retry these automatically. if ($response->getStatusCode() == '200') { $filesystem = \Illuminate\FileSystem\FileSystem(); + $filename = $this->createFilenameFromURL($source); + //backup file first + if ($filesystem->exists($filename)) { + $filesystem->copy($filename, $filename . '.' . date('Y-m-d') . '.backup'); + } $filesystem->put( - $this->createFilenameFromURL($source), + $filename, (string) $response->getBody() ); } From 54079e407c69848fa10a46761692f55737795c22 Mon Sep 17 00:00:00 2001 From: Jonny Barnes Date: Mon, 19 Sep 2016 17:19:04 +0100 Subject: [PATCH 5/8] =?UTF-8?q?Don=E2=80=99t=20parse=20backups?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/Console/Commands/ParseCachedWebMentions.php | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/app/Console/Commands/ParseCachedWebMentions.php b/app/Console/Commands/ParseCachedWebMentions.php index d271dd15..f375f213 100644 --- a/app/Console/Commands/ParseCachedWebMentions.php +++ b/app/Console/Commands/ParseCachedWebMentions.php @@ -41,13 +41,15 @@ class ParseCachedWebMentions extends Command { $HTMLfiles = $filesystem->allFiles(storage_path() . '/HTML'); foreach ($HTMLfiles as $file) { - $filepath = $file->getPathname(); - $html = $filesystem->get($filepath); - $url = $this->URLFromFilename($filepath); - $microformats = \Mf2\parse($html, $url); - $webmention = WebMention::where('source', $url)->firstOrFail(); - $webmention->mf2 = json_encode($microformats); - $webmention->save(); + if ($file->getExtension() != 'backup') { //we don’t want to parse.backup files + $filepath = $file->getPathname(); + $html = $filesystem->get($filepath); + $url = $this->URLFromFilename($filepath); + $microformats = \Mf2\parse($html, $url); + $webmention = WebMention::where('source', $url)->firstOrFail(); + $webmention->mf2 = json_encode($microformats); + $webmention->save(); + } } } From cdfb437dd345527957a9699c7c2beb0aaeca1764 Mon Sep 17 00:00:00 2001 From: Jonny Barnes Date: Mon, 19 Sep 2016 17:25:01 +0100 Subject: [PATCH 6/8] Delete unnecesary backups --- app/Jobs/DownloadWebMention.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app/Jobs/DownloadWebMention.php b/app/Jobs/DownloadWebMention.php index 71894270..63d8d307 100644 --- a/app/Jobs/DownloadWebMention.php +++ b/app/Jobs/DownloadWebMention.php @@ -43,13 +43,19 @@ class DownloadWebMention implements ShouldQueue $filesystem = \Illuminate\FileSystem\FileSystem(); $filename = $this->createFilenameFromURL($source); //backup file first + $filenameBackup = $filename . '.' . date('Y-m-d') . '.backup'; if ($filesystem->exists($filename)) { - $filesystem->copy($filename, $filename . '.' . date('Y-m-d') . '.backup'); + $filesystem->copy($filename, $filenameBackup); } + //save new HTML $filesystem->put( $filename, (string) $response->getBody() ); + //remove backup if the same + if ($filesystem->get($filename) == $filesystem->get($filenameBackup)) { + $filesystem->delete($filenameBackup); + } } } From 4555abb50124142f9f2005e75fbb75f4a84a0062 Mon Sep 17 00:00:00 2001 From: Jonny Barnes Date: Mon, 19 Sep 2016 17:27:49 +0100 Subject: [PATCH 7/8] Update changelog --- changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog.md b/changelog.md index c372fe20..ad95f79f 100644 --- a/changelog.md +++ b/changelog.md @@ -2,6 +2,7 @@ ## Version {next} - Simplify how we filter/cache reply html + - Better handling of webmention reply HTML cache ## Version 0.0.11.2 (2016-09-19) - Update Typekit’s js sri hash From 914de4405770c30d27c1b71a33445a57cda7e6cc Mon Sep 17 00:00:00 2001 From: Jonny Barnes Date: Mon, 19 Sep 2016 17:28:57 +0100 Subject: [PATCH 8/8] Bump version number --- changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index ad95f79f..be67ec53 100644 --- a/changelog.md +++ b/changelog.md @@ -1,6 +1,6 @@ # Changelog -## Version {next} +## Version 0.0.11.3 (2016-09-19) - Simplify how we filter/cache reply html - Better handling of webmention reply HTML cache