From 9f193904b57dddd40c929bf2417e31e4bffb8c73 Mon Sep 17 00:00:00 2001 From: Andy Miller Date: Tue, 21 Feb 2017 08:23:26 -0700 Subject: [PATCH 1/7] Initial work to genericize direct-install (#1321) --- system/src/Grav/Common/GPM/GPM.php | 141 ++++++++++++ system/src/Grav/Common/GPM/Response.php | 11 + .../Grav/Console/Gpm/DirectInstallCommand.php | 213 ++---------------- 3 files changed, 173 insertions(+), 192 deletions(-) diff --git a/system/src/Grav/Common/GPM/GPM.php b/system/src/Grav/Common/GPM/GPM.php index 857dd8fc0..e66e5bc47 100644 --- a/system/src/Grav/Common/GPM/GPM.php +++ b/system/src/Grav/Common/GPM/GPM.php @@ -9,6 +9,7 @@ namespace Grav\Common\GPM; use Grav\Common\Grav; +use Grav\Common\Filesystem\Folder; use Grav\Common\Inflector; use Grav\Common\Iterator; use Grav\Common\Utils; @@ -499,6 +500,146 @@ class GPM extends Iterator return false; } + /** + * Download the zip package via the URL + * + * @param $package_file + * @param $tmp + * @return null|string + */ + public static function downloadPackage($package_file, $tmp) + { + $package = parse_url($package_file); + $filename = basename($package['path']); + $output = Response::get($package_file, []); + + if ($output) { + Folder::mkdir($tmp); + file_put_contents($tmp . DS . $filename, $output); + return $tmp . DS . $filename; + } + + return null; + } + + /** + * Copy the local zip package to tmp + * + * @param $package_file + * @param $tmp + * @return null|string + */ + public static function copyPackage($package_file, $tmp) + { + $package_file = realpath($package_file); + + if (file_exists($package_file)) { + $filename = basename($package_file); + Folder::mkdir($tmp); + copy(realpath($package_file), $tmp . DS . $filename); + return $tmp . DS . $filename; + } + + return null; + } + + /** + * Try to guess the package type from the source files + * + * @param $source + * @return bool|string + */ + public static function getPackageType($source) + { + $plugin_regex = '/^class\\s{1,}[a-zA-Z0-9]{1,}\\s{1,}extends.+Plugin/m'; + $theme_regex = '/^class\\s{1,}[a-zA-Z0-9]{1,}\\s{1,}extends.+Theme/m'; + + if ( + file_exists($source . 'system/defines.php') && + file_exists($source . 'system/config/system.yaml') + ) { + return 'grav'; + } else { + // must have a blueprint + if (!file_exists($source . 'blueprints.yaml')) { + return false; + } + + // either theme or plugin + $name = basename($source); + if (Utils::contains($name, 'theme')) { + return 'theme'; + } elseif (Utils::contains($name, 'plugin')) { + return 'plugin'; + } + foreach (glob($source . "*.php") as $filename) { + $contents = file_get_contents($filename); + if (preg_match($theme_regex, $contents)) { + return 'theme'; + } elseif (preg_match($plugin_regex, $contents)) { + return 'plugin'; + } + } + + // Assume it's a theme + return 'theme'; + } + } + + /** + * Try to guess the package name from the source files + * + * @param $source + * @return bool|string + */ + public static function getPackageName($source) + { + foreach (glob($source . "*.yaml") as $filename) { + $name = strtolower(basename($filename, '.yaml')); + if ($name == 'blueprints') { + continue; + } + return $name; + } + return false; + } + + /** + * Find/Parse the blueprint file + * + * @param $source + * @return array|bool + */ + public static function getBlueprints($source) + { + $blueprint_file = $source . 'blueprints.yaml'; + if (!file_exists($blueprint_file)) { + return false; + } + + $blueprint = (array)Yaml::parse(file_get_contents($blueprint_file)); + return $blueprint; + } + + /** + * Get the install path for a name and a particular type of package + * + * @param $type + * @param $name + * @return string + */ + public static function getInstallPath($type, $name) + { + $locator = Grav::instance()['locator']; + + if ($type == 'theme') { + $install_path = $locator->findResource('themes://', false) . DS . $name; + } else { + $install_path = $locator->findResource('plugins://', false) . DS . $name; + } + return $install_path; + } + /** * Searches for a list of Packages in the repository * @param array $searches An array of either slugs or names diff --git a/system/src/Grav/Common/GPM/Response.php b/system/src/Grav/Common/GPM/Response.php index 6b3036ff7..f046352ac 100644 --- a/system/src/Grav/Common/GPM/Response.php +++ b/system/src/Grav/Common/GPM/Response.php @@ -189,6 +189,17 @@ class Response return preg_match('/1|yes|on|true/i', ini_get('allow_url_fopen')); } + /** + * Is this a remote file or not + * + * @param $file + * @return bool + */ + public static function isRemote($file) + { + return (bool) filter_var($file, FILTER_VALIDATE_URL); + } + /** * Progress normalized for cURL and Fopen * Accepts a variable length of arguments passed in by stream method diff --git a/system/src/Grav/Console/Gpm/DirectInstallCommand.php b/system/src/Grav/Console/Gpm/DirectInstallCommand.php index e23712e9f..1a4170e96 100644 --- a/system/src/Grav/Console/Gpm/DirectInstallCommand.php +++ b/system/src/Grav/Console/Gpm/DirectInstallCommand.php @@ -9,14 +9,13 @@ namespace Grav\Console\Gpm; use Grav\Common\Grav; -use Grav\Common\Utils; use Grav\Common\Filesystem\Folder; +use Grav\Common\GPM\GPM; use Grav\Common\GPM\Installer; use Grav\Common\GPM\Response; use Grav\Console\ConsoleCommand; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Question\ConfirmationQuestion; -use Symfony\Component\Yaml\Yaml; class DirectInstallCommand extends ConsoleCommand { @@ -63,10 +62,22 @@ class DirectInstallCommand extends ConsoleCommand $this->output->writeln("Preparing to install " . $package_file . ""); - if ($this->isRemote($package_file)) { - $zip = $this->downloadPackage($package_file, $tmp_zip); + if (Response::isRemote($package_file)) { + $this->output->write(" |- Downloading package... 0%"); + $zip = GPM::downloadPackage($package_file, $tmp_zip); + if ($zip) { + $this->output->write("\x0D"); + $this->output->write(" |- Downloading package... 100%"); + $this->output->writeln(''); + } } else { - $zip = $this->copyPackage($package_file, $tmp_zip); + $this->output->write(" |- Copying package... 0%"); + $zip = GPM::copyPackage($package_file, $tmp_zip); + if ($zip) { + $this->output->write("\x0D"); + $this->output->write(" |- Copying package... 100%"); + $this->output->writeln(''); + } } if (file_exists($zip)) { @@ -85,7 +96,7 @@ class DirectInstallCommand extends ConsoleCommand $this->output->writeln(" |- Extracting package... ok"); - $type = $this->getPackageType($extracted); + $type = GPM::getPackageType($extracted); if (!$type) { $this->output->writeln(" '- ERROR: Not a valid Grav package"); @@ -93,7 +104,7 @@ class DirectInstallCommand extends ConsoleCommand exit; } - $blueprint = $this->getBlueprints($extracted); + $blueprint = GPM::getBlueprints($extracted); if ($blueprint) { if (isset($blueprint['dependencies'])) { $depencencies = []; @@ -137,7 +148,7 @@ class DirectInstallCommand extends ConsoleCommand $this->output->write(" |- Installing package... "); Installer::install($zip, GRAV_ROOT, ['sophisticated' => true, 'overwrite' => true, 'ignore_symlinks' => true], $extracted); } else { - $name = $this->getPackageName($extracted); + $name = GPM::getPackageName($extracted); if (!$name) { $this->output->writeln("ERROR: Name could not be determined. Please specify with --name|-n"); @@ -145,7 +156,7 @@ class DirectInstallCommand extends ConsoleCommand exit; } - $install_path = $this->getInstallPath($type, $name); + $install_path = GPM::getInstallPath($type, $name); $is_update = file_exists($install_path); $this->output->write(" |- Checking destination... "); @@ -173,7 +184,7 @@ class DirectInstallCommand extends ConsoleCommand $this->output->write("\x0D"); if(Installer::lastErrorCode()) { - $this->output->writeln(" '- Installation failed or aborted."); + $this->output->writeln(" '- " . Installer::lastErrorMsg() . ""); $this->output->writeln(''); } else { $this->output->writeln(" |- Installing package... ok"); @@ -193,186 +204,4 @@ class DirectInstallCommand extends ConsoleCommand return true; } - - /** - * Get the install path for a name and a particular type of package - * - * @param $type - * @param $name - * @return string - */ - protected function getInstallPath($type, $name) - { - $locator = Grav::instance()['locator']; - - if ($type == 'theme') { - $install_path = $locator->findResource('themes://', false) . DS . $name; - } else { - $install_path = $locator->findResource('plugins://', false) . DS . $name; - } - return $install_path; - } - - /** - * Try to guess the package name from the source files - * - * @param $source - * @return bool|string - */ - protected function getPackageName($source) - { - foreach (glob($source . "*.yaml") as $filename) { - $name = strtolower(basename($filename, '.yaml')); - if ($name == 'blueprints') { - continue; - } - return $name; - } - return false; - } - - /** - * Try to guess the package type from the source files - * - * @param $source - * @return bool|string - */ - protected function getPackageType($source) - { - $plugin_regex = '/^class\\s{1,}[a-zA-Z0-9]{1,}\\s{1,}extends.+Plugin/m'; - $theme_regex = '/^class\\s{1,}[a-zA-Z0-9]{1,}\\s{1,}extends.+Theme/m'; - - if ( - file_exists($source . 'system/defines.php') && - file_exists($source . 'system/config/system.yaml') - ) { - return 'grav'; - } else { - // must have a blueprint - if (!file_exists($source . 'blueprints.yaml')) { - return false; - } - - // either theme or plugin - $name = basename($source); - if (Utils::contains($name, 'theme')) { - return 'theme'; - } elseif (Utils::contains($name, 'plugin')) { - return 'plugin'; - } - foreach (glob($source . "*.php") as $filename) { - $contents = file_get_contents($filename); - if (preg_match($theme_regex, $contents)) { - return 'theme'; - } elseif (preg_match($plugin_regex, $contents)) { - return 'plugin'; - } - } - - // Assume it's a theme - return 'theme'; - } - } - - /** - * Determine if this is a local or a remote file - * - * @param $file - * @return bool - */ - protected function isRemote($file) - { - return (bool) filter_var($file, FILTER_VALIDATE_URL); - } - - /** - * Find/Parse the blueprint file - * - * @param $source - * @return array|bool - */ - protected function getBlueprints($source) - { - $blueprint_file = $source . 'blueprints.yaml'; - if (!file_exists($blueprint_file)) { - return false; - } - - $blueprint = (array)Yaml::parse(file_get_contents($blueprint_file)); - return $blueprint; - } - - /** - * Download the zip package via the URL - * - * @param $package_file - * @param $tmp - * @return null|string - */ - private function downloadPackage($package_file, $tmp) - { - $this->output->write(" |- Downloading package... 0%"); - - $package = parse_url($package_file); - - - $filename = basename($package['path']); - $output = Response::get($package_file, [], [$this, 'progress']); - - if ($output) { - Folder::mkdir($tmp); - - $this->output->write("\x0D"); - $this->output->write(" |- Downloading package... 100%"); - $this->output->writeln(''); - - file_put_contents($tmp . DS . $filename, $output); - - return $tmp . DS . $filename; - } - - return null; - - } - - /** - * Copy the local zip package to tmp - * - * @param $package_file - * @param $tmp - * @return null|string - */ - private function copyPackage($package_file, $tmp) - { - $this->output->write(" |- Copying package... 0%"); - - $package_file = realpath($package_file); - - if (file_exists($package_file)) { - $filename = basename($package_file); - - Folder::mkdir($tmp); - - $this->output->write("\x0D"); - $this->output->write(" |- Copying package... 100%"); - $this->output->writeln(''); - - copy(realpath($package_file), $tmp . DS . $filename); - - return $tmp . DS . $filename; - } - - return null; - - } - - /** - * @param $progress - */ - public function progress($progress) - { - $this->output->write("\x0D"); - $this->output->write(" |- Downloading package... " . str_pad($progress['percent'], 5, " ", - STR_PAD_LEFT) . '%'); - } } From 389cc296c239a2429bc74b4a57e43be05db05ed1 Mon Sep 17 00:00:00 2001 From: Andy Miller Date: Tue, 21 Feb 2017 18:08:08 -0700 Subject: [PATCH 2/7] Added `Uri::isValidUrl()` --- system/src/Grav/Common/Uri.php | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/system/src/Grav/Common/Uri.php b/system/src/Grav/Common/Uri.php index dd8308db6..e1383688c 100644 --- a/system/src/Grav/Common/Uri.php +++ b/system/src/Grav/Common/Uri.php @@ -727,7 +727,7 @@ class Uri * * @return boolean is eternal state */ - public function isExternal($url) + public static function isExternal($url) { if (Utils::startsWith($url, 'http')) { return true; @@ -1086,4 +1086,20 @@ class Uri return $urlWithNonce; } + + /** + * Is the passed in URL a valid URL? + * + * @param $url + * @return bool + */ + public static function isValidUrl($url) + { + $regex = '/^(?:(https?|ftp|telnet):)?\/\/((?:[a-z0-9@:.-]|%[0-9A-F]{2}){3,})(?::(\d+))?((?:\/(?:[a-z0-9-._~!$&\'\(\)\*\+\,\;\=\:\@]|%[0-9A-F]{2})*)*)(?:\?((?:[a-z0-9-._~!$&\'\(\)\*\+\,\;\=\:\/?@]|%[0-9A-F]{2})*))?(?:#((?:[a-z0-9-._~!$&\'\(\)\*\+\,\;\=\:\/?@]|%[0-9A-F]{2})*))?/'; + if (preg_match($regex, $url)) { + return true; + } else { + return false; + } + } } From af304f14f4af443d472125bdc925c7a498709919 Mon Sep 17 00:00:00 2001 From: Andy Miller Date: Wed, 22 Feb 2017 13:26:40 -0700 Subject: [PATCH 3/7] Added offical_gpm_only check --- CHANGELOG.md | 9 +++++++++ system/blueprints/config/system.yaml | 13 +++++++++++++ system/config/system.yaml | 1 + system/src/Grav/Common/GPM/GPM.php | 5 +++++ .../src/Grav/Console/Gpm/DirectInstallCommand.php | 10 +++++++++- 5 files changed, 37 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2996b161..57403a4d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +# v1.1.18 +## 02/xx/2017 + +1. [](#new) + * Added default setting to only allow `direct-installs` from official GPM. Can be configured in `system.yaml` + * Added a new `Utils::isValidUrl()` method +1. [](#improved) + * Genericized `direct-install` so it can be called via Admin plugin + # v1.1.17 ## 02/17/2017 diff --git a/system/blueprints/config/system.yaml b/system/blueprints/config/system.yaml index 6aa0d6012..03f6b898e 100644 --- a/system/blueprints/config/system.yaml +++ b/system/blueprints/config/system.yaml @@ -1057,6 +1057,19 @@ form: fopen: PLUGIN_ADMIN.FOPEN curl: PLUGIN_ADMIN.CURL + gpm.official_gpm_only: + type: toggle + label: PLUGIN_ADMIN.GPM_OFFICIAL_ONLY + highlight: auto + help: PLUGIN_ADMIN.GPM_OFFICIAL_ONLY_HELP + highlight: 1 + options: + 1: PLUGIN_ADMIN.YES + 0: PLUGIN_ADMIN.NO + default: true + validate: + type: bool + gpm.verify_peer: type: toggle label: PLUGIN_ADMIN.GPM_VERIFY_PEER diff --git a/system/config/system.yaml b/system/config/system.yaml index 7d4de30e0..99628151c 100644 --- a/system/config/system.yaml +++ b/system/config/system.yaml @@ -138,3 +138,4 @@ gpm: proxy_url: # Configure a manual proxy URL for GPM (eg 127.0.0.1:3128) method: 'auto' # Either 'curl', 'fopen' or 'auto'. 'auto' will try fopen first and if not available cURL verify_peer: true # Sometimes on some systems (Windows most commonly) GPM is unable to connect because the SSL certificate cannot be verified. Disabling this setting might help. + official_gpm_only: true # By default GPM direct-install will only allow URLs via the official GPM proxy to ensure security diff --git a/system/src/Grav/Common/GPM/GPM.php b/system/src/Grav/Common/GPM/GPM.php index e66e5bc47..584adbdd7 100644 --- a/system/src/Grav/Common/GPM/GPM.php +++ b/system/src/Grav/Common/GPM/GPM.php @@ -511,6 +511,11 @@ class GPM extends Iterator { $package = parse_url($package_file); $filename = basename($package['path']); + + if (Grav::instance()['config']->get('system.gpm.official_gpm_only') && $package['host'] !== 'getgrav.org') { + throw new \RuntimeException("Only offical GPM URLs are allowed. You can modify this behavior in the System configuration."); + } + $output = Response::get($package_file, []); if ($output) { diff --git a/system/src/Grav/Console/Gpm/DirectInstallCommand.php b/system/src/Grav/Console/Gpm/DirectInstallCommand.php index 1a4170e96..202840289 100644 --- a/system/src/Grav/Console/Gpm/DirectInstallCommand.php +++ b/system/src/Grav/Console/Gpm/DirectInstallCommand.php @@ -64,7 +64,15 @@ class DirectInstallCommand extends ConsoleCommand if (Response::isRemote($package_file)) { $this->output->write(" |- Downloading package... 0%"); - $zip = GPM::downloadPackage($package_file, $tmp_zip); + try { + $zip = GPM::downloadPackage($package_file, $tmp_zip); + } catch (\RuntimeException $e) { + $this->output->writeln(''); + $this->output->writeln(" `- ERROR: " . $e->getMessage() . ""); + $this->output->writeln(''); + exit; + } + if ($zip) { $this->output->write("\x0D"); $this->output->write(" |- Downloading package... 100%"); From 08e2bb558ac54c99517eb893a513a193b817f1e3 Mon Sep 17 00:00:00 2001 From: Josh Weiss Date: Sat, 25 Feb 2017 23:51:05 -0800 Subject: [PATCH 4/7] Fixing a minor bug in Number validation. (#1329) The function validateNumber only checks for numeric values. **By PHP isNumeric** "42", 1337, 0x539, 02471, 0b10100111001, 1337e0, "not numeric", array(), 9.1 **Will evaluate respectively** '42' is numeric '1337' is numeric '1337' is numeric '1337' is numeric '1337' is numeric '1337' is numeric 'not numeric' is NOT numeric 'Array' is NOT numeric '9.1' is numeric Grav though does not support all value types for a variety of reasons. One being YAML Blueprint definitions, where it makes sense to make a new type value for a more specialized format. Specifically for numbers if a more advance number format it would make sense to make a new type for that number format. YAML spec specifically allows both Integer or Float contexts, as seen in the validate class validateInt and validateFloat. This is useful when the output formats explicitly needs to be a certain format. However, in the case of generic numeric contexts, which numbers could be floats or ints dynamically, the values are cast back to an int currently in Grav numeric validation. Having dynamic primitive number formats is important, true in an interpreted language like JavaScript where 1 and 1.0 will both work for a number value. The reason to cast the type of the variable still is to prevent wide selection of number formats, and to keep Grav in line with primitive YAML field formats. --- system/src/Grav/Common/Data/Validation.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/system/src/Grav/Common/Data/Validation.php b/system/src/Grav/Common/Data/Validation.php index b597cd642..fd886b017 100644 --- a/system/src/Grav/Common/Data/Validation.php +++ b/system/src/Grav/Common/Data/Validation.php @@ -339,7 +339,7 @@ class Validation protected static function filterNumber($value, array $params, array $field) { - return (int) $value; + return self::validateFloat($value, $params) ? (float) $value : (int) $value; } protected static function filterDateTime($value, array $params, array $field) @@ -585,7 +585,7 @@ class Validation $values[$key] = array_map('trim', explode(',', $value)); } else { $values[$key] = trim($value); - } + } } } From e771938672a6177a9da3deb841d3d64c74a8a86e Mon Sep 17 00:00:00 2001 From: Flavio Copes Date: Sun, 26 Feb 2017 08:55:22 +0100 Subject: [PATCH 5/7] Changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57403a4d4..45d65406c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ * Added a new `Utils::isValidUrl()` method 1. [](#improved) * Genericized `direct-install` so it can be called via Admin plugin +1. [](#bugfix) + * Fix a minor bug in Number validation [#1329](https://github.com/getgrav/grav/issues/1329) # v1.1.17 ## 02/17/2017 From da2a0f507b8b75218502482b0329c0f1e57cb6b0 Mon Sep 17 00:00:00 2001 From: Andy Miller Date: Sun, 26 Feb 2017 14:10:43 -0700 Subject: [PATCH 6/7] Added a fragment + query test --- tests/unit/Grav/Common/Markdown/ParsedownTest.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/Grav/Common/Markdown/ParsedownTest.php b/tests/unit/Grav/Common/Markdown/ParsedownTest.php index bbd1cad3a..e73e11f01 100644 --- a/tests/unit/Grav/Common/Markdown/ParsedownTest.php +++ b/tests/unit/Grav/Common/Markdown/ParsedownTest.php @@ -675,6 +675,8 @@ class ParsedownTest extends \Codeception\TestCase\Test { $this->uri->initializeWithURL('http://testing.dev/item2/item2-2')->init(); + $this->assertSame('

Anchor Class

', + $this->parsedown->text('[Anchor Class](?classes=button#something)')); $this->assertSame('

Relative Class

', $this->parsedown->text('[Relative Class](../item2-3?classes=button)')); $this->assertSame('

Relative ID

', From 175fcb9415201d8065015497af83e303b4c58f8e Mon Sep 17 00:00:00 2001 From: Matias Griese Date: Mon, 27 Feb 2017 12:53:24 +0200 Subject: [PATCH 7/7] Fixed exception when trying to find user account and there is no `user://accounts` folder --- CHANGELOG.md | 3 ++- system/src/Grav/Common/User/User.php | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45d65406c..620be3db5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,8 @@ 1. [](#improved) * Genericized `direct-install` so it can be called via Admin plugin 1. [](#bugfix) - * Fix a minor bug in Number validation [#1329](https://github.com/getgrav/grav/issues/1329) + * Fixed a minor bug in Number validation [#1329](https://github.com/getgrav/grav/issues/1329) + * Fixed exception when trying to find user account and there is no `user://accounts` folder # v1.1.17 ## 02/17/2017 diff --git a/system/src/Grav/Common/User/User.php b/system/src/Grav/Common/User/User.php index fcc8f6c43..d5c108ff8 100644 --- a/system/src/Grav/Common/User/User.php +++ b/system/src/Grav/Common/User/User.php @@ -57,14 +57,14 @@ class User extends Data /** * Find a user by username, email, etc * - * @param $query the query to search for + * @param string $query the query to search for * @param array $fields the fields to search * @return User */ public static function find($query, $fields = ['username', 'email']) { $account_dir = Grav::instance()['locator']->findResource('account://'); - $files = array_diff(scandir($account_dir), ['.', '..']); + $files = $account_dir ? array_diff(scandir($account_dir), ['.', '..']) : []; // Try with username first, you never know! if (in_array('username', $fields)) { @@ -100,7 +100,7 @@ class User extends Data public static function remove($username) { $file_path = Grav::instance()['locator']->findResource('account://' . $username . YAML_EXT); - if (file_exists($file_path) && unlink($file_path)) { + if ($file_path && unlink($file_path)) { return true; }