diff --git a/bin/crawler b/bin/crawler new file mode 100755 index 0000000..eb8689d --- /dev/null +++ b/bin/crawler @@ -0,0 +1,13 @@ +#!/usr/bin/env php + +run(); diff --git a/composer.json b/composer.json index 2aadb8b..8f7b336 100644 --- a/composer.json +++ b/composer.json @@ -3,7 +3,9 @@ "description": "generate full sitemap report", "type": "project", "require": { + "php": "^7.1", "spatie/crawler": "^4.6", + "symfony/console": "^4.0|^5.0", "cweagans/composer-patches": "~1.0" }, "require-dev": { diff --git a/composer.lock b/composer.lock index 2fe9042..3bf549f 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "635f636b861c8e79694a12db07fa6800", + "content-hash": "0f0e1a413fb5eda56634b65a8abfdbf4", "packages": [ { "name": "cweagans/composer-patches", @@ -490,6 +490,55 @@ "description": "A basic but flexible php tree data structure and a fluent tree builder implementation.", "time": "2015-08-20T20:50:35+00:00" }, + { + "name": "psr/container", + "version": "1.0.0", + "source": { + "type": "git", + "url": "https://github.com/php-fig/container.git", + "reference": "b7ce3b176482dbbc1245ebf52b181af44c2cf55f" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/container/zipball/b7ce3b176482dbbc1245ebf52b181af44c2cf55f", + "reference": "b7ce3b176482dbbc1245ebf52b181af44c2cf55f", + "shasum": "" + }, + "require": { + "php": ">=5.3.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\Container\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "http://www.php-fig.org/" + } + ], + "description": "Common Container Interface (PHP FIG PSR-11)", + "homepage": "https://github.com/php-fig/container", + "keywords": [ + "PSR-11", + "container", + "container-interface", + "container-interop", + "psr" + ], + "time": "2017-02-14T16:28:37+00:00" + }, { "name": "psr/http-message", "version": "1.0.1", @@ -942,6 +991,82 @@ ], "time": "2019-12-15T18:52:09+00:00" }, + { + "name": "symfony/console", + "version": "v5.0.4", + "source": { + "type": "git", + "url": "https://github.com/symfony/console.git", + "reference": "91c294166c38d8c0858a86fad76d8c14dc1144c8" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/console/zipball/91c294166c38d8c0858a86fad76d8c14dc1144c8", + "reference": "91c294166c38d8c0858a86fad76d8c14dc1144c8", + "shasum": "" + }, + "require": { + "php": "^7.2.5", + "symfony/polyfill-mbstring": "~1.0", + "symfony/polyfill-php73": "^1.8", + "symfony/service-contracts": "^1.1|^2" + }, + "conflict": { + "symfony/dependency-injection": "<4.4", + "symfony/event-dispatcher": "<4.4", + "symfony/lock": "<4.4", + "symfony/process": "<4.4" + }, + "provide": { + "psr/log-implementation": "1.0" + }, + "require-dev": { + "psr/log": "~1.0", + "symfony/config": "^4.4|^5.0", + "symfony/dependency-injection": "^4.4|^5.0", + "symfony/event-dispatcher": "^4.4|^5.0", + "symfony/lock": "^4.4|^5.0", + "symfony/process": "^4.4|^5.0", + "symfony/var-dumper": "^4.4|^5.0" + }, + "suggest": { + "psr/log": "For using the console logger", + "symfony/event-dispatcher": "", + "symfony/lock": "", + "symfony/process": "" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "5.0-dev" + } + }, + "autoload": { + "psr-4": { + "Symfony\\Component\\Console\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Fabien Potencier", + "email": "fabien@symfony.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony Console Component", + "homepage": "https://symfony.com", + "time": "2020-01-25T15:56:29+00:00" + }, { "name": "symfony/dom-crawler", "version": "v5.0.4", @@ -1120,6 +1245,64 @@ ], "time": "2020-01-13T11:15:53+00:00" }, + { + "name": "symfony/polyfill-php73", + "version": "v1.14.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-php73.git", + "reference": "5e66a0fa1070bf46bec4bea7962d285108edd675" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-php73/zipball/5e66a0fa1070bf46bec4bea7962d285108edd675", + "reference": "5e66a0fa1070bf46bec4bea7962d285108edd675", + "shasum": "" + }, + "require": { + "php": ">=5.3.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.14-dev" + } + }, + "autoload": { + "psr-4": { + "Symfony\\Polyfill\\Php73\\": "" + }, + "files": [ + "bootstrap.php" + ], + "classmap": [ + "Resources/stubs" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill backporting some PHP 7.3+ features to lower PHP versions", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "polyfill", + "portable", + "shim" + ], + "time": "2020-01-13T11:15:53+00:00" + }, { "name": "symfony/process", "version": "v5.0.4", @@ -1169,6 +1352,64 @@ "homepage": "https://symfony.com", "time": "2020-01-09T09:53:06+00:00" }, + { + "name": "symfony/service-contracts", + "version": "v2.0.1", + "source": { + "type": "git", + "url": "https://github.com/symfony/service-contracts.git", + "reference": "144c5e51266b281231e947b51223ba14acf1a749" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/service-contracts/zipball/144c5e51266b281231e947b51223ba14acf1a749", + "reference": "144c5e51266b281231e947b51223ba14acf1a749", + "shasum": "" + }, + "require": { + "php": "^7.2.5", + "psr/container": "^1.0" + }, + "suggest": { + "symfony/service-implementation": "" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.0-dev" + } + }, + "autoload": { + "psr-4": { + "Symfony\\Contracts\\Service\\": "" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Generic abstractions related to writing services", + "homepage": "https://symfony.com", + "keywords": [ + "abstractions", + "contracts", + "decoupling", + "interfaces", + "interoperability", + "standards" + ], + "time": "2019-11-18T17:27:11+00:00" + }, { "name": "symfony/var-dumper", "version": "v5.0.4", @@ -2761,6 +3002,8 @@ "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, - "platform": [], + "platform": { + "php": "^7.1" + }, "platform-dev": [] } diff --git a/src/ConsoleApplication.php b/src/ConsoleApplication.php new file mode 100644 index 0000000..ddb5fef --- /dev/null +++ b/src/ConsoleApplication.php @@ -0,0 +1,22 @@ +add(new CrawlCommand()); + } + + public function getLongVersion() + { + return parent::getLongVersion().' by JHodges'; + } +} diff --git a/src/CrawlCommand.php b/src/CrawlCommand.php new file mode 100644 index 0000000..8a64de1 --- /dev/null +++ b/src/CrawlCommand.php @@ -0,0 +1,54 @@ +setName('crawl') + ->setDescription('Crawl and generate sitemap for the website.') + ->addArgument( + 'url', + InputArgument::REQUIRED, + 'The url to check' + )->addOption( + 'found-on', + 'f', + InputOption::VALUE_NONE, + 'Display found on URLs' + ); + } + + /** + * @param \Symfony\Component\Console\Input\InputInterface $input + * @param \Symfony\Component\Console\Output\OutputInterface $output + * + * @return int + */ + protected function execute(InputInterface $input, OutputInterface $output) + { + $baseUrl = $input->getArgument('url'); + $crawler=new Crawler($baseUrl); + $crawler->crawl($baseUrl); + + foreach($crawler->getResults() as $url=>$result){ + $output->writeln("{$result['code']} {$url}"); + if($input->getOption('found-on')){ + foreach($result['foundOn'] as $url=>$count){ + $output->writeln(" -> ($count) $url"); + } + } + } + + return 0; + } +}