Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
ac9e614086 | ||
|
|
bea673807c | ||
|
35aeeeca72 | ||
|
|
5f71eeb6a5 | ||
|
|
2ece7c9e99 | ||
| f739042cfd |
@ -5,7 +5,7 @@ name: testsuite
|
||||
steps:
|
||||
|
||||
- name: composer install
|
||||
image: chialab/php:7.4
|
||||
image: chialab/php:7.3
|
||||
commands:
|
||||
- composer install
|
||||
volumes:
|
||||
@ -20,14 +20,14 @@ steps:
|
||||
- echo "Ready!"
|
||||
|
||||
- name: run tests
|
||||
image: chialab/php:7.4
|
||||
image: chialab/php:7.3
|
||||
commands:
|
||||
- URL=http://testserver:8080 vendor/bin/phpunit tests --testdox --color=always --no-interaction
|
||||
|
||||
services:
|
||||
|
||||
- name: testserver
|
||||
image: node:15.2.0-alpine3.12
|
||||
image: node:15.2
|
||||
detach: true
|
||||
commands:
|
||||
- cd tests/server/
|
||||
|
||||
@ -3,7 +3,7 @@ Similar project to [spatie/http-status-check](https://github.com/spatie/http-sta
|
||||
## Install
|
||||
|
||||
```plain
|
||||
composer config repositories.jhodges composer https://git.jhodges.co.uk/composer
|
||||
composer config repositories.jhodges composer https://composer.jhodges.co.uk/
|
||||
composer require jhodges/sitemap
|
||||
```
|
||||
|
||||
|
||||
@ -3,9 +3,10 @@
|
||||
"description": "Generate full sitemap report using spatie/crawler",
|
||||
"type": "library",
|
||||
"require": {
|
||||
"php": "^7.4",
|
||||
"spatie/crawler": "^5.0",
|
||||
"cweagans/composer-patches": "~1.0"
|
||||
"php": "^7.1",
|
||||
"spatie/crawler": "4.6.6",
|
||||
"cweagans/composer-patches": "~1.0",
|
||||
"guzzlehttp/guzzle": "6.5.2"
|
||||
},
|
||||
"require-dev": {
|
||||
"phpunit/phpunit": "^8.5"
|
||||
@ -25,6 +26,9 @@
|
||||
"patches": {
|
||||
"spatie/crawler": {
|
||||
"add crawled again observer": "https://patch-diff.githubusercontent.com/raw/spatie/crawler/pull/280.patch"
|
||||
},
|
||||
"guzzlehttp/guzzle": {
|
||||
"Status code must be an integer value between 1xx and 5xx": "https://github.com/guzzle/guzzle/commit/f81cd6cdff1213f90de8f012489017510e3d6ff4.patch"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
556
composer.lock
generated
556
composer.lock
generated
@ -4,7 +4,7 @@
|
||||
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
|
||||
"This file is @generated automatically"
|
||||
],
|
||||
"content-hash": "c06e067a5805658d93962a86f0d985dd",
|
||||
"content-hash": "7ab60eca38a75bfd4220b46a7b45a582",
|
||||
"packages": [
|
||||
{
|
||||
"name": "cweagans/composer-patches",
|
||||
@ -52,43 +52,40 @@
|
||||
},
|
||||
{
|
||||
"name": "guzzlehttp/guzzle",
|
||||
"version": "7.2.0",
|
||||
"version": "6.5.2",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/guzzle/guzzle.git",
|
||||
"reference": "0aa74dfb41ae110835923ef10a9d803a22d50e79"
|
||||
"reference": "43ece0e75098b7ecd8d13918293029e555a50f82"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/guzzle/guzzle/zipball/0aa74dfb41ae110835923ef10a9d803a22d50e79",
|
||||
"reference": "0aa74dfb41ae110835923ef10a9d803a22d50e79",
|
||||
"url": "https://api.github.com/repos/guzzle/guzzle/zipball/43ece0e75098b7ecd8d13918293029e555a50f82",
|
||||
"reference": "43ece0e75098b7ecd8d13918293029e555a50f82",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"ext-json": "*",
|
||||
"guzzlehttp/promises": "^1.4",
|
||||
"guzzlehttp/psr7": "^1.7",
|
||||
"php": "^7.2.5 || ^8.0",
|
||||
"psr/http-client": "^1.0"
|
||||
},
|
||||
"provide": {
|
||||
"psr/http-client-implementation": "1.0"
|
||||
"guzzlehttp/promises": "^1.0",
|
||||
"guzzlehttp/psr7": "^1.6.1",
|
||||
"php": ">=5.5"
|
||||
},
|
||||
"require-dev": {
|
||||
"ext-curl": "*",
|
||||
"php-http/client-integration-tests": "^3.0",
|
||||
"phpunit/phpunit": "^8.5.5 || ^9.3.5",
|
||||
"phpunit/phpunit": "^4.8.35 || ^5.7 || ^6.4 || ^7.0",
|
||||
"psr/log": "^1.1"
|
||||
},
|
||||
"suggest": {
|
||||
"ext-curl": "Required for CURL handler support",
|
||||
"ext-intl": "Required for Internationalized Domain Name (IDN) support",
|
||||
"psr/log": "Required for using the Log middleware"
|
||||
},
|
||||
"type": "library",
|
||||
"extra": {
|
||||
"branch-alias": {
|
||||
"dev-master": "7.1-dev"
|
||||
"dev-master": "6.5-dev"
|
||||
},
|
||||
"patches_applied": {
|
||||
"Status code must be an integer value between 1xx and 5xx": "https://github.com/guzzle/guzzle/commit/f81cd6cdff1213f90de8f012489017510e3d6ff4.patch"
|
||||
}
|
||||
},
|
||||
"autoload": {
|
||||
@ -108,11 +105,6 @@
|
||||
"name": "Michael Dowling",
|
||||
"email": "mtdowling@gmail.com",
|
||||
"homepage": "https://github.com/mtdowling"
|
||||
},
|
||||
{
|
||||
"name": "Márk Sági-Kazár",
|
||||
"email": "mark.sagikazar@gmail.com",
|
||||
"homepage": "https://sagikazarmark.hu"
|
||||
}
|
||||
],
|
||||
"description": "Guzzle is a PHP HTTP client library",
|
||||
@ -123,30 +115,10 @@
|
||||
"framework",
|
||||
"http",
|
||||
"http client",
|
||||
"psr-18",
|
||||
"psr-7",
|
||||
"rest",
|
||||
"web service"
|
||||
],
|
||||
"funding": [
|
||||
{
|
||||
"url": "https://github.com/GrahamCampbell",
|
||||
"type": "github"
|
||||
},
|
||||
{
|
||||
"url": "https://github.com/Nyholm",
|
||||
"type": "github"
|
||||
},
|
||||
{
|
||||
"url": "https://github.com/alexeyshockov",
|
||||
"type": "github"
|
||||
},
|
||||
{
|
||||
"url": "https://github.com/gmponos",
|
||||
"type": "github"
|
||||
}
|
||||
],
|
||||
"time": "2020-10-10T11:47:56+00:00"
|
||||
"time": "2019-12-23T11:57:10+00:00"
|
||||
},
|
||||
{
|
||||
"name": "guzzlehttp/promises",
|
||||
@ -270,142 +242,6 @@
|
||||
],
|
||||
"time": "2020-09-30T07:37:11+00:00"
|
||||
},
|
||||
{
|
||||
"name": "illuminate/collections",
|
||||
"version": "v8.14.0",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/illuminate/collections.git",
|
||||
"reference": "e61e796ff98e19eac5e6735ea6abfa66f4a2f6db"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/illuminate/collections/zipball/e61e796ff98e19eac5e6735ea6abfa66f4a2f6db",
|
||||
"reference": "e61e796ff98e19eac5e6735ea6abfa66f4a2f6db",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"illuminate/contracts": "^8.0",
|
||||
"illuminate/macroable": "^8.0",
|
||||
"php": "^7.3|^8.0"
|
||||
},
|
||||
"suggest": {
|
||||
"symfony/var-dumper": "Required to use the dump method (^5.1)."
|
||||
},
|
||||
"type": "library",
|
||||
"extra": {
|
||||
"branch-alias": {
|
||||
"dev-master": "8.x-dev"
|
||||
}
|
||||
},
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"Illuminate\\Support\\": ""
|
||||
},
|
||||
"files": [
|
||||
"helpers.php"
|
||||
]
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "Taylor Otwell",
|
||||
"email": "taylor@laravel.com"
|
||||
}
|
||||
],
|
||||
"description": "The Illuminate Collections package.",
|
||||
"homepage": "https://laravel.com",
|
||||
"time": "2020-10-27T15:20:30+00:00"
|
||||
},
|
||||
{
|
||||
"name": "illuminate/contracts",
|
||||
"version": "v8.14.0",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/illuminate/contracts.git",
|
||||
"reference": "c9c9ae3e00526010b9b46c644287765ef60d6147"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/illuminate/contracts/zipball/c9c9ae3e00526010b9b46c644287765ef60d6147",
|
||||
"reference": "c9c9ae3e00526010b9b46c644287765ef60d6147",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"php": "^7.3|^8.0",
|
||||
"psr/container": "^1.0",
|
||||
"psr/simple-cache": "^1.0"
|
||||
},
|
||||
"type": "library",
|
||||
"extra": {
|
||||
"branch-alias": {
|
||||
"dev-master": "8.x-dev"
|
||||
}
|
||||
},
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"Illuminate\\Contracts\\": ""
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "Taylor Otwell",
|
||||
"email": "taylor@laravel.com"
|
||||
}
|
||||
],
|
||||
"description": "The Illuminate Contracts package.",
|
||||
"homepage": "https://laravel.com",
|
||||
"time": "2020-11-04T19:55:57+00:00"
|
||||
},
|
||||
{
|
||||
"name": "illuminate/macroable",
|
||||
"version": "v8.14.0",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/illuminate/macroable.git",
|
||||
"reference": "300aa13c086f25116b5f3cde3ca54ff5c822fb05"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/illuminate/macroable/zipball/300aa13c086f25116b5f3cde3ca54ff5c822fb05",
|
||||
"reference": "300aa13c086f25116b5f3cde3ca54ff5c822fb05",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"php": "^7.3|^8.0"
|
||||
},
|
||||
"type": "library",
|
||||
"extra": {
|
||||
"branch-alias": {
|
||||
"dev-master": "8.x-dev"
|
||||
}
|
||||
},
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"Illuminate\\Support\\": ""
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "Taylor Otwell",
|
||||
"email": "taylor@laravel.com"
|
||||
}
|
||||
],
|
||||
"description": "The Illuminate Macroable package.",
|
||||
"homepage": "https://laravel.com",
|
||||
"time": "2020-10-27T15:20:30+00:00"
|
||||
},
|
||||
{
|
||||
"name": "intervention/image",
|
||||
"version": "2.5.1",
|
||||
@ -559,6 +395,12 @@
|
||||
"sftp",
|
||||
"storage"
|
||||
],
|
||||
"funding": [
|
||||
{
|
||||
"url": "https://offset.earth/frankdejonge",
|
||||
"type": "other"
|
||||
}
|
||||
],
|
||||
"time": "2020-08-23T07:39:11+00:00"
|
||||
},
|
||||
{
|
||||
@ -675,23 +517,20 @@
|
||||
},
|
||||
{
|
||||
"name": "nicmart/tree",
|
||||
"version": "0.3.0",
|
||||
"version": "v0.2.7",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/nicmart/Tree.git",
|
||||
"reference": "049bad40787ad2bc44a916c7bf0ccdd3d3a113b1"
|
||||
"reference": "0616b54bb49938e1a816141d7943db48ebf76938"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/nicmart/Tree/zipball/049bad40787ad2bc44a916c7bf0ccdd3d3a113b1",
|
||||
"reference": "049bad40787ad2bc44a916c7bf0ccdd3d3a113b1",
|
||||
"url": "https://api.github.com/repos/nicmart/Tree/zipball/0616b54bb49938e1a816141d7943db48ebf76938",
|
||||
"reference": "0616b54bb49938e1a816141d7943db48ebf76938",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"php": "^5.4 || ^7.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"phpunit/phpunit": "^4.8"
|
||||
"php": ">=5.4"
|
||||
},
|
||||
"type": "library",
|
||||
"autoload": {
|
||||
@ -710,105 +549,7 @@
|
||||
}
|
||||
],
|
||||
"description": "A basic but flexible php tree data structure and a fluent tree builder implementation.",
|
||||
"time": "2020-06-18T07:18:00+00:00"
|
||||
},
|
||||
{
|
||||
"name": "psr/container",
|
||||
"version": "1.0.0",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/php-fig/container.git",
|
||||
"reference": "b7ce3b176482dbbc1245ebf52b181af44c2cf55f"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/php-fig/container/zipball/b7ce3b176482dbbc1245ebf52b181af44c2cf55f",
|
||||
"reference": "b7ce3b176482dbbc1245ebf52b181af44c2cf55f",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"php": ">=5.3.0"
|
||||
},
|
||||
"type": "library",
|
||||
"extra": {
|
||||
"branch-alias": {
|
||||
"dev-master": "1.0.x-dev"
|
||||
}
|
||||
},
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"Psr\\Container\\": "src/"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "PHP-FIG",
|
||||
"homepage": "http://www.php-fig.org/"
|
||||
}
|
||||
],
|
||||
"description": "Common Container Interface (PHP FIG PSR-11)",
|
||||
"homepage": "https://github.com/php-fig/container",
|
||||
"keywords": [
|
||||
"PSR-11",
|
||||
"container",
|
||||
"container-interface",
|
||||
"container-interop",
|
||||
"psr"
|
||||
],
|
||||
"time": "2017-02-14T16:28:37+00:00"
|
||||
},
|
||||
{
|
||||
"name": "psr/http-client",
|
||||
"version": "1.0.1",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/php-fig/http-client.git",
|
||||
"reference": "2dfb5f6c5eff0e91e20e913f8c5452ed95b86621"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/php-fig/http-client/zipball/2dfb5f6c5eff0e91e20e913f8c5452ed95b86621",
|
||||
"reference": "2dfb5f6c5eff0e91e20e913f8c5452ed95b86621",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"php": "^7.0 || ^8.0",
|
||||
"psr/http-message": "^1.0"
|
||||
},
|
||||
"type": "library",
|
||||
"extra": {
|
||||
"branch-alias": {
|
||||
"dev-master": "1.0.x-dev"
|
||||
}
|
||||
},
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"Psr\\Http\\Client\\": "src/"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "PHP-FIG",
|
||||
"homepage": "http://www.php-fig.org/"
|
||||
}
|
||||
],
|
||||
"description": "Common interface for HTTP clients",
|
||||
"homepage": "https://github.com/php-fig/http-client",
|
||||
"keywords": [
|
||||
"http",
|
||||
"http-client",
|
||||
"psr",
|
||||
"psr-18"
|
||||
],
|
||||
"time": "2020-06-29T06:28:15+00:00"
|
||||
"time": "2015-08-20T20:50:35+00:00"
|
||||
},
|
||||
{
|
||||
"name": "psr/http-message",
|
||||
@ -907,54 +648,6 @@
|
||||
],
|
||||
"time": "2020-03-23T09:12:05+00:00"
|
||||
},
|
||||
{
|
||||
"name": "psr/simple-cache",
|
||||
"version": "1.0.1",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/php-fig/simple-cache.git",
|
||||
"reference": "408d5eafb83c57f6365a3ca330ff23aa4a5fa39b"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/php-fig/simple-cache/zipball/408d5eafb83c57f6365a3ca330ff23aa4a5fa39b",
|
||||
"reference": "408d5eafb83c57f6365a3ca330ff23aa4a5fa39b",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"php": ">=5.3.0"
|
||||
},
|
||||
"type": "library",
|
||||
"extra": {
|
||||
"branch-alias": {
|
||||
"dev-master": "1.0.x-dev"
|
||||
}
|
||||
},
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"Psr\\SimpleCache\\": "src/"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "PHP-FIG",
|
||||
"homepage": "http://www.php-fig.org/"
|
||||
}
|
||||
],
|
||||
"description": "Common interfaces for simple caching",
|
||||
"keywords": [
|
||||
"cache",
|
||||
"caching",
|
||||
"psr",
|
||||
"psr-16",
|
||||
"simple-cache"
|
||||
],
|
||||
"time": "2017-10-23T01:57:42+00:00"
|
||||
},
|
||||
{
|
||||
"name": "ralouphie/getallheaders",
|
||||
"version": "3.0.3",
|
||||
@ -997,27 +690,27 @@
|
||||
},
|
||||
{
|
||||
"name": "spatie/browsershot",
|
||||
"version": "3.40.3",
|
||||
"version": "3.40.2",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/spatie/browsershot.git",
|
||||
"reference": "321f0d8ec8e303d145968ebece9fa1ebf2497864"
|
||||
"reference": "3e55eaf5ab8cee65d1661a567e89b3374afb9116"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/spatie/browsershot/zipball/321f0d8ec8e303d145968ebece9fa1ebf2497864",
|
||||
"reference": "321f0d8ec8e303d145968ebece9fa1ebf2497864",
|
||||
"url": "https://api.github.com/repos/spatie/browsershot/zipball/3e55eaf5ab8cee65d1661a567e89b3374afb9116",
|
||||
"reference": "3e55eaf5ab8cee65d1661a567e89b3374afb9116",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"php": "^7.4|^8.0",
|
||||
"php": "^7.1",
|
||||
"spatie/image": "^1.5.3",
|
||||
"spatie/temporary-directory": "^1.1",
|
||||
"symfony/process": "^5.0"
|
||||
"symfony/process": "^4.2|^5.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"phpunit/phpunit": "^9.0",
|
||||
"spatie/phpunit-snapshot-assertions": "^4.2.3"
|
||||
"phpunit/phpunit": "^6.1|^7.5",
|
||||
"spatie/phpunit-snapshot-assertions": "^1.0"
|
||||
},
|
||||
"type": "library",
|
||||
"autoload": {
|
||||
@ -1055,38 +748,44 @@
|
||||
"type": "github"
|
||||
}
|
||||
],
|
||||
"time": "2020-11-12T16:31:59+00:00"
|
||||
"time": "2020-11-11T22:18:15+00:00"
|
||||
},
|
||||
{
|
||||
"name": "spatie/crawler",
|
||||
"version": "5.0.1",
|
||||
"version": "4.6.6",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/spatie/crawler.git",
|
||||
"reference": "d94b5b651bcd85566863b25fec1e8e498ac3c30a"
|
||||
"reference": "6f814a401cee20b24a1ffe53d8deddac90d1a42d"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/spatie/crawler/zipball/d94b5b651bcd85566863b25fec1e8e498ac3c30a",
|
||||
"reference": "d94b5b651bcd85566863b25fec1e8e498ac3c30a",
|
||||
"url": "https://api.github.com/repos/spatie/crawler/zipball/6f814a401cee20b24a1ffe53d8deddac90d1a42d",
|
||||
"reference": "6f814a401cee20b24a1ffe53d8deddac90d1a42d",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"guzzlehttp/guzzle": "^6.3|^7.0",
|
||||
"guzzlehttp/guzzle": "^6.3 || ^7.0",
|
||||
"guzzlehttp/psr7": "^1.4",
|
||||
"illuminate/collections": "^8.6",
|
||||
"nicmart/tree": "^0.3.0",
|
||||
"php": "^7.4",
|
||||
"nicmart/tree": "^0.2.7",
|
||||
"php": "^7.1",
|
||||
"spatie/browsershot": "^3.14",
|
||||
"spatie/robots-txt": "^1.0.1",
|
||||
"symfony/dom-crawler": "^5.0"
|
||||
"symfony/dom-crawler": "^4.0 || ^5.0",
|
||||
"tightenco/collect": "^5.6|^6.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"phpunit/phpunit": "^9.3"
|
||||
"larapack/dd": "^1.1",
|
||||
"phpunit/phpunit": "^7.0"
|
||||
},
|
||||
"type": "library",
|
||||
"extra": {
|
||||
"patches_applied": []
|
||||
"branch-alias": {
|
||||
"dev-master": "0.0.x-dev"
|
||||
},
|
||||
"patches_applied": {
|
||||
"add crawled again observer": "https://patch-diff.githubusercontent.com/raw/spatie/crawler/pull/280.patch"
|
||||
}
|
||||
},
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
@ -1111,13 +810,7 @@
|
||||
"spatie",
|
||||
"website"
|
||||
],
|
||||
"funding": [
|
||||
{
|
||||
"url": "https://github.com/spatie",
|
||||
"type": "github"
|
||||
}
|
||||
],
|
||||
"time": "2020-10-09T09:13:50+00:00"
|
||||
"time": "2020-01-31T10:51:58+00:00"
|
||||
},
|
||||
{
|
||||
"name": "spatie/image",
|
||||
@ -1688,6 +1381,141 @@
|
||||
}
|
||||
],
|
||||
"time": "2020-10-24T12:01:57+00:00"
|
||||
},
|
||||
{
|
||||
"name": "symfony/var-dumper",
|
||||
"version": "v5.1.8",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/symfony/var-dumper.git",
|
||||
"reference": "4e13f3fcefb1fcaaa5efb5403581406f4e840b9a"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/symfony/var-dumper/zipball/4e13f3fcefb1fcaaa5efb5403581406f4e840b9a",
|
||||
"reference": "4e13f3fcefb1fcaaa5efb5403581406f4e840b9a",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"php": ">=7.2.5",
|
||||
"symfony/polyfill-mbstring": "~1.0",
|
||||
"symfony/polyfill-php80": "^1.15"
|
||||
},
|
||||
"conflict": {
|
||||
"phpunit/phpunit": "<5.4.3",
|
||||
"symfony/console": "<4.4"
|
||||
},
|
||||
"require-dev": {
|
||||
"ext-iconv": "*",
|
||||
"symfony/console": "^4.4|^5.0",
|
||||
"symfony/process": "^4.4|^5.0",
|
||||
"twig/twig": "^2.4|^3.0"
|
||||
},
|
||||
"suggest": {
|
||||
"ext-iconv": "To convert non-UTF-8 strings to UTF-8 (or symfony/polyfill-iconv in case ext-iconv cannot be used).",
|
||||
"ext-intl": "To show region name in time zone dump",
|
||||
"symfony/console": "To use the ServerDumpCommand and/or the bin/var-dump-server script"
|
||||
},
|
||||
"bin": [
|
||||
"Resources/bin/var-dump-server"
|
||||
],
|
||||
"type": "library",
|
||||
"autoload": {
|
||||
"files": [
|
||||
"Resources/functions/dump.php"
|
||||
],
|
||||
"psr-4": {
|
||||
"Symfony\\Component\\VarDumper\\": ""
|
||||
},
|
||||
"exclude-from-classmap": [
|
||||
"/Tests/"
|
||||
]
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "Nicolas Grekas",
|
||||
"email": "p@tchwork.com"
|
||||
},
|
||||
{
|
||||
"name": "Symfony Community",
|
||||
"homepage": "https://symfony.com/contributors"
|
||||
}
|
||||
],
|
||||
"description": "Symfony mechanism for exploring and dumping PHP variables",
|
||||
"homepage": "https://symfony.com",
|
||||
"keywords": [
|
||||
"debug",
|
||||
"dump"
|
||||
],
|
||||
"funding": [
|
||||
{
|
||||
"url": "https://symfony.com/sponsor",
|
||||
"type": "custom"
|
||||
},
|
||||
{
|
||||
"url": "https://github.com/fabpot",
|
||||
"type": "github"
|
||||
},
|
||||
{
|
||||
"url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
|
||||
"type": "tidelift"
|
||||
}
|
||||
],
|
||||
"time": "2020-10-27T10:11:13+00:00"
|
||||
},
|
||||
{
|
||||
"name": "tightenco/collect",
|
||||
"version": "v6.18.34",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/tighten/collect.git",
|
||||
"reference": "624b86cde21f8d9ba14ec0e4974ef0aca3922a1c"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/tighten/collect/zipball/624b86cde21f8d9ba14ec0e4974ef0aca3922a1c",
|
||||
"reference": "624b86cde21f8d9ba14ec0e4974ef0aca3922a1c",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"php": "^7.1.3",
|
||||
"symfony/var-dumper": "^3.4 || ^4.0 || ^5.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"mockery/mockery": "^1.0",
|
||||
"nesbot/carbon": "^2.23.0",
|
||||
"phpunit/phpunit": "^7.0"
|
||||
},
|
||||
"type": "library",
|
||||
"autoload": {
|
||||
"files": [
|
||||
"src/Collect/Support/helpers.php",
|
||||
"src/Collect/Support/alias.php"
|
||||
],
|
||||
"psr-4": {
|
||||
"Tightenco\\Collect\\": "src/Collect"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "Taylor Otwell",
|
||||
"email": "taylorotwell@gmail.com"
|
||||
}
|
||||
],
|
||||
"description": "Collect - Illuminate Collections as a separate package.",
|
||||
"keywords": [
|
||||
"collection",
|
||||
"laravel"
|
||||
],
|
||||
"time": "2020-08-21T22:25:58+00:00"
|
||||
}
|
||||
],
|
||||
"packages-dev": [
|
||||
@ -3120,6 +2948,12 @@
|
||||
}
|
||||
],
|
||||
"description": "A small library for converting tokenized PHP source code into XML and potentially other formats",
|
||||
"funding": [
|
||||
{
|
||||
"url": "https://github.com/theseer",
|
||||
"type": "github"
|
||||
}
|
||||
],
|
||||
"time": "2020-07-12T23:59:07+00:00"
|
||||
},
|
||||
{
|
||||
@ -3178,7 +3012,7 @@
|
||||
"prefer-stable": false,
|
||||
"prefer-lowest": false,
|
||||
"platform": {
|
||||
"php": "^7.4"
|
||||
"php": "^7.1"
|
||||
},
|
||||
"platform-dev": [],
|
||||
"plugin-api-version": "1.1.0"
|
||||
|
||||
17
docker/Dockerfile
Normal file
17
docker/Dockerfile
Normal file
@ -0,0 +1,17 @@
|
||||
FROM markizano/devuan:beowulf-amd64
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y php-cli git unzip php-xml
|
||||
|
||||
COPY get_composer.sh /root/
|
||||
RUN bash /root/get_composer.sh && rm /root/get_composer.sh
|
||||
|
||||
RUN mkdir /app
|
||||
WORKDIR /app
|
||||
RUN composer init && composer config repositories.jhodges composer https://composer.jhodges.co.uk && composer require jhodges/sitemap
|
||||
|
||||
COPY crawl.php /app/
|
||||
|
||||
RUN apt clean
|
||||
|
||||
ENTRYPOINT php crawl.php
|
||||
45
docker/crawl.php
Normal file
45
docker/crawl.php
Normal file
@ -0,0 +1,45 @@
|
||||
<?php
|
||||
require_once(__DIR__.'/vendor/autoload.php');
|
||||
|
||||
use \JHodges\Sitemap\Crawler;
|
||||
|
||||
if($url=getenv('CRAWL_URL')){
|
||||
$urls=array_filter(array_map('trim',explode(',',$url)));
|
||||
}else{
|
||||
die("No env: CRAWL_URL\n");
|
||||
}
|
||||
|
||||
if($code=getenv('CRAWL_CODE')){
|
||||
$codes=array_filter(array_map('trim',explode(',',$code)));
|
||||
}else{
|
||||
$codes=[];
|
||||
}
|
||||
|
||||
$crawler=new Crawler();
|
||||
foreach($urls as $url){
|
||||
$crawler->crawl($url);
|
||||
}
|
||||
|
||||
$summary=[];
|
||||
$details='';
|
||||
foreach($crawler->getResults() as $url=>$result){
|
||||
$summary[$result['code']]++;
|
||||
|
||||
if( count($codes)==0 || in_array($result['code'],$codes) ){
|
||||
$details.="{$result['code']} {$url}\n";
|
||||
foreach($result['foundOn'] as $url=>$count){
|
||||
$details.=" <- ($count) $url\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ksort($summary);
|
||||
echo '|code|count|'."\n";
|
||||
echo '|----|-----|'."\n";
|
||||
foreach($summary as $code=>$count){
|
||||
echo "| $code | $count |\n";
|
||||
}
|
||||
|
||||
if($details){
|
||||
echo "\n\n----\n\n```\n$details\n```\n";
|
||||
}
|
||||
17
docker/get_composer.sh
Normal file
17
docker/get_composer.sh
Normal file
@ -0,0 +1,17 @@
|
||||
#!/bin/sh
|
||||
|
||||
EXPECTED_CHECKSUM="$(wget -q -O - https://composer.github.io/installer.sig)"
|
||||
php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
|
||||
ACTUAL_CHECKSUM="$(php -r "echo hash_file('sha384', 'composer-setup.php');")"
|
||||
|
||||
if [ "$EXPECTED_CHECKSUM" != "$ACTUAL_CHECKSUM" ]
|
||||
then
|
||||
>&2 echo 'ERROR: Invalid installer checksum'
|
||||
rm composer-setup.php
|
||||
exit 1
|
||||
fi
|
||||
|
||||
php composer-setup.php --1 --filename=composer --install-dir=/usr/local/bin
|
||||
RESULT=$?
|
||||
rm composer-setup.php
|
||||
exit $RESULT
|
||||
@ -10,13 +10,14 @@ use Psr\Http\Message\UriInterface;
|
||||
use Spatie\Crawler\Crawler as SpatieCrawler;
|
||||
use Spatie\Crawler\CrawlUrl;
|
||||
use Spatie\Crawler\CrawlAllUrls;
|
||||
use Spatie\Crawler\CrawlProfile;
|
||||
|
||||
class Crawler{
|
||||
|
||||
private $observer;
|
||||
private $crawler;
|
||||
|
||||
public function __construct($reqOps=[]){
|
||||
public function __construct(array $reqOps=[]){
|
||||
$this->crawler = SpatieCrawler::create(array_merge($reqOps, [
|
||||
RequestOptions::ALLOW_REDIRECTS => [
|
||||
'track_redirects' => true,
|
||||
@ -28,15 +29,21 @@ class Crawler{
|
||||
$this->crawler->setCrawlProfile(new CrawlAllUrls());
|
||||
}
|
||||
|
||||
public function setUserAgent($agent){
|
||||
public function setUserAgent(String $agent) : self{
|
||||
$this->crawler->setUserAgent($agent);
|
||||
return $this;
|
||||
}
|
||||
|
||||
public function crawl($url){
|
||||
public function setCrawlProfile(CrawlProfile $p) : self{
|
||||
$this->crawler->setCrawlProfile($p);
|
||||
return $this;
|
||||
}
|
||||
|
||||
public function crawl(String $url) : void{
|
||||
$this->crawler->startCrawling($url);
|
||||
}
|
||||
|
||||
public function getResults(){
|
||||
public function getResults() : array{
|
||||
return $this->observer->results;
|
||||
}
|
||||
|
||||
|
||||
@ -34,7 +34,7 @@ class CrawlerTest extends TestCase{
|
||||
$this->url.'/notFound' => ['code' => 404],
|
||||
$this->url.'/redirect1' => ['code' => 302],
|
||||
$this->url.'/redirect2' => ['code' => 302],
|
||||
$this->url.'/redirectLoop' => ['code' => '302'],
|
||||
$this->url.'/redirectLoop' => ['code' => '---'],
|
||||
$this->url.'/redirectToFound' => ['code' => 302 ],
|
||||
$this->url.'/redirectToNotFound' => ['code' => 302 ],
|
||||
$this->url.'/redirectToRedirectToNotFound' => ['code' => 302],
|
||||
@ -178,7 +178,7 @@ class CrawlerTest extends TestCase{
|
||||
$crawler->crawl($this->url.'/redirectLoop');
|
||||
$sitemap=$crawler->getResults();
|
||||
$this->assertTreeContains($sitemap,[
|
||||
$this->url.'/redirectLoop' => ['code' => '302'],
|
||||
$this->url.'/redirectLoop' => ['code' => '---'],
|
||||
], print_r($sitemap,true));
|
||||
}
|
||||
|
||||
|
||||
@ -25,7 +25,7 @@ app.get('/', function (request, response) {
|
||||
});
|
||||
|
||||
app.get('/externalLink', function (request, response) {
|
||||
response.end('<a href="http://example.com/">ext</a>');
|
||||
response.end('<a href="http://example.com/"</a>');
|
||||
});
|
||||
|
||||
app.get('/deeplink1', function (request, response) {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user