update deps properly

fix redirect loop test (now returns 302) may not be good :/
fix broken external link test
2020-11-17 08:37:20 +00:00 · 2020-11-17 08:26:47 +00:00 · 2020-11-16 19:46:37 +00:00 · 2020-11-16 16:01:32 +00:00 · 2020-11-16 15:47:38 +00:00
10 changed files with 375 additions and 299 deletions
--- a/.drone.yml
+++ b/.drone.yml
@ -5,7 +5,7 @@ name: testsuite
 steps:

 - name: composer install
-  image: chialab/php:7.3
+  image: chialab/php:7.4
  commands:
  - composer install
  volumes:
@ -20,14 +20,14 @@ steps:
  - echo "Ready!"

 - name: run tests
-  image: chialab/php:7.3
+  image: chialab/php:7.4
  commands:
  - URL=http://testserver:8080 vendor/bin/phpunit tests --testdox --color=always --no-interaction

 services:

 - name: testserver
-  image: node:15.2
+  image: node:15.2.0-alpine3.12
  detach: true
  commands:
  - cd tests/server/
--- a/README.md
+++ b/README.md
@ -3,7 +3,7 @@ Similar project to [spatie/http-status-check](https://github.com/spatie/http-sta
 ## Install

 ```plain
-composer config repositories.jhodges composer https://composer.jhodges.co.uk/
+composer config repositories.jhodges composer https://git.jhodges.co.uk/composer
 composer require jhodges/sitemap
 ```

--- a/composer.json
+++ b/composer.json
@ -3,10 +3,9 @@
    "description": "Generate full sitemap report using spatie/crawler",
    "type": "library",
    "require": {
-        "php": "^7.1",
-        "spatie/crawler": "4.6.6",
-        "cweagans/composer-patches": "~1.0",
-        "guzzlehttp/guzzle": "6.5.2"
+        "php": "^7.4",
+        "spatie/crawler": "^5.0",
+        "cweagans/composer-patches": "~1.0"
    },
    "require-dev": {
        "phpunit/phpunit": "^8.5"
@ -26,9 +25,6 @@
        "patches": {
            "spatie/crawler": {
                "add crawled again observer": "https://patch-diff.githubusercontent.com/raw/spatie/crawler/pull/280.patch"
-            },
-            "guzzlehttp/guzzle": {
-                "Status code must be an integer value between 1xx and 5xx": "https://github.com/guzzle/guzzle/commit/f81cd6cdff1213f90de8f012489017510e3d6ff4.patch"
            }
        }
    }
--- a/composer.lock
+++ b/composer.lock
@ -4,7 +4,7 @@
        "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
        "This file is @generated automatically"
    ],
-    "content-hash": "7ab60eca38a75bfd4220b46a7b45a582",
+    "content-hash": "c06e067a5805658d93962a86f0d985dd",
    "packages": [
        {
            "name": "cweagans/composer-patches",
@ -52,40 +52,43 @@
        },
        {
            "name": "guzzlehttp/guzzle",
-            "version": "6.5.2",
+            "version": "7.2.0",
            "source": {
                "type": "git",
                "url": "https://github.com/guzzle/guzzle.git",
-                "reference": "43ece0e75098b7ecd8d13918293029e555a50f82"
+                "reference": "0aa74dfb41ae110835923ef10a9d803a22d50e79"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/guzzle/guzzle/zipball/43ece0e75098b7ecd8d13918293029e555a50f82",
-                "reference": "43ece0e75098b7ecd8d13918293029e555a50f82",
+                "url": "https://api.github.com/repos/guzzle/guzzle/zipball/0aa74dfb41ae110835923ef10a9d803a22d50e79",
+                "reference": "0aa74dfb41ae110835923ef10a9d803a22d50e79",
                "shasum": ""
            },
            "require": {
                "ext-json": "*",
-                "guzzlehttp/promises": "^1.0",
-                "guzzlehttp/psr7": "^1.6.1",
-                "php": ">=5.5"
+                "guzzlehttp/promises": "^1.4",
+                "guzzlehttp/psr7": "^1.7",
+                "php": "^7.2.5 || ^8.0",
+                "psr/http-client": "^1.0"
+            },
+            "provide": {
+                "psr/http-client-implementation": "1.0"
            },
            "require-dev": {
                "ext-curl": "*",
-                "phpunit/phpunit": "^4.8.35 || ^5.7 || ^6.4 || ^7.0",
+                "php-http/client-integration-tests": "^3.0",
+                "phpunit/phpunit": "^8.5.5 || ^9.3.5",
                "psr/log": "^1.1"
            },
            "suggest": {
+                "ext-curl": "Required for CURL handler support",
                "ext-intl": "Required for Internationalized Domain Name (IDN) support",
                "psr/log": "Required for using the Log middleware"
            },
            "type": "library",
            "extra": {
                "branch-alias": {
-                    "dev-master": "6.5-dev"
-                },
-                "patches_applied": {
-                    "Status code must be an integer value between 1xx and 5xx": "https://github.com/guzzle/guzzle/commit/f81cd6cdff1213f90de8f012489017510e3d6ff4.patch"
+                    "dev-master": "7.1-dev"
                }
            },
            "autoload": {
@ -105,6 +108,11 @@
                    "name": "Michael Dowling",
                    "email": "mtdowling@gmail.com",
                    "homepage": "https://github.com/mtdowling"
+                },
+                {
+                    "name": "Márk Sági-Kazár",
+                    "email": "mark.sagikazar@gmail.com",
+                    "homepage": "https://sagikazarmark.hu"
                }
            ],
            "description": "Guzzle is a PHP HTTP client library",
@ -115,10 +123,30 @@
                "framework",
                "http",
                "http client",
+                "psr-18",
+                "psr-7",
                "rest",
                "web service"
            ],
-            "time": "2019-12-23T11:57:10+00:00"
+            "funding": [
+                {
+                    "url": "https://github.com/GrahamCampbell",
+                    "type": "github"
+                },
+                {
+                    "url": "https://github.com/Nyholm",
+                    "type": "github"
+                },
+                {
+                    "url": "https://github.com/alexeyshockov",
+                    "type": "github"
+                },
+                {
+                    "url": "https://github.com/gmponos",
+                    "type": "github"
+                }
+            ],
+            "time": "2020-10-10T11:47:56+00:00"
        },
        {
            "name": "guzzlehttp/promises",
@ -242,6 +270,142 @@
            ],
            "time": "2020-09-30T07:37:11+00:00"
        },
+        {
+            "name": "illuminate/collections",
+            "version": "v8.14.0",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/illuminate/collections.git",
+                "reference": "e61e796ff98e19eac5e6735ea6abfa66f4a2f6db"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/illuminate/collections/zipball/e61e796ff98e19eac5e6735ea6abfa66f4a2f6db",
+                "reference": "e61e796ff98e19eac5e6735ea6abfa66f4a2f6db",
+                "shasum": ""
+            },
+            "require": {
+                "illuminate/contracts": "^8.0",
+                "illuminate/macroable": "^8.0",
+                "php": "^7.3|^8.0"
+            },
+            "suggest": {
+                "symfony/var-dumper": "Required to use the dump method (^5.1)."
+            },
+            "type": "library",
+            "extra": {
+                "branch-alias": {
+                    "dev-master": "8.x-dev"
+                }
+            },
+            "autoload": {
+                "psr-4": {
+                    "Illuminate\\Support\\": ""
+                },
+                "files": [
+                    "helpers.php"
+                ]
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Taylor Otwell",
+                    "email": "taylor@laravel.com"
+                }
+            ],
+            "description": "The Illuminate Collections package.",
+            "homepage": "https://laravel.com",
+            "time": "2020-10-27T15:20:30+00:00"
+        },
+        {
+            "name": "illuminate/contracts",
+            "version": "v8.14.0",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/illuminate/contracts.git",
+                "reference": "c9c9ae3e00526010b9b46c644287765ef60d6147"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/illuminate/contracts/zipball/c9c9ae3e00526010b9b46c644287765ef60d6147",
+                "reference": "c9c9ae3e00526010b9b46c644287765ef60d6147",
+                "shasum": ""
+            },
+            "require": {
+                "php": "^7.3|^8.0",
+                "psr/container": "^1.0",
+                "psr/simple-cache": "^1.0"
+            },
+            "type": "library",
+            "extra": {
+                "branch-alias": {
+                    "dev-master": "8.x-dev"
+                }
+            },
+            "autoload": {
+                "psr-4": {
+                    "Illuminate\\Contracts\\": ""
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Taylor Otwell",
+                    "email": "taylor@laravel.com"
+                }
+            ],
+            "description": "The Illuminate Contracts package.",
+            "homepage": "https://laravel.com",
+            "time": "2020-11-04T19:55:57+00:00"
+        },
+        {
+            "name": "illuminate/macroable",
+            "version": "v8.14.0",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/illuminate/macroable.git",
+                "reference": "300aa13c086f25116b5f3cde3ca54ff5c822fb05"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/illuminate/macroable/zipball/300aa13c086f25116b5f3cde3ca54ff5c822fb05",
+                "reference": "300aa13c086f25116b5f3cde3ca54ff5c822fb05",
+                "shasum": ""
+            },
+            "require": {
+                "php": "^7.3|^8.0"
+            },
+            "type": "library",
+            "extra": {
+                "branch-alias": {
+                    "dev-master": "8.x-dev"
+                }
+            },
+            "autoload": {
+                "psr-4": {
+                    "Illuminate\\Support\\": ""
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Taylor Otwell",
+                    "email": "taylor@laravel.com"
+                }
+            ],
+            "description": "The Illuminate Macroable package.",
+            "homepage": "https://laravel.com",
+            "time": "2020-10-27T15:20:30+00:00"
+        },
        {
            "name": "intervention/image",
            "version": "2.5.1",
@ -395,12 +559,6 @@
                "sftp",
                "storage"
            ],
-            "funding": [
-                {
-                    "url": "https://offset.earth/frankdejonge",
-                    "type": "other"
-                }
-            ],
            "time": "2020-08-23T07:39:11+00:00"
        },
        {
@ -517,20 +675,23 @@
        },
        {
            "name": "nicmart/tree",
-            "version": "v0.2.7",
+            "version": "0.3.0",
            "source": {
                "type": "git",
                "url": "https://github.com/nicmart/Tree.git",
-                "reference": "0616b54bb49938e1a816141d7943db48ebf76938"
+                "reference": "049bad40787ad2bc44a916c7bf0ccdd3d3a113b1"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/nicmart/Tree/zipball/0616b54bb49938e1a816141d7943db48ebf76938",
-                "reference": "0616b54bb49938e1a816141d7943db48ebf76938",
+                "url": "https://api.github.com/repos/nicmart/Tree/zipball/049bad40787ad2bc44a916c7bf0ccdd3d3a113b1",
+                "reference": "049bad40787ad2bc44a916c7bf0ccdd3d3a113b1",
                "shasum": ""
            },
            "require": {
-                "php": ">=5.4"
+                "php": "^5.4 || ^7.0"
+            },
+            "require-dev": {
+                "phpunit/phpunit": "^4.8"
            },
            "type": "library",
            "autoload": {
@ -549,7 +710,105 @@
                }
            ],
            "description": "A basic but flexible php tree data structure and a fluent tree builder implementation.",
-            "time": "2015-08-20T20:50:35+00:00"
+            "time": "2020-06-18T07:18:00+00:00"
+        },
+        {
+            "name": "psr/container",
+            "version": "1.0.0",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/php-fig/container.git",
+                "reference": "b7ce3b176482dbbc1245ebf52b181af44c2cf55f"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/php-fig/container/zipball/b7ce3b176482dbbc1245ebf52b181af44c2cf55f",
+                "reference": "b7ce3b176482dbbc1245ebf52b181af44c2cf55f",
+                "shasum": ""
+            },
+            "require": {
+                "php": ">=5.3.0"
+            },
+            "type": "library",
+            "extra": {
+                "branch-alias": {
+                    "dev-master": "1.0.x-dev"
+                }
+            },
+            "autoload": {
+                "psr-4": {
+                    "Psr\\Container\\": "src/"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "PHP-FIG",
+                    "homepage": "http://www.php-fig.org/"
+                }
+            ],
+            "description": "Common Container Interface (PHP FIG PSR-11)",
+            "homepage": "https://github.com/php-fig/container",
+            "keywords": [
+                "PSR-11",
+                "container",
+                "container-interface",
+                "container-interop",
+                "psr"
+            ],
+            "time": "2017-02-14T16:28:37+00:00"
+        },
+        {
+            "name": "psr/http-client",
+            "version": "1.0.1",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/php-fig/http-client.git",
+                "reference": "2dfb5f6c5eff0e91e20e913f8c5452ed95b86621"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/php-fig/http-client/zipball/2dfb5f6c5eff0e91e20e913f8c5452ed95b86621",
+                "reference": "2dfb5f6c5eff0e91e20e913f8c5452ed95b86621",
+                "shasum": ""
+            },
+            "require": {
+                "php": "^7.0 || ^8.0",
+                "psr/http-message": "^1.0"
+            },
+            "type": "library",
+            "extra": {
+                "branch-alias": {
+                    "dev-master": "1.0.x-dev"
+                }
+            },
+            "autoload": {
+                "psr-4": {
+                    "Psr\\Http\\Client\\": "src/"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "PHP-FIG",
+                    "homepage": "http://www.php-fig.org/"
+                }
+            ],
+            "description": "Common interface for HTTP clients",
+            "homepage": "https://github.com/php-fig/http-client",
+            "keywords": [
+                "http",
+                "http-client",
+                "psr",
+                "psr-18"
+            ],
+            "time": "2020-06-29T06:28:15+00:00"
        },
        {
            "name": "psr/http-message",
@ -648,6 +907,54 @@
            ],
            "time": "2020-03-23T09:12:05+00:00"
        },
+        {
+            "name": "psr/simple-cache",
+            "version": "1.0.1",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/php-fig/simple-cache.git",
+                "reference": "408d5eafb83c57f6365a3ca330ff23aa4a5fa39b"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/php-fig/simple-cache/zipball/408d5eafb83c57f6365a3ca330ff23aa4a5fa39b",
+                "reference": "408d5eafb83c57f6365a3ca330ff23aa4a5fa39b",
+                "shasum": ""
+            },
+            "require": {
+                "php": ">=5.3.0"
+            },
+            "type": "library",
+            "extra": {
+                "branch-alias": {
+                    "dev-master": "1.0.x-dev"
+                }
+            },
+            "autoload": {
+                "psr-4": {
+                    "Psr\\SimpleCache\\": "src/"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "PHP-FIG",
+                    "homepage": "http://www.php-fig.org/"
+                }
+            ],
+            "description": "Common interfaces for simple caching",
+            "keywords": [
+                "cache",
+                "caching",
+                "psr",
+                "psr-16",
+                "simple-cache"
+            ],
+            "time": "2017-10-23T01:57:42+00:00"
+        },
        {
            "name": "ralouphie/getallheaders",
            "version": "3.0.3",
@ -690,27 +997,27 @@
        },
        {
            "name": "spatie/browsershot",
-            "version": "3.40.2",
+            "version": "3.40.3",
            "source": {
                "type": "git",
                "url": "https://github.com/spatie/browsershot.git",
-                "reference": "3e55eaf5ab8cee65d1661a567e89b3374afb9116"
+                "reference": "321f0d8ec8e303d145968ebece9fa1ebf2497864"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/spatie/browsershot/zipball/3e55eaf5ab8cee65d1661a567e89b3374afb9116",
-                "reference": "3e55eaf5ab8cee65d1661a567e89b3374afb9116",
+                "url": "https://api.github.com/repos/spatie/browsershot/zipball/321f0d8ec8e303d145968ebece9fa1ebf2497864",
+                "reference": "321f0d8ec8e303d145968ebece9fa1ebf2497864",
                "shasum": ""
            },
            "require": {
-                "php": "^7.1",
+                "php": "^7.4|^8.0",
                "spatie/image": "^1.5.3",
                "spatie/temporary-directory": "^1.1",
-                "symfony/process": "^4.2|^5.0"
+                "symfony/process": "^5.0"
            },
            "require-dev": {
-                "phpunit/phpunit": "^6.1|^7.5",
-                "spatie/phpunit-snapshot-assertions": "^1.0"
+                "phpunit/phpunit": "^9.0",
+                "spatie/phpunit-snapshot-assertions": "^4.2.3"
            },
            "type": "library",
            "autoload": {
@ -748,44 +1055,38 @@
                    "type": "github"
                }
            ],
-            "time": "2020-11-11T22:18:15+00:00"
+            "time": "2020-11-12T16:31:59+00:00"
        },
        {
            "name": "spatie/crawler",
-            "version": "4.6.6",
+            "version": "5.0.1",
            "source": {
                "type": "git",
                "url": "https://github.com/spatie/crawler.git",
-                "reference": "6f814a401cee20b24a1ffe53d8deddac90d1a42d"
+                "reference": "d94b5b651bcd85566863b25fec1e8e498ac3c30a"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/spatie/crawler/zipball/6f814a401cee20b24a1ffe53d8deddac90d1a42d",
-                "reference": "6f814a401cee20b24a1ffe53d8deddac90d1a42d",
+                "url": "https://api.github.com/repos/spatie/crawler/zipball/d94b5b651bcd85566863b25fec1e8e498ac3c30a",
+                "reference": "d94b5b651bcd85566863b25fec1e8e498ac3c30a",
                "shasum": ""
            },
            "require": {
-                "guzzlehttp/guzzle": "^6.3 || ^7.0",
+                "guzzlehttp/guzzle": "^6.3|^7.0",
                "guzzlehttp/psr7": "^1.4",
-                "nicmart/tree": "^0.2.7",
-                "php": "^7.1",
+                "illuminate/collections": "^8.6",
+                "nicmart/tree": "^0.3.0",
+                "php": "^7.4",
                "spatie/browsershot": "^3.14",
                "spatie/robots-txt": "^1.0.1",
-                "symfony/dom-crawler": "^4.0 || ^5.0",
-                "tightenco/collect": "^5.6|^6.0"
+                "symfony/dom-crawler": "^5.0"
            },
            "require-dev": {
-                "larapack/dd": "^1.1",
-                "phpunit/phpunit": "^7.0"
+                "phpunit/phpunit": "^9.3"
            },
            "type": "library",
            "extra": {
-                "branch-alias": {
-                    "dev-master": "0.0.x-dev"
-                },
-                "patches_applied": {
-                    "add crawled again observer": "https://patch-diff.githubusercontent.com/raw/spatie/crawler/pull/280.patch"
-                }
+                "patches_applied": []
            },
            "autoload": {
                "psr-4": {
@ -810,7 +1111,13 @@
                "spatie",
                "website"
            ],
-            "time": "2020-01-31T10:51:58+00:00"
+            "funding": [
+                {
+                    "url": "https://github.com/spatie",
+                    "type": "github"
+                }
+            ],
+            "time": "2020-10-09T09:13:50+00:00"
        },
        {
            "name": "spatie/image",
@ -1381,141 +1688,6 @@
                }
            ],
            "time": "2020-10-24T12:01:57+00:00"
-        },
-        {
-            "name": "symfony/var-dumper",
-            "version": "v5.1.8",
-            "source": {
-                "type": "git",
-                "url": "https://github.com/symfony/var-dumper.git",
-                "reference": "4e13f3fcefb1fcaaa5efb5403581406f4e840b9a"
-            },
-            "dist": {
-                "type": "zip",
-                "url": "https://api.github.com/repos/symfony/var-dumper/zipball/4e13f3fcefb1fcaaa5efb5403581406f4e840b9a",
-                "reference": "4e13f3fcefb1fcaaa5efb5403581406f4e840b9a",
-                "shasum": ""
-            },
-            "require": {
-                "php": ">=7.2.5",
-                "symfony/polyfill-mbstring": "~1.0",
-                "symfony/polyfill-php80": "^1.15"
-            },
-            "conflict": {
-                "phpunit/phpunit": "<5.4.3",
-                "symfony/console": "<4.4"
-            },
-            "require-dev": {
-                "ext-iconv": "*",
-                "symfony/console": "^4.4|^5.0",
-                "symfony/process": "^4.4|^5.0",
-                "twig/twig": "^2.4|^3.0"
-            },
-            "suggest": {
-                "ext-iconv": "To convert non-UTF-8 strings to UTF-8 (or symfony/polyfill-iconv in case ext-iconv cannot be used).",
-                "ext-intl": "To show region name in time zone dump",
-                "symfony/console": "To use the ServerDumpCommand and/or the bin/var-dump-server script"
-            },
-            "bin": [
-                "Resources/bin/var-dump-server"
-            ],
-            "type": "library",
-            "autoload": {
-                "files": [
-                    "Resources/functions/dump.php"
-                ],
-                "psr-4": {
-                    "Symfony\\Component\\VarDumper\\": ""
-                },
-                "exclude-from-classmap": [
-                    "/Tests/"
-                ]
-            },
-            "notification-url": "https://packagist.org/downloads/",
-            "license": [
-                "MIT"
-            ],
-            "authors": [
-                {
-                    "name": "Nicolas Grekas",
-                    "email": "p@tchwork.com"
-                },
-                {
-                    "name": "Symfony Community",
-                    "homepage": "https://symfony.com/contributors"
-                }
-            ],
-            "description": "Symfony mechanism for exploring and dumping PHP variables",
-            "homepage": "https://symfony.com",
-            "keywords": [
-                "debug",
-                "dump"
-            ],
-            "funding": [
-                {
-                    "url": "https://symfony.com/sponsor",
-                    "type": "custom"
-                },
-                {
-                    "url": "https://github.com/fabpot",
-                    "type": "github"
-                },
-                {
-                    "url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
-                    "type": "tidelift"
-                }
-            ],
-            "time": "2020-10-27T10:11:13+00:00"
-        },
-        {
-            "name": "tightenco/collect",
-            "version": "v6.18.34",
-            "source": {
-                "type": "git",
-                "url": "https://github.com/tighten/collect.git",
-                "reference": "624b86cde21f8d9ba14ec0e4974ef0aca3922a1c"
-            },
-            "dist": {
-                "type": "zip",
-                "url": "https://api.github.com/repos/tighten/collect/zipball/624b86cde21f8d9ba14ec0e4974ef0aca3922a1c",
-                "reference": "624b86cde21f8d9ba14ec0e4974ef0aca3922a1c",
-                "shasum": ""
-            },
-            "require": {
-                "php": "^7.1.3",
-                "symfony/var-dumper": "^3.4 || ^4.0 || ^5.0"
-            },
-            "require-dev": {
-                "mockery/mockery": "^1.0",
-                "nesbot/carbon": "^2.23.0",
-                "phpunit/phpunit": "^7.0"
-            },
-            "type": "library",
-            "autoload": {
-                "files": [
-                    "src/Collect/Support/helpers.php",
-                    "src/Collect/Support/alias.php"
-                ],
-                "psr-4": {
-                    "Tightenco\\Collect\\": "src/Collect"
-                }
-            },
-            "notification-url": "https://packagist.org/downloads/",
-            "license": [
-                "MIT"
-            ],
-            "authors": [
-                {
-                    "name": "Taylor Otwell",
-                    "email": "taylorotwell@gmail.com"
-                }
-            ],
-            "description": "Collect - Illuminate Collections as a separate package.",
-            "keywords": [
-                "collection",
-                "laravel"
-            ],
-            "time": "2020-08-21T22:25:58+00:00"
        }
    ],
    "packages-dev": [
@ -2948,12 +3120,6 @@
                }
            ],
            "description": "A small library for converting tokenized PHP source code into XML and potentially other formats",
-            "funding": [
-                {
-                    "url": "https://github.com/theseer",
-                    "type": "github"
-                }
-            ],
            "time": "2020-07-12T23:59:07+00:00"
        },
        {
@ -3012,7 +3178,7 @@
    "prefer-stable": false,
    "prefer-lowest": false,
    "platform": {
-        "php": "^7.1"
+        "php": "^7.4"
    },
    "platform-dev": [],
    "plugin-api-version": "1.1.0"
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@ -1,17 +0,0 @@
-FROM markizano/devuan:beowulf-amd64
-
-RUN apt-get update
-RUN apt-get install -y php-cli git unzip php-xml
-
-COPY get_composer.sh /root/
-RUN bash /root/get_composer.sh && rm /root/get_composer.sh
-
-RUN mkdir /app
-WORKDIR /app
-RUN composer init && composer config repositories.jhodges composer https://composer.jhodges.co.uk && composer require jhodges/sitemap
-
-COPY crawl.php /app/
-
-RUN apt clean
-
-ENTRYPOINT php crawl.php
--- a/docker/crawl.php
+++ b/docker/crawl.php
@ -1,45 +0,0 @@
-<?php
-require_once(__DIR__.'/vendor/autoload.php');
-
-use \JHodges\Sitemap\Crawler;
-
-if($url=getenv('CRAWL_URL')){
-    $urls=array_filter(array_map('trim',explode(',',$url)));
-}else{
-	die("No env: CRAWL_URL\n");
-}
-
-if($code=getenv('CRAWL_CODE')){
-	$codes=array_filter(array_map('trim',explode(',',$code)));
-}else{
-    $codes=[];
-}
-
-$crawler=new Crawler();
-foreach($urls as $url){
-    $crawler->crawl($url);
-}
-
-$summary=[];
-$details='';
-foreach($crawler->getResults() as $url=>$result){
-    $summary[$result['code']]++;
-
-    if( count($codes)==0 || in_array($result['code'],$codes) ){
-        $details.="{$result['code']} {$url}\n";
-        foreach($result['foundOn'] as $url=>$count){
-            $details.="    <- ($count) $url\n";
-        }
-    }
-}
-
-ksort($summary);
-echo '|code|count|'."\n";
-echo '|----|-----|'."\n";
-foreach($summary as $code=>$count){
-    echo "| $code | $count |\n";
-}
-
-if($details){
-    echo "\n\n----\n\n```\n$details\n```\n";
-}
--- a/docker/get_composer.sh
+++ b/docker/get_composer.sh
@ -1,17 +0,0 @@
-#!/bin/sh
-
-EXPECTED_CHECKSUM="$(wget -q -O - https://composer.github.io/installer.sig)"
-php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
-ACTUAL_CHECKSUM="$(php -r "echo hash_file('sha384', 'composer-setup.php');")"
-
-if [ "$EXPECTED_CHECKSUM" != "$ACTUAL_CHECKSUM" ]
-then
-    >&2 echo 'ERROR: Invalid installer checksum'
-    rm composer-setup.php
-    exit 1
-fi
-
-php composer-setup.php --1 --filename=composer --install-dir=/usr/local/bin
-RESULT=$?
-rm composer-setup.php
-exit $RESULT
--- a/src/Crawler.php
+++ b/src/Crawler.php
@ -10,14 +10,13 @@ use Psr\Http\Message\UriInterface;
 use Spatie\Crawler\Crawler as SpatieCrawler;
 use Spatie\Crawler\CrawlUrl;
 use Spatie\Crawler\CrawlAllUrls;
-use Spatie\Crawler\CrawlProfile;

 class Crawler{

    private $observer;
    private $crawler;

-    public function __construct(array $reqOps=[]){
+    public function __construct($reqOps=[]){
        $this->crawler = SpatieCrawler::create(array_merge($reqOps, [
            RequestOptions::ALLOW_REDIRECTS => [
                'track_redirects' => true,
@ -29,21 +28,15 @@ class Crawler{
        $this->crawler->setCrawlProfile(new CrawlAllUrls());
    }

-    public function setUserAgent(String $agent) : self{
+    public function setUserAgent($agent){
        $this->crawler->setUserAgent($agent);
-	return $this;
    }

-    public function setCrawlProfile(CrawlProfile $p) : self{
-        $this->crawler->setCrawlProfile($p);
-	return $this;
-    }
-
-    public function crawl(String $url) : void{
+    public function crawl($url){
        $this->crawler->startCrawling($url);
    }

-    public function getResults() : array{
+    public function getResults(){
        return $this->observer->results;
    }

--- a/tests/CrawlerTest.php
+++ b/tests/CrawlerTest.php
@ -34,7 +34,7 @@ class CrawlerTest extends TestCase{
            $this->url.'/notFound' => ['code' => 404],
            $this->url.'/redirect1' => ['code' => 302],
            $this->url.'/redirect2' => ['code' => 302],
-            $this->url.'/redirectLoop' => ['code' => '---'],
+            $this->url.'/redirectLoop' => ['code' => '302'],
            $this->url.'/redirectToFound' => ['code' => 302 ],
            $this->url.'/redirectToNotFound' => ['code' => 302 ],
            $this->url.'/redirectToRedirectToNotFound' => ['code' => 302],
@ -178,7 +178,7 @@ class CrawlerTest extends TestCase{
        $crawler->crawl($this->url.'/redirectLoop');
        $sitemap=$crawler->getResults();
        $this->assertTreeContains($sitemap,[
-            $this->url.'/redirectLoop' => ['code' => '---'],
+            $this->url.'/redirectLoop' => ['code' => '302'],
        ], print_r($sitemap,true));
    }

--- a/tests/server/server.js
+++ b/tests/server/server.js
@ -25,7 +25,7 @@ app.get('/', function (request, response) {
 });

 app.get('/externalLink', function (request, response) {
-    response.end('<a href="http://example.com/"</a>');
+    response.end('<a href="http://example.com/">ext</a>');
 });

 app.get('/deeplink1', function (request, response) {
Author	SHA1	Message	Date
James	0714a891fe	update deps properly Some checks failed continuous-integration/drone/push Build is failing Details	2020-11-17 08:37:20 +00:00
James	20b260233a	fix redirect loop test (now returns 302) may not be good :/ All checks were successful continuous-integration/drone/push Build is passing Details	2020-11-17 08:26:47 +00:00
James	0c30a268b8	fix broken external link test Some checks failed continuous-integration/drone/push Build is failing Details	2020-11-16 19:46:37 +00:00
James	e8ebef4cc4	update deps even more :) Some checks failed continuous-integration/drone/push Build is failing Details	2020-11-16 16:01:32 +00:00
James	f696e28e49	update deps Some checks failed continuous-integration/drone/push Build is failing Details	2020-11-16 15:47:38 +00:00