update deps properly

.drone.yml

@@ -5,7 +5,7 @@ name: testsuite
 steps:
 
 - name: composer install
-  image: chialab/php
+  image: chialab/php:7.4
   commands:
   - composer install
   volumes:
@@ -13,21 +13,21 @@ steps:
   path: /root/.composer/cache/
 
 - name: wait for test server
-  image: alpine
+  image: alpine:3.12
   commands:
   - echo "Waiting for server to launch on testserver:8080..."
   - while ! nc -z testserver 8080; do sleep 0.1 ; done
   - echo "Ready!"
 
 - name: run tests
-  image: chialab/php
+  image: chialab/php:7.4
   commands:
-  - URL=http://testserver:8080 vendor/bin/phpunit tests
+  - URL=http://testserver:8080 vendor/bin/phpunit tests --testdox --color=always --no-interaction
 
 services:
 
 - name: testserver
-  image: node
+  image: node:15.2.0-alpine3.12
   detach: true
   commands:
   - cd tests/server/

composer.json

@@ -3,8 +3,8 @@
     "description": "Generate full sitemap report using spatie/crawler",
     "type": "library",
     "require": {
-        "php": "^7.1",
-        "spatie/crawler": "^4.6",
+        "php": "^7.4",
+        "spatie/crawler": "^5.0",
         "cweagans/composer-patches": "~1.0"
     },
     "require-dev": {
@@ -25,9 +25,6 @@
         "patches": {
             "spatie/crawler": {
                 "add crawled again observer": "https://patch-diff.githubusercontent.com/raw/spatie/crawler/pull/280.patch"
-            },
-            "guzzlehttp/guzzle": {
-                "Status code must be an integer value between 1xx and 5xx": "https://patch-diff.githubusercontent.com/raw/guzzle/guzzle/pull/2591.patch"
             }
         }
     }

src/Crawler.php

@@ -9,7 +9,7 @@ use Psr\Http\Message\UriInterface;
 
 use Spatie\Crawler\Crawler as SpatieCrawler;
 use Spatie\Crawler\CrawlUrl;
-use Spatie\Crawler\CrawlInternalUrls;
+use Spatie\Crawler\CrawlAllUrls;
 
 class Crawler{
 
@@ -25,6 +25,11 @@ class Crawler{
 
         $this->observer = new CrawlObserver();
         $this->crawler->setCrawlObserver($this->observer);
+        $this->crawler->setCrawlProfile(new CrawlAllUrls());
+    }
+
+    public function setUserAgent($agent){
+        $this->crawler->setUserAgent($agent);
     }
 
     public function crawl($url){

tests/CrawlerTest.php

@@ -34,7 +34,7 @@ class CrawlerTest extends TestCase{
             $this->url.'/notFound' => ['code' => 404],
             $this->url.'/redirect1' => ['code' => 302],
             $this->url.'/redirect2' => ['code' => 302],
-            $this->url.'/redirectLoop' => ['code' => '---'],
+            $this->url.'/redirectLoop' => ['code' => '302'],
             $this->url.'/redirectToFound' => ['code' => 302 ],
             $this->url.'/redirectToNotFound' => ['code' => 302 ],
             $this->url.'/redirectToRedirectToNotFound' => ['code' => 302],
@@ -178,7 +178,7 @@ class CrawlerTest extends TestCase{
         $crawler->crawl($this->url.'/redirectLoop');
         $sitemap=$crawler->getResults();
         $this->assertTreeContains($sitemap,[
-            $this->url.'/redirectLoop' => ['code' => '---'],
+            $this->url.'/redirectLoop' => ['code' => '302'],
         ], print_r($sitemap,true));
     }
 

tests/server/server.js

@@ -25,7 +25,7 @@ app.get('/', function (request, response) {
 });
 
 app.get('/externalLink', function (request, response) {
-    response.end('<a href="http://example.com/"</a>');
+    response.end('<a href="http://example.com/">ext</a>');
 });
 
 app.get('/deeplink1', function (request, response) {