add setUserAgent support

.drone.yml

@@ -22,7 +22,7 @@ steps:
 - name: run tests
   image: chialab/php
   commands:
-  - URL=http://testserver:8080 vendor/bin/phpunit tests
+  - URL=http://testserver:8080 vendor/bin/phpunit tests --testdox --color=always --no-interaction
 
 services:
 

composer.json

@@ -4,8 +4,9 @@
     "type": "library",
     "require": {
         "php": "^7.1",
-        "spatie/crawler": "^4.6",
-        "cweagans/composer-patches": "~1.0"
+        "spatie/crawler": "4.6.6",
+        "cweagans/composer-patches": "~1.0",
+        "guzzlehttp/guzzle": "6.5.2"
     },
     "require-dev": {
         "phpunit/phpunit": "^8.5"
@@ -27,7 +28,7 @@
                 "add crawled again observer": "https://patch-diff.githubusercontent.com/raw/spatie/crawler/pull/280.patch"
             },
             "guzzlehttp/guzzle": {
-                "Status code must be an integer value between 1xx and 5xx": "https://patch-diff.githubusercontent.com/raw/guzzle/guzzle/pull/2591.patch"
+                "Status code must be an integer value between 1xx and 5xx": "https://github.com/guzzle/guzzle/commit/f81cd6cdff1213f90de8f012489017510e3d6ff4.patch"
             }
         }
     }

src/Crawler.php

@@ -9,7 +9,7 @@ use Psr\Http\Message\UriInterface;
 
 use Spatie\Crawler\Crawler as SpatieCrawler;
 use Spatie\Crawler\CrawlUrl;
-use Spatie\Crawler\CrawlInternalUrls;
+use Spatie\Crawler\CrawlAllUrls;
 
 class Crawler{
 
@@ -25,6 +25,11 @@ class Crawler{
 
         $this->observer = new CrawlObserver();
         $this->crawler->setCrawlObserver($this->observer);
+        $this->crawler->setCrawlProfile(new CrawlAllUrls());
+    }
+
+    public function setUserAgent($agent){
+        $this->crawler->setUserAgent($agent);
     }
 
     public function crawl($url){