update deps properly

init drone file
allow testing on custom host/port

Co-authored-by: James <inbox.dev@jhodges.co.uk>
Reviewed-on: #1
Co-Authored-By: James <jhodges@noreply@git.jhodges.co.uk>
Co-Committed-By: James <jhodges@noreply@git.jhodges.co.uk>

.drone.yml

@@ -0,0 +1,35 @@
+kind: pipeline
+type: docker
+name: testsuite
+
+steps:
+
+- name: composer install
+  image: chialab/php:7.4
+  commands:
+  - composer install
+  volumes:
+  - name: composer-cache
+  path: /root/.composer/cache/
+
+- name: wait for test server
+  image: alpine:3.12
+  commands:
+  - echo "Waiting for server to launch on testserver:8080..."
+  - while ! nc -z testserver 8080; do sleep 0.1 ; done
+  - echo "Ready!"
+
+- name: run tests
+  image: chialab/php:7.4
+  commands:
+  - URL=http://testserver:8080 vendor/bin/phpunit tests --testdox --color=always --no-interaction
+
+services:
+
+- name: testserver
+  image: node:15.2.0-alpine3.12
+  detach: true
+  commands:
+  - cd tests/server/
+  - npm install
+  - node server.js

README.md

@@ -37,7 +37,8 @@ Start the test server, will listen on localhost:8080
 
 ```plain
 cd tests/server
-./start_server.sh
+npm install
+node server.js
 ```
 
 Run the tests:

composer.json

@@ -3,8 +3,8 @@
     "description": "Generate full sitemap report using spatie/crawler",
     "type": "library",
     "require": {
-        "php": "^7.1",
+        "php": "^7.4",
-        "spatie/crawler": "^4.6",
+        "spatie/crawler": "^5.0",
         "cweagans/composer-patches": "~1.0"
     },
     "require-dev": {
@@ -25,9 +25,6 @@
         "patches": {
             "spatie/crawler": {
                 "add crawled again observer": "https://patch-diff.githubusercontent.com/raw/spatie/crawler/pull/280.patch"
-            },
-            "guzzlehttp/guzzle": {
-                "Status code must be an integer value between 1xx and 5xx": "https://patch-diff.githubusercontent.com/raw/guzzle/guzzle/pull/2591.patch"
             }
         }
     }

example/crawl.php

@@ -0,0 +1,19 @@
+#!/usr/bin/php
+<?php
+require_once(__DIR__.'/../vendor/autoload.php');
+
+if(!$url=$argv[1]??null){
+	die("url?\n");
+}
+
+use \JHodges\Sitemap\Crawler;
+
+$crawler=new Crawler();
+$crawler->crawl($argv[1]);
+
+foreach($crawler->getResults() as $url=>$result){
+        echo("{$result['code']} {$url}\n");
+        foreach($result['foundOn'] as $url=>$count){
+            echo("    -> ($count) $url\n");
+        }
+}

src/Crawler.php

@@ -9,7 +9,7 @@ use Psr\Http\Message\UriInterface;
 
 use Spatie\Crawler\Crawler as SpatieCrawler;
 use Spatie\Crawler\CrawlUrl;
-use Spatie\Crawler\CrawlInternalUrls;
+use Spatie\Crawler\CrawlAllUrls;
 
 class Crawler{
 
@@ -25,6 +25,11 @@ class Crawler{
 
         $this->observer = new CrawlObserver();
         $this->crawler->setCrawlObserver($this->observer);
+        $this->crawler->setCrawlProfile(new CrawlAllUrls());
+    }
+
+    public function setUserAgent($agent){
+        $this->crawler->setUserAgent($agent);
     }
 
     public function crawl($url){

tests/CrawlerTest.php

@@ -6,188 +6,197 @@ use GuzzleHttp\RequestOptions;
 
 class CrawlerTest extends TestCase{
 
+    private $url='http://localhost:8080';
+
+    public function __construct(){
+        parent::__construct();
+        if(getenv('URL')){
+            $this->url=getenv('URL');
+        }
+    }
+
     public function testFullSite(){
         $crawler=new Crawler([RequestOptions::CONNECT_TIMEOUT => 3, RequestOptions::TIMEOUT => 3]);
-        $crawler->crawl('http://localhost:8080/');
+        $crawler->crawl($this->url);
         $sitemap=$crawler->getResults();
         $this->assertTreeContains($sitemap,[
             'http://example.com/' => ['code' => 200],
-            'http://localhost:8080/deeplink1' => ['code' => 200],
+            $this->url.'/deeplink1' => ['code' => 200],
-            'http://localhost:8080/deeplink2' => ['code' => 200],
+            $this->url.'/deeplink2' => ['code' => 200],
-            'http://localhost:8080/deeplink3' => ['code' => 200],
+            $this->url.'/deeplink3' => ['code' => 200],
-            'http://localhost:8080/externalLink' => ['code' => 200],
+            $this->url.'/externalLink' => ['code' => 200],
-            'http://localhost:8080/found' => ['code' => 200],
+            $this->url.'/found' => ['code' => 200],
-            'http://localhost:8080/interlinked1' => ['code' => 200],
+            $this->url.'/interlinked1' => ['code' => 200],
-            'http://localhost:8080/interlinked2' => ['code' => 200],
+            $this->url.'/interlinked2' => ['code' => 200],
-            'http://localhost:8080/interlinked3' => ['code' => 200],
+            $this->url.'/interlinked3' => ['code' => 200],
-            'http://localhost:8080/internalServerError' => ['code' => 500],
+            $this->url.'/internalServerError' => ['code' => 500],
-            'http://localhost:8080/invalidStatusCode' => ['code' => '---'],
+            $this->url.'/invalidStatusCode' => ['code' => '---'],
-            'http://localhost:8080/notFound' => ['code' => 404],
+            $this->url.'/notFound' => ['code' => 404],
-            'http://localhost:8080/redirect1' => ['code' => 302],
+            $this->url.'/redirect1' => ['code' => 302],
-            'http://localhost:8080/redirect2' => ['code' => 302],
+            $this->url.'/redirect2' => ['code' => 302],
-            'http://localhost:8080/redirectLoop' => ['code' => '---'],
+            $this->url.'/redirectLoop' => ['code' => '302'],
-            'http://localhost:8080/redirectToFound' => ['code' => 302 ],
+            $this->url.'/redirectToFound' => ['code' => 302 ],
-            'http://localhost:8080/redirectToNotFound' => ['code' => 302 ],
+            $this->url.'/redirectToNotFound' => ['code' => 302 ],
-            'http://localhost:8080/redirectToRedirectToNotFound' => ['code' => 302],
+            $this->url.'/redirectToRedirectToNotFound' => ['code' => 302],
-            'http://localhost:8080/timeout' => ['code' => '---'],
+            $this->url.'/timeout' => ['code' => '---'],
-            'http://localhost:8080/twoRedirectsToSameLocation' => ['code' => 200],
+            $this->url.'/twoRedirectsToSameLocation' => ['code' => 200],
         ], print_r($sitemap,true));
     }
 
     public function testFound(){
         $crawler=new Crawler();
-        $crawler->crawl('http://localhost:8080/found');
+        $crawler->crawl($this->url.'/found');
         $sitemap=$crawler->getResults();
         $this->assertTreeContains($sitemap,[
-            'http://localhost:8080/found' => ['code' => 200],
+            $this->url.'/found' => ['code' => 200],
         ], print_r($sitemap,true));
     }
 
     public function testNotFound(){
         $crawler=new Crawler();
-        $crawler->crawl('http://localhost:8080/notFound');
+        $crawler->crawl($this->url.'/notFound');
         $sitemap=$crawler->getResults();
         $this->assertTreeContains($sitemap,[
-            'http://localhost:8080/notFound' => ['code' => 404],
+            $this->url.'/notFound' => ['code' => 404],
         ], print_r($sitemap,true));
     }
 
     public function testExternalLink(){
         $crawler=new Crawler();
-        $crawler->crawl('http://localhost:8080/externalLink');
+        $crawler->crawl($this->url.'/externalLink');
         $sitemap=$crawler->getResults();
         $this->assertTreeContains($sitemap,[
-            'http://localhost:8080/externalLink' => ['code' => 200],
+            $this->url.'/externalLink' => ['code' => 200],
             'http://example.com/' => ['code' => 200],
         ], print_r($sitemap,true));
     }
 
     public function testDeeplink(){
         $crawler=new Crawler();
-        $crawler->crawl('http://localhost:8080/deeplink1');
+        $crawler->crawl($this->url.'/deeplink1');
         $sitemap=$crawler->getResults();
         $this->assertTreeContains($sitemap,[
-            'http://localhost:8080/deeplink1' => ['code' => 200],
+            $this->url.'/deeplink1' => ['code' => 200],
-            'http://localhost:8080/deeplink2' => ['code' => 200],
+            $this->url.'/deeplink2' => ['code' => 200],
-            'http://localhost:8080/deeplink3' => ['code' => 200],
+            $this->url.'/deeplink3' => ['code' => 200],
         ], print_r($sitemap,true));
     }
 
     public function testInterlinked(){
         $crawler=new Crawler();
-        $crawler->crawl('http://localhost:8080/interlinked1');
+        $crawler->crawl($this->url.'/interlinked1');
-        $crawler->crawl('http://localhost:8080/interlinked4'); //this ensures the order or results for the URL tracking test PART2
+        $crawler->crawl($this->url.'/interlinked4'); //this ensures the order or results for the URL tracking test PART2
         $sitemap=$crawler->getResults();
         $this->assertTreeContains($sitemap,[
-            'http://localhost:8080/interlinked1' => ['code' => 200 , 'foundOn' => [
+            $this->url.'/interlinked1' => ['code' => 200 , 'foundOn' => [
-                'http://localhost:8080/interlinked1' => 1,
+                $this->url.'/interlinked1' => 1,
-                'http://localhost:8080/interlinked2' => 1,
+                $this->url.'/interlinked2' => 1,
-                'http://localhost:8080/interlinked3' => 1,
+                $this->url.'/interlinked3' => 1,
-                'http://localhost:8080/interlinked4' => 1,
+                $this->url.'/interlinked4' => 1,
             ]],
-            'http://localhost:8080/interlinked2' => ['code' => 200 , 'foundOn' => [
+            $this->url.'/interlinked2' => ['code' => 200 , 'foundOn' => [
-                'http://localhost:8080/interlinked1' => 1,
+                $this->url.'/interlinked1' => 1,
-                'http://localhost:8080/interlinked2' => 1,
+                $this->url.'/interlinked2' => 1,
-                'http://localhost:8080/interlinked3' => 1,
+                $this->url.'/interlinked3' => 1,
-                'http://localhost:8080/interlinked4' => 1,
+                $this->url.'/interlinked4' => 1,
             ]],
-            'http://localhost:8080/interlinked3' => ['code' => 200 , 'foundOn' => [
+            $this->url.'/interlinked3' => ['code' => 200 , 'foundOn' => [
-                'http://localhost:8080/interlinked1' => 1,
+                $this->url.'/interlinked1' => 1,
-                'http://localhost:8080/interlinked2' => 1,
+                $this->url.'/interlinked2' => 1,
-                'http://localhost:8080/interlinked3' => 1,
+                $this->url.'/interlinked3' => 1,
-                'http://localhost:8080/interlinked4' => 1,
+                $this->url.'/interlinked4' => 1,
             ]],
-            'http://localhost:8080/found' => ['code' => 200 , 'foundOn' => [
+            $this->url.'/found' => ['code' => 200 , 'foundOn' => [
-                'http://localhost:8080/interlinked1' => 1,
+                $this->url.'/interlinked1' => 1,
-                'http://localhost:8080/interlinked2' => 1,
+                $this->url.'/interlinked2' => 1,
-                'http://localhost:8080/interlinked3' => 1,
+                $this->url.'/interlinked3' => 1,
-                'http://localhost:8080/interlinked4' => 1,
+                $this->url.'/interlinked4' => 1,
             ]],
-            'http://localhost:8080/redirectToFound' => ['code' => 302 , 'foundOn' => [
+            $this->url.'/redirectToFound' => ['code' => 302 , 'foundOn' => [
-                'http://localhost:8080/interlinked1' => 1,
+                $this->url.'/interlinked1' => 1,
-                'http://localhost:8080/interlinked2' => 1,
+                $this->url.'/interlinked2' => 1,
-                'http://localhost:8080/interlinked3' => 1,
+                $this->url.'/interlinked3' => 1,
-                'http://localhost:8080/interlinked4' => 1,
+                $this->url.'/interlinked4' => 1,
             ]],
         ], print_r($sitemap,true));
     }
 
     public function testRedirectToFound(){
         $crawler=new Crawler();
-        $crawler->crawl('http://localhost:8080/redirectToFound');
+        $crawler->crawl($this->url.'/redirectToFound');
         $sitemap=$crawler->getResults();
         $this->assertTreeContains($sitemap,[
-            'http://localhost:8080/redirectToFound' => ['code' => 302],
+            $this->url.'/redirectToFound' => ['code' => 302],
-            'http://localhost:8080/found' => ['code' => 200 ],
+            $this->url.'/found' => ['code' => 200 ],
         ], print_r($sitemap,true));
     }
 
     public function testRedirectToNotFound(){
         $crawler=new Crawler();
-        $crawler->crawl('http://localhost:8080/redirectToNotFound');
+        $crawler->crawl($this->url.'/redirectToNotFound');
         $sitemap=$crawler->getResults();
         $this->assertTreeContains($sitemap,[
-            'http://localhost:8080/redirectToNotFound' => ['code' => 302],
+            $this->url.'/redirectToNotFound' => ['code' => 302],
-            'http://localhost:8080/notFound' => ['code' => 404 ],
+            $this->url.'/notFound' => ['code' => 404 ],
         ], print_r($sitemap,true));
     }
 
     public function testRedirectToRedirectToNotFound(){
         $crawler=new Crawler();
-        $crawler->crawl('http://localhost:8080/redirectToRedirectToNotFound');
+        $crawler->crawl($this->url.'/redirectToRedirectToNotFound');
         $sitemap=$crawler->getResults();
         $this->assertTreeContains($sitemap,[
-            'http://localhost:8080/redirectToRedirectToNotFound' => ['code' => 302],
+            $this->url.'/redirectToRedirectToNotFound' => ['code' => 302],
-            'http://localhost:8080/redirectToNotFound' => ['code' => 302],
+            $this->url.'/redirectToNotFound' => ['code' => 302],
-            'http://localhost:8080/notFound' => ['code' => 404],
+            $this->url.'/notFound' => ['code' => 404],
         ], print_r($sitemap,true));
     }
 
     public function testTwoRedirectsToSameLocation(){
         $crawler=new Crawler();
-        $crawler->crawl('http://localhost:8080/twoRedirectsToSameLocation');
+        $crawler->crawl($this->url.'/twoRedirectsToSameLocation');
         $sitemap=$crawler->getResults();
         $this->assertTreeContains($sitemap,[
-            'http://localhost:8080/twoRedirectsToSameLocation' => ['code' => 200],
+            $this->url.'/twoRedirectsToSameLocation' => ['code' => 200],
-            'http://localhost:8080/redirect1' => ['code' => 302],
+            $this->url.'/redirect1' => ['code' => 302],
-            'http://localhost:8080/redirect2' => ['code' => 302],
+            $this->url.'/redirect2' => ['code' => 302],
-            'http://localhost:8080/found' => ['code' => 200],
+            $this->url.'/found' => ['code' => 200],
         ], print_r($sitemap,true));
     }
 
     public function testTimeout(){
         $crawler=new Crawler([RequestOptions::CONNECT_TIMEOUT => 3, RequestOptions::TIMEOUT => 3]);
-        $crawler->crawl('http://localhost:8080/timeout');
+        $crawler->crawl($this->url.'/timeout');
         $sitemap=$crawler->getResults();
         $this->assertTreeContains($sitemap,[
-            'http://localhost:8080/timeout' => ['code' => '---'],
+            $this->url.'/timeout' => ['code' => '---'],
         ], print_r($sitemap,true));
     }
 
     public function testRedirectLoop(){
         $crawler=new Crawler();
-        $crawler->crawl('http://localhost:8080/redirectLoop');
+        $crawler->crawl($this->url.'/redirectLoop');
         $sitemap=$crawler->getResults();
         $this->assertTreeContains($sitemap,[
-            'http://localhost:8080/redirectLoop' => ['code' => '---'],
+            $this->url.'/redirectLoop' => ['code' => '302'],
         ], print_r($sitemap,true));
     }
 
     public function testInternalServerError(){
         $crawler=new Crawler();
-        $crawler->crawl('http://localhost:8080/internalServerError');
+        $crawler->crawl($this->url.'/internalServerError');
         $sitemap=$crawler->getResults();
         $this->assertTreeContains($sitemap,[
-            'http://localhost:8080/internalServerError' => ['code' => 500],
+            $this->url.'/internalServerError' => ['code' => 500],
         ], print_r($sitemap,true));
     }
 
     public function testInvalidStatusCode(){
         $crawler=new Crawler();
-        $crawler->crawl('http://localhost:8080/invalidStatusCode');
+        $crawler->crawl($this->url.'/invalidStatusCode');
         $sitemap=$crawler->getResults();
         $this->assertTreeContains($sitemap,[
-            'http://localhost:8080/invalidStatusCode' => ['code' => '---'],
+            $this->url.'/invalidStatusCode' => ['code' => '---'],
         ], print_r($sitemap,true));
     }
 

tests/server/server.js

@@ -25,7 +25,7 @@ app.get('/', function (request, response) {
 });
 
 app.get('/externalLink', function (request, response) {
-    response.end('<a href="http://example.com/"</a>');
+    response.end('<a href="http://example.com/">ext</a>');
 });
 
 app.get('/deeplink1', function (request, response) {

tests/server/start_server.sh

@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-
-npm install
-
-if [ -z ${TRAVIS_JOB_ID} ]; then
-    # not running under travis, stay in foreground until stopped
-    node server.js
-else
-    cd tests/server
-
-    # running under travis, daemonize
-    (node server.js &) || /bin/true
-fi