lock pacakge version to fix dep breakage

init drone file
allow testing on custom host/port

Co-authored-by: James <inbox.dev@jhodges.co.uk>
Reviewed-on: #1
Co-Authored-By: James <jhodges@noreply@git.jhodges.co.uk>
Co-Committed-By: James <jhodges@noreply@git.jhodges.co.uk>

.drone.yml

@@ -0,0 +1,35 @@
+kind: pipeline
+type: docker
+name: testsuite
+
+steps:
+
+- name: composer install
+  image: chialab/php:7.3
+  commands:
+  - composer install
+  volumes:
+  - name: composer-cache
+  path: /root/.composer/cache/
+
+- name: wait for test server
+  image: alpine:3.12
+  commands:
+  - echo "Waiting for server to launch on testserver:8080..."
+  - while ! nc -z testserver 8080; do sleep 0.1 ; done
+  - echo "Ready!"
+
+- name: run tests
+  image: chialab/php:7.3
+  commands:
+  - URL=http://testserver:8080 vendor/bin/phpunit tests --testdox --color=always --no-interaction
+
+services:
+
+- name: testserver
+  image: node:15.2
+  detach: true
+  commands:
+  - cd tests/server/
+  - npm install
+  - node server.js

README.md

@@ -3,7 +3,7 @@ Similar project to [spatie/http-status-check](https://github.com/spatie/http-sta
 ## Install
 
 ```plain
-composer config repositories.jhodges composer https://git.jhodges.co.uk/composer
+composer config repositories.jhodges composer https://composer.jhodges.co.uk/
 composer require jhodges/sitemap
 ```
 
@@ -37,7 +37,8 @@ Start the test server, will listen on localhost:8080
 
 ```plain
 cd tests/server
-./start_server.sh
+npm install
+node server.js
 ```
 
 Run the tests:

composer.json

@@ -4,8 +4,9 @@
     "type": "library",
     "require": {
         "php": "^7.1",
-        "spatie/crawler": "^4.6",
+        "spatie/crawler": "4.6.6",
-        "cweagans/composer-patches": "~1.0"
+        "cweagans/composer-patches": "~1.0",
+        "guzzlehttp/guzzle": "6.5.2"
     },
     "require-dev": {
         "phpunit/phpunit": "^8.5"
@@ -27,7 +28,7 @@
                 "add crawled again observer": "https://patch-diff.githubusercontent.com/raw/spatie/crawler/pull/280.patch"
             },
             "guzzlehttp/guzzle": {
-                "Status code must be an integer value between 1xx and 5xx": "https://patch-diff.githubusercontent.com/raw/guzzle/guzzle/pull/2591.patch"
+                "Status code must be an integer value between 1xx and 5xx": "https://github.com/guzzle/guzzle/commit/f81cd6cdff1213f90de8f012489017510e3d6ff4.patch"
             }
         }
     }

docker/Dockerfile

@@ -0,0 +1,17 @@
+FROM markizano/devuan:beowulf-amd64
+
+RUN apt-get update
+RUN apt-get install -y php-cli git unzip php-xml
+
+COPY get_composer.sh /root/
+RUN bash /root/get_composer.sh && rm /root/get_composer.sh
+
+RUN mkdir /app
+WORKDIR /app
+RUN composer init && composer config repositories.jhodges composer https://composer.jhodges.co.uk && composer require jhodges/sitemap
+
+COPY crawl.php /app/
+
+RUN apt clean
+
+ENTRYPOINT php crawl.php

docker/crawl.php

@@ -0,0 +1,45 @@
+<?php
+require_once(__DIR__.'/vendor/autoload.php');
+
+use \JHodges\Sitemap\Crawler;
+
+if($url=getenv('CRAWL_URL')){
+    $urls=array_filter(array_map('trim',explode(',',$url)));
+}else{
+	die("No env: CRAWL_URL\n");
+}
+
+if($code=getenv('CRAWL_CODE')){
+	$codes=array_filter(array_map('trim',explode(',',$code)));
+}else{
+    $codes=[];
+}
+
+$crawler=new Crawler();
+foreach($urls as $url){
+    $crawler->crawl($url);
+}
+
+$summary=[];
+$details='';
+foreach($crawler->getResults() as $url=>$result){
+    $summary[$result['code']]++;
+
+    if( count($codes)==0 || in_array($result['code'],$codes) ){
+        $details.="{$result['code']} {$url}\n";
+        foreach($result['foundOn'] as $url=>$count){
+            $details.="    <- ($count) $url\n";
+        }
+    }
+}
+
+ksort($summary);
+echo '|code|count|'."\n";
+echo '|----|-----|'."\n";
+foreach($summary as $code=>$count){
+    echo "| $code | $count |\n";
+}
+
+if($details){
+    echo "\n\n----\n\n```\n$details\n```\n";
+}

docker/get_composer.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+EXPECTED_CHECKSUM="$(wget -q -O - https://composer.github.io/installer.sig)"
+php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
+ACTUAL_CHECKSUM="$(php -r "echo hash_file('sha384', 'composer-setup.php');")"
+
+if [ "$EXPECTED_CHECKSUM" != "$ACTUAL_CHECKSUM" ]
+then
+    >&2 echo 'ERROR: Invalid installer checksum'
+    rm composer-setup.php
+    exit 1
+fi
+
+php composer-setup.php --1 --filename=composer --install-dir=/usr/local/bin
+RESULT=$?
+rm composer-setup.php
+exit $RESULT

example/crawl.php

@@ -0,0 +1,19 @@
+#!/usr/bin/php
+<?php
+require_once(__DIR__.'/../vendor/autoload.php');
+
+if(!$url=$argv[1]??null){
+	die("url?\n");
+}
+
+use \JHodges\Sitemap\Crawler;
+
+$crawler=new Crawler();
+$crawler->crawl($argv[1]);
+
+foreach($crawler->getResults() as $url=>$result){
+        echo("{$result['code']} {$url}\n");
+        foreach($result['foundOn'] as $url=>$count){
+            echo("    -> ($count) $url\n");
+        }
+}

src/Crawler.php

@@ -9,14 +9,15 @@ use Psr\Http\Message\UriInterface;
 
 use Spatie\Crawler\Crawler as SpatieCrawler;
 use Spatie\Crawler\CrawlUrl;
-use Spatie\Crawler\CrawlInternalUrls;
+use Spatie\Crawler\CrawlAllUrls;
+use Spatie\Crawler\CrawlProfile;
 
 class Crawler{
 
     private $observer;
     private $crawler;
 
-    public function __construct($reqOps=[]){
+    public function __construct(array $reqOps=[]){
         $this->crawler = SpatieCrawler::create(array_merge($reqOps, [
             RequestOptions::ALLOW_REDIRECTS => [
                 'track_redirects' => true,
@@ -25,13 +26,24 @@ class Crawler{
 
         $this->observer = new CrawlObserver();
         $this->crawler->setCrawlObserver($this->observer);
+        $this->crawler->setCrawlProfile(new CrawlAllUrls());
     }
 
-    public function crawl($url){
+    public function setUserAgent(String $agent) : self{
+        $this->crawler->setUserAgent($agent);
+	return $this;
+    }
+
+    public function setCrawlProfile(CrawlProfile $p) : self{
+        $this->crawler->setCrawlProfile($p);
+	return $this;
+    }
+
+    public function crawl(String $url) : void{
         $this->crawler->startCrawling($url);
     }
 
-    public function getResults(){
+    public function getResults() : array{
         return $this->observer->results;
     }
 

tests/CrawlerTest.php

@@ -6,188 +6,197 @@ use GuzzleHttp\RequestOptions;
 
 class CrawlerTest extends TestCase{
 
+    private $url='http://localhost:8080';
+
+    public function __construct(){
+        parent::__construct();
+        if(getenv('URL')){
+            $this->url=getenv('URL');
+        }
+    }
+
     public function testFullSite(){
         $crawler=new Crawler([RequestOptions::CONNECT_TIMEOUT => 3, RequestOptions::TIMEOUT => 3]);
-        $crawler->crawl('http://localhost:8080/');
+        $crawler->crawl($this->url);
         $sitemap=$crawler->getResults();
         $this->assertTreeContains($sitemap,[
             'http://example.com/' => ['code' => 200],
-            'http://localhost:8080/deeplink1' => ['code' => 200],
+            $this->url.'/deeplink1' => ['code' => 200],
-            'http://localhost:8080/deeplink2' => ['code' => 200],
+            $this->url.'/deeplink2' => ['code' => 200],
-            'http://localhost:8080/deeplink3' => ['code' => 200],
+            $this->url.'/deeplink3' => ['code' => 200],
-            'http://localhost:8080/externalLink' => ['code' => 200],
+            $this->url.'/externalLink' => ['code' => 200],
-            'http://localhost:8080/found' => ['code' => 200],
+            $this->url.'/found' => ['code' => 200],
-            'http://localhost:8080/interlinked1' => ['code' => 200],
+            $this->url.'/interlinked1' => ['code' => 200],
-            'http://localhost:8080/interlinked2' => ['code' => 200],
+            $this->url.'/interlinked2' => ['code' => 200],
-            'http://localhost:8080/interlinked3' => ['code' => 200],
+            $this->url.'/interlinked3' => ['code' => 200],
-            'http://localhost:8080/internalServerError' => ['code' => 500],
+            $this->url.'/internalServerError' => ['code' => 500],
-            'http://localhost:8080/invalidStatusCode' => ['code' => '---'],
+            $this->url.'/invalidStatusCode' => ['code' => '---'],
-            'http://localhost:8080/notFound' => ['code' => 404],
+            $this->url.'/notFound' => ['code' => 404],
-            'http://localhost:8080/redirect1' => ['code' => 302],
+            $this->url.'/redirect1' => ['code' => 302],
-            'http://localhost:8080/redirect2' => ['code' => 302],
+            $this->url.'/redirect2' => ['code' => 302],
-            'http://localhost:8080/redirectLoop' => ['code' => '---'],
+            $this->url.'/redirectLoop' => ['code' => '---'],
-            'http://localhost:8080/redirectToFound' => ['code' => 302 ],
+            $this->url.'/redirectToFound' => ['code' => 302 ],
-            'http://localhost:8080/redirectToNotFound' => ['code' => 302 ],
+            $this->url.'/redirectToNotFound' => ['code' => 302 ],
-            'http://localhost:8080/redirectToRedirectToNotFound' => ['code' => 302],
+            $this->url.'/redirectToRedirectToNotFound' => ['code' => 302],
-            'http://localhost:8080/timeout' => ['code' => '---'],
+            $this->url.'/timeout' => ['code' => '---'],
-            'http://localhost:8080/twoRedirectsToSameLocation' => ['code' => 200],
+            $this->url.'/twoRedirectsToSameLocation' => ['code' => 200],
         ], print_r($sitemap,true));
     }
 
     public function testFound(){
         $crawler=new Crawler();
-        $crawler->crawl('http://localhost:8080/found');
+        $crawler->crawl($this->url.'/found');
         $sitemap=$crawler->getResults();
         $this->assertTreeContains($sitemap,[
-            'http://localhost:8080/found' => ['code' => 200],
+            $this->url.'/found' => ['code' => 200],
         ], print_r($sitemap,true));
     }
 
     public function testNotFound(){
         $crawler=new Crawler();
-        $crawler->crawl('http://localhost:8080/notFound');
+        $crawler->crawl($this->url.'/notFound');
         $sitemap=$crawler->getResults();
         $this->assertTreeContains($sitemap,[
-            'http://localhost:8080/notFound' => ['code' => 404],
+            $this->url.'/notFound' => ['code' => 404],
         ], print_r($sitemap,true));
     }
 
     public function testExternalLink(){
         $crawler=new Crawler();
-        $crawler->crawl('http://localhost:8080/externalLink');
+        $crawler->crawl($this->url.'/externalLink');
         $sitemap=$crawler->getResults();
         $this->assertTreeContains($sitemap,[
-            'http://localhost:8080/externalLink' => ['code' => 200],
+            $this->url.'/externalLink' => ['code' => 200],
             'http://example.com/' => ['code' => 200],
         ], print_r($sitemap,true));
     }
 
     public function testDeeplink(){
         $crawler=new Crawler();
-        $crawler->crawl('http://localhost:8080/deeplink1');
+        $crawler->crawl($this->url.'/deeplink1');
         $sitemap=$crawler->getResults();
         $this->assertTreeContains($sitemap,[
-            'http://localhost:8080/deeplink1' => ['code' => 200],
+            $this->url.'/deeplink1' => ['code' => 200],
-            'http://localhost:8080/deeplink2' => ['code' => 200],
+            $this->url.'/deeplink2' => ['code' => 200],
-            'http://localhost:8080/deeplink3' => ['code' => 200],
+            $this->url.'/deeplink3' => ['code' => 200],
         ], print_r($sitemap,true));
     }
 
     public function testInterlinked(){
         $crawler=new Crawler();
-        $crawler->crawl('http://localhost:8080/interlinked1');
+        $crawler->crawl($this->url.'/interlinked1');
-        $crawler->crawl('http://localhost:8080/interlinked4'); //this ensures the order or results for the URL tracking test PART2
+        $crawler->crawl($this->url.'/interlinked4'); //this ensures the order or results for the URL tracking test PART2
         $sitemap=$crawler->getResults();
         $this->assertTreeContains($sitemap,[
-            'http://localhost:8080/interlinked1' => ['code' => 200 , 'foundOn' => [
+            $this->url.'/interlinked1' => ['code' => 200 , 'foundOn' => [
-                'http://localhost:8080/interlinked1' => 1,
+                $this->url.'/interlinked1' => 1,
-                'http://localhost:8080/interlinked2' => 1,
+                $this->url.'/interlinked2' => 1,
-                'http://localhost:8080/interlinked3' => 1,
+                $this->url.'/interlinked3' => 1,
-                'http://localhost:8080/interlinked4' => 1,
+                $this->url.'/interlinked4' => 1,
             ]],
-            'http://localhost:8080/interlinked2' => ['code' => 200 , 'foundOn' => [
+            $this->url.'/interlinked2' => ['code' => 200 , 'foundOn' => [
-                'http://localhost:8080/interlinked1' => 1,
+                $this->url.'/interlinked1' => 1,
-                'http://localhost:8080/interlinked2' => 1,
+                $this->url.'/interlinked2' => 1,
-                'http://localhost:8080/interlinked3' => 1,
+                $this->url.'/interlinked3' => 1,
-                'http://localhost:8080/interlinked4' => 1,
+                $this->url.'/interlinked4' => 1,
             ]],
-            'http://localhost:8080/interlinked3' => ['code' => 200 , 'foundOn' => [
+            $this->url.'/interlinked3' => ['code' => 200 , 'foundOn' => [
-                'http://localhost:8080/interlinked1' => 1,
+                $this->url.'/interlinked1' => 1,
-                'http://localhost:8080/interlinked2' => 1,
+                $this->url.'/interlinked2' => 1,
-                'http://localhost:8080/interlinked3' => 1,
+                $this->url.'/interlinked3' => 1,
-                'http://localhost:8080/interlinked4' => 1,
+                $this->url.'/interlinked4' => 1,
             ]],
-            'http://localhost:8080/found' => ['code' => 200 , 'foundOn' => [
+            $this->url.'/found' => ['code' => 200 , 'foundOn' => [
-                'http://localhost:8080/interlinked1' => 1,
+                $this->url.'/interlinked1' => 1,
-                'http://localhost:8080/interlinked2' => 1,
+                $this->url.'/interlinked2' => 1,
-                'http://localhost:8080/interlinked3' => 1,
+                $this->url.'/interlinked3' => 1,
-                'http://localhost:8080/interlinked4' => 1,
+                $this->url.'/interlinked4' => 1,
             ]],
-            'http://localhost:8080/redirectToFound' => ['code' => 302 , 'foundOn' => [
+            $this->url.'/redirectToFound' => ['code' => 302 , 'foundOn' => [
-                'http://localhost:8080/interlinked1' => 1,
+                $this->url.'/interlinked1' => 1,
-                'http://localhost:8080/interlinked2' => 1,
+                $this->url.'/interlinked2' => 1,
-                'http://localhost:8080/interlinked3' => 1,
+                $this->url.'/interlinked3' => 1,
-                'http://localhost:8080/interlinked4' => 1,
+                $this->url.'/interlinked4' => 1,
             ]],
         ], print_r($sitemap,true));
     }
 
     public function testRedirectToFound(){
         $crawler=new Crawler();
-        $crawler->crawl('http://localhost:8080/redirectToFound');
+        $crawler->crawl($this->url.'/redirectToFound');
         $sitemap=$crawler->getResults();
         $this->assertTreeContains($sitemap,[
-            'http://localhost:8080/redirectToFound' => ['code' => 302],
+            $this->url.'/redirectToFound' => ['code' => 302],
-            'http://localhost:8080/found' => ['code' => 200 ],
+            $this->url.'/found' => ['code' => 200 ],
         ], print_r($sitemap,true));
     }
 
     public function testRedirectToNotFound(){
         $crawler=new Crawler();
-        $crawler->crawl('http://localhost:8080/redirectToNotFound');
+        $crawler->crawl($this->url.'/redirectToNotFound');
         $sitemap=$crawler->getResults();
         $this->assertTreeContains($sitemap,[
-            'http://localhost:8080/redirectToNotFound' => ['code' => 302],
+            $this->url.'/redirectToNotFound' => ['code' => 302],
-            'http://localhost:8080/notFound' => ['code' => 404 ],
+            $this->url.'/notFound' => ['code' => 404 ],
         ], print_r($sitemap,true));
     }
 
     public function testRedirectToRedirectToNotFound(){
         $crawler=new Crawler();
-        $crawler->crawl('http://localhost:8080/redirectToRedirectToNotFound');
+        $crawler->crawl($this->url.'/redirectToRedirectToNotFound');
         $sitemap=$crawler->getResults();
         $this->assertTreeContains($sitemap,[
-            'http://localhost:8080/redirectToRedirectToNotFound' => ['code' => 302],
+            $this->url.'/redirectToRedirectToNotFound' => ['code' => 302],
-            'http://localhost:8080/redirectToNotFound' => ['code' => 302],
+            $this->url.'/redirectToNotFound' => ['code' => 302],
-            'http://localhost:8080/notFound' => ['code' => 404],
+            $this->url.'/notFound' => ['code' => 404],
         ], print_r($sitemap,true));
     }
 
     public function testTwoRedirectsToSameLocation(){
         $crawler=new Crawler();
-        $crawler->crawl('http://localhost:8080/twoRedirectsToSameLocation');
+        $crawler->crawl($this->url.'/twoRedirectsToSameLocation');
         $sitemap=$crawler->getResults();
         $this->assertTreeContains($sitemap,[
-            'http://localhost:8080/twoRedirectsToSameLocation' => ['code' => 200],
+            $this->url.'/twoRedirectsToSameLocation' => ['code' => 200],
-            'http://localhost:8080/redirect1' => ['code' => 302],
+            $this->url.'/redirect1' => ['code' => 302],
-            'http://localhost:8080/redirect2' => ['code' => 302],
+            $this->url.'/redirect2' => ['code' => 302],
-            'http://localhost:8080/found' => ['code' => 200],
+            $this->url.'/found' => ['code' => 200],
         ], print_r($sitemap,true));
     }
 
     public function testTimeout(){
         $crawler=new Crawler([RequestOptions::CONNECT_TIMEOUT => 3, RequestOptions::TIMEOUT => 3]);
-        $crawler->crawl('http://localhost:8080/timeout');
+        $crawler->crawl($this->url.'/timeout');
         $sitemap=$crawler->getResults();
         $this->assertTreeContains($sitemap,[
-            'http://localhost:8080/timeout' => ['code' => '---'],
+            $this->url.'/timeout' => ['code' => '---'],
         ], print_r($sitemap,true));
     }
 
     public function testRedirectLoop(){
         $crawler=new Crawler();
-        $crawler->crawl('http://localhost:8080/redirectLoop');
+        $crawler->crawl($this->url.'/redirectLoop');
         $sitemap=$crawler->getResults();
         $this->assertTreeContains($sitemap,[
-            'http://localhost:8080/redirectLoop' => ['code' => '---'],
+            $this->url.'/redirectLoop' => ['code' => '---'],
         ], print_r($sitemap,true));
     }
 
     public function testInternalServerError(){
         $crawler=new Crawler();
-        $crawler->crawl('http://localhost:8080/internalServerError');
+        $crawler->crawl($this->url.'/internalServerError');
         $sitemap=$crawler->getResults();
         $this->assertTreeContains($sitemap,[
-            'http://localhost:8080/internalServerError' => ['code' => 500],
+            $this->url.'/internalServerError' => ['code' => 500],
         ], print_r($sitemap,true));
     }
 
     public function testInvalidStatusCode(){
         $crawler=new Crawler();
-        $crawler->crawl('http://localhost:8080/invalidStatusCode');
+        $crawler->crawl($this->url.'/invalidStatusCode');
         $sitemap=$crawler->getResults();
         $this->assertTreeContains($sitemap,[
-            'http://localhost:8080/invalidStatusCode' => ['code' => '---'],
+            $this->url.'/invalidStatusCode' => ['code' => '---'],
         ], print_r($sitemap,true));
     }
 

tests/server/start_server.sh

@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-
-npm install
-
-if [ -z ${TRAVIS_JOB_ID} ]; then
-    # not running under travis, stay in foreground until stopped
-    node server.js
-else
-    cd tests/server
-
-    # running under travis, daemonize
-    (node server.js &) || /bin/true
-fi