lock pacakge version to fix dep breakage

.drone.yml

@@ -5,7 +5,7 @@ name: testsuite
 steps:
 
 - name: composer install
-  image: chialab/php
+  image: chialab/php:7.3
   commands:
   - composer install
   volumes:
@@ -13,21 +13,21 @@ steps:
   path: /root/.composer/cache/
 
 - name: wait for test server
-  image: alpine
+  image: alpine:3.12
   commands:
   - echo "Waiting for server to launch on testserver:8080..."
   - while ! nc -z testserver 8080; do sleep 0.1 ; done
   - echo "Ready!"
 
 - name: run tests
-  image: chialab/php
+  image: chialab/php:7.3
   commands:
-  - URL=http://testserver:8080 vendor/bin/phpunit tests
+  - URL=http://testserver:8080 vendor/bin/phpunit tests --testdox --color=always --no-interaction
 
 services:
 
 - name: testserver
-  image: node
+  image: node:15.2
   detach: true
   commands:
   - cd tests/server/

README.md

@@ -3,7 +3,7 @@ Similar project to [spatie/http-status-check](https://github.com/spatie/http-sta
 ## Install
 
 ```plain
-composer config repositories.jhodges composer https://git.jhodges.co.uk/composer
+composer config repositories.jhodges composer https://composer.jhodges.co.uk/
 composer require jhodges/sitemap
 ```
 

composer.json

@@ -4,8 +4,9 @@
     "type": "library",
     "require": {
         "php": "^7.1",
-        "spatie/crawler": "^4.6",
+        "spatie/crawler": "4.6.6",
-        "cweagans/composer-patches": "~1.0"
+        "cweagans/composer-patches": "~1.0",
+        "guzzlehttp/guzzle": "6.5.2"
     },
     "require-dev": {
         "phpunit/phpunit": "^8.5"
@@ -27,7 +28,7 @@
                 "add crawled again observer": "https://patch-diff.githubusercontent.com/raw/spatie/crawler/pull/280.patch"
             },
             "guzzlehttp/guzzle": {
-                "Status code must be an integer value between 1xx and 5xx": "https://patch-diff.githubusercontent.com/raw/guzzle/guzzle/pull/2591.patch"
+                "Status code must be an integer value between 1xx and 5xx": "https://github.com/guzzle/guzzle/commit/f81cd6cdff1213f90de8f012489017510e3d6ff4.patch"
             }
         }
     }

docker/Dockerfile

@@ -0,0 +1,17 @@
+FROM markizano/devuan:beowulf-amd64
+
+RUN apt-get update
+RUN apt-get install -y php-cli git unzip php-xml
+
+COPY get_composer.sh /root/
+RUN bash /root/get_composer.sh && rm /root/get_composer.sh
+
+RUN mkdir /app
+WORKDIR /app
+RUN composer init && composer config repositories.jhodges composer https://composer.jhodges.co.uk && composer require jhodges/sitemap
+
+COPY crawl.php /app/
+
+RUN apt clean
+
+ENTRYPOINT php crawl.php

docker/crawl.php

@@ -0,0 +1,45 @@
+<?php
+require_once(__DIR__.'/vendor/autoload.php');
+
+use \JHodges\Sitemap\Crawler;
+
+if($url=getenv('CRAWL_URL')){
+    $urls=array_filter(array_map('trim',explode(',',$url)));
+}else{
+	die("No env: CRAWL_URL\n");
+}
+
+if($code=getenv('CRAWL_CODE')){
+	$codes=array_filter(array_map('trim',explode(',',$code)));
+}else{
+    $codes=[];
+}
+
+$crawler=new Crawler();
+foreach($urls as $url){
+    $crawler->crawl($url);
+}
+
+$summary=[];
+$details='';
+foreach($crawler->getResults() as $url=>$result){
+    $summary[$result['code']]++;
+
+    if( count($codes)==0 || in_array($result['code'],$codes) ){
+        $details.="{$result['code']} {$url}\n";
+        foreach($result['foundOn'] as $url=>$count){
+            $details.="    <- ($count) $url\n";
+        }
+    }
+}
+
+ksort($summary);
+echo '|code|count|'."\n";
+echo '|----|-----|'."\n";
+foreach($summary as $code=>$count){
+    echo "| $code | $count |\n";
+}
+
+if($details){
+    echo "\n\n----\n\n```\n$details\n```\n";
+}

docker/get_composer.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+EXPECTED_CHECKSUM="$(wget -q -O - https://composer.github.io/installer.sig)"
+php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
+ACTUAL_CHECKSUM="$(php -r "echo hash_file('sha384', 'composer-setup.php');")"
+
+if [ "$EXPECTED_CHECKSUM" != "$ACTUAL_CHECKSUM" ]
+then
+    >&2 echo 'ERROR: Invalid installer checksum'
+    rm composer-setup.php
+    exit 1
+fi
+
+php composer-setup.php --1 --filename=composer --install-dir=/usr/local/bin
+RESULT=$?
+rm composer-setup.php
+exit $RESULT

src/Crawler.php

@@ -9,14 +9,15 @@ use Psr\Http\Message\UriInterface;
 
 use Spatie\Crawler\Crawler as SpatieCrawler;
 use Spatie\Crawler\CrawlUrl;
-use Spatie\Crawler\CrawlInternalUrls;
+use Spatie\Crawler\CrawlAllUrls;
+use Spatie\Crawler\CrawlProfile;
 
 class Crawler{
 
     private $observer;
     private $crawler;
 
-    public function __construct($reqOps=[]){
+    public function __construct(array $reqOps=[]){
         $this->crawler = SpatieCrawler::create(array_merge($reqOps, [
             RequestOptions::ALLOW_REDIRECTS => [
                 'track_redirects' => true,
@@ -25,13 +26,24 @@ class Crawler{
 
         $this->observer = new CrawlObserver();
         $this->crawler->setCrawlObserver($this->observer);
+        $this->crawler->setCrawlProfile(new CrawlAllUrls());
     }
 
-    public function crawl($url){
+    public function setUserAgent(String $agent) : self{
+        $this->crawler->setUserAgent($agent);
+	return $this;
+    }
+
+    public function setCrawlProfile(CrawlProfile $p) : self{
+        $this->crawler->setCrawlProfile($p);
+	return $this;
+    }
+
+    public function crawl(String $url) : void{
         $this->crawler->startCrawling($url);
     }
 
-    public function getResults(){
+    public function getResults() : array{
         return $this->observer->results;
     }