diff --git a/src/Crawler.php b/src/Crawler.php index 6ca04b7..35b8e6e 100644 --- a/src/Crawler.php +++ b/src/Crawler.php @@ -22,8 +22,8 @@ class Crawler{ RequestOptions::ALLOW_REDIRECTS => [ 'track_redirects' => true, ], - RequestOptions::CONNECT_TIMEOUT => 10, - RequestOptions::TIMEOUT => 10, + RequestOptions::CONNECT_TIMEOUT => 3, + RequestOptions::TIMEOUT => 3, ]) //->setMaximumDepth(1) ->setCrawlObserver($this->observer) diff --git a/tests/CrawlerTest.php b/tests/CrawlerTest.php index bf894ef..43a8660 100644 --- a/tests/CrawlerTest.php +++ b/tests/CrawlerTest.php @@ -5,6 +5,33 @@ use JHodges\Sitemap\Crawler; class CrawlerTest extends TestCase{ + public function testFullSite(){ + $crawler=new Crawler(); + $crawler->crawl('http://localhost:8080/'); + $sitemap=$crawler->getResults(); + $this->assertTreeContains($sitemap,[ + 'http://example.com/' => ['code' => 200], + 'http://localhost:8080/deeplink1' => ['code' => 200], + 'http://localhost:8080/deeplink2' => ['code' => 200], + 'http://localhost:8080/deeplink3' => ['code' => 200], + 'http://localhost:8080/externalLink' => ['code' => 200], + 'http://localhost:8080/found' => ['code' => 200], + 'http://localhost:8080/interlinked1' => ['code' => 200], + 'http://localhost:8080/interlinked2' => ['code' => 200], + 'http://localhost:8080/interlinked3' => ['code' => 200], + 'http://localhost:8080/internalServerError' => ['code' => 500], + 'http://localhost:8080/notFound' => ['code' => 404], + 'http://localhost:8080/redirect1' => ['code' => 302], + 'http://localhost:8080/redirect2' => ['code' => 302], + 'http://localhost:8080/redirectLoop' => ['code' => '---'], + 'http://localhost:8080/redirectToFound' => ['code' => 302 ], + 'http://localhost:8080/redirectToNotFound' => ['code' => 302 ], + 'http://localhost:8080/redirectToRedirectToNotFound' => ['code' => 302], + 'http://localhost:8080/timeout' => ['code' => '---'], + 'http://localhost:8080/twoRedirectsToSameLocation' => ['code' => 200], + ], print_r($sitemap,true)); + } + public function testFound(){ $crawler=new Crawler(); $crawler->crawl('http://localhost:8080/found'); @@ -44,7 +71,6 @@ class CrawlerTest extends TestCase{ ], print_r($sitemap,true)); } - public function testInterlinked(){ $crawler=new Crawler(); $crawler->crawl('http://localhost:8080/interlinked1'); @@ -74,7 +100,7 @@ class CrawlerTest extends TestCase{ $crawler->crawl('http://localhost:8080/redirectToFound'); $sitemap=$crawler->getResults(); $this->assertTreeContains($sitemap,[ - 'http://localhost:8080/redirectToFound' => ['code' => 302 ], + 'http://localhost:8080/redirectToFound' => ['code' => 302], 'http://localhost:8080/found' => ['code' => 200 ], ]); } @@ -84,7 +110,7 @@ class CrawlerTest extends TestCase{ $crawler->crawl('http://localhost:8080/redirectToNotFound'); $sitemap=$crawler->getResults(); $this->assertTreeContains($sitemap,[ - 'http://localhost:8080/redirectToNotFound' => ['code' => 302 ], + 'http://localhost:8080/redirectToNotFound' => ['code' => 302], 'http://localhost:8080/notFound' => ['code' => 404 ], ]); } @@ -94,9 +120,9 @@ class CrawlerTest extends TestCase{ $crawler->crawl('http://localhost:8080/redirectToRedirectToNotFound'); $sitemap=$crawler->getResults(); $this->assertTreeContains($sitemap,[ - 'http://localhost:8080/redirectToRedirectToNotFound' => ['code' => 302 ], - 'http://localhost:8080/redirectToNotFound' => ['code' => 302 ], - 'http://localhost:8080/notFound' => ['code' => 404 ], + 'http://localhost:8080/redirectToRedirectToNotFound' => ['code' => 302], + 'http://localhost:8080/redirectToNotFound' => ['code' => 302], + 'http://localhost:8080/notFound' => ['code' => 404], ]); } @@ -105,10 +131,10 @@ class CrawlerTest extends TestCase{ $crawler->crawl('http://localhost:8080/twoRedirectsToSameLocation'); $sitemap=$crawler->getResults(); $this->assertTreeContains($sitemap,[ - 'http://localhost:8080/twoRedirectsToSameLocation' => ['code' => 200 ], - 'http://localhost:8080/redirect1' => ['code' => 302 ], - 'http://localhost:8080/redirect2' => ['code' => 302 ], - 'http://localhost:8080/found' => ['code' => 200 ], + 'http://localhost:8080/twoRedirectsToSameLocation' => ['code' => 200], + 'http://localhost:8080/redirect1' => ['code' => 302], + 'http://localhost:8080/redirect2' => ['code' => 302], + 'http://localhost:8080/found' => ['code' => 200], ]); } @@ -117,7 +143,16 @@ class CrawlerTest extends TestCase{ $crawler->crawl('http://localhost:8080/timeout'); $sitemap=$crawler->getResults(); $this->assertTreeContains($sitemap,[ - 'http://localhost:8080/timeout' => ['code' => '---' ], + 'http://localhost:8080/timeout' => ['code' => '---'], + ]); + } + + public function testRedirectLoop(){ + $crawler=new Crawler(); + $crawler->crawl('http://localhost:8080/redirectLoop'); + $sitemap=$crawler->getResults(); + $this->assertTreeContains($sitemap,[ + 'http://localhost:8080/redirectLoop' => ['code' => '---'], ]); } @@ -126,7 +161,7 @@ class CrawlerTest extends TestCase{ $crawler->crawl('http://localhost:8080/internalServerError'); $sitemap=$crawler->getResults(); $this->assertTreeContains($sitemap,[ - 'http://localhost:8080/internalServerError' => ['code' => 500 ], + 'http://localhost:8080/internalServerError' => ['code' => 500], ]); } diff --git a/tests/server/server.js b/tests/server/server.js index aca2cf6..f90e79c 100644 --- a/tests/server/server.js +++ b/tests/server/server.js @@ -12,6 +12,7 @@ app.get('/', function (request, response) { '