add tests

This commit is contained in:
James
2020-02-21 10:59:58 +00:00
parent ac7b849036
commit c81551e2bf
7 changed files with 551 additions and 10 deletions

View File

@@ -21,7 +21,9 @@ class Crawler{
$this->crawler = SpatieCrawler::create([
RequestOptions::ALLOW_REDIRECTS => [
'track_redirects' => true,
]
],
RequestOptions::CONNECT_TIMEOUT => 10,
RequestOptions::TIMEOUT => 10,
])
//->setMaximumDepth(1)
->setCrawlProfile(new CrawlInternalUrls($baseUrl))