make observer collate all found on urls

This commit is contained in:
James
2020-02-21 10:24:34 +00:00
parent 6a986a62c0
commit ac7b849036
4 changed files with 977 additions and 909 deletions

View File

@@ -13,21 +13,28 @@ use Spatie\Crawler\CrawlInternalUrls;
class Crawler{
public function Crawl($url){
$observer=new CrawlObserver();
private $observer;
private $crawler;
SpatieCrawler::create([
public function __construct($baseUrl){
$this->observer = new CrawlObserver();
$this->crawler = SpatieCrawler::create([
RequestOptions::ALLOW_REDIRECTS => [
'track_redirects' => true,
]
])
//->setMaximumDepth(1)
->setCrawlObserver($observer)
->setCrawlProfile(new CrawlInternalUrls($url))
//->addToCrawlQueue( CrawlUrl::create(new Uri('https://hudevad.com/en/')) )
->startCrawling($url)
->setCrawlProfile(new CrawlInternalUrls($baseUrl))
->setCrawlObserver($this->observer)
;
return $observer->results;
}
public function crawl($url){
$this->crawler->startCrawling($url);
}
public function getResults(){
return $this->observer->results;
}
}