FM / src

/home/techb158/ileadtechnology.com/vendor/spatie/crawler/src/

/home/techb158/ileadtechnology.com/vendor/spatie/crawler/src

Name	Size	Mode	Actions
CrawlQueue/	-	0755	rm
Exception/	-	0755	rm
Handlers/	-	0755	rm
CrawlAllUrls.php	204	0644	edit dl rm
Crawler.php	13484	0644	edit dl rm
CrawlerRobots.php	1242	0644	edit dl rm
CrawlInternalUrls.php	493	0644	edit dl rm
CrawlObserver.php	1333	0644	edit dl rm
CrawlObserverCollection.php	2144	0644	edit dl rm
CrawlProfile.php	322	0644	edit dl rm
CrawlSubdomains.php	658	0644	edit dl rm
CrawlUrl.php	858	0644	edit dl rm
LinkAdder.php	3723	0644	edit dl rm

Edit: /home/techb158/ileadtechnology.com/vendor/spatie/crawler/src/CrawlerRobots.php (1242B)

<?php

namespace Spatie\Crawler;

use Spatie\Robots\RobotsHeaders;
use Spatie\Robots\RobotsMeta;

class CrawlerRobots
{
    /** @var \Spatie\Robots\RobotsHeaders */
    protected $robotsHeaders;

/** @var \Spatie\Robots\RobotsMeta */
    protected $robotsMeta;

/** @var bool */
    protected $mustRespectRobots;

public function __construct(array $headers, string $body, bool $mustRespectRobots)
    {
        $this->robotsHeaders = RobotsHeaders::create($headers);

$this->robotsMeta = RobotsMeta::create($body);

$this->mustRespectRobots = $mustRespectRobots;
    }

public function mayIndex(): bool
    {
        if (! $this->mustRespectRobots) {
            return true;
        }

if (! $this->robotsHeaders->mayIndex()) {
            return false;
        }

if (! $this->robotsMeta->mayIndex()) {
            return false;
        }

return true;
    }

public function mayFollow(): bool
    {
        if (! $this->mustRespectRobots) {
            return true;
        }

if (! $this->robotsHeaders->mayFollow()) {
            return false;
        }

if (! $this->robotsMeta->mayFollow()) {
            return false;
        }

return true;
    }
}

robotsHeaders = RobotsHeaders::create($headers); $this->robotsMeta = RobotsMeta::create($body); $this->mustRespectRobots = $mustRespectRobots; } public function mayIndex(): bool { if (! $this->mustRespectRobots) { return true; } if (! $this->robotsHeaders->mayIndex()) { return false; } if (! $this->robotsMeta->mayIndex()) { return false; } return true; } public function mayFollow(): bool { if (! $this->mustRespectRobots) { return true; } if (! $this->robotsHeaders->mayFollow()) { return false; } if (! $this->robotsMeta->mayFollow()) { return false; } return true; } }