-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathcrawl.php
More file actions
executable file
·36 lines (26 loc) · 854 Bytes
/
crawl.php
File metadata and controls
executable file
·36 lines (26 loc) · 854 Bytes
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
#!/usr/bin/php
<?php
require 'vendor/autoload.php';
if (count($argv) < 2) {
die("You need to specify a URL as the parameter to this script\n");
}
try {
$url = new SiteCrawler\Url($argv[1]);
} catch (SiteCrawler\Exceptions\InvalidUrlException $e) {
die("Invalid URL: {$e->getMessage()}");
} catch (Exception $e) {
die($e->getMessage());
}
$startTime = time();
$httpClient = new GuzzleHttp\Client;
$htmlParser = new SiteCrawler\HtmlParser(
new SiteCrawler\Factories\DOMFactory,
new SiteCrawler\Factories\UrlFactory
);
$scraper = new SiteCrawler\Scraper($httpClient, $htmlParser);
$crawler = new SiteCrawler\Crawler($url, $scraper);
$crawler->start();
$mapMaker = new SiteCrawler\Utils\MapMaker($url);
$mapMaker->createMap($crawler->getVisitedUrls());
$duration = time() - $startTime;
echo "Duration: {$duration} seconds";