Pada artikel berikut, kami akan memberikan beberapa jawaban untuk pertanyaan Anda yaitu Cara Crawling website menggunakan Laravel Dusk Spider dengan Contoh . Saya sangat berharap ini adalah jawaban yang Anda cari dan ini akan membantu Anda menghemat waktu. penegakan
Cara Merayap situs web menggunakan Laravel Dusk Spider dengan Contoh - Merayapi situs web menggunakan Laravel Dusk Spider
<?php
namespace Tests\Browser;
use App\Page;
use Facebook\WebDriver\WebDriverBy;
use Tests\DuskTestCase;
use Laravel\Dusk\Browser;
use Illuminate\Foundation\Testing\DatabaseMigrations;
class duskSpiderTest extends DuskTestCase
{
protected static $domain = 'laravel.com';
protected static $startUrl = '//laravel.com/';
public function setUp(): void{
parent::setUp();
$this->artisan('migrate:fresh');
}
/** @test */
public function urlSpider()
{
$startingLink = Page::create([
'url' => self::$startUrl,
'isCrawled' => false,
]);
$this->browse(function (Browser $browser) use ($startingLink) {
$this->getLinks($browser, $startingLink);
});
}
protected function getLinks(Browser $browser, $currentUrl){
$this->processCurrentUrl($browser, $currentUrl);
try{
foreach(Page::where('isCrawled', false)->get() as $link) {
$this->getLinks($browser, $link);
}
}catch(Exception $e){
}
}
protected function processCurrentUrl(Browser $browser, $currentUrl){
//Check if already crawled
if(Page::where('url', $currentUrl->url)->first()->isCrawled == true)
return;
//Visit URL
$browser->visit($currentUrl->url);
//Get Links and Save to DB if Valid
$linkElements = $browser->driver->findElements(WebDriverBy::tagName('a'));
foreach($linkElements as $element){
$href = $element->getAttribute('href');
$href = $this->trimUrl($href);
if($this->isValidUrl($href)){
//var_dump($href);
Page::create([
'url' => $href,
'isCrawled' => false,
]);
}
}
//Update current url status to crawled
$currentUrl->isCrawled = true;
$currentUrl->status = $this->getHttpStatus($currentUrl->url);
$currentUrl->title = $browser->driver->getTitle();
$currentUrl->save();
}
protected function isValidUrl($url){
$parsed_url = parse_url($url);
if(isset($parsed_url['host'])){
if(strpos($parsed_url['host'], self::$domain) !== false && !Page::where('url', $url)->exists()){
return true;
}
}
return false;
}
protected function trimUrl($url){
$url = strtok($url, '#');
$url = rtrim($url,"/");
return $url;
}
protected function getHttpStatus($url){
$headers = get_headers($url, 1);
return intval(substr($headers[0], 9, 3));
}
}
Jika Anda tidak puas dengan jawaban saya untuk Cara Merayap situs web menggunakan Laravel Dusk Spider dengan Contoh. Lihat lebih banyak topik serupa, atau beri saya pertanyaan baru
tidak terdefinisi