from scrapling.fetchers import Fetcher, FetcherSession
with FetcherSession(impersonate='chrome') as session: # Use latest version of Chrome's TLS fingerprint
page = session.get('https://quotes.toscrape.com/', stealthy_headers=True)
quotes = page.css('.quote .text::text').getall()
# Or use one-off requests
page = Fetcher.get('https://quotes.toscrape.com/')
quotes = page.css('.quote .text::text').getall()
高级隐身模式
from scrapling.fetchers import StealthyFetcher, StealthySession
with StealthySession(headless=True, solve_cloudflare=True) as session: # Keep the browser open until you finish
page = session.fetch('https://nopecha.com/demo/cloudflare', google_search=False)
data = page.css('#padded_content a').getall()
# Or use one-off request style, it opens the browser for this request, then closes it after finishing
page = StealthyFetcher.fetch('https://nopecha.com/demo/cloudflare', solve_cloudflare=True)
data = page.css('#padded_content a').getall()
完整的浏览器自动化
from scrapling.fetchers import StealthyFetcher, StealthySession
with StealthySession(headless=True, disable_resources=False, network_idle=True) as session: # Keep the browser open until you finish
page = session.fetch('https://quotes.toscrape.com/', load_dom=False)
data = page.xpath('//span[@class="text"]/text()').getall() # XPath selector if you prefer it
# Or use one-off request style, it opens the browser for this request, then closes it after finishing
page = StealthyFetcher.fetch('https://quotes.toscrape.com/')
data = page.css('.quote .text::text').getall()
爬虫
构建支持并发请求、多种会话类型以及暂停 / 恢复的完整爬虫：
from scrapling.spiders import Spider, Response
class QuotesSpider(Spider):
name = "quotes"
start_urls = ["https://quotes.toscrape.com/"]
concurrent_requests = 10
async def parse(self, response: Response):
for quote in response.css('.quote'):
yield {
"text": quote.css('.text::text').get(),
"author": quote.css('.author::text').get(),
}
next_page = response.css('.next a')
if next_page:
yield response.follow(next_page[0].attrib['href'])
result = QuotesSpider().start()
print(f"Scraped {len(result.items)} quotes")
result.items.to_json("quotes.json")
在单个爬虫中使用多种会话类型：
from scrapling.spiders import Spider, Request, Response
from scrapling.fetchers import FetcherSession, AsyncStealthySession
class MultiSessionSpider(Spider):
name = "multi"
start_urls = ["https://example.com/"]
def configure_sessions(self, manager):
manager.add("fast", FetcherSession(impersonate="chrome"))
manager.add("stealth", AsyncStealthySession(headless=True), lazy=True)
async def parse(self, response: Response):
for link in response.css('a::attr(href)').getall():
# Route protected pages through the stealth session
if "protected" in link:
yield Request(link, sid="stealth")
else:
yield Request(link, sid="fast", callback=self.parse) # explicit callback
from scrapling.spiders import ShopifySpider
class MyStore(ShopifySpider):
target_website = "example.com"
result = MyStore().start() # Every product in the store, one item per variant
该模板会读取通过公开集合端点提供的商品，每个变体对应一件商品。
高级解析与导航
from scrapling.fetchers import Fetcher
# Rich element selection and navigation
page = Fetcher.get('https://quotes.toscrape.com/')
# Get quotes with multiple selection methods
quotes = page.css('.quote') # CSS selector
quotes = page.xpath('//div[@class="quote"]') # XPath
quotes = page.find_all('div', {'class': 'quote'}) # BeautifulSoup-style
# Same as
quotes = page.find_all('div', class_='quote')
quotes = page.find_all(['div'], class_='quote')
quotes = page.find_all(class_='quote') # and so on...
# Find element by text content
quotes = page.find_by_text('Albert Einstein', first_match=False)
# Advanced navigation
quote_text = page.css('.quote')[0].css('.text::text').get()
quote_texts = page.css('.quote').css('.text::text').getall() # Chained selectors
first_quote = page.css('.quote')[0]
author = first_quote.css('.author::text').get()
next_quote = first_quote.next
parent_container = first_quote.parent
# Element relationships and similarity
similar_elements = first_quote.find_similar()
below_elements = first_quote.below_elements
如果你不想像下面这样获取网站，也可以直接使用解析器：
from scrapling.parser import Selector
page = Selector("<html>...</html>")
效果完全相同！
异步会话管理示例
import asyncio
from scrapling.fetchers import FetcherSession, AsyncStealthySession
async def main():
async with FetcherSession(http3=True) as http_session: # `FetcherSession` is context-aware and can work in both sync/async patterns
page1 = await http_session.get('https://quotes.toscrape.com/')
page2 = await http_session.get('https://quotes.toscrape.com/page/2/', impersonate='chrome')
print(len(page1.css('.quote')), len(page2.css('.quote')))
async with AsyncStealthySession(max_pages=2) as browser_session:
urls = ['https://quotes.toscrape.com/js/', 'https://quotes.toscrape.com/js/page/2/']
pages = await asyncio.gather(*(browser_session.fetch(url, wait_selector='.quote') for url in urls))
print([len(page.css('.quote')) for page in pages])
asyncio.run(main())
@misc{scrapling,
author = {Karim Shoair},
title = {Scrapling},
year = {2024},
url = {https://github.com/D4Vinci/Scrapling},
note = {A Python Web Scraping framework for everything from a single request to a full-scale crawl!}
}
-DFE0E5){kind=icon}
