網頁抓取是資料收集、監控、自動化的基礎技能。Python 標準庫 urllib 提供無第三方依賴的 HTTP 客戶端功能。本文詳細說明 urllib.request 核心用法、錯誤處理、標頭偽裝、串流下載、編碼處理與最佳實務，並對比 requests 庫優劣。
import urllib.request
import urllib.error
url = "https://www.baidu.com"
# 簡單請求
try:
response = urllib.request.urlopen(url)
html = response.read().decode('utf-8', errors='ignore')
print(f"Status: {response.status}")
print(f"Length: {len(html)}")
except urllib.error.URLError as e:
print(f"網路錯誤: {e.reason}")
except urllib.error.HTTPError as e:
print(f"HTTP 錯誤: {e.code} {e.reason}")
# 建立 Request 物件（支援自訂標頭、POST 資料等）
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
'Accept': 'text/html,application/xhtml+xml',
'Accept-Language': 'zh-TW,zh;q=0.9,en;q=0.8',
}
req = urllib.request.Request(url, headers=headers)
try:
with urllib.request.urlopen(req, timeout=10) as response:
html = response.read().decode('utf-8')
print(response.headers.get('Content-Type'))
except Exception as e:
print(f"請求失敗: {e}")
import urllib.parse
post_url = "https://httpbin.org/post"
data = urllib.parse.urlencode({'key1': 'value1', 'key2': 'value2'}).encode('utf-8')
req = urllib.request.Request(post_url, data=data, method='POST')
req.add_header('Content-Type', 'application/x-www-form-urlencoded')
with urllib.request.urlopen(req) as response:
print(response.read().decode())
def download_file(url, filepath, chunk_size=8192):
"""串流下載，支援大檔案"""
req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
try:
with urllib.request.urlopen(req, timeout=30) as response, \
open(filepath, 'wb') as f:
total = response.length or 0
downloaded = 0
while True:
chunk = response.read(chunk_size)
if not chunk:
break
f.write(chunk)
downloaded += len(chunk)
if total:
print(f"\r下載進度: {downloaded}/{total} ({downloaded*100/total:.1f}%)", end="")
print("\n下載完成")
except Exception as e:
print(f"下載失敗: {e}")
download_file("https://speedtest.ftp.otenet.gr/files/test1Mb.db", "test1Mb.db")
import chardet # pip install chardet
def fetch_with_encoding(url):
req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
with urllib.request.urlopen(req) as response:
raw = response.read()
# 1. 優先用 HTTP 標頭 charset
content_type = response.headers.get('Content-Type', '')
encoding = None
if 'charset=' in content_type:
encoding = content_type.split('charset=')[-1].split(';')[0].strip()
# 2. 次優先用 chardet 偵測
if not encoding:
detected = chardet.detect(raw)
encoding = detected['encoding']
confidence = detected['confidence']
print(f"偵測編碼: {encoding} (信心度: {confidence})")
# 3. 最後 fallback utf-8
encoding = encoding or 'utf-8'
return raw.decode(encoding, errors='replace')
html = fetch_with_encoding("https://example.com")
import socket
from urllib.error import URLError, HTTPError
def robust_fetch(url, retries=3, timeout=10):
for attempt in range(retries):
try:
req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
with urllib.request.urlopen(req, timeout=timeout) as response:
return response.read().decode('utf-8', errors='ignore')
except HTTPError as e:
if e.code in (429, 500, 502, 503, 504):
print(f"HTTP {e.code}, 重試 {attempt+1}/{retries}")
time.sleep(2 ** attempt) # 指數退避
else:
raise
except URLError as e:
print(f"網路錯誤: {e.reason}, 重試 {attempt+1}/{retries}")
time.sleep(2 ** attempt)
except socket.timeout:
print(f"逾時, 重試 {attempt+1}/{retries}")
time.sleep(2 ** attempt)
raise Exception(f"重試 {retries} 次失敗: {url}")
import http.cookiejar
# 建立 Cookie 容器
cookie_jar = http.cookiejar.CookieJar()
opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookie_jar))
urllib.request.install_opener(opener) # 全域生效
# 登入
login_data = urllib.parse.urlencode({'username': 'user', 'password': 'pass'}).encode()
urllib.request.urlopen('https://example.com/login', login_data)
# 後續請求自動攜帶 Cookie
response = urllib.request.urlopen('https://example.com/dashboard')
print(response.read().decode())| 特性 | urllib (標準庫) | requests (第三方) |
|---|---|---|
| 安裝 | 無需安裝 | pip install requests |
| API 設計 | 較冗長、物件導向 | 簡潔、人性化 |
| 自動編碼偵測 | 需手動處理 | 內建 .encoding、.text |
| Session/Cookie | 手動建立 Opener | requests.Session() 自動 |
| JSON 支援 | 手動 json.loads | .json() 直接回傳 dict |
| 串流下載 | 支援 | 支援 stream=True |
| 代理、認證、重試 | 手動實作 | 內建/擴充套件豐富 |
| 效能 | 略快（較輕量） | 略慢（功能豐富） |
建議： 簡單腳本、無依賴環境用 urllib；專案開發、複雜爬蟲、需維護性用 requests。
# 1. 旋轉 User-Agent
USER_AGENTS = [
'Mozilla/5.0 (Windows NT 10.0; Win64; x64)...',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)...',
# 更多...
]
headers = {'User-Agent': random.choice(USER_AGENTS)}
# 2. 適當延遲
time.sleep(random.uniform(1, 3))
# 3. 處理重導向、Referer
req.add_header('Referer', 'https://www.google.com/')
# 4. 使用代理 IP（進階）
proxy = urllib.request.ProxyHandler({'http': 'https://proxy_ip:port'})
opener = urllib.request.build_opener(proxy)urllib 是 Python 標準庫 HTTP 客戶端，功能完整但 API 較繁瑣。掌握 Request 物件、串流下載、編碼偵測、錯誤重試、Cookie 維持等核心技巧，能無第三方依賴完成大多數抓取任務。實際專案建議搭配 requests + BeautifulSoup/lxml + selenium/playwright 構建完整爬蟲生態。遵守 robots.txt、控制請求頻率、處理反爬機制，是負責任爬蟲開發者的基本素養。
