爬虫下载网页pdf文件
import os
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin
from urllib.parse import urljoin, unquote
from tqdm import tqdm
# 设置网页的URL
base_url = "http://119/download/dzz/pdf/"
# 创建保存文件的文件夹,如果文件夹不存在则创建
download_folder = "downloaded_pdfs"
os.makedirs(download_folder, exist_ok=True)
def get_name(base_url):
name_list=[]
# 获取网页内容
response = requests.get(base_url) # 发送请求获取网页内容
response.raise_for_status() # 检查请求是否成功,若不成功则抛出异常
soup = BeautifulSoup(response.text, "html.parser") # 解析HTML内容
# 查找所有文件链接
for link in soup.find_all("a"): # 遍历网页中所有的<a>标签,找到每个链接
name_list.append(link.get("href"))
return name_list
f1_list=get_name(base_url)
for i1 in f1_list:
if i1 and i1 not in ["../"]:
print(i1)
f1_url = urljoin(base_url, i1) # 将相对链接转为完整的URL
f2_list=get_name(f1_url)
for i2 in f2_list:
if i2 and i2 not in ["../"]:
f2_url = urljoin(f1_url, i2) # 将相对链接转为完整的URL
f3_list=get_name(f2_url)
for i3 in tqdm(f3_list):
if i3 and i3 not in ["../"]:
f3_url = urljoin(f2_url, i3) # 将相对链接转为完整的URL
response = requests.get(f3_url) # 发送请求获取网页内容
response.raise_for_status() # 检查请求是否成功,若不成功则抛出异常
file_response = requests.get(f3_url) # 发送请求下载文件内容
download_folder_new = download_folder+'/'+i1+i2
download_folder_new=unquote(download_folder_new)
os.makedirs(download_folder_new, exist_ok=True)
download_folder_path=download_folder_new+i3
download_folder_path = unquote(download_folder_path)
with open(download_folder_path, "wb") as f: # 以二进制写入方式保存文件
f.write(file_response.content) # 写入下载的文件内容到本地
# print(f"{download_folder_path}下载完成") # 打印下载完成信息