爬虫 on 南国遗梦的小站

国家大学生就业服务平台岗位爬取

Wed, 20 Aug 2025 17:47:51 +0800

获取Cookie

起初在进入到岗位页面时，发现无需登录也能访问岗位列表，因此打算直接从网络中找到返回工作列表的请求，通过侧边栏搜索相关字段，定位请求位置:

获取其中的请求URL和Cookie等参数，构造请求头并发起请求。在此过程中发现请求URL中最后一段数字是在不断变化的：

查询网页内部的js文件发现，该串数字和时间有关，因此大胆猜测这段数字可能是时间戳，采用代码测试：

import time
def generate_timestamp():
    """生成13位时间戳"""
    return str(int(time.time() * 1000))
print(generate_timestamp())
#output->1755764369564

代码测试结果与URL的末端数字相似，因此得证。但在爬取的过程中，发现在未登录状态下只能浏览前10页内容。因此需要解决登录问题，找到相关请求：

根据相关字段向该URL提交表单并获取登录后的Cookie，但用Python获取Cookie后发现依旧无法获取，返回的结果：

一开始我怀疑是获取的Cookie有问题，因此使用代码分析直接从网页上获取的Cookie的变化，试图找到规律：

from difflib import  SequenceMatcher
#将Cookie进行切割
def Cookie_split(Cookie):
    fragments = Cookie.split("; ")
    dic = {}
    for fragment in fragments:
        key,value = fragment.split('=',1)
        dic[key] = value
    return dic


def compare_strings(str1,str2):
    matcher = SequenceMatcher(None,str1,str2)
    for op , i1 , i2 ,j1 ,j2 in matcher.get_opcodes():
        if op == 'replace':
            print(f"在位置{i1}替换了“{str1[i1:i2]}”为“{str2[j1:j2]}”")
        elif op == 'delete':
            print(f"在位置{i1}删除了“{str1[i1:i2]}”")
        elif op == 'insert':
            print(f"在位置{i1}插入了“{str1[i1:i2]}”")

#Cookie比较
def Cookie_compare(Cookie1,Cookie2):
    """比较两个Cookie的差别"""
    cookie_dic1 = Cookie_split(Cookie1)
    cookie_dic2 = Cookie_split(Cookie2)

    key_list1 = cookie_dic1.keys()
    key_list2 = cookie_dic2.keys()
    print(key_list1)
    print(key_list2)

    # Cookie中属性的比较
    diff = list(set(key_list1) ^ set(key_list2))
    if diff == []:
        print("两个Cookie在属性上一致")
        for key in key_list1:
            print(f"|--------------------------{key}的差异点-------------------------|")
            compare_strings(cookie_dic1[key],cookie_dic2[key])
            print(f"|--------------------------------------------------------------|")

    else:
        print(f"两个Cookie的属性差异：{diff}")
        same_attribute = list(set(key_list1) & set(key_list2))
        for key in same_attribute:
            print(f"|--------------------------{key}的差异点-------------------------|")
            compare_strings(cookie_dic1[key],cookie_dic2[key])
            print(f"|--------------------------------------------------------------|")


if __name__ == "__main__":
    Cookie_compare(Cookie6,Cookie7)

在比较的过程中发现，当网页保持登录状态时，从网页获取的Cookie可以正常使用，而当关闭网页后，之前获取的Cookie就失效了，因此怀疑整个流程如图：

所以如果要保持Cookie的有效性，就必须保持登录状态。因此考虑使用Selenium进行模拟登录，再从其中获取组成Cookie所需的相关字段。事实上，能直接获取的字段并不全，因此通过上面的比较代码，找到Cookie中的不变（有时不变）部分和变动（每次登录都会改变）部分，幸运的是变动部分都能在直接获取的字段中找到。

def keep_connect(chrome_location,username,password):
    """使用selenium模拟登录，获取Cookie，并维持登录状态防止服务器端将Cookie注销"""
    options = webdriver.ChromeOptions()
    options.binary_location = chrome_location
    options.add_argument("ignore-certificate-errors")
    options.add_experimental_option("detach", True)
    driver = webdriver.Chrome(options=options, executable_path='../chromedriver.exe')

    login_url = 'https://account.chsi.com.cn/passport/login?service=https://www.ncss.cn/student/connect/chsi&entrytype=stu'
    driver.get(login_url)
    time.sleep(1)

    acc_input = driver.find_element(By.XPATH, r'/html/body/div/div[2]/div[2]/div/div[2]/form/div[1]/input')
    acc_input.send_keys(username)

    pwd_input = driver.find_element(By.XPATH, f'/html/body/div/div[2]/div[2]/div/div[2]/form/div[2]/input')
    pwd_input.send_keys(password)
    driver.find_element(By.XPATH, r'/html/body/div/div[2]/div[2]/div/div[2]/form/div[4]').click()

    element = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, r"/html/body/div/span/span/a[1]")))
    element.click()

    dic = {}

    for i in driver.get_cookies():
        dic[i['name']] = i['value']

    return dic


def Cookie_concat(got_property):
    """将从keep_connect函数中获取到的Cookie数据进行重新包装"""
    changed_property_list = ['SESSION', 'Hm_lpvt_378ff17a1ac046691cf78376632d1ed4', '_ga','_ga_6CXWRD3K0D']

    no_change = {
        '_ga_1ESVLDHDYL': 'GS1.1.1726503021.3.0.1726503021.0.0.0',
        '_abfpc': 'd7760a7e8d00953eaa18b111273b00d6ad83f1cc_2.0',
        'cna': 'eac1e178d67b00529df2b287b5842f2c',
        '_gid': 'GA1.2.673435621.1755336171',
        'aliyungf_tc': 'bfc4565f794c9e6ac253a5c09f4a937f2afb69e01fc7b413d486aaeaadf06249',
        'XSRF-CCKTOKEN': 'a12b3436b08549ba2f33a4aefa9a1454',
        'CHSICC_CLIENTFLAGNCSS': 'aba5b818eaa8bc94d6fb1ddf17d1df4f',
        'CHSICC01': '!DzrVNB/pHD1H78bzYxYLahOzddj6Y4XQ6NJ5RnOPIOyzHzKixC+5X5WINIjztT+S4x5PGaf/cowaI/Q=',
        'CHSICC_CLIENTFLAGSTUDENT': '5d0ab9cce044f18a699886e7d6705555',
        'Hm_lvt_378ff17a1ac046691cf78376632d1ed4': '1754926580,1754968150,1755336169,1755411478',
        'HMACCOUNT': 'CEB955474E107530',
        'acw_tc': 'ac11000117554230396983911ee7b259a823321b52a20a878c0b42ee677273',
        '_gat_gtag_UA_105074615_1': '1'
    }

    property_all = no_change.copy()

    for i in changed_property_list:
        property_all[i] = got_property[i]
    property_rank = ['SESSION','_ga_1ESVLDHDYL','_abfpc','cna','_gid','aliyungf_tc','acw_tc','XSRF-CCKTOKEN','CHSICC_CLIENTFLAGNCSS','CHSICC01','CHSICC_CLIENTFLAGSTUDENT','Hm_lvt_378ff17a1ac046691cf78376632d1ed4','HMACCOUNT','_gat_gtag_UA_105074615_1','Hm_lpvt_378ff17a1ac046691cf78376632d1ed4','_ga','_ga_6CXWRD3K0D']

    Cookie=''

    for i in property_rank:
        Cookie = Cookie + i + '=' + property_all[i] + '; '
    print('完成Cookie创建')

    return Cookie


def simulate_login_get_cookie(chrome_location,username,password):
    """将模拟登录部分和包装Cookie部分整合在一起"""
    dic = keep_connect(chrome_location=chrome_location,username=username,password=password)
    return Cookie_concat(dic)

获取数据

数据分为两类，一类是岗位列表数据，另一类是岗位详情页数据。

岗位列表数据

岗位列表数据即如图所示：

该类数据可以直接从返回的响应中的"data"–>“list"中找到，相关代码为：

def get_job_data(http,Cookie,page=1,page_size=10):
    """获取岗位数据"""
    baseurl = "https://www.ncss.cn/student/jobs/jobslist/ajax/"
    #构造请求头
    headers = {
        'User-Agent':ua.random,
        'Connection':'keep-alive',
        'Cookie':Cookie,
        "Accept": "application/json,*/*",
        'Referer':"https://account.chsi.com.cn/passport/login?service=https://job.ncss.cn/student/connect/chsi&entrytype=stu"
    }
    
    # 查询参数
    params = {
        "jobType": "",
        "areaCode": "",
        "jobName": "",
        "monthPay": "",
        "industrySectors": "",
        "property": "",
        "categoryCode": "",
        "memberLevel": "",
        "recruitType": "",
        "offset": page,
        "limit": page_size,
        "keyUnits": "",
        "degreeCode": "",
        "sourcesName": "0",
        "sourcesType": "",
        "_": generate_timestamp()  # 动态时间戳
    }

    try:
        resp = http.request(
            method='GET',
            url=baseurl,
            fields=params,
            headers=headers
        )
        if resp.status in [403,401]:
            print(f"请求第{page}页时登录过期")
            return None
        elif resp.status == 200:
            data = json.loads(resp.data.decode('UTF-8'))
            return data['data']['list']
        else:
            print(f"请求第{page}页时错误，状态码{resp.status}")
            return None

    except Exception as e:
        print(f"请求第{page}页时发生错误:{e}")
        return None

详情页数据

详情页数据主要为详情页中的岗位介绍部分：

该部分主要是使用BeautifulSoup对返回的结果进行解析，找到相关属性内容的位置并进行解析，代码如下：

def fetch_detail_page(job_id,http,Cookie):
    """获取详情页数据"""

    headers = {
        'User-Agent':ua.random,
        'Cookie':Cookie,
        'Referer':'https://job.ncss.cn/student/jobs/index.html'
    }

    detail_url = f"https://www.ncss.cn/student/jobs/{job_id}/detail.html"
    try:
        resp = http.request(
            'GET',
            detail_url,
            headers = headers
        )
        if resp.status == 200:
            return resp.data.decode('utf-8')
        elif resp.status in [401,403]:
            print(f"获取{job_id}详情页时登录过期")
            return None
        else:
            print(f"请求失败，状态码{resp.status}")
            return None
    except Exception as e:
        print(f"请求失败，{str(e)}")
        return None



def pares_detail_job_info(html,job_info):
    """将爬取到的详情页进行解析，获取“岗位介绍”部分的数据"""
    soup = BeautifulSoup(html,'html.parser')

    job_detail_describe_div =soup.find(name='pre',attrs={'class':"mainContent mainContent"})
    job_detail_describe = job_detail_describe_div.getText()

    job_info.update({
        "岗位介绍":job_detail_describe
    })

    return job_info

效率与反爬

考虑到爬取的数据量稍多，因此使用Urllib3进行分布式爬取，一个主进程，两个工作线程（岗位列表和详情页），采用消费者-生产者模型，同时使用fake_useragent模块不断改变User-Agent的值，代码如下：

def process_manager(total_pages,http,Cookie,num_list_workers=2,num_detail_workers=8,output_address=""):
    """进程管理器（主进程）"""
    global  enqueued_jobs_count
    start_time = time.time()
    print(f"开始爬取任务，总页数: {total_pages}")

    #创建Manager
    with Manager() as manager:
        #创建队列
        list_page_queue = manager.Queue()
        detail_task_queue = manager.Queue(maxsize=5000)
        result_queue = manager.Queue()

        #添加列表页任务
        for page in range(1,total_pages+1):
            list_page_queue.put(page)

        expected_total_list_pages = total_pages
        expected_total_detail_jobs = total_pages * 10

        #进度条
        list_pbar = tqdm(total=expected_total_list_pages,desc='列表页',unit='页',dynamic_ncols=True,file=sys.stdout)
        detail_pbar = tqdm(total=expected_total_detail_jobs,desc='详情页',unit='项',dynamic_ncols=True,file=sys.stdout)

        #创建并启动线程
        threads = []

        #列表页工作线程
        for i in range(num_list_workers):
            t = threading.Thread(
                target=list_page_worker,
                args=(list_page_queue,http,Cookie,detail_task_queue,list_pbar),
                name=f"ListWorker-{i+1}",
                daemon=True
            )
            t.start()
            threads.append(t)

        #详情页工作线程
        for i in range(num_detail_workers):
            t = threading.Thread(
                target=detail_page_worker,
                args=(detail_task_queue,result_queue,http,Cookie,detail_pbar),
                name=f"DetailWorker-{i+1}",
                daemon=True
            )
            t.start()
            threads.append(t)

        #结果写入线程
        writer_thread = threading.Thread(
            target=result_writer,
            args=(result_queue,output_address),
            name="ResultWriter",
            daemon=True
        )
        writer_thread.start()

        #等待列表页队列为空（等待完成列表页任务）
        list_page_queue.join()

        #通知列表页任务线程退出
        for _ in range(num_list_workers):
            list_page_queue.put(None)
        print(f"所有列表页任务已完成,{num_list_workers}个工作线程已退出")


        #调整详情任务总数
        with progress_lock:
            actual_enqueued = enqueued_jobs_count
        if actual_enqueued > 0 and actual_enqueued < detail_pbar.total:
            detail_pbar.total = actual_enqueued
            detail_pbar.refresh()


        #等待详情页任务队列为空（详情页任务完成）
        detail_task_queue.join()

        #通知详情页工作线程退出
        for _ in range(num_detail_workers):
            detail_task_queue.put(None)
        print(f"所有详情页任务已完成,{num_detail_workers}个工作线程已退出")


        #通知结果写入线程退出
        result_queue.put(None)

        for t in threads:
            t.join(timeout=10)

        writer_thread.join(timeout=10)

        elapsed_time = time.time() - start_time

        list_pbar.close()
        detail_pbar.close()

        print(f"爬取完成! 总耗时: {elapsed_time:.2f} 秒")

项目流程图

完整代码

import queue
import random
import threading
import time
from multiprocessing import Manager
from bs4 import BeautifulSoup
import urllib3
import json
from tqdm.auto import tqdm
import sys
from  fake_useragent import UserAgent
from selenium import  webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait


#抑制认证警告
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)


def keep_connect(chrome_location,username,password):
    """使用selenium模拟登录，获取Cookie，并维持登录状态防止服务器端将Cookie注销"""
    options = webdriver.ChromeOptions()
    options.binary_location = chrome_location
    options.add_argument("ignore-certificate-errors")
    options.add_argument("--headless")
    options.add_argument("--disable-gpu")
    options.add_argument("--disable-software-rasterizer")
    options.add_experimental_option("detach", True)
    driver = webdriver.Chrome(options=options,executable_path='../chromedriver.exe')

    login_url = 'https://account.chsi.com.cn/passport/login?service=https://www.ncss.cn/student/connect/chsi&entrytype=stu'
    driver.get(login_url)
    time.sleep(1)

    acc_input = driver.find_element(By.XPATH, r'/html/body/div/div[2]/div[2]/div/div[2]/form/div[1]/input')
    acc_input.send_keys(username)

    pwd_input = driver.find_element(By.XPATH, f'/html/body/div/div[2]/div[2]/div/div[2]/form/div[2]/input')
    pwd_input.send_keys(password)
    driver.find_element(By.XPATH, r'/html/body/div/div[2]/div[2]/div/div[2]/form/div[4]').click()

    element = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, r"/html/body/div/span/span/a[1]")))
    element.click()

    dic = {}

    for i in driver.get_cookies():
        dic[i['name']] = i['value']

    return dic


def Cookie_concat(got_property):
    """将从keep_connect函数中获取到的Cookie数据进行重新包装"""
    changed_property_list = ['SESSION', 'Hm_lpvt_378ff17a1ac046691cf78376632d1ed4', '_ga','_ga_6CXWRD3K0D']

    no_change = {
        '_ga_1ESVLDHDYL': 'GS1.1.1726503021.3.0.1726503021.0.0.0',
        '_abfpc': 'd7760a7e8d00953eaa18b111273b00d6ad83f1cc_2.0',
        'cna': 'eac1e178d67b00529df2b287b5842f2c',
        '_gid': 'GA1.2.673435621.1755336171',
        'aliyungf_tc': 'bfc4565f794c9e6ac253a5c09f4a937f2afb69e01fc7b413d486aaeaadf06249',
        'XSRF-CCKTOKEN': 'a12b3436b08549ba2f33a4aefa9a1454',
        'CHSICC_CLIENTFLAGNCSS': 'aba5b818eaa8bc94d6fb1ddf17d1df4f',
        'CHSICC01': '!DzrVNB/pHD1H78bzYxYLahOzddj6Y4XQ6NJ5RnOPIOyzHzKixC+5X5WINIjztT+S4x5PGaf/cowaI/Q=',
        'CHSICC_CLIENTFLAGSTUDENT': '5d0ab9cce044f18a699886e7d6705555',
        'Hm_lvt_378ff17a1ac046691cf78376632d1ed4': '1754926580,1754968150,1755336169,1755411478',
        'HMACCOUNT': 'CEB955474E107530',
        'acw_tc': 'ac11000117554230396983911ee7b259a823321b52a20a878c0b42ee677273',
        '_gat_gtag_UA_105074615_1': '1'
    }

    property_all = no_change.copy()

    for i in changed_property_list:
        property_all[i] = got_property[i]
    property_rank = ['SESSION','_ga_1ESVLDHDYL','_abfpc','cna','_gid','aliyungf_tc','acw_tc','XSRF-CCKTOKEN','CHSICC_CLIENTFLAGNCSS','CHSICC01','CHSICC_CLIENTFLAGSTUDENT','Hm_lvt_378ff17a1ac046691cf78376632d1ed4','HMACCOUNT','_gat_gtag_UA_105074615_1','Hm_lpvt_378ff17a1ac046691cf78376632d1ed4','_ga','_ga_6CXWRD3K0D']

    Cookie=''

    for i in property_rank:
        Cookie = Cookie + i + '=' + property_all[i] + '; '
    print("------------------完成Cookie创建------------------")

    return Cookie


def simulate_login_get_cookie(chrome_location,username,password):
    """将模拟登录部分和包装Cookie部分整合在一起"""
    dic = keep_connect(chrome_location=chrome_location,username=username,password=password)
    return Cookie_concat(dic)


ua = UserAgent()


def generate_timestamp():
    """生成13位时间戳"""
    return str(int(time.time() * 1000))



def get_job_data(http,Cookie,page=1,page_size=10):
    """获取岗位数据"""
    baseurl = "https://www.ncss.cn/student/jobs/jobslist/ajax/"
    #构造请求头
    headers = {
        'User-Agent':ua.random,
        'Connection':'keep-alive',
        'Cookie':Cookie,
        "Accept": "application/json,*/*",
        'Referer':"https://account.chsi.com.cn/passport/login?service=https://job.ncss.cn/student/connect/chsi&entrytype=stu"
    }
    
    # 查询参数
    params = {
        "jobType": "",
        "areaCode": "",
        "jobName": "",
        "monthPay": "",
        "industrySectors": "",
        "property": "",
        "categoryCode": "",
        "memberLevel": "",
        "recruitType": "",
        "offset": page,
        "limit": page_size,
        "keyUnits": "",
        "degreeCode": "",
        "sourcesName": "0",
        "sourcesType": "",
        "_": generate_timestamp()  # 动态时间戳
    }

    try:
        resp = http.request(
            method='GET',
            url=baseurl,
            fields=params,
            headers=headers
        )
        if resp.status in [403,401]:
            tqdm.write(f"请求第{page}页时登录过期")
            return None
        elif resp.status == 200:
            data = json.loads(resp.data.decode('UTF-8'))
            return data['data']['list']
        else:
            tqdm.write(f"请求第{page}页时错误，状态码{resp.status}")
            return None

    except Exception as e:
        tqdm.write(f"请求第{page}页时发生错误:{e}")
        return None




def fetch_detail_page(job_id,http,Cookie):
    """获取详情页数据"""

    headers = {
        'User-Agent':ua.random,
        'Cookie':Cookie,
        'Referer':'https://job.ncss.cn/student/jobs/index.html'
    }

    detail_url = f"https://www.ncss.cn/student/jobs/{job_id}/detail.html"
    try:
        resp = http.request(
            'GET',
            detail_url,
            headers = headers
        )
        if resp.status == 200:
            return resp.data.decode('utf-8')
        elif resp.status in [401,403]:
            tqdm.write(f"获取{job_id}详情页时登录过期")
            return None
        else:
            tqdm.write(f"请求失败，状态码{resp.status}")
            return None
    except Exception as e:
        tqdm.write(f"请求失败，{str(e)}")
        return None



def pares_detail_job_info(html,job_info):
    """将爬取到的详情页进行解析，获取“岗位介绍”部分的数据"""
    soup = BeautifulSoup(html,'html.parser')

    job_detail_describe_div =soup.find(name='div',attrs={'class':"details"})
    if job_detail_describe_div is not None:
        job_detail_describe = job_detail_describe_div.getText()
    else:
        tqdm.write(f"未找到岗位描述元素，岗位ID:{job_info.get('岗位ID')}")
        job_detail_describe = "未知"

    job_info.update({
        "岗位介绍":job_detail_describe
    })

    return job_info



def parse_job_info(job):
    """解析岗位信息"""
    id = job.get("jobId")
    timeStamp = job.get("updateDate")
    timeArray = time.localtime(float(timeStamp)/1000)

    return {
        "岗位ID":id,
       "职位名称":job.get("jobName"),
        "薪资水平":str(job.get("lowMonthPay"))+'k-'+str(job.get('highMonthPay'))+'k',
        "招聘人数":job.get("headCount"),
        "学历要求":job.get("degreeName"),
        "招聘方":job.get("recName"),
        "公司规模":job.get("recScale"),
        "地区":job.get("areaCodeName"),
        "福利":job.get("recTags"),
        "专业要求":job.get("major"),
        "岗位更新时间":time.strftime("%Y-%m-%d %H:%M:%S",timeArray),
        "详情网址":f"https://www.ncss.cn/student/jobs/{id}/detail.html"
    }





def list_page_worker(list_page_queue, http, Cookie, detail_task_queue, list_pbar):
    """获取岗位列表的工作流程"""
    global list_get_wrong, list_pages_done, enqueued_jobs_count

    while True:
        page = None
        try:
            page = list_page_queue.get(timeout=30)

            if page is None:
                list_page_queue.task_done()   # 对应主线程 put(None)
                break

            # 爬取列表页
            jobs = get_job_data(http=http, Cookie=Cookie, page=page, page_size=10)
            if jobs:
                for job in jobs:
                    job_info = parse_job_info(job)
                    detail_task_queue.put(job_info)
                    with progress_lock:
                        enqueued_jobs_count += 1
            time.sleep(random.uniform(0.5, 1.5))

        except queue.Empty:
            thread_name = threading.current_thread().name
            tqdm.write(f"列表页任务队列空，线程 {thread_name} 准备退出")
            continue

        except Exception as e:
            tqdm.write(f"列表页工作线程异常: {str(e)}")
            if page:
                list_get_wrong.append(page)
            time.sleep(5)

        finally:
            if page is not None:  # 只在真实任务时更新
                list_pages_done += 1
                list_pbar.update(1)
                list_page_queue.task_done()




def detail_page_worker(detail_task_queue,result_queue,http,Cookie,detail_pbar):
    """详情页工作流程"""

    global detail_get_wrong,detail_jobs_done

    while True:
        job_info = None
        job_id =None
        try:
            job_info = detail_task_queue.get(timeout=30)

            if job_info is None:
                detail_task_queue.task_done()
                break

            job_id = job_info.get("岗位ID")

            # print(f"开始爬取详情页{job_id}")

            html = fetch_detail_page(job_id, http, Cookie)

            if html:
                new_job_info = pares_detail_job_info(html, job_info)
                result_queue.put(new_job_info)
            # print(f"详情页{job_id}解析完成")
            else:
                tqdm.write(f"详情页{job_id}爬取失败")

            time.sleep(random.uniform(0.3, 0.8))

        except queue.Empty:
            thread_name = threading.current_thread().name
            tqdm.write(f"详情页任务队列空，线程{thread_name}准备退出")
            continue

        except Exception as e:
            tqdm.write(f"详情页工作线程异常: {str(e)}")
            if job_id:
                detail_get_wrong.append(job_id)
            time.sleep(3)
        finally:
            if job_info is not None:
                detail_jobs_done += 1
                detail_pbar.update(1)
                detail_task_queue.task_done()


def result_writer(result_queue, output_address):
    """结果写入线程"""
    struc_time = time.localtime()
    time_year = struc_time.tm_year
    time_month = struc_time.tm_mon
    time_day = struc_time.tm_mday

    output_file = output_address + f'{time_year}_{time_month}_{time_day}_jobs'

    count = 0
    with open(output_file, "a", encoding='utf-8', newline='') as f:
        while True:
            try:
                # 获取结果
                result = result_queue.get(timeout=120)
                if result is None:
                    break

                f.write(json.dumps(result, ensure_ascii=False) + "\n")
                f.flush()

                count += 1

                if count % 100 == 0:
                    tqdm.write(f"已写入 {count} 条结果")

                result_queue.task_done()
            except queue.Empty:
                tqdm.write("结果写入线程超时退出")
                break
            except Exception as e:
                tqdm.write(f"结果写入异常: {str(e)}")

    tqdm.write(f"结果写入完成，共写入 {count} 条数据")



def process_manager(total_pages,http,Cookie,num_list_workers=2,num_detail_workers=8,output_address=""):
    """进程管理器（主进程）"""
    global  enqueued_jobs_count
    start_time = time.time()
    tqdm.write(f"开始爬取任务，总页数: {total_pages}")

    #创建Manager
    with (Manager() as manager):
        #创建队列
        list_page_queue = manager.Queue()
        detail_task_queue = manager.Queue(maxsize=5000)
        result_queue = manager.Queue()

        #添加列表页任务
        for page in range(1,total_pages+1):
            list_page_queue.put(page)

        expected_total_list_pages = total_pages
        expected_total_detail_jobs = total_pages * 10

        #进度条
        list_pbar = tqdm(total=expected_total_list_pages,desc='列表页',unit='页',dynamic_ncols=True,file=sys.stdout)
        detail_pbar = tqdm(total=expected_total_detail_jobs,desc='详情页',unit='项',dynamic_ncols=True,file=sys.stdout)

        #创建并启动线程
        threads = []

        #列表页工作线程
        for i in range(num_list_workers):
            t = threading.Thread(
                target=list_page_worker,
                args=(list_page_queue,http,Cookie,detail_task_queue,list_pbar),
                name=f"ListWorker-{i+1}",
                daemon=True
            )
            t.start()
            threads.append(t)

        #详情页工作线程
        for i in range(num_detail_workers):
            t = threading.Thread(
                target=detail_page_worker,
                args=(detail_task_queue,result_queue,http,Cookie,detail_pbar),
                name=f"DetailWorker-{i+1}",
                daemon=True
            )
            t.start()
            threads.append(t)

        #结果写入线程
        writer_thread = threading.Thread(
            target=result_writer,
            args=(result_queue,output_address),
            name="ResultWriter",
            daemon=True
        )
        writer_thread.start()

        #等待列表页队列为空（等待完成列表页任务）
        list_page_queue.join()

        #通知列表页任务线程退出
        for _ in range(num_list_workers):
            list_page_queue.put(None)
        tqdm.write(f"所有列表页任务已完成,{num_list_workers}个工作线程已退出")


        #调整详情任务总数
        with progress_lock:
            actual_enqueued = enqueued_jobs_count
        if actual_enqueued > 0 and actual_enqueued < detail_pbar.total:
            detail_pbar.total = actual_enqueued
            detail_pbar.refresh()


        #等待详情页任务队列为空（详情页任务完成）
        detail_task_queue.join()

        #通知详情页工作线程退出
        for _ in range(num_detail_workers):
            detail_task_queue.put(None)
        tqdm.write(f"所有详情页任务已完成,{num_detail_workers}个工作线程已退出")


        #通知结果写入线程退出
        result_queue.put(None)

        for t in threads:
            t.join(timeout=10)

        writer_thread.join(timeout=10)

        elapsed_time = time.time() - start_time

        list_pbar.close()
        detail_pbar.close()

        print(f"爬取完成! 总耗时: {elapsed_time:.2f} 秒")


if __name__ == "__main__":
    #全局变量
    progress_lock = threading.Lock()
    list_pages_done = 0
    detail_jobs_done =0
    enqueued_jobs_count = 0
    # 配置爬取参数
    TOTAL_PAGES = 200  # 要爬取的列表页总数
    NUM_LIST_WORKERS = 2  # 列表页工作线程数
    NUM_DETAIL_WORKERS = 5  # 详情页工作线程数
    OUTPUT_ADDRESS = ""  # 输出文件的地址，默认为项目地址
    #配置模拟登录参数
    chrome_location = r'D:\python_project\Google\Chrome\Application\chrome.exe' #chrome启动器的位置
    username = "" #模拟登录的账号
    password = ""#模拟登陆的密码

    detail_get_wrong = [] #获取详情失败的job_id
    list_get_wrong = [] #获取岗位失败的页码


    #构建连接池
    print("------------------开始构建连接池------------------")
    http = urllib3.PoolManager(
        num_pools=50,
        maxsize=50,
        cert_reqs='CERT_NONE',
        assert_hostname=False,
        block=True,
        timeout=urllib3.Timeout(connect=5.0, read=10.0),
        retries=urllib3.Retry(total=3, backoff_factor=0.5),
    )
    print("------------------完成构建连接池------------------")

    #模拟登录，维持登录状态，获取组成Cookie的必要的参数
    print("-------------------开始模拟登录-------------------")
    Cookie = simulate_login_get_cookie(chrome_location=chrome_location,username=username,password=password)
    print("-------------------完成模拟登录-------------------")

    # 启动爬虫
    print("-------------------开始网络爬取-------------------")
    process_manager(
        total_pages=TOTAL_PAGES,
        num_list_workers=NUM_LIST_WORKERS,
        num_detail_workers=NUM_DETAIL_WORKERS,
        output_address=OUTPUT_ADDRESS,
        http = http,
        Cookie = Cookie
    )
    print("-------------------完成网络爬取-------------------")

问题 OR 改进

当前的爬取仍需要登录才能获取，如何才能以非登录状态获取所需的数据
可以增加代理，从而不断变动IP使得不易被锁IP

BeautifulSoup

Tue, 05 Aug 2025 15:27:58 +0800

BeautifulSoup

简介

BeautifulSoup 是一个可以把 HTML 或 XML 文档“变成结构化树”，方便你用Python 来查找、遍历、提取内容的工具。

解析器

解析器	使用方法	优势	劣势
Python标准库	BeautifulSoup(markup, ‘html.parser’)	python内置的标准库，执行速度适中	Python3.2.2之前的版本容错能力差
lxml HTML解析器	BeautifulSoup(markup, ’lxml')	速度快、文档容错能力强	需要安装C语言库
lxml XML解析器	BeautifulSoup(markup ‘xml’)	速度快，唯一支持XML的解析器	需要安装C语言库
html5lib	BeautifulSoup(markup, ‘html5lib’)	最好的容错性、以浏览器的方式解析文档、生成HTML5格式的文档	速度慢，不依赖外部拓展

具体使用代码展示如下：

from bs4 import BeautifulSoup

soup = BeautifulSoup('<p>Hello world</p>', 'lxml')
print(soup.p)

基本用法

当传入不标准的HTML字符串，BeautifulSoup可以自动更正格式。

soup.prettify(): 用于将 HTML 或 XML 文档格式化输出为带缩进的字符串,即会把原始网页内容重新排版，输出一个带有层级缩进的漂亮 HTML 字符串，例：

from bs4 import BeautifulSoup

html = '<html><head><title>Test</title></head><body><h1>Hi</h1><p>Hello</p></body></html>'
soup = BeautifulSoup(html, 'html.parser')

pretty_html = soup.prettify()
print(pretty_html)

输出：

<html>
 <head>
  <title>
   Test
  </title>
 </head>
 <body>
  <h1>
   Hi
  </h1>
  <p>
   Hello
  </p>
 </body>
</html>

.title：获取title标签

html_doc="""
<html><head><title>The Dormouse's story</title></head>
<body>
<p class="title"><b>The Dormouse's story</b></p>

<p class="story">Once upon a time there were three little sisters; and their names were
<a href="http://example.com/elsie" class="sister" id="link1">Elsie</a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>

<p class="story">...</p>
""""
# 创建beautifulsoup对象 解析器为lxml
soup = BeautifulSoup(html_doc, 'lxml')
print(soup.title)
#output-><title>The Dormouse's story</title>

.name：返回的是当前节点的“标签名称”

soup = BeautifulSoup(html_doc, 'lxml')
print(soup.title.name)
print(soup.name)
#output->title
#[document]

.string/.text：获取标签中的文字内容

soup = BeautifulSoup(html_doc, 'lxml')
print(soup.title.string)
print(soup.title.text)
#output->The Dormouse's story
#The Dormouse's story

.p：访问HTML中第一个

标签

1
2
3

soup = BeautifulSoup(html_doc, 'lxml')
print(soup.p)
#output-><p class="title"><b>The Dormouse's story</b></p>

.find_all()：查找文档中所有符合条件的标签元素，返回一个列表

用法	示例	说明
查找所有某种标签	soup.find_all(‘p’)	找出所有 `<p>`
根据 class 属性	soup.find_all(‘p’, class_=‘story’)	class 要用 `class_` 表示
查找多个标签	soup.find_all([‘p’, ‘a’])	所有 `<p>` 和 `<a>` 标签
查找包含特定字符串的标签	soup.find_all(string=‘Hello’)	内容匹配为 ‘Hello’ 的标签
使用属性字典	soup.find_all(‘a’, {‘id’: ’link1’})	属性筛选
限制返回数量	soup.find_all(‘p’, limit=2)	最多返回 2 个 `<p>` 标签

'''
基本语法：
soup.find_all(name=None, attrs={}, recursive=True, string=None, limit=None, **kwargs)
'''

.find()：获取第一次匹配条件的元素

1
2
3

soup = BeautifulSoup(html_doc, 'lxml')
print(soup.find(id="link1"))
#output-><a class="sister" href="https://example.com/elsie" id="link1">Elsie</a>

.parent：获取父级标签

soup = BeautifulSoup(html_doc, 'lxml')
print(soup.title)
print(soup.title.parnt)
#output-><title>The Dormouse's story</title>
#<head><title>The Dormouse's story</title></head>

.p[‘class’]：获取某个 <p> 标签的 class 属性的值

1
2
3

soup = BeautifulSoup(html_doc, 'lxml')
print(soup.p["class"])
#output->['title']

.get_text()：获取文档中所有文字内容

soup = BeautifulSoup(html_doc, 'lxml')
print(soup.get_text())
'''
output->
The Dormouse's story

The Dormouse's story
Once upon a time there were three little sisters; and their names were
Elsie,
Lacie and
Tillie;
and they lived at the bottom of a well.
...
'''

.get()：用于安全地获取某个属性的值

'''
tag.get('属性名') 相当于 tag['属性名']，但如果属性不存在，不会报错，而是返回 None
'''
a_tags = soup.find_all('a')
for a_tag in a_tags:
    print(a_tag.get("href"))
#output->https://example.com/elsie
#https://example.com/lacie
#https://example.com/tillie

BeautifulSoup的对象种类

主要有四种主要类型：Tag，NavigableString，BeautifulSoup，Comment。

Tag：该对象与HTML或XML原生文档中的标签相同，该对象可以包含其他标签，文本内容和属性。

soup = BeautifulSoup(html_doc, 'lxml')
tag = soup.title
print(type(tag))
#output-><class 'bs4.element.Tag'>

NavigableString：标签中的文本内容，是一个不可变字符串，可以由Tag对象的.string获取

soup = BeautifulSoup(html_doc, 'lxml')
tag = soup.title
print(type(tag.string))
#output-> <class 'bs4.element.NavigableString'>

BeautifulSoup：整个文档的根对象，即整个文档的根内容，可以被视为一个特殊的Tag对象，但没有名称和属性，其提供对整个文档的遍历，搜索和修改
1 2 3

soup = BeautifulSoup(html_doc, 'lxml') print(type(soup)) #output-> <class 'bs4.BeautifulSoup'>

Comment：对象是一个特殊类型的NavigableString对象,表示HTML和XML中的注释部分

# <b><!--This is a comment--></b>
soup = BeautifulSoup(html_doc, 'lxml')
print(type(soup.b.string))
#output-> <class 'bs4.element.NavigableString'>

BeautifulSoup遍历文档树

BeautifulSoup提供很多方法来遍历解析后的文档树

导航父节点：.parent和.parents。其中.parent可以获取当前节点的上一级父节点，.parents可以遍历获取当前节点的所有父辈节点

soup = BeautifulSoup(html_doc, 'lxml')
title_tag = soup.title
print(title_tag.parent)
#<head><title>The Dormouse's story</title></head>

soup = BeautifulSoup(html_doc, 'lxml')
body_tag = soup.body
for parent in body_tag.parents:
    print(parent)
#<html><head><title>The Dormouse's story</title></head>
#<body>
#<p class="title"><b>The Dormouse's story</b></p>
#<p class="story">Once upon a time there were three little sisters; and their names were
#<a class="sister" href="https://example.com/elsie" id="link1">Elsie</a>,
#....

导航子节点：.contents可以获取当前节点的所有子节点；.children可以遍历当前节点的所有子节点，返回一个list。字符串没有这两个属性。这两个仅包含tag直接子结点，字符串”The Dormouse’s story”是<head>标签的子孙结点

soup = BeautifulSoup(html_doc, 'lxml')
head_contents = soup.head.contents
print(head_contents)
#output-> [<title>The Dormouse's story</title>]

soup = BeautifulSoup(html_doc, 'lxml')
body_children = soup.body.children

for child in body_children:
    print(child)
#output-><p class="title"><b>The Dormouse's story</b></p>
#<a class="sister" href="https://example.com/elsie" id="link1">Elsie</a>,
#<a class="sister" href="https://example.com/tillie" id="link3">Tillie</a>;
#and they lived at the bottom of a well.</p>
#.....

.descendants：可以遍历当前节点的所有后代节点（层遍历）

1
2
3

soup = BeautifulSoup(html_doc, 'lxml')
for descendant in soup.descendants:
    print(descendant)

节点内容：.string，.strings，.stripped_strings。.string如果如果tag只有一个NavigableString类型子节点,那么这个tag可以使用.string得到其子节点。但若tag中包含了多个子节点,tag就无法确定string方法应该调用哪一个字节的内容,则会输出None

soup = BeautifulSoup(html_doc, 'lxml')
print(soup.head.string)
#The Dormouse's story
print(soup.title.string)
#The Dormouse's story

1
2
3

soup = BeautifulSoup(html_doc, 'lxml')
print(soup.body.string)
#None

.strings可以遍历获取标签中的所有文本内容，.stripped_strings可以去除多余的空白字符

soup = BeautifulSoup(html_doc, 'lxml')
for string in soup.strings:
    print(string)
#The Dormouse's story
......

#The Dormouse's story

BeautifulSoup搜索文档树

find_all()：搜索当前tag的所有tag子节点

soup = BeautifulSoup(html_doc, 'lxml')
print(soup.find_all("title"))  # 查找所有的title标签
print(soup.find_all("p", "title"))  # 查找p标签中class为title的标签

print(soup.find_all("a"))  # 查找所有的a标签

print(soup.find_all(id="link2"))  # 查找id为link2的标签
#[<title>The Dormouse's story</title>]
#[<p class="title"><b>The Dormouse's story</b></p>]
#[<a class="sister" href="https://example.com/elsie" id="link1">Elsie</a>, <a class="sister" href="https://example.com/lacie" id="link2">Lacie</a>, <a class="sister" href="https://example.com/tillie" id="link3">Tillie</a>]
#[<a class="sister" href="https://example.com/lacie" id="link2">Lacie</a>]

find_parent()：只返回最接近的父标签

soup = BeautifulSoup(html_doc, 'lxml')
a_string = soup.find(string='Lacie')
print(a_string.find_parent())  # 查找父节点
#<a class="sister" href="https://example.com/lacie" id="link2">Lacie</a>

find_parents()：返回所有符合条件的祖先标签，按从近到远的顺序排列

soup = BeautifulSoup(html_doc, 'lxml')
a_string = soup.find(string='Lacie')
print(a_string.find_parents())  # 查找父节点
#[<a class="sister" href="https://example.com/lacie" id="link2">Lacie</a>, <p class="story">Once upon a time there were three little sisters; and their names were
#<a class="sister" href="https://example.com/elsie" id="link1">Elsie</a>,
#<a class="sister" href="https://example.com/lacie" id="link2">Lacie</a> and
#and they lived at the bottom of a well.</p>, <body>....]

BeautifulSoup的CSS选择器

我们在写CSS时,标签名不加任何修饰,类名前加点,id名前加#,BeautifulSoup中也可以使用类似的方法来筛选元素。BeautifulSoup中的select()方法允许使用CSS选择器来查找HTML文档元素,其返回一个包含所有匹配元素的列表类似于find_all()方法。

通过标签名查找：

1
2
3

soup = BeautifulSoup(html_doc, 'lxml')
print(soup.select('b'))
#[<b>The Dormouse's story</b>, <b><!--This is a comment--></b>]

通过类名查找：

1
2
3

soup = BeautifulSoup(html_doc, 'lxml')
print(soup.select('.title'))
#[<p class="title"><b>The Dormouse's story</b></p>]

id名查找：

1
2
3

soup = BeautifulSoup(html_doc, 'lxml')
print(soup.select('#link1'))
#[<a class="sister" href="https://example.com/elsie" id="link1">Elsie</a>]

组合查找：组合查找即与写class时一致,标签名与类名id名进行组合的原理一样

1
2
3

soup = BeautifulSoup(html_doc, 'lxml')
print(soup.select('p #link1'))
#[<a class="sister" href="https://example.com/elsie" id="link1">Elsie</a>]

属性查找：选择具有特定属性或属性值的标签

简单属性选择器：

1
2
3

soup = BeautifulSoup(html_doc, 'lxml')
print(soup.select("a[href='https://example.com/elsie']"))  # 选择a标签中href属性为https://example.com/elsie的标签
#[<a class="sister" href="https://example.com/elsie" id="link1">Elsie</a>]

属性值选择器


'''
精确匹配:[attribute="value"]
部分匹配
包含特定值:[attribute~="value"] 选择属性值包含特定单词的标签。
以特定值开头:[attribute^="value"] 选择属性值以特定字符串开头的标签
以特定值结尾:[attribute$="value"] 选择属性值以特定字符串结尾的标签。
包含特定子字符串:[attribute*="value"] 选择属性值包含特定子字符串的标签
'''

soup = BeautifulSoup(html_doc, 'lxml')
print(soup.select('a[href^="https://example.com"]'))  # 选择href以https://example.com开头的a标签
#[<a class="sister" href="https://example.com/elsie" id="link1">Elsie</a>, <a class="sister" href="https://example.com/lacie" id="link2">Lacie</a>, <a class="sister" href="https://example.com/tillie" id="link3">Tillie</a>]

Urllib3

Tue, 05 Aug 2025 12:10:50 +0800

Urllib3

特点：

连接池管理：在客户端和服务器之间复用已有连接，避免每次请求都重新建立新连接，核心是PoolManager,内部维护一个或多个ConnectionPool；
线程安全：适合在多线程环境下进行并发请求；
重试机制：请求失败自动重试；
SSL/TLS验证：建立 HTTPS 连接时，客户端校验服务器提供的数字证书是否可信，并通过 TLS 协议完成数据加密通道的协商；
代理支持：在客户端（你）访问目标网站时，通过一个中间服务器（代理服务器）中转请求和响应，而不是直接访问目标网站；
文件上传：支持 multipart 文件上传；
编码处理：自动处理响应内容的编码问题；

核心类与方法：

PoolManager：最核心类，负责管理连接池和所有请求。

http = urllib3.PoolManager(
	num_pools = 50,          #连接池数量
    maxsize = 10,            #每个连接池最大连接数
    block = True,            #连接池满时是否阻塞等待
    timeout = 30.0,	         #请求超时时间
    retries = 3,  			 #默认重试次数
    headers={'User-Agent':''}
)

GET请求：

response = http.request(
	'GET',
    'http://',
    fields = {'arg':'value'} #查询参数
)

POST请求：

#表单数据
response = http.request(
	'POST',
    'http://',
    fields = {'field':'value'}
)

#JSON数据
import json
response = http.request(
	'POST',
    'http://',
    body = json.dumps({'key':'value'}).encode('utf-8'),
    headers = {'Content-Type':'application/json'}
)

PUT/DELETE请求：

#PUT请求
response = http.request(
	'PUT',
    'http://',
    body = b 'data to put'
)

#DELETE请求
response = http.request(
	'DELETE',
    'http://'
)

文件上传：

with open('example.txt','rb') as f:
    file_data = f.read()

response = http.request(
	'POST',
    'http://',
    fields = {
        'filefield':('example.txt',file_data,'text/plain'),
        'description':'File upload example'
    }
)

响应处理与重要属性

响应对象属性：

response = http.request('GET','http://exmaple.com')

#状态码
print(response.status)
#响应头
print(reponse.headers)
#响应体
print(response.data)
print(response.data.decode('utf-8'))#解码为字符串
#重新定向历史
print(response.redirect_location)
#消耗时间
print(response.elapsed)

响应内容处理：

#JSON响应处理
import json
json_response = json.loads(response.data.decode('utf-8'))

#流式响应处理
response = http.request(
	'GET',
    'http://exmaple.com/largefile',
    preload_content=False
)

try:
    for chunk in response.stream(1024):#每次读1024字节
        process_chunk(chunk)
finally:
     response.release_conn()#释放连接

高级特性与配置

重试机制：

from urllib3.util.retry import Retry

retry_strategy = Retry(
	total = 3, 			#重试总次数
    backoff_factor = 1, #重试间隔增长因子
    status_forcelist = [500,502,503,504]#指定哪些状态码会触发自动重试
)
http = urllib3.PoolManager(retries = retry_strategy)

超时设置：

#全局超时
http = urllib3.PoolManager(timeout=2.0)

#单个请求超时 
response = http.request(
    'GET',
    'http://example.com',
    timeout=5.0 
)

#分别设置连接和读取超时 
response = http.request(
    'GET',
    'http://example.com',
    timeout=urllib3.Timeout(connect=2.0, read=10.0)
)

SSL/TLS配置：

#禁用证书验证(不推荐生产环境使用)
http = urllib3.PoolManager(
    cert_reqs='CERT_NONE',#不校验证书有效性
    assert_hostname=False #不检查证书是否匹配域名
)


#自定义CA证书 
http = urllib3.PoolManager(
    cert_reqs='CERT_REQUIRED',
    ca_certs='/path/to/certificate.pem')#自定义CA

#客户端证书认证 
http = urllib3.PoolManager(
    #证书文件路径
    cert_file='/path/to/client_cert.pem',
    #证书对应私钥文件路径
    key_file='/path/to/client_key.pem')

代理配置：


#HTTP代理 
http = urllib3.ProxyManager(
    #代理服务器地址
    'http://proxy.example.com:8080/',
    #身份认证信息
    proxy_headers={'Proxy-Authorization': 'Basic ...'})

#SOCKS代理(需要安装PySocks)
pip install pysocks 

from urllib3.contrib.socks import SOCKSProxyManager 
proxy = SOCKSProxyManager(
    'socks5://user:password@127.0.0.1:1080/'
)

性能优化技巧

连接池调优：

#根据应用场景调整连接池参数 
http = urllib3.PoolManager(
    num_pools=10,      # 适合大多数应用 
    maxsize=10,        # 每个连接池最大连接数 
    block=True,        # 连接池满时阻塞而非创建新连接 
    timeout=60.0       # 适当延长超时时间 
)

连接重用：

#使用上下文管理器确保连接正确释放 
with http.request('GET', 'http://example.com', preload_content=False) as response:
    process_response(response)
#连接自动返回到连接池

批处理请求（使用线程池并发发起请求，并收集响应对象）：

from concurrent.futures import ThreadPoolExecutor 

urls = ['http://example.com/1',
        'http://example.com/2',
        'http://example.com/3']

def fetch(url):
    return http.request('GET', url)

with ThreadPoolExecutor(max_workers=5) as executor:
    results = list(executor.map(fetch, urls))

6.常见的应用场景

Web API调用：

import json 
from urllib.parse import urlencode 

base_url = "https://api.example.com/v1"

def get_user(user_id):
    response = http.request(
        'GET',
        f"{base_url}/users/{user_id}",
        headers={'Authorization': 'Bearer token123'}
    )
    return json.loads(response.data.decode('utf-8'))

def search_users(query, limit=10):
    params = {'q': query, 'limit': limit}
    response = http.request(
        'GET',
        f"{base_url}/users/search?{urlencode(params)}"
    )
    return json.loads(response.data.decode('utf-8'))

网页抓取：

from bs4 import BeautifulSoup 

def scrape_website(url):
    response = http.request('GET', url)
    if response.status == 200:
        soup = BeautifulSoup(response.data, 'html.parser')
        # 提取数据...
        return {
            'title': soup.title.string,
            #遍历所有的a标签，并提取其中的href属性
            'links': [a['href'] for a in soup.find_all('a')]
        }
    return None

文件下载：

def download_file(url, save_path):
    with http.request('GET', url, preload_content=False) as response:
        if response.status == 200:
            with open(save_path, 'wb') as f:
                for chunk in response.stream(1024):
                    f.write(chunk)
            return True 
    return False

微服务通信：

import json 

def call_service(service_url, method, payload=None):
    headers = {
        'Content-Type': 'application/json',
        'X-Request-ID': 'unique-id-123'
    }

    body = json.dumps(payload).encode('utf-8') if payload else None 

    response = http.request(
        method.upper(),
        service_url,
        headers=headers,
        body=body 
    )

    if response.status >= 400:
        raise Exception(f"Service error: {response.status}")

    return json.loads(response.data.decode('utf-8'))

常见问题

异常处理：

import urllib3.exceptions 

try:
    response = http.request('GET', 'http://example.com')
except urllib3.exceptions.HTTPError as e:
    print(f"HTTP错误: {e}")
except urllib3.exceptions.SSLError as e:
    print(f"SSL错误: {e}")
except urllib3.exceptions.TimeoutError as e:
    print(f"请求超时: {e}")
except urllib3.exceptions.RequestError as e:
    print(f"请求错误: {e}")
except Exception as e:
    print(f"其他错误: {e}")

调试技巧：

#启用调试日志 
import logging 
logging.basicConfig(level=logging.DEBUG)

#或者只启用urllib3的调试日志 
logger = logging.getLogger('urllib3')
logger.setLevel(logging.DEBUG)

#查看连接池状态 
print(http.connection_pool_kw)
print(http.pools)

与request库的对比

更底层的控制：直接访问连接池和底层配置
更小的内存占用：没有 requests 的额外抽象层
更早的错误检测：在请求发送前就能检测到某些问题
更灵活的流处理：对大型文件或流式API更友好
大多数情况用request，精细控制或高性能用urllib3

一些问题

Q	当我向https://www.xiaohongshu.com/发起请求失败时，urllib3会自动重新定向到https://www.xiaohongshu.com/explore，这是为什么，程序为什么会知道https://www.xiaohongshu.com/explore是正确的
A	不是urllib3“知道正确地址”，重新定向是服务端的行为，当向https://www.xiaohongshu.com/发起请求后，小红书的服务器会返回一个HTTP 301或302响应，将正确的网址返回给客户端，同时urllib会默认自动跟随重定向。

Q	JSON响应处理和流式响应处理的区别
A	对于JSON响应处理，response.data一次性读取全部内容，再将其解析为Python对象；流式响应处理，不会把整个响应体加载进内存。JSON响应处理适用于REST API、结构化数据；流式响应处理适用于下载大文件、音视频流等场景

Q	线程与并发的关系
A	线程是并发的一种实现方式，但并发并不限于线程，线程像“工人”，并发像“工人排班的调度方式”，一个线程处理一个任务，多个线程合作是实现

Q	PoolManager和ProxyManager的区别
A	ProxyManager是继承自PoolManager，专门用来处理从代理服务器发出的请求，其重写了PoolManager的部分方法，确保所有的请求都从预定的代理进行，并且它能根据目标的URL的方案和代理类型，智能的选择使用HTTP转发或HTTP CONNECT通道

一些测试代码

一些库导入

import urllib3
import threading
import time
from urllib3.exceptions import EmptyPoolError
import json
import logging
logging.basicConfig(level=logging.DEBUG)

测试连接复用：

# 创建连接池，最大连接数设置为 1（便于观察复用）
http = urllib3.HTTPConnectionPool("httpbin.org", maxsize=1)

# 第一次请求
print("第1次请求")
r1 = http.request("GET", "/get")
print("响应状态码:", r1.status)

# 暂停 1 秒
time.sleep(1)
# 第二次请求
print("\n第2次请求")
r2 = http.request("GET", "/get")
print("响应状态码:", r2.status)

'''
#通过如下方式，检查底层 socket 是否复用
print("连接池中连接数量:", len(http.pool))
for conn in http.pool:
    print("连接对象:", conn)
'''

'''
#测试“不复用的情况”，可以设置 retries=0，并且让服务器关闭连接
r = http.request("GET", "/get", headers={"Connection": "close"})
'''

测试限制连接数+阻塞模式:

# 设置连接池为 maxsize=1，block=False
http = urllib3.HTTPConnectionPool("httpbin.org", maxsize=1, block=False)

# 请求函数：保持连接占用一段时间
def make_request(name):
    print(f"[{name}] 准备发送请求")
    try:
        r = http.request("GET", "/delay/5", preload_content=False, timeout=10.0)
        print(f"[{name}] 已发送请求，延迟读取内容")
        time.sleep(6)  # 保持连接被占用
        r.release_conn()  # 主动释放连接
        print(f"[{name}] 释放连接")
    except EmptyPoolError as e:
        print(f"[{name}] 连接池耗尽异常: {e}")
    except Exception as e:
        print(f"[{name}] 其他异常: {e}")

# 第一个线程：占住唯一连接
t1 = threading.Thread(target=make_request, args=("线程1",))
# 第二个线程：争抢连接
t2 = threading.Thread(target=make_request, args=("线程2",))

t1.start()
time.sleep(0.2)  # 保证线程1先占住连接
t2.start()

t1.join()
t2.join()
'''
连接池已满且不允许阻塞新的请求时，新的连接请求将会被丢弃。这意味着连接池无法处理所有的请求，可能会导致一些请求失败或被延迟处理。
所以在触发连接池已满的情况下不会报错且能正常运行，但在DEBUG日志下会有警告。
'''

验证连接池（num_pool）作用:

'num_pool控制的是最多允许缓存多少个不同主机的连接池，每一个host:port组合会对应一个connectionpool实例，当请求的host数超过num_pools时，旧的连接池将会被清理(LRU策略)'
# 创建连接池管理器，最多2个主机连接池
http = urllib3.PoolManager(num_pools=2, maxsize=1)

hosts = [
    "http://httpbin.org/get",
    "https://www.xiaohongshu.com/explore",
    "http://baidu.com"  # 任意有效网址也可
]

def fetch(url):
    try:
        r = http.request("GET", url)
        print(f"访问 {url} 状态码: {r.status}")
    except Exception as e:
        print(f"访问 {url} 出错: {e}")

# 访问3个不同host，超出num_pools限制
for i in range(2):
    for url in hosts:
        fetch(url)
        time.sleep(0.5)
'''
可以观察到每次发送请求前都得先建立连接，这是因为只能允许存在两个连接池，当继续往里添加连接池时，会把旧的清理，使得再次连接时需要重新建立连接
'''