国家大学生就业服务平台岗位爬取

Wed, 20 Aug 2025 17:47:51 +0800

获取Cookie

起初在进入到岗位页面时，发现无需登录也能访问岗位列表，因此打算直接从网络中找到返回工作列表的请求，通过侧边栏搜索相关字段，定位请求位置:

获取其中的请求URL和Cookie等参数，构造请求头并发起请求。在此过程中发现请求URL中最后一段数字是在不断变化的：

查询网页内部的js文件发现，该串数字和时间有关，因此大胆猜测这段数字可能是时间戳，采用代码测试：

import time
def generate_timestamp():
    """生成13位时间戳"""
    return str(int(time.time() * 1000))
print(generate_timestamp())
#output->1755764369564

代码测试结果与URL的末端数字相似，因此得证。但在爬取的过程中，发现在未登录状态下只能浏览前10页内容。因此需要解决登录问题，找到相关请求：

根据相关字段向该URL提交表单并获取登录后的Cookie，但用Python获取Cookie后发现依旧无法获取，返回的结果：

一开始我怀疑是获取的Cookie有问题，因此使用代码分析直接从网页上获取的Cookie的变化，试图找到规律：

from difflib import  SequenceMatcher
#将Cookie进行切割
def Cookie_split(Cookie):
    fragments = Cookie.split("; ")
    dic = {}
    for fragment in fragments:
        key,value = fragment.split('=',1)
        dic[key] = value
    return dic


def compare_strings(str1,str2):
    matcher = SequenceMatcher(None,str1,str2)
    for op , i1 , i2 ,j1 ,j2 in matcher.get_opcodes():
        if op == 'replace':
            print(f"在位置{i1}替换了“{str1[i1:i2]}”为“{str2[j1:j2]}”")
        elif op == 'delete':
            print(f"在位置{i1}删除了“{str1[i1:i2]}”")
        elif op == 'insert':
            print(f"在位置{i1}插入了“{str1[i1:i2]}”")

#Cookie比较
def Cookie_compare(Cookie1,Cookie2):
    """比较两个Cookie的差别"""
    cookie_dic1 = Cookie_split(Cookie1)
    cookie_dic2 = Cookie_split(Cookie2)

    key_list1 = cookie_dic1.keys()
    key_list2 = cookie_dic2.keys()
    print(key_list1)
    print(key_list2)

    # Cookie中属性的比较
    diff = list(set(key_list1) ^ set(key_list2))
    if diff == []:
        print("两个Cookie在属性上一致")
        for key in key_list1:
            print(f"|--------------------------{key}的差异点-------------------------|")
            compare_strings(cookie_dic1[key],cookie_dic2[key])
            print(f"|--------------------------------------------------------------|")

    else:
        print(f"两个Cookie的属性差异：{diff}")
        same_attribute = list(set(key_list1) & set(key_list2))
        for key in same_attribute:
            print(f"|--------------------------{key}的差异点-------------------------|")
            compare_strings(cookie_dic1[key],cookie_dic2[key])
            print(f"|--------------------------------------------------------------|")


if __name__ == "__main__":
    Cookie_compare(Cookie6,Cookie7)

在比较的过程中发现，当网页保持登录状态时，从网页获取的Cookie可以正常使用，而当关闭网页后，之前获取的Cookie就失效了，因此怀疑整个流程如图：

所以如果要保持Cookie的有效性，就必须保持登录状态。因此考虑使用Selenium进行模拟登录，再从其中获取组成Cookie所需的相关字段。事实上，能直接获取的字段并不全，因此通过上面的比较代码，找到Cookie中的不变（有时不变）部分和变动（每次登录都会改变）部分，幸运的是变动部分都能在直接获取的字段中找到。

def keep_connect(chrome_location,username,password):
    """使用selenium模拟登录，获取Cookie，并维持登录状态防止服务器端将Cookie注销"""
    options = webdriver.ChromeOptions()
    options.binary_location = chrome_location
    options.add_argument("ignore-certificate-errors")
    options.add_experimental_option("detach", True)
    driver = webdriver.Chrome(options=options, executable_path='../chromedriver.exe')

    login_url = 'https://account.chsi.com.cn/passport/login?service=https://www.ncss.cn/student/connect/chsi&entrytype=stu'
    driver.get(login_url)
    time.sleep(1)

    acc_input = driver.find_element(By.XPATH, r'/html/body/div/div[2]/div[2]/div/div[2]/form/div[1]/input')
    acc_input.send_keys(username)

    pwd_input = driver.find_element(By.XPATH, f'/html/body/div/div[2]/div[2]/div/div[2]/form/div[2]/input')
    pwd_input.send_keys(password)
    driver.find_element(By.XPATH, r'/html/body/div/div[2]/div[2]/div/div[2]/form/div[4]').click()

    element = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, r"/html/body/div/span/span/a[1]")))
    element.click()

    dic = {}

    for i in driver.get_cookies():
        dic[i['name']] = i['value']

    return dic


def Cookie_concat(got_property):
    """将从keep_connect函数中获取到的Cookie数据进行重新包装"""
    changed_property_list = ['SESSION', 'Hm_lpvt_378ff17a1ac046691cf78376632d1ed4', '_ga','_ga_6CXWRD3K0D']

    no_change = {
        '_ga_1ESVLDHDYL': 'GS1.1.1726503021.3.0.1726503021.0.0.0',
        '_abfpc': 'd7760a7e8d00953eaa18b111273b00d6ad83f1cc_2.0',
        'cna': 'eac1e178d67b00529df2b287b5842f2c',
        '_gid': 'GA1.2.673435621.1755336171',
        'aliyungf_tc': 'bfc4565f794c9e6ac253a5c09f4a937f2afb69e01fc7b413d486aaeaadf06249',
        'XSRF-CCKTOKEN': 'a12b3436b08549ba2f33a4aefa9a1454',
        'CHSICC_CLIENTFLAGNCSS': 'aba5b818eaa8bc94d6fb1ddf17d1df4f',
        'CHSICC01': '!DzrVNB/pHD1H78bzYxYLahOzddj6Y4XQ6NJ5RnOPIOyzHzKixC+5X5WINIjztT+S4x5PGaf/cowaI/Q=',
        'CHSICC_CLIENTFLAGSTUDENT': '5d0ab9cce044f18a699886e7d6705555',
        'Hm_lvt_378ff17a1ac046691cf78376632d1ed4': '1754926580,1754968150,1755336169,1755411478',
        'HMACCOUNT': 'CEB955474E107530',
        'acw_tc': 'ac11000117554230396983911ee7b259a823321b52a20a878c0b42ee677273',
        '_gat_gtag_UA_105074615_1': '1'
    }

    property_all = no_change.copy()

    for i in changed_property_list:
        property_all[i] = got_property[i]
    property_rank = ['SESSION','_ga_1ESVLDHDYL','_abfpc','cna','_gid','aliyungf_tc','acw_tc','XSRF-CCKTOKEN','CHSICC_CLIENTFLAGNCSS','CHSICC01','CHSICC_CLIENTFLAGSTUDENT','Hm_lvt_378ff17a1ac046691cf78376632d1ed4','HMACCOUNT','_gat_gtag_UA_105074615_1','Hm_lpvt_378ff17a1ac046691cf78376632d1ed4','_ga','_ga_6CXWRD3K0D']

    Cookie=''

    for i in property_rank:
        Cookie = Cookie + i + '=' + property_all[i] + '; '
    print('完成Cookie创建')

    return Cookie


def simulate_login_get_cookie(chrome_location,username,password):
    """将模拟登录部分和包装Cookie部分整合在一起"""
    dic = keep_connect(chrome_location=chrome_location,username=username,password=password)
    return Cookie_concat(dic)

获取数据

数据分为两类，一类是岗位列表数据，另一类是岗位详情页数据。

岗位列表数据

岗位列表数据即如图所示：

该类数据可以直接从返回的响应中的"data"–>“list"中找到，相关代码为：

def get_job_data(http,Cookie,page=1,page_size=10):
    """获取岗位数据"""
    baseurl = "https://www.ncss.cn/student/jobs/jobslist/ajax/"
    #构造请求头
    headers = {
        'User-Agent':ua.random,
        'Connection':'keep-alive',
        'Cookie':Cookie,
        "Accept": "application/json,*/*",
        'Referer':"https://account.chsi.com.cn/passport/login?service=https://job.ncss.cn/student/connect/chsi&entrytype=stu"
    }
    
    # 查询参数
    params = {
        "jobType": "",
        "areaCode": "",
        "jobName": "",
        "monthPay": "",
        "industrySectors": "",
        "property": "",
        "categoryCode": "",
        "memberLevel": "",
        "recruitType": "",
        "offset": page,
        "limit": page_size,
        "keyUnits": "",
        "degreeCode": "",
        "sourcesName": "0",
        "sourcesType": "",
        "_": generate_timestamp()  # 动态时间戳
    }

    try:
        resp = http.request(
            method='GET',
            url=baseurl,
            fields=params,
            headers=headers
        )
        if resp.status in [403,401]:
            print(f"请求第{page}页时登录过期")
            return None
        elif resp.status == 200:
            data = json.loads(resp.data.decode('UTF-8'))
            return data['data']['list']
        else:
            print(f"请求第{page}页时错误，状态码{resp.status}")
            return None

    except Exception as e:
        print(f"请求第{page}页时发生错误:{e}")
        return None

详情页数据

详情页数据主要为详情页中的岗位介绍部分：

该部分主要是使用BeautifulSoup对返回的结果进行解析，找到相关属性内容的位置并进行解析，代码如下：

def fetch_detail_page(job_id,http,Cookie):
    """获取详情页数据"""

    headers = {
        'User-Agent':ua.random,
        'Cookie':Cookie,
        'Referer':'https://job.ncss.cn/student/jobs/index.html'
    }

    detail_url = f"https://www.ncss.cn/student/jobs/{job_id}/detail.html"
    try:
        resp = http.request(
            'GET',
            detail_url,
            headers = headers
        )
        if resp.status == 200:
            return resp.data.decode('utf-8')
        elif resp.status in [401,403]:
            print(f"获取{job_id}详情页时登录过期")
            return None
        else:
            print(f"请求失败，状态码{resp.status}")
            return None
    except Exception as e:
        print(f"请求失败，{str(e)}")
        return None



def pares_detail_job_info(html,job_info):
    """将爬取到的详情页进行解析，获取“岗位介绍”部分的数据"""
    soup = BeautifulSoup(html,'html.parser')

    job_detail_describe_div =soup.find(name='pre',attrs={'class':"mainContent mainContent"})
    job_detail_describe = job_detail_describe_div.getText()

    job_info.update({
        "岗位介绍":job_detail_describe
    })

    return job_info

效率与反爬

考虑到爬取的数据量稍多，因此使用Urllib3进行分布式爬取，一个主进程，两个工作线程（岗位列表和详情页），采用消费者-生产者模型，同时使用fake_useragent模块不断改变User-Agent的值，代码如下：

def process_manager(total_pages,http,Cookie,num_list_workers=2,num_detail_workers=8,output_address=""):
    """进程管理器（主进程）"""
    global  enqueued_jobs_count
    start_time = time.time()
    print(f"开始爬取任务，总页数: {total_pages}")

    #创建Manager
    with Manager() as manager:
        #创建队列
        list_page_queue = manager.Queue()
        detail_task_queue = manager.Queue(maxsize=5000)
        result_queue = manager.Queue()

        #添加列表页任务
        for page in range(1,total_pages+1):
            list_page_queue.put(page)

        expected_total_list_pages = total_pages
        expected_total_detail_jobs = total_pages * 10

        #进度条
        list_pbar = tqdm(total=expected_total_list_pages,desc='列表页',unit='页',dynamic_ncols=True,file=sys.stdout)
        detail_pbar = tqdm(total=expected_total_detail_jobs,desc='详情页',unit='项',dynamic_ncols=True,file=sys.stdout)

        #创建并启动线程
        threads = []

        #列表页工作线程
        for i in range(num_list_workers):
            t = threading.Thread(
                target=list_page_worker,
                args=(list_page_queue,http,Cookie,detail_task_queue,list_pbar),
                name=f"ListWorker-{i+1}",
                daemon=True
            )
            t.start()
            threads.append(t)

        #详情页工作线程
        for i in range(num_detail_workers):
            t = threading.Thread(
                target=detail_page_worker,
                args=(detail_task_queue,result_queue,http,Cookie,detail_pbar),
                name=f"DetailWorker-{i+1}",
                daemon=True
            )
            t.start()
            threads.append(t)

        #结果写入线程
        writer_thread = threading.Thread(
            target=result_writer,
            args=(result_queue,output_address),
            name="ResultWriter",
            daemon=True
        )
        writer_thread.start()

        #等待列表页队列为空（等待完成列表页任务）
        list_page_queue.join()

        #通知列表页任务线程退出
        for _ in range(num_list_workers):
            list_page_queue.put(None)
        print(f"所有列表页任务已完成,{num_list_workers}个工作线程已退出")


        #调整详情任务总数
        with progress_lock:
            actual_enqueued = enqueued_jobs_count
        if actual_enqueued > 0 and actual_enqueued < detail_pbar.total:
            detail_pbar.total = actual_enqueued
            detail_pbar.refresh()


        #等待详情页任务队列为空（详情页任务完成）
        detail_task_queue.join()

        #通知详情页工作线程退出
        for _ in range(num_detail_workers):
            detail_task_queue.put(None)
        print(f"所有详情页任务已完成,{num_detail_workers}个工作线程已退出")


        #通知结果写入线程退出
        result_queue.put(None)

        for t in threads:
            t.join(timeout=10)

        writer_thread.join(timeout=10)

        elapsed_time = time.time() - start_time

        list_pbar.close()
        detail_pbar.close()

        print(f"爬取完成! 总耗时: {elapsed_time:.2f} 秒")

项目流程图

完整代码

import queue
import random
import threading
import time
from multiprocessing import Manager
from bs4 import BeautifulSoup
import urllib3
import json
from tqdm.auto import tqdm
import sys
from  fake_useragent import UserAgent
from selenium import  webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait


#抑制认证警告
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)


def keep_connect(chrome_location,username,password):
    """使用selenium模拟登录，获取Cookie，并维持登录状态防止服务器端将Cookie注销"""
    options = webdriver.ChromeOptions()
    options.binary_location = chrome_location
    options.add_argument("ignore-certificate-errors")
    options.add_argument("--headless")
    options.add_argument("--disable-gpu")
    options.add_argument("--disable-software-rasterizer")
    options.add_experimental_option("detach", True)
    driver = webdriver.Chrome(options=options,executable_path='../chromedriver.exe')

    login_url = 'https://account.chsi.com.cn/passport/login?service=https://www.ncss.cn/student/connect/chsi&entrytype=stu'
    driver.get(login_url)
    time.sleep(1)

    acc_input = driver.find_element(By.XPATH, r'/html/body/div/div[2]/div[2]/div/div[2]/form/div[1]/input')
    acc_input.send_keys(username)

    pwd_input = driver.find_element(By.XPATH, f'/html/body/div/div[2]/div[2]/div/div[2]/form/div[2]/input')
    pwd_input.send_keys(password)
    driver.find_element(By.XPATH, r'/html/body/div/div[2]/div[2]/div/div[2]/form/div[4]').click()

    element = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, r"/html/body/div/span/span/a[1]")))
    element.click()

    dic = {}

    for i in driver.get_cookies():
        dic[i['name']] = i['value']

    return dic


def Cookie_concat(got_property):
    """将从keep_connect函数中获取到的Cookie数据进行重新包装"""
    changed_property_list = ['SESSION', 'Hm_lpvt_378ff17a1ac046691cf78376632d1ed4', '_ga','_ga_6CXWRD3K0D']

    no_change = {
        '_ga_1ESVLDHDYL': 'GS1.1.1726503021.3.0.1726503021.0.0.0',
        '_abfpc': 'd7760a7e8d00953eaa18b111273b00d6ad83f1cc_2.0',
        'cna': 'eac1e178d67b00529df2b287b5842f2c',
        '_gid': 'GA1.2.673435621.1755336171',
        'aliyungf_tc': 'bfc4565f794c9e6ac253a5c09f4a937f2afb69e01fc7b413d486aaeaadf06249',
        'XSRF-CCKTOKEN': 'a12b3436b08549ba2f33a4aefa9a1454',
        'CHSICC_CLIENTFLAGNCSS': 'aba5b818eaa8bc94d6fb1ddf17d1df4f',
        'CHSICC01': '!DzrVNB/pHD1H78bzYxYLahOzddj6Y4XQ6NJ5RnOPIOyzHzKixC+5X5WINIjztT+S4x5PGaf/cowaI/Q=',
        'CHSICC_CLIENTFLAGSTUDENT': '5d0ab9cce044f18a699886e7d6705555',
        'Hm_lvt_378ff17a1ac046691cf78376632d1ed4': '1754926580,1754968150,1755336169,1755411478',
        'HMACCOUNT': 'CEB955474E107530',
        'acw_tc': 'ac11000117554230396983911ee7b259a823321b52a20a878c0b42ee677273',
        '_gat_gtag_UA_105074615_1': '1'
    }

    property_all = no_change.copy()

    for i in changed_property_list:
        property_all[i] = got_property[i]
    property_rank = ['SESSION','_ga_1ESVLDHDYL','_abfpc','cna','_gid','aliyungf_tc','acw_tc','XSRF-CCKTOKEN','CHSICC_CLIENTFLAGNCSS','CHSICC01','CHSICC_CLIENTFLAGSTUDENT','Hm_lvt_378ff17a1ac046691cf78376632d1ed4','HMACCOUNT','_gat_gtag_UA_105074615_1','Hm_lpvt_378ff17a1ac046691cf78376632d1ed4','_ga','_ga_6CXWRD3K0D']

    Cookie=''

    for i in property_rank:
        Cookie = Cookie + i + '=' + property_all[i] + '; '
    print("------------------完成Cookie创建------------------")

    return Cookie


def simulate_login_get_cookie(chrome_location,username,password):
    """将模拟登录部分和包装Cookie部分整合在一起"""
    dic = keep_connect(chrome_location=chrome_location,username=username,password=password)
    return Cookie_concat(dic)


ua = UserAgent()


def generate_timestamp():
    """生成13位时间戳"""
    return str(int(time.time() * 1000))



def get_job_data(http,Cookie,page=1,page_size=10):
    """获取岗位数据"""
    baseurl = "https://www.ncss.cn/student/jobs/jobslist/ajax/"
    #构造请求头
    headers = {
        'User-Agent':ua.random,
        'Connection':'keep-alive',
        'Cookie':Cookie,
        "Accept": "application/json,*/*",
        'Referer':"https://account.chsi.com.cn/passport/login?service=https://job.ncss.cn/student/connect/chsi&entrytype=stu"
    }
    
    # 查询参数
    params = {
        "jobType": "",
        "areaCode": "",
        "jobName": "",
        "monthPay": "",
        "industrySectors": "",
        "property": "",
        "categoryCode": "",
        "memberLevel": "",
        "recruitType": "",
        "offset": page,
        "limit": page_size,
        "keyUnits": "",
        "degreeCode": "",
        "sourcesName": "0",
        "sourcesType": "",
        "_": generate_timestamp()  # 动态时间戳
    }

    try:
        resp = http.request(
            method='GET',
            url=baseurl,
            fields=params,
            headers=headers
        )
        if resp.status in [403,401]:
            tqdm.write(f"请求第{page}页时登录过期")
            return None
        elif resp.status == 200:
            data = json.loads(resp.data.decode('UTF-8'))
            return data['data']['list']
        else:
            tqdm.write(f"请求第{page}页时错误，状态码{resp.status}")
            return None

    except Exception as e:
        tqdm.write(f"请求第{page}页时发生错误:{e}")
        return None




def fetch_detail_page(job_id,http,Cookie):
    """获取详情页数据"""

    headers = {
        'User-Agent':ua.random,
        'Cookie':Cookie,
        'Referer':'https://job.ncss.cn/student/jobs/index.html'
    }

    detail_url = f"https://www.ncss.cn/student/jobs/{job_id}/detail.html"
    try:
        resp = http.request(
            'GET',
            detail_url,
            headers = headers
        )
        if resp.status == 200:
            return resp.data.decode('utf-8')
        elif resp.status in [401,403]:
            tqdm.write(f"获取{job_id}详情页时登录过期")
            return None
        else:
            tqdm.write(f"请求失败，状态码{resp.status}")
            return None
    except Exception as e:
        tqdm.write(f"请求失败，{str(e)}")
        return None



def pares_detail_job_info(html,job_info):
    """将爬取到的详情页进行解析，获取“岗位介绍”部分的数据"""
    soup = BeautifulSoup(html,'html.parser')

    job_detail_describe_div =soup.find(name='div',attrs={'class':"details"})
    if job_detail_describe_div is not None:
        job_detail_describe = job_detail_describe_div.getText()
    else:
        tqdm.write(f"未找到岗位描述元素，岗位ID:{job_info.get('岗位ID')}")
        job_detail_describe = "未知"

    job_info.update({
        "岗位介绍":job_detail_describe
    })

    return job_info



def parse_job_info(job):
    """解析岗位信息"""
    id = job.get("jobId")
    timeStamp = job.get("updateDate")
    timeArray = time.localtime(float(timeStamp)/1000)

    return {
        "岗位ID":id,
       "职位名称":job.get("jobName"),
        "薪资水平":str(job.get("lowMonthPay"))+'k-'+str(job.get('highMonthPay'))+'k',
        "招聘人数":job.get("headCount"),
        "学历要求":job.get("degreeName"),
        "招聘方":job.get("recName"),
        "公司规模":job.get("recScale"),
        "地区":job.get("areaCodeName"),
        "福利":job.get("recTags"),
        "专业要求":job.get("major"),
        "岗位更新时间":time.strftime("%Y-%m-%d %H:%M:%S",timeArray),
        "详情网址":f"https://www.ncss.cn/student/jobs/{id}/detail.html"
    }





def list_page_worker(list_page_queue, http, Cookie, detail_task_queue, list_pbar):
    """获取岗位列表的工作流程"""
    global list_get_wrong, list_pages_done, enqueued_jobs_count

    while True:
        page = None
        try:
            page = list_page_queue.get(timeout=30)

            if page is None:
                list_page_queue.task_done()   # 对应主线程 put(None)
                break

            # 爬取列表页
            jobs = get_job_data(http=http, Cookie=Cookie, page=page, page_size=10)
            if jobs:
                for job in jobs:
                    job_info = parse_job_info(job)
                    detail_task_queue.put(job_info)
                    with progress_lock:
                        enqueued_jobs_count += 1
            time.sleep(random.uniform(0.5, 1.5))

        except queue.Empty:
            thread_name = threading.current_thread().name
            tqdm.write(f"列表页任务队列空，线程 {thread_name} 准备退出")
            continue

        except Exception as e:
            tqdm.write(f"列表页工作线程异常: {str(e)}")
            if page:
                list_get_wrong.append(page)
            time.sleep(5)

        finally:
            if page is not None:  # 只在真实任务时更新
                list_pages_done += 1
                list_pbar.update(1)
                list_page_queue.task_done()




def detail_page_worker(detail_task_queue,result_queue,http,Cookie,detail_pbar):
    """详情页工作流程"""

    global detail_get_wrong,detail_jobs_done

    while True:
        job_info = None
        job_id =None
        try:
            job_info = detail_task_queue.get(timeout=30)

            if job_info is None:
                detail_task_queue.task_done()
                break

            job_id = job_info.get("岗位ID")

            # print(f"开始爬取详情页{job_id}")

            html = fetch_detail_page(job_id, http, Cookie)

            if html:
                new_job_info = pares_detail_job_info(html, job_info)
                result_queue.put(new_job_info)
            # print(f"详情页{job_id}解析完成")
            else:
                tqdm.write(f"详情页{job_id}爬取失败")

            time.sleep(random.uniform(0.3, 0.8))

        except queue.Empty:
            thread_name = threading.current_thread().name
            tqdm.write(f"详情页任务队列空，线程{thread_name}准备退出")
            continue

        except Exception as e:
            tqdm.write(f"详情页工作线程异常: {str(e)}")
            if job_id:
                detail_get_wrong.append(job_id)
            time.sleep(3)
        finally:
            if job_info is not None:
                detail_jobs_done += 1
                detail_pbar.update(1)
                detail_task_queue.task_done()


def result_writer(result_queue, output_address):
    """结果写入线程"""
    struc_time = time.localtime()
    time_year = struc_time.tm_year
    time_month = struc_time.tm_mon
    time_day = struc_time.tm_mday

    output_file = output_address + f'{time_year}_{time_month}_{time_day}_jobs'

    count = 0
    with open(output_file, "a", encoding='utf-8', newline='') as f:
        while True:
            try:
                # 获取结果
                result = result_queue.get(timeout=120)
                if result is None:
                    break

                f.write(json.dumps(result, ensure_ascii=False) + "\n")
                f.flush()

                count += 1

                if count % 100 == 0:
                    tqdm.write(f"已写入 {count} 条结果")

                result_queue.task_done()
            except queue.Empty:
                tqdm.write("结果写入线程超时退出")
                break
            except Exception as e:
                tqdm.write(f"结果写入异常: {str(e)}")

    tqdm.write(f"结果写入完成，共写入 {count} 条数据")



def process_manager(total_pages,http,Cookie,num_list_workers=2,num_detail_workers=8,output_address=""):
    """进程管理器（主进程）"""
    global  enqueued_jobs_count
    start_time = time.time()
    tqdm.write(f"开始爬取任务，总页数: {total_pages}")

    #创建Manager
    with (Manager() as manager):
        #创建队列
        list_page_queue = manager.Queue()
        detail_task_queue = manager.Queue(maxsize=5000)
        result_queue = manager.Queue()

        #添加列表页任务
        for page in range(1,total_pages+1):
            list_page_queue.put(page)

        expected_total_list_pages = total_pages
        expected_total_detail_jobs = total_pages * 10

        #进度条
        list_pbar = tqdm(total=expected_total_list_pages,desc='列表页',unit='页',dynamic_ncols=True,file=sys.stdout)
        detail_pbar = tqdm(total=expected_total_detail_jobs,desc='详情页',unit='项',dynamic_ncols=True,file=sys.stdout)

        #创建并启动线程
        threads = []

        #列表页工作线程
        for i in range(num_list_workers):
            t = threading.Thread(
                target=list_page_worker,
                args=(list_page_queue,http,Cookie,detail_task_queue,list_pbar),
                name=f"ListWorker-{i+1}",
                daemon=True
            )
            t.start()
            threads.append(t)

        #详情页工作线程
        for i in range(num_detail_workers):
            t = threading.Thread(
                target=detail_page_worker,
                args=(detail_task_queue,result_queue,http,Cookie,detail_pbar),
                name=f"DetailWorker-{i+1}",
                daemon=True
            )
            t.start()
            threads.append(t)

        #结果写入线程
        writer_thread = threading.Thread(
            target=result_writer,
            args=(result_queue,output_address),
            name="ResultWriter",
            daemon=True
        )
        writer_thread.start()

        #等待列表页队列为空（等待完成列表页任务）
        list_page_queue.join()

        #通知列表页任务线程退出
        for _ in range(num_list_workers):
            list_page_queue.put(None)
        tqdm.write(f"所有列表页任务已完成,{num_list_workers}个工作线程已退出")


        #调整详情任务总数
        with progress_lock:
            actual_enqueued = enqueued_jobs_count
        if actual_enqueued > 0 and actual_enqueued < detail_pbar.total:
            detail_pbar.total = actual_enqueued
            detail_pbar.refresh()


        #等待详情页任务队列为空（详情页任务完成）
        detail_task_queue.join()

        #通知详情页工作线程退出
        for _ in range(num_detail_workers):
            detail_task_queue.put(None)
        tqdm.write(f"所有详情页任务已完成,{num_detail_workers}个工作线程已退出")


        #通知结果写入线程退出
        result_queue.put(None)

        for t in threads:
            t.join(timeout=10)

        writer_thread.join(timeout=10)

        elapsed_time = time.time() - start_time

        list_pbar.close()
        detail_pbar.close()

        print(f"爬取完成! 总耗时: {elapsed_time:.2f} 秒")


if __name__ == "__main__":
    #全局变量
    progress_lock = threading.Lock()
    list_pages_done = 0
    detail_jobs_done =0
    enqueued_jobs_count = 0
    # 配置爬取参数
    TOTAL_PAGES = 200  # 要爬取的列表页总数
    NUM_LIST_WORKERS = 2  # 列表页工作线程数
    NUM_DETAIL_WORKERS = 5  # 详情页工作线程数
    OUTPUT_ADDRESS = ""  # 输出文件的地址，默认为项目地址
    #配置模拟登录参数
    chrome_location = r'D:\python_project\Google\Chrome\Application\chrome.exe' #chrome启动器的位置
    username = "" #模拟登录的账号
    password = ""#模拟登陆的密码

    detail_get_wrong = [] #获取详情失败的job_id
    list_get_wrong = [] #获取岗位失败的页码


    #构建连接池
    print("------------------开始构建连接池------------------")
    http = urllib3.PoolManager(
        num_pools=50,
        maxsize=50,
        cert_reqs='CERT_NONE',
        assert_hostname=False,
        block=True,
        timeout=urllib3.Timeout(connect=5.0, read=10.0),
        retries=urllib3.Retry(total=3, backoff_factor=0.5),
    )
    print("------------------完成构建连接池------------------")

    #模拟登录，维持登录状态，获取组成Cookie的必要的参数
    print("-------------------开始模拟登录-------------------")
    Cookie = simulate_login_get_cookie(chrome_location=chrome_location,username=username,password=password)
    print("-------------------完成模拟登录-------------------")

    # 启动爬虫
    print("-------------------开始网络爬取-------------------")
    process_manager(
        total_pages=TOTAL_PAGES,
        num_list_workers=NUM_LIST_WORKERS,
        num_detail_workers=NUM_DETAIL_WORKERS,
        output_address=OUTPUT_ADDRESS,
        http = http,
        Cookie = Cookie
    )
    print("-------------------完成网络爬取-------------------")

问题 OR 改进

当前的爬取仍需要登录才能获取，如何才能以非登录状态获取所需的数据
可以增加代理，从而不断变动IP使得不易被锁IP

Selenium on 南国遗梦的小站