使用多线程获取某招聘网站的信息，使用环境py3，话不多说直接上代码
该导的包你可就不能少了
import threading
import requests
from pyquery import PyQuery as pq
import json
from pymongo import MongoClient
from fake_useragent import UserAgent
import time
from config import *
我这里用到的是Mongodb储存的，fake_useragent是一个‘User-Agent’一个共享库，看着用，我这边写的是一个配置文件，大家可以看将**文件写自己的文件。
client = MongoClient(().mogodb()["IP"],().mogodb()["port"])
u = UserAgent()
class data():
def __init__(self,startpage,page,citynum,hangyenum):
self.startpage=startpage
self.page=page
self.citynum=citynum
self.hangyenum=hangyenum
def datature(self):
if self.startpage==0:
self.startpage=1
for i in range(self.startpage,self.page):
try:
headers = {
'User-Agent': u.random,
}
yu = requests.get('这里是一个代理url').text
ip = yu.split(':')[0]
port = yu.split(':')[1]
proxy_http = "http://{0}:{1}".format(ip, port)
proxy_dict = {"http": proxy_http}
print(proxy_dict)
url='https://fe-api.zhaopin.com/c/i/sou?start={0}&pageSize=60&cityId={1}&industry={2}&workExperience=-1&education=-1&companyType=-1&employmentType=-1&jobWelfareTag=-1&kt=3&_v=0.64455588&x-zp-page-request-id=2fb2f4e53284471aa21894bf291db027-1543801568582-396367'.format(str(i*60),self.citynum,self.hangyenum)
print(self.startpage,self.page)
r=requests.get(url,headers=headers,proxies=proxy_dict)
dict1=json.loads(r.text)
for j in dict1['data']['results']:
href='https://jobs.zhaopin.com/'+j['number']+'.htm'
href=j['positionURL']
print(href)
r1 = requests.get(href, headers=headers,proxies=proxy_dict)
doc = pq(r1.text)
title = doc('.new-info h1.l.info-h3').text()
xinzhi = doc('.new-info div.l.info-money strong').text()
company = doc('li.clearfix div.company.l a').text()
jianjie = doc('li.clearfix div.info-three.l').text()
zhiweixinxi = doc('div.l.pos-info-in div.pos-info-tit p.r ').text()
zhiweimiaoshu = doc('div.responsibility.pos-common div.pos-ul').text()
gongsigaikuang = doc('div.intro-content div[align="left"]').text()
lianxifangshi = doc('div.intro-content div[style="font-family: 宋体;"]').text()
list1=jianjie.split(' ')
try:
list2=zhiweixinxi.split(' ')[0]
except:
list2=''
try:
jianjie1=list1[0]
jianjie2=list1[1]
jianjie3=list1[2]
jianjie4=list1[3]
except:pass
if title!=""and len(list1)==4:
db = client.自己的数据库名1# 创建数据库名
post = {'title': title,
'xinzhi': xinzhi,
'conmpany': company,
'dizhi': jianjie1,
'jingyan':jianjie2,
'xueli':jianjie3,
'zhaopinrenshu':jianjie4,
'zhiweixinxi': list2,
'zhiweimiaoshu': zhiweimiaoshu,
'gongsigaikuang': gongsigaikuang,
'lianxifangshi': lianxifangshi,
'html':href
}
posts = db.zhaopin # 创建表名
post_id = posts.insert_one(post).inserted_id # 创建id将数据提交
else:
pass
print(i)
self.startpage+=1
except:
print("请求次数过多或程序出错错误页：",self.startpage)
self.startpage += 1
self.datature()下面是使用多线程执行上面这段请求解析
def action(a,b,c,d):
start = data(int(a),int(b),c,d)
start.datature()
if __name__=="__main__":
while True:
print('88888888此程序启动为默认程序8888888888')
for i in range(26):
citynum = zhilianzhaopin().sheng(i)["citynum"]
for j in range(12):
hangyenum = zhilianzhaopin().hangye(j)["hangyenum"]
print('点击查看多少页,默认程序是101页：', end='')
print('https://sou.zhaopin.com/?jl={0}&in={1}'.format(citynum, hangyenum))
page = zhilianzhaopin().zongyeshu(101)[1]
time.sleep(5)
fenyepage = int(int(page) / 10)
duandianpage = fenyepage * 10
if fenyepage!=0:
for i in range(11):
if i <= 9:
t = threading.Thread(target=action, args=(i * fenyepage, (i + 1) * fenyepage,citynum,hangyenum))
else:
t = threading.Thread(target=action, args=(duandianpage, page,citynum,hangyenum))
t.start()
elif fenyepage==0:
action(1, page,citynum,hangyenum)
这下面是配置文件，可供参考：
class zhilianzhaopin():
def mogodb(self):
a={}
#mongodbIp
IP=''
#mongodb端口号
port=
a["IP"]=IP
a["port"]=port
if a["IP"]!="" and a["port"]!="" :
return a
else:
print("配置文件mongodb参数缺失")
#省会名
def sheng(self,cityxuhao):
list1={0: '北京', 1: '上海', 2: '深圳', 3: '广州', 4: '天津', 5: '成都', 6: '杭州', 7: '武汉', 8: '大连', 9: '长春', 10: '南京', 11: '济南',
12: '青岛', 13: '苏州', 14: '沈阳', 15: '西安', 16: '郑州', 17: '长沙', 18: '重庆', 19: '哈尔滨', 20: '无锡', 21: '宁波', 22: '福州',
23: '厦门', 24: '石家庄', 25: '合肥', 26: '惠州'}
a='''0: '北京', 1: '上海', 2: '深圳', 3: '广州', 4: '天津', 5: '成都', 6: '杭州', 7: '武汉', 8: '大连', 9: '长春' '''
b=''' 10: '南京', 11: '济南',12: '青岛', 13: '苏州', 14: '沈阳', 15: '西安', 16: '郑州', 17: '长沙', 18: '重庆', '''
c='''19: '哈尔滨', 20: '无锡', 21: '宁波', 22: '福州',23: '厦门', 24: '石家庄', 25: '合肥', 26: '惠州' '''
# print('%s\n%s\n%s'%(a,b,c))
# try:
# cityxuhao=int(input('输入序号，整数：'))
# except:
# print('输入错误！！！')
# self.sheng()
#城市名
city=list1[cityxuhao]
print(city)
#城市编码
dict1={}
citynum={'北京':'530','上海':'538','深圳':'765','广州':'763','天津':'537','成都':'801','杭州':'653','武汉':'736','大连':'600','长春':'631','南京':'635','济南':'702','青岛':'703','苏州':'639','沈阳':'599','西安':'854','郑州':'719','长沙':'749','重庆':'551','哈尔滨':'622','无锡':'636','宁波':'654','福州':'681','厦门':'682','石家庄':'515','合肥':'664','惠州':'773'}
dict1['citynum']=citynum[city]
if dict1['citynum']!="":
return dict1
else:
print("配置文件省会名参数出错")
#行业
def hangye(self,hangyexuhao):
#参考行业
list1={0: '互联网/IT', 1: '金融', 2: '房地产/建筑', 3: '商业服务', 4: '贸易/批发/零售', 5: '教育/艺术', 6: '服务业', 7: '文化/传媒/娱乐', 8: '制造业', 9: '物流运输', 10: '能源/环保', 11: '政府/非盈利', 12: '农林牧渔'}
a=''' 0: '互联网/IT', 1: '金融', 2: '房地产/建筑', 3: '商业服务', 4: '贸易/批发/零售', 5: '教育/艺术', 6: '服务业', 7: '文化/传媒/娱乐', 8: '制造业', 9: '物流运输' '''
b=''' 10: '能源/环保', 11: '政府/非盈利', 12: '农林牧渔' '''
# print('%s\n%s'%(a,b))
# hanyexuhao=hanyexuhao
# try:
# hangyexuhao=int(input('输入对应序号：'))
# except:
# print("输入错误!!!")
# self.hangye()
dict1={}
hangye = list1[hangyexuhao]
print(hangye)
# 行业编码
hangyenum = {"互联网/IT": "10100", "金融": "10200", "房地产/建筑": "10800", "商业服务": "10900", "贸易/批发/零售": "10300",
"教育/艺术": "10400", "服务业": "10000", "文化/传媒/娱乐": "11300", "制造业": "10500", "物流运输": "11500",
"能源/环保": "11600", "政府/非盈利": "11100", "农林牧渔": "11400"}
dict1["hangyenum"] = hangyenum[hangye]
if dict1["hangyenum"]!="":
return dict1
else:
print("配置文件行业中参数缺失")
#总页数
def zongyeshu(self,page):
startpage=1
try:
# page=int(input('输入总页数：'))
page=int(page)
except:
print('输入有误！！！')
self.zongyeshu()
if page!="":
return startpage,page
else:
print("配置文件总页数缺失")
有啥可以帮助的可以联系1307761253
