江苏龙网

  1. 主页 > 生活百科 > >

源码 用python一天爬取20万条企业信息( 二 )

Python


try:
# 对响应体进行解析
soup = BeautifulSoup(respo, "lxml")
# 找到页面子链接，进入子页面，对子页面进行抓取
# 用select函数抽取需要的内容，单击需要的内容》检查》copy select
lianjie = soup.select('#main > div.main > div.intro > div.intros > div.text > p > a')
lianjie = lianjie[0].get('href')
wzs1.append(lianjie)
print(lianjie)
company = soup.select("#main > div.aside > div.info > div.info_c > p:nth-child(1) > strong") # 标题
company = company[0].text
# 匹配电话号码
dianhua = soup.select("#main > div.aside > div.info > div.info_c > p:nth-child(3)") # 地址
dianhua = dianhua[0].text.split("：")[1]
# 匹配手机号码
phone = soup.select("#main > div.aside > div.info > div.info_c > p:nth-child(4)") # 日租价格
phone = phone[0].text.split("：")[1]
# 匹配传真
chuanzhen = soup.select("#main > div.aside > div.info > div.info_c > p:nth-child(5)") # 月租价格
chuanzhen = chuanzhen[0].text.split("：")[1]
# 经营模式
jingying = soup.select("#main > div.aside > div.info > div.info_c > p:nth-child(8)") # 面积大小
jingying = jingying[0].text.split("：")[1]
# 公司地址
address = soup.select('#main > div.aside > div.info > div.info_c > p:nth-child(9)') # 抽取建造年份
address = address[0].text.split("：")[1]
# 公司简介
# introduction = soup.select("#main > div.main > div.intro > div.intros > div.text > p") # 楼层属性
# introduction = introduction[0].text.strip()
data = https://www.isolves.com/it/cxkf/yy/Python/2019-08-14/[company, address, dianhua, phone, chuanzhen, jingying]
print(data)
with open('首富网企业7.csv', 'a+', newline='', encoding='GB2312', errors='ignore') as csvfile:
w1 = csv.writer(csvfile)
w1.writerow(data, [1]) except:
print("出错！")
async def main(loop):
async with aiohttp.ClientSession() as sess:
tasks = []
for ii in wzs:
ur = ii
try:
tasks.append(loop.create_task(get_html(sess, ur)))
except:
print('error')
# 设置0.1的网络延迟增加爬取效率
await asyncio.sleep(0.1)
finished, unfinised = await asyncio.wait(tasks)
for i1 in finished:
await parser(i1.result())
if __name__ == '__main__':
t1 = time.time()
loop = asyncio.get_event_loop()
loop.run_until_complete(main(loop))
print("花费时间", time.time() - t1)
print('详细页链接抓取完毕！')
"""
########################################################################################################################
# 使用并发法获取详细页的内容
########################################################################################################################
# 定义函数获取每个网页需要爬取的内容
def parser(url):
global data
try:
res = requests.get(url, headers=headers)
# 对响应体进行解析
soup = BeautifulSoup(res.text, 'lxml')
# 找到页面子链接，进入子页面，对子页面进行抓取
# 用select函数抽取需要的内容，单击需要的内容》检查》copy select
company = soup.select('#main > div.aside > div.info > div.info_c > p:nth-child(1) > strong')
company = company[0].text
name = soup.select('#main > div.aside > div.info > div.info_c > p:nth-child(2)')
name = name[0].text
dianhua = soup.select('#main > div.aside > div.info > div.info_c > p:nth-child(3)')
dianhua = dianhua[0].text.split('：')[1]
shouji = soup.select('#main > div.aside > div.info > div.info_c > p:nth-child(4)')
shouji = shouji[0].text.split('：')[1]
chuanzhen = soup.select('#main > div.aside > div.info > div.info_c > p:nth-child(5)')
chuanzhen = chuanzhen[0].text.split('：')[1]
product = soup.select('tr:nth-child(1) > td:nth-child(2)')
product = product[0].text
company_type = soup.select('tr:nth-child(2) > td:nth-child(2) > span')
company_type = company_type[0].text.strip()
legal_person = soup.select('tr:nth-child(3) > td:nth-child(2)')
legal_person = legal_person[0].text
main_address = soup.select('tr:nth-child(5) > td:nth-child(2) > span')
main_address = main_address[0].text
brand = soup.select('tr:nth-child(6) > td:nth-child(2) > span')


推荐阅读


上一篇：Java内存映射，上G大文件轻松处理

下一篇：2021年幼儿园大班班级工作总结 幼儿园大班教学总结