Python爬虫+数据分析之影评分析( 二 ) _Python爬虫

三、数据解析

构造请求头，模拟数据请求

def get_film_data(offset = 0, filename="film"):url = f'aHR0cHM6Ly9tLm1hb3lhbi5jb20vYXBvbGxvL2Fwb2xsb2FwaS9tbWRiL3JlcGxpZXMvY29tbWVudC8xMTY3MTI5MDg5Lmpzb24/X3ZfPXllcyZvZmZzZXQ9NDA='headers = {'User-Agent': 'Mozilla/5.0 (iphone; CPU iPhone OS 11_0 like mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1'}cookies = {'uuid_n_v':'v1','iuuid':'942C12B0DF4311E9ADA9C1C3B540BA45F066B2B3028841B8A0BC3544E4C0AD17','ci':'1%2C%E5%8C%97%E4%BA%AC','_lxsdk_cuid':'16d6c9b401ec8-0c6c86354bd8a9-5b123211-100200-16d6c9b401ec8','webp':'true','_lxsdk':'942C12B0DF4311E9ADA9C1C3B540BA45F066B2B3028841B8A0BC3544E4C0AD17'}# 开始页面请求，返回响应内容response = requests.get(url,headers=headers,cookies=cookies).json()# 总评论数total = response['total']print(total)# 评论信息列表cmts = response['cmts']pprint(cmts)for comment in cmts:data = https://www.isolves.com/it/cxkf/yy/Python/2023-03-24/[]# 评论id# id = comment['id']# 评论内容content = comment['content']# 用户昵称nickName = comment['nickName']# 用户评分score = comment['score']# 评论时间# startTime = comment['time']# 用户iduserId = comment['userId']# 用户等级userLevel = comment['userLevel']# 用户性别gender = comment.get('gender',None)data['nickName '] = nickNamedata['gender'] = genderdata['score'] = scoredata['content'] = contentdata['userId '] = userIddata['userLevel'] = userLevelsave_data_csv(data,filename)return total

2. 数据存储（这里为以csv演示）

def save_data_csv(data, file_name):with open(file_name,'a',encoding='utf-8-sig',newline='')as fp:# 创建写对象writer = csv.writer(fp)title = ['nickName ','gender','score','content','userId ','userLevel']# 解决循环存储，表头重复问题with open(file_name,'r',encoding='utf-8-sig',newline='')as fp:# 创建读对象reader = csv.reader(fp)if not [row for row in reader]:writer.writerow(title)writer.writerow([data[i] for i in title])else:writer.writerow([data[i] for i in title])print('*'*10+'保存完毕'+'*'*10)

文章插图
影评结果
四、数据可视化

影评分词

def wordcloud_analysis(file_name):df = pd.read_csv(file_name, encoding='utf-8')content = df['content'].to_string()# 开始分词 使用jieba进行精确分词获取词语列表words = jieba.lcut(content)# 使用空格拼接获得字符串words = ' '.join(words)# 生成词云# 读取图片，生成图片形状mask_pic = np.array(Image.open('1.jpg'))words_cloud = WordCloud(background_color='white',# 词云图片的背景颜色width=800, height=600,# 词云图片的宽度，默认400像素;词云图片的高度，默认200像素font_path='msyh.ttf',# 词云指定字体文件的完整路径max_words=200,# 词云图中最大词数，默认200max_font_size=80,# 词云图中最大的字体字号，默认None，根据高度自动调节 min_font_size# 词云图中最小的字体字号，默认４号font_step=1,# 词云图中字号步进间隔，默认１random_state=30,# 设置有多少种随机生成状态，即有多少种配色方案mask=mask_pic# 词云形状，默认None，即方形图).generate(words)# 有jieba分词拼接的字符串生成词云words_cloud.to_file('comment.png')# 保存词云为图片# 使用plt显示词云plt.imshow(words_cloud, interpolation='bilinear')# 消除坐标轴plt.axis('off')plt.show()

文章插图
分词
2. 观看人群性别及评分占比分析（由于取得部分数据，不代表最终现实结果，勿纠）

def gender_pie_analysis(file_name):df = pd.read_csv(file_name, encoding='utf-8')print(df)## # 1.观看人群性别gender = df['gender'].value_counts()print(gender)# 饼图，标题：观看人群性别占比# 调用自定义饼图函数# 创建画布和轴fig, ax = plt.subplots(figsize=(6, 6), dpi=100)# plt.figure()size = 0.5# labels = data.indexax.pie(gender, labels=['女','男','未知'], startangle=90, autopct='%.1f%%', colors=sns.color_palette('husl', len(gender)),radius=1,# 饼图半径，默认为1pctdistance=0.75,# 控制百分比显示位置wedgeprops=dict(width=size, edgecolor='w'),# 控制甜甜圈的宽度textprops=dict(fontsize=10)# 控制字号及颜色)ax.set_title("【满江红】观看人群性别占比", fontsize=15)# plt.title(title)plt.show()