Files
2022-11-09 10:41:47 +08:00

805 lines
280 KiB
HTML
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<!-- saved from url=(0048)https://edu.shuidilab.cn/console/expdesktop/5665 -->
<html lang="zh-cn" delegate-events="click,change"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><style class="vjs-styles-defaults">
.video-js {
width: 300px;
height: 150px;
}
.vjs-fluid {
padding-top: 56.25%
}
</style>
<meta http-equiv="Cache-Control" content="max-age=7200">
<meta name="viewport" content="width=device-width, user-scalable=no, initial-scale=1.0, maximum-scale=1.0, minimum-scale=1.0">
<meta http-equiv="X-UA-Compatible" content="ie=edge">
<meta name="description" content="“水滴实验营”是一个专门为高校师生、科研人员及IT从业人员提供计算机技术相关实验课程的在线教学实训服务平台。平台支持按需定制的实验环境、秒级就绪的实验桌面、互动互助的实验教学模式,并专门想高校教师提供实验监控、作业考试、学情分析等精准教学服务,同时也可为高等院校、科研机构以及IT培训机构定制专属的在线实验教学门户。">
<meta name="keywords" content="水滴实验营,水滴,实验,实验营,教学实训,大数据,云计算,人工智能,高性能计算,信息技术,实验室,大数据实验室,云计算实验室,IT在线教育,编程,在线编程,Linux,Python,在线实验,软件开发">
<link href="https://edu.shuidilab.cn/static/public/img/shuidi_favicon.ico" rel="shortcut icon" type="image/x-icon">
<title>水滴实验营</title>
<link rel="stylesheet" href="./水滴实验营_files/font_877519_0ywdpr3rijo.css">
<link href="./水滴实验营_files/toastr.css" rel="stylesheet">
<link rel="stylesheet" href="./水滴实验营_files/desktop.19d8939c.css">
<script>
var ua = navigator.userAgent.toLowerCase();
var s;
(s = ua.match(/rv:([\d.]+)\) like gecko/)) ? s = parseFloat(s[1]) >= 11:
(s = ua.match(/msie ([\d.]+)/)) ? s = false :
(s = ua.match(/firefox\/([\d.]+)/)) ? s = parseFloat(s[1]) >= 30 :
(s = ua.match(/chrome\/([\d.]+)/)) ? s = parseFloat(s[1]) >= 28 :
(s = ua.match(/version\/([\d.]+).*safari/)) ? s = parseFloat(s[1]) >= 7 : false;
if (!s) {
window.location.href = '/msg/update-browser';
}
</script>
<script>
window.SDC = {"cas_url": "https://sso.shuidilab.cn/login?service=https://edu.shuidilab.cn/auth/?next=/slab/userpanel/", "cas_server_url": "https://sso.shuidilab.cn/", "slab_url": "https://edu.shuidilab.cn/", "tlab_url": "/", "mlab_url": "/", "realtime_num": 20, "proxyBase": "/api/proxy", "menuspread": true, "t": {"LANG": "zh-hans"}, "env": {"showSdcLink": true}, "websockets": ["RegionOne"], "msgsocket": "", "SOCKJS_URL": "wsconsole.shuidilab.cn/mcenter", "DATASET_SOCK_URL": "/ws", "vm_id": null, "guacHost": "wsconsole.shuidilab.cn/gws/", "sock_type": "wss", "domain": {"user_domain": "https://jn1.is.shanhe.com/shuidilab/", "pcourse_domain": "https://jn1.is.shanhe.com/shuidilab/", "vcourse_domain": "https://jn1.is.shanhe.com/shuidilab/", "icourse_domain": "https://jn1.is.shanhe.com/shuidilab/", "report_domain": "https://jn1.is.shanhe.com/shuidilab/", "step_domain": "https://jn1.is.shanhe.com/shuidilab/", "step_att_domain": "https://jn1.is.shanhe.com/shuidilab/", "idauth_att_domain": "https://jn1.is.shanhe.com/shuidilab/", "invite_domain": "https://jn1.is.shanhe.com/shuidilab/"}, "user": {"project_id": "", "name": "18247649623", "user_pic_path": "file/defaultavatar.png ", "email": "This_weiyuqing@qq.com", "nickname": "student20221107012836", "user_id": "805f8fa8-5e3b-11ed-aaad-0242ac110002", "mobile": "18247649623", "address": "", "industry": "", "manage_spec": {"allow_custom_template": 0}, "id": "9732", "auth_token": "df3fb84d-cd76-4b9c-9b13-20ce42f1a38e", "roles": ["student"], "isAdmin": true, "user_type": 10, "userquota": {"lab_env_count": {"value": 20, "type": "", "number": "", "enabled": true, "quota_name": "\u53ef\u521b\u5efa\u5b9e\u9a8c\u73af\u5883\u6570\u91cf", "unit": "count"}, "vclass_count": {"value": 20, "type": "", "number": "", "enabled": true, "quota_name": "\u53ef\u521b\u5efa\u865a\u62df\u8bfe\u5802\u6570\u91cf", "unit": "count"}, "stu_class_student_count": {"value": 200, "type": "", "number": "", "enabled": true, "quota_name": "\u73ed\u7ea7\u5b66\u751f\u6570\u91cf", "unit": "count"}, "vclass_student_count": {"value": 50, "type": "", "number": "", "enabled": true, "quota_name": "\u865a\u62df\u8bfe\u5802\u5b66\u751f\u6570\u91cf", "unit": "count"}, "study_spec_count": {"value": 1, "type": "", "number": "", "enabled": true, "quota_name": "\u53ef\u521b\u5efa\u4e3b\u673a\u6570", "unit": "count"}, "software_info_count": {"value": 10, "type": "", "number": "", "enabled": true, "quota_name": "\u53ef\u5b89\u88c5\u8f6f\u4ef6\u6570\u91cf", "unit": "count"}, "lab_template_count": {"value": 20, "type": "", "number": "", "enabled": true, "quota_name": "\u53ef\u521b\u5efa\u5b9e\u9a8c\u6a21\u677f\u6570\u91cf", "unit": "count"}}}}
t = SDC.t;
</script>
<link id="layuicss-layer" rel="stylesheet" href="./水滴实验营_files/layer.css" media="all"><script type="text/javascript" charset="utf-8" async="" data-requirecontext="_" data-requiremodule="layermsg" src="./水滴实验营_files/layer.js.下载"></script><link type="text/css" rel="stylesheet" href="./水滴实验营_files/katex.min.css"></head>
<body style="background: rgb(0, 0, 0);"><input type="hidden" name="csrfmiddlewaretoken" value="ztx4Z2bG9Qv1B981SDZFa8hzxKS59fGJFkwhCBBAoHO0gKNXvbWO0YsiDodoCc4u"><style>
body{
overflow: hidden;
font-size: 12px;
}
body {
background: black
}
.topbar {
background: #222;
height: 22px;
position: relative;
overflow: hidden
}
.tip {
position: absolute;
top: 0;
width: 100%;
z-index: 2
}
.tip p {
padding: 3px 15px;
background: #658BA0;
color: white;
text-align: center;
width: 370px;
margin: 0 auto;
}
.pipe {
margin: 3px 0;
height: 1px;
background: #666
}
.dropdown {
width: 140px;
height: 22px;
z-index: 3
}
.message {
position: fixed;
left: 0;
top: 0;
width: 100%;
height: 22px;
color: #CEC17E;
line-height: 22px;
vertical-align: middle;
text-align: center;
display: none;
z-index: 0
}
.fullscreen {
position: fixed;
top: 2px;
right: 220px;
cursor: pointer;
display: block
}
.modal {
transform: translate(-50%, -50%);
}
.modal-header {
color: #000000;
background: #ebecef;
}
.modal-header .close {
color: #000000;
}
.form-horizontal .controls {
margin-right: 30px;
}
.mid-canvas {
display: block;
margin: 0 auto
}
input.dropdown-toggle {
height: 22px
}
.dropdown-text {
line-height: 22px;
border: 0 none
}
.dropdown-text::after {
top: 10px
}
.dropdown-content:hover, .dropdown-toggle:focus ~ .dropdown-content {
top: 24px
}
.dropdown-content {
top: 24px;
width: 140px;
}
#noVNC_screen {
position: absolute;
margin: 0px;
padding: 0px;
bottom: 0px;
top: 36px; /* the height of the control bar */
left: 0px;
right: 0px;
width: auto;
height: auto;
display: flex;
display: -webkit-flex;
align-items: center;
justify-content: center;
}
</style><div style="height: 100%;"><div class="console_residue_time hide" change-class="console_residue_time hide" style="display: none;"><div><span>00</span>:
<span>00</span>:
<span>00</span></div></div><div class="expDesktop clearfix"><div class="expDesktop_tool" style="right: 0px;"><div class="tools-toggle show_menu"><span title="隐藏工具" class="show-tools" avalon-events="click:eclick_0_events46desktop95tools95hide4041" style="display: block;"><img src="./水滴实验营_files/tools.png"></span><span title="显示工具" class="hide-tools" avalon-events="click:eclick_0_events46desktop95tools95show4041" style="display: none;"><img src="./水滴实验营_files/tools.png"></span></div><!--if--><div class="right-toolbar os-host os-theme-dark os-host-resize-disabled os-host-scrollbar-horizontal-hidden os-host-scrollbar-vertical-hidden os-host-transition"><div class="os-resize-observer-host"><div class="os-resize-observer observed" style="left: 0px; right: auto;"></div></div><div class="os-size-auto-observer" style="height: calc(100% + 1px); float: left;"><div class="os-resize-observer observed"></div></div><div class="os-content-glue" style="width: 139px; margin: 0px; height: 881px;"></div><div class="os-padding"><div class="os-viewport os-viewport-native-scrollbars-invisible" style="right: 0px; bottom: 0px;"><div class="os-content" style="padding: 0px; height: 100%; width: 100%;"><ul class="layui-nav layui-nav-tree tool_button" lay-filter="desktop"><li class="layui-nav-item layui-nav-itemed" style="display: none;"><a href="javascript:;">主机名<span class="layui-nav-more"></span></a><dl class="layui-nav-child"><!-- 二级菜单 --><!--ms-for:option in step.vm_name.data--><!--ms-for-end:--><dd><!--if--><!--if--></dd></dl></li><li class="layui-nav-item layui-nav-itemed" style="display: none;"><a href="javascript:;">实验<span class="layui-nav-more"></span></a><dl class="layui-nav-child"><!-- 二级菜单 --><!--ms-for:option in step.interacts.data--><!--ms-for-end:--></dl></li><li class="layui-nav-item layui-nav-itemed"><a href="javascript:;">操作<span class="layui-nav-more"></span></a><dl class="layui-nav-child"><!--ms-for:field in step.fields--><dd><button name="forced_close" class="hide consoleRipple" change-class="hide consoleRipple"><i class="fa fa-play-circle" change-class="fa fa-play-circle"></i><span>强制关机</span></button></dd><!--for0069487838--><dd><button name="create" class="consoleRipple" change-class="consoleRipple"><i class="fa fa-play-circle" change-class="fa fa-play-circle"></i><span>创建</span></button></dd><!--for0069487838--><dd><button name="delay" class="hide consoleRipple" change-class="hide consoleRipple"><i class="fa fa-clock-o" change-class="fa fa-clock-o"></i><span>延时</span></button></dd><!--for0069487838--><dd><button name="saveexp" class="hide consoleRipple" change-class="hide consoleRipple"><i class="fa fa-save" change-class="fa fa-save"></i><span>保存实验</span></button></dd><!--for0069487838--><dd><button name="lookrecover" class="consoleRipple" change-class="consoleRipple"><i class="fa fa-cloud" change-class="fa fa-cloud"></i><span>恢复实验</span></button></dd><!--for0069487838--><dd><button name="help" class="hide consoleRipple" change-class="hide consoleRipple"><i class="fa fa-share-alt" change-class="fa fa-share-alt"></i><span>请求协助</span></button></dd><!--for0069487838--><dd><button name="unhelp" class="hide consoleRipple" change-class="hide consoleRipple"><i class="fa fa-share-square" change-class="fa fa-share-square"></i><span>取消协助</span></button></dd><!--for0069487838--><dd><button name="viewtype" class="hide consoleRipple" change-class="hide consoleRipple"><i class="fa fa-columns" change-class="fa fa-columns"></i><span>显示方式</span></button></dd><!--for0069487838--><dd><button name="copytext" class="hide consoleRipple" change-class="hide consoleRipple"><i class="fa fa-copy" change-class="fa fa-copy"></i><span></span></button></dd><!--for0069487838--><dd><button name="scrapbooking" class="hide consoleRipple" change-class="hide consoleRipple"><i class="fa fa-scissors" change-class="fa fa-scissors"></i><span>剪贴板</span></button></dd><!--for0069487838--><dd><button name="fullscreen" class="hide consoleRipple" change-class="hide consoleRipple"><i class="fa fa-arrows-alt" change-class="fa fa-arrows-alt"></i><span>全屏</span></button></dd><!--for0069487838--><dd><button name="refresh" class="hide consoleRipple" change-class="hide consoleRipple"><i class="fa fa-refresh" change-class="fa fa-refresh"></i><span>刷新</span></button></dd><!--for0069487838--><dd><button name="close" class="hide consoleRipple" change-class="hide consoleRipple"><i class="fa fa-close" change-class="fa fa-close"></i><span>关闭</span></button></dd><!--for0069487838--><dd><button name="exception" class="consoleRipple" change-class="consoleRipple"><i class="fa fa-bug" change-class="fa fa-bug"></i><span>报告异常</span></button></dd><!--for0069487838--><!--ms-for-end:--></dl></li><span class="layui-nav-bar" style="top: 18px; height: 0px; opacity: 0;"></span></ul><ul class="layui-nav layui-nav-tree tool_button"><li class="layui-nav-item layui-nav-itemed" style="display: none;"><a href="javascript:;">扩展功能<span class="layui-nav-more"></span></a><dl class="layui-nav-child"><!-- 二级菜单 --><!--ms-for:option in step.extension.data--><dd change-class=""><!--if--></dd><!--for92837868529--><!--ms-for-end:--><!--if--><dd><!--if--></dd></dl></li><span class="layui-nav-bar"></span></ul><ul class="layui-nav layui-nav-tree tool_button" style="display: none"><li class="layui-nav-item"><a href="javascript:;">按键操作<span class="layui-nav-more"></span></a><dl class="layui-nav-child"><!-- 二级菜单 --><dd class="" id="sendCtrlAltDelButton"><button>Ctrl-Alt-Del</button></dd><dd class="" id="sendCtrlAltBackspaceButton"><button>Ctrl-Alt-Backspace</button></dd><dd class="" id="sendMemoryDumpButton"><button>RCtrl-Scroll-Scroll</button></dd><dd class="sendCtrlAltFunctionButton" data-n="1"><button>Ctrl-Alt-F1</button></dd><dd class="sendCtrlAltFunctionButton" data-n="2"><button>Ctrl-Alt-F2</button></dd><dd class="sendCtrlAltFunctionButton" data-n="3"><button>Ctrl-Alt-F3</button></dd><dd class="sendCtrlAltFunctionButton" data-n="4"><button>Ctrl-Alt-F4</button></dd><dd class="sendCtrlAltFunctionButton" data-n="5"><button>Ctrl-Alt-F5</button></dd><dd class="sendCtrlAltFunctionButton" data-n="6"><button>Ctrl-Alt-F6</button></dd><dd class="sendCtrlAltFunctionButton" data-n="7"><button>Ctrl-Alt-F7</button></dd><dd class="sendCtrlAltFunctionButton" data-n="8"><button>Ctrl-Alt-F8</button></dd><dd class="sendCtrlAltFunctionButton" data-n="9"><button>Ctrl-Alt-F9</button></dd><dd class="sendCtrlAltFunctionButton" data-n="10"><button>Ctrl-Alt-F10</button></dd><dd class="sendCtrlAltFunctionButton" data-n="11"><button>Ctrl-Alt-F11</button></dd></dl></li><span class="layui-nav-bar"></span></ul></div></div></div><div class="os-scrollbar os-scrollbar-horizontal os-scrollbar-unusable os-scrollbar-auto-hidden"><div class="os-scrollbar-track os-scrollbar-track-off"><div class="os-scrollbar-handle" style="width: 100%; transform: translate(0px, 0px);"></div></div></div><div class="os-scrollbar os-scrollbar-vertical os-scrollbar-unusable os-scrollbar-auto-hidden"><div class="os-scrollbar-track os-scrollbar-track-off"><div class="os-scrollbar-handle" style="height: 100%; transform: translate(0px, 0px);"></div></div></div><div class="os-scrollbar-corner"></div></div><div id="realtime_msg" data-placement="top-left"><span>消息</span><span style="margin-left: 6px;" id="msg_num" class="layui-badge">0</span></div></div><div id="editormd_view" class="expDesktop_editormd_box" style="width: 360px;"><div style="left: 334px; top: 182px;" class="editormd-toggle show_menu"><span title="移动图标" class="editormd-toggle-move"><img src="./水滴实验营_files/arrows_v.png"></span><span title="隐藏文档" class="layui-icon layui-icon-shrink-right show-editormd sd-font-awefontt" avalon-events="click:eclick_0_events46desktop95editormd95hide4041" style="display: block;"><!--
<img src="/static/public/img/desktop/edit.png" />
--></span><span class="layui-icon layui-icon-spread-left hide-editormd sd-font-awefontt" title="显示文档" change-class="layui-icon layui-icon-spread-left hide-editormd sd-font-awefontt" avalon-events="click:eclick_0_events46desktop95editormd95show4041" style="display: none;"><img style="display: none;" src="./水滴实验营_files/arrows_all.png" class="hide" change-class="hide"></span><span title="最大化" class="layui-icon layui-icon-layer maximize-editormd sd-font-awefontt sd-font-awefontt" avalon-events="click:eclick_0_events46desktop95editormd95maximize4041" style="display: block;"><!--
<img src="/static/public/img/desktop/arrows_all.png" />
--></span></div><div class="expDesktop_editormd" style="width: 360px; left: 0px;"><div class="shuidi-left-drag"></div><ul class="layui-nav layui-bg-cyan desktop_menu"><!--ms-for:(index,i) in doctabs--><li class="layui-nav-item layui-this" change-class="layui-this" avalon-events="click:eclick_0_events46changetabs40i44index41">文档</li><!--for303271319807--><li class="layui-nav-item" change-class="" avalon-events="click:eclick_0_events46changetabs40i44index41">评价</li><!--for303271319807--><li class="layui-nav-item" change-class="" avalon-events="click:eclick_0_events46changetabs40i44index41">我的报告</li><!--for303271319807--><li class="layui-nav-item" change-class="" avalon-events="click:eclick_0_events46changetabs40i44index41">随材</li><!--for303271319807--><li class="layui-nav-item" change-class="" avalon-events="click:eclick_0_events46changetabs40i44index41">视频</li><!--for303271319807--><li class="layui-nav-item" change-class="" avalon-events="click:eclick_0_events46changetabs40i44index41">文件</li><!--for303271319807--><!--ms-for-end:--><span class="layui-nav-bar"></span></ul><h4 class="expDesktop_title"><span title="案例实训-基于Python+MySQL+Kettle+R的某点评网数据采集分析">案例实训-基于Python+MySQL+Kettle+R的某点评网数据采集分析</span></h4><div class="expDesktop_editormd_console"><!--ms-for:(index,d) in doctabs--><div id="docmarkdown" style="height: 100%;"><div style="width: 100%;height: 100%;"><div style="width: 100%;height: 100%;overflow-y: auto"><div id="docmarkdown_doc" class="markdown-body editormd-html-preview"><div class="markdown-toc editormd-markdown-toc"><ul class="markdown-toc-list"><li><a class="toc-level-1" href="https://edu.shuidilab.cn/console/expdesktop/5665#%E5%9F%BA%E4%BA%8EPython+MySQL+Kettle+R%E7%9A%84%E6%9F%90%E7%82%B9%E8%AF%84%E7%BD%91%E6%95%B0%E6%8D%AE%E9%87%87%E9%9B%86%E5%88%86%E6%9E%90" level="1">基于Python+MySQL+Kettle+R的某点评网数据采集分析</a></li><li><a class="toc-level-1" href="https://edu.shuidilab.cn/console/expdesktop/5665#0.%20%E6%A1%88%E4%BE%8B%E7%AE%80%E4%BB%8B" level="1">0. 案例简介</a><ul><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#0.1%20%E6%A1%88%E4%BE%8B%E7%9B%AE%E7%9A%84" level="2">0.1 案例目的</a></li><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#0.2%20%E9%80%82%E7%94%A8%E5%AF%B9%E8%B1%A1" level="2">0.2 适用对象</a></li><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#0.3%20%E6%97%B6%E9%97%B4%E5%AE%89%E6%8E%92" level="2">0.3 时间安排</a></li><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#0.4%20%E7%A1%AC%E4%BB%B6%E8%A6%81%E6%B1%82" level="2">0.4 硬件要求</a></li><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#0.5%20%E8%BD%AF%E4%BB%B6%E5%B7%A5%E5%85%B7" level="2">0.5 软件工具</a></li><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#0.6%20%E6%A1%88%E4%BE%8B%E4%BB%BB%E5%8A%A1" level="2">0.6 案例任务</a></li><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#0.7%20%E5%AE%9E%E9%AA%8C%E6%AD%A5%E9%AA%A4%E6%A6%82%E8%BF%B0" level="2">0.7 实验步骤概述</a></li></ul></li><li><a class="toc-level-1" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.%20Python%E7%BC%96%E7%A8%8B%E7%88%AC%E5%8F%96%E6%9F%90%E7%82%B9%E8%AF%84%E7%BD%91%E6%95%B0%E6%8D%AE" level="1">1. Python编程爬取某点评网数据</a><ul><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.1%20%E4%BB%80%E4%B9%88%E6%98%AF%E7%88%AC%E8%99%AB%EF%BC%9F" level="2">1.1 什么是爬虫?</a></li><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.2%20%E7%88%AC%E8%99%AB%E5%9F%BA%E6%9C%AC%E5%8E%9F%E7%90%86" level="2">1.2 爬虫基本原理</a></li><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.3%20%E7%88%AC%E8%99%AB%E5%9F%BA%E6%9C%AC%E6%AD%A5%E9%AA%A4" level="2">1.3 爬虫基本步骤</a></li><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4%20%E7%88%AC%E8%99%AB%E5%AE%9E%E6%88%98%E6%A1%88%E4%BE%8B%E2%80%94%E2%80%94%E6%9F%90%E7%82%B9%E8%AF%84%E7%BD%91%E6%95%B0%E6%8D%AE%E7%88%AC%E5%8F%96" level="2">1.4 爬虫实战案例——某点评网数据爬取</a><ul><li><a class="toc-level-3" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4.1%20%E6%8B%9F%E7%88%AC%E5%8F%96%E7%BD%91%E9%A1%B5%E5%88%9D%E6%AD%A5%E5%88%86%E6%9E%90" level="3">1.4.1 拟爬取网页初步分析</a></li><li><a class="toc-level-3" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4.2%20%E5%88%9B%E5%BB%BAMySQL%E6%95%B0%E6%8D%AE%E5%BA%93" level="3">1.4.2 创建MySQL数据库</a></li><li><a class="toc-level-3" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4.3%20Python%E7%BC%96%E7%A8%8B%E7%88%AC%E8%99%AB" level="3">1.4.3 Python编程爬虫</a><ul><li><a class="toc-level-4" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4.3.1%20%E6%96%B0%E5%BB%BA%E9%A1%B9%E7%9B%AE" level="4">1.4.3.1 新建项目</a></li><li><a class="toc-level-4" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4.3.2%E8%AE%BE%E7%BD%AE%E7%88%AC%E8%99%AB%E4%BB%A3%E7%A0%81%E7%9B%AE%E5%BD%95" level="4">1.4.3.2设置爬虫代码目录</a></li><li><a class="toc-level-4" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4.3.3%20python%E7%BC%96%E7%A8%8B%E6%95%B0%E6%8D%AE%E5%AD%98%E5%82%A8" level="4">1.4.3.3 python编程数据存储</a></li><li><a class="toc-level-4" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4.3.4%20Python%E7%88%AC%E8%99%AB%E7%BC%96%E7%A8%8B" level="4">1.4.3.4 Python爬虫编程</a></li><li><a class="toc-level-4" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4.3.5%20%E8%AE%BE%E7%BD%AE%E4%BE%9D%E8%B5%96%E5%8C%85" level="4">1.4.3.5 设置依赖包</a></li><li><a class="toc-level-4" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4.3.6%20%E6%89%A7%E8%A1%8C%E7%88%AC%E8%99%AB%E7%88%AC%E5%8F%96%E6%95%B0%E6%8D%AE" level="4">1.4.3.6 执行爬虫爬取数据</a></li><li><a class="toc-level-4" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4.3.7%20%E6%9F%A5%E7%9C%8B%E7%88%AC%E5%8F%96%E7%9A%84%E6%95%B0%E6%8D%AE" level="4">1.4.3.7 查看爬取的数据</a></li><li><a class="toc-level-4" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4.3.8%20%E5%AD%A6%E7%94%9F%E5%AE%9E%E8%AE%AD%E4%BB%BB%E5%8A%A1" level="4">1.4.3.8 学生实训任务</a><ul><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4.3.8.1%20%E5%A2%9E%E5%8A%A0%E5%85%B6%E4%BB%9620%E4%B8%AA%E5%9F%8E%E5%B8%82%E8%BF%9B%E8%A1%8C%E7%88%AC%E8%99%AB" level="5">1.4.3.8.1 增加其他20个城市进行爬虫</a></li><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4.3.8.2%20%E7%88%AC%E5%8F%96%E6%95%B0%E6%8D%AE%E5%A2%9E%E5%8A%A0%E7%94%B5%E8%AF%9D%E5%8F%B7%E7%A0%81%E2%80%9CphoneNo%E2%80%9D%E5%AD%97%E6%AE%B5" level="5">1.4.3.8.2 爬取数据增加电话号码“phoneNo”字段</a></li><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4.3.8.3%20%E7%88%AC%E8%99%AB%E8%B1%86%E7%93%A3%E7%94%B5%E5%BD%B1%E6%AD%A3%E5%88%99%E8%A1%A8%E8%BE%BE%E5%BC%8F%E5%88%86%E6%9E%90" level="5">1.4.3.8.3 爬虫豆瓣电影正则表达式分析</a></li><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4.3.8.4%20Scrapy%E7%88%AC%E8%99%AB%E6%A1%86%E6%9E%B6" level="5">1.4.3.8.4 Scrapy爬虫框架</a></li></ul></li></ul></li></ul></li></ul></li><li><a class="toc-level-1" href="https://edu.shuidilab.cn/console/expdesktop/5665#2.%20%E5%88%A9%E7%94%A8Kettle%E8%BF%9B%E8%A1%8C%E6%95%B0%E6%8D%AE%E6%B8%85%E6%B4%97%E8%BD%AC%E6%8D%A2" level="1">2. 利用Kettle进行数据清洗转换</a><ul><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#2.1%20Kettle%20%E5%9F%BA%E6%9C%AC%E4%BD%BF%E7%94%A8" level="2">2.1 Kettle 基本使用</a></li><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#2.2%20%E6%95%B0%E6%8D%AE%E6%B8%85%E6%B4%97%E5%86%85%E5%AE%B9" level="2">2.2 数据清洗内容</a></li><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#2.3%20%E5%BB%BA%E7%AB%8BMySQL%E6%96%B0%E8%A1%A8%E5%AD%98%E5%82%A8%E6%B8%85%E6%B4%97%E5%90%8E%E6%95%B0%E6%8D%AE" level="2">2.3 建立MySQL新表存储清洗后数据</a></li><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#2.4%20Kettle%E6%95%B0%E6%8D%AE%E6%B8%85%E6%B4%97%E3%80%81%E8%BD%AC%E6%8D%A2" level="2">2.4 Kettle数据清洗、转换</a><ul><li><a class="toc-level-3" href="https://edu.shuidilab.cn/console/expdesktop/5665#2.4.1%20%E5%88%9B%E5%BB%BA%E8%BD%AC%E6%8D%A2%E4%BB%BB%E5%8A%A1%E8%AE%BE%E8%AE%A1%E8%BE%93%E5%85%A5%E7%BB%84%E4%BB%B6" level="3">2.4.1 创建转换任务设计输入组件</a></li><li><a class="toc-level-3" href="https://edu.shuidilab.cn/console/expdesktop/5665#2.4.2%20%E8%AE%BE%E8%AE%A1%E6%95%B0%E6%8D%AE%E6%B8%85%E6%B4%97%E7%BB%84%E4%BB%B6" level="3">2.4.2 设计数据清洗组件</a><ul><li><a class="toc-level-4" href="https://edu.shuidilab.cn/console/expdesktop/5665#2.4.2.1%E8%BF%87%E6%BB%A4%E7%A9%BA%E5%80%BC" level="4">2.4.2.1过滤空值</a></li><li><a class="toc-level-4" href="https://edu.shuidilab.cn/console/expdesktop/5665#2.4.2.2%E8%BF%87%E6%BB%A40%E5%80%BC" level="4">2.4.2.2过滤0值</a></li><li><a class="toc-level-4" href="https://edu.shuidilab.cn/console/expdesktop/5665#2.4.2.3%E6%8E%92%E5%BA%8F" level="4">2.4.2.3排序</a></li></ul></li><li><a class="toc-level-3" href="https://edu.shuidilab.cn/console/expdesktop/5665#2.4.3%20%E8%AE%BE%E8%AE%A1%E8%BE%93%E5%87%BA%E7%BB%84%E4%BB%B6" level="3">2.4.3 设计输出组件</a></li><li><a class="toc-level-3" href="https://edu.shuidilab.cn/console/expdesktop/5665#2.4.4%20%E6%89%A7%E8%A1%8C%E6%B8%85%E6%B4%97%E5%92%8C%E8%BD%AC%E6%8D%A2" level="3">2.4.4 执行清洗和转换</a></li></ul></li><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#2.5%20%E5%AD%A6%E7%94%9F%E5%AE%9E%E8%AE%AD%E4%BB%BB%E5%8A%A1" level="2">2.5 学生实训任务</a></li></ul></li><li><a class="toc-level-1" href="https://edu.shuidilab.cn/console/expdesktop/5665#3%20%E6%95%B0%E6%8D%AE%E7%BB%9F%E8%AE%A1%E5%88%86%E6%9E%90" level="1">3 数据统计分析</a><ul><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1%20Python+MySQL%E6%95%B0%E6%8D%AE%E7%BB%9F%E8%AE%A1%E5%88%86%E6%9E%90" level="2">3.1 Python+MySQL数据统计分析</a><ul><li><a class="toc-level-3" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.1%20%E5%9F%BA%E6%9C%ACSQL%E8%AF%AD%E5%8F%A5%E5%AD%A6%E4%B9%A0" level="3">3.1.1 基本SQL语句学习</a></li><li><a class="toc-level-3" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.2%20%E6%89%93%E5%BC%80%E9%A1%B9%E7%9B%AE" level="3">3.1.2 打开项目</a></li><li><a class="toc-level-3" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.3%20%E8%AE%BE%E7%BD%AE%E4%BB%A3%E7%A0%81%E7%9B%AE%E5%BD%95" level="3">3.1.3 设置代码目录</a></li><li><a class="toc-level-3" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4%20Python%E7%BC%96%E7%A8%8B%E8%BF%9B%E8%A1%8C%E6%95%B0%E6%8D%AE%E7%BB%9F%E8%AE%A1%E5%8F%AF%E8%A7%86%E5%8C%96" level="3">3.1.4 Python编程进行数据统计可视化</a><ul><li><a class="toc-level-4" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.1%2020%E4%B8%AA%E5%A4%A7%E4%B8%AD%E5%9F%8E%E5%B8%82%E6%98%9F%E7%BA%A7%E5%95%86%E9%93%BA%E7%BB%9F%E8%AE%A1%E5%88%86%E6%9E%90" level="4">3.1.4.1 20个大中城市星级商铺统计分析</a><ul><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.1.1%20%E6%96%B0%E5%BB%BA%E4%BB%A3%E7%A0%81%E6%96%87%E4%BB%B6" level="5">3.1.4.1.1 新建代码文件</a></li><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.1.2%20%E7%BC%96%E5%86%99python%E7%A8%8B%E5%BA%8F%E4%BB%A3%E7%A0%81" level="5">3.1.4.1.2 编写python程序代码</a></li><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.1.3%20%E8%AE%BE%E7%BD%AE%E4%BE%9D%E8%B5%96%E5%8C%85" level="5">3.1.4.1.3 设置依赖包</a></li><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.1.4%20%E6%89%A7%E8%A1%8C%E7%A8%8B%E5%BA%8F" level="5">3.1.4.1.4 执行程序</a></li><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.1.5%20%E6%9F%A5%E7%9C%8B%E5%8F%AF%E8%A7%86%E5%8C%96%E6%95%B0%E6%8D%AE" level="5">3.1.4.1.5 查看可视化数据</a></li></ul></li><li><a class="toc-level-4" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.2%20%E5%B1%B1%E4%B8%9C3%E5%A4%A7%E5%9F%8E%E5%B8%82%E8%8F%9C%E5%93%81%E5%88%86%E7%B1%BB%E7%BB%9F%E8%AE%A1" level="4">3.1.4.2 山东3大城市菜品分类统计</a><ul><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.2.1%20%E6%96%B0%E5%BB%BA%E4%BB%A3%E7%A0%81" level="5">3.1.4.2.1 新建代码</a></li><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.2.2%20%E7%BC%96%E5%86%99python%E7%A8%8B%E5%BA%8F%E4%BB%A3%E7%A0%81" level="5">3.1.4.2.2 编写python程序代码</a></li><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.2.3%20%E8%AE%BE%E7%BD%AE%E4%BE%9D%E8%B5%96%E5%8C%85" level="5">3.1.4.2.3 设置依赖包</a></li><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.2.4%20%E6%89%A7%E8%A1%8C%E7%A8%8B%E5%BA%8F" level="5">3.1.4.2.4 执行程序</a></li><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.2.5%20%E6%9F%A5%E7%9C%8B%E5%8F%AF%E8%A7%86%E5%8C%96%E6%95%B0%E6%8D%AE" level="5">3.1.4.2.5 查看可视化数据</a></li></ul></li><li><a class="toc-level-4" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.3%20%E5%B1%B1%E4%B8%9C3%E5%A4%A7%E5%9F%8E%E5%B8%82TOP20%E9%A5%AD%E5%BA%97%E6%9C%8D%E5%8A%A1%E7%9A%84%E7%BB%BC%E5%90%88%E8%AF%84%E4%BB%B7" level="4">3.1.4.3 山东3大城市TOP20饭店服务的综合评价</a><ul><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.3.1%20%E6%96%B0%E5%BB%BA%E4%BB%A3%E7%A0%81" level="5">3.1.4.3.1 新建代码</a></li><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.3.2%20%E7%BC%96%E5%86%99python%E7%A8%8B%E5%BA%8F%E4%BB%A3%E7%A0%81" level="5">3.1.4.3.2 编写python程序代码</a></li><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.3.3%20%E8%AE%BE%E7%BD%AE%E4%BE%9D%E8%B5%96%E5%8C%85" level="5">3.1.4.3.3 设置依赖包</a></li><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.3.4%20%E6%89%A7%E8%A1%8C%E7%A8%8B%E5%BA%8F" level="5">3.1.4.3.4 执行程序</a></li><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.3.5%20%E6%9F%A5%E7%9C%8B%E5%8F%AF%E8%A7%86%E5%8C%96%E6%95%B0%E6%8D%AE" level="5">3.1.4.3.5 查看可视化数据</a></li></ul></li></ul></li></ul></li><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.2%20%20R%E7%BC%96%E7%A8%8B+CSV%E6%96%87%E4%BB%B6%E6%95%B0%E6%8D%AE%E7%BB%9F%E8%AE%A1%E5%88%86%E6%9E%90%E5%8F%AF%E8%A7%86%E5%8C%96" level="2">3.2 R编程+CSV文件数据统计分析可视化</a><ul><li><a class="toc-level-3" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.2.1%20%E6%96%B0%E5%BB%BA%E4%BB%A3%E7%A0%81" level="3">3.2.1 新建代码</a></li><li><a class="toc-level-3" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.2.2%20%E6%89%A7%E8%A1%8CR%E8%84%9A%E6%9C%AC" level="3">3.2.2 执行R脚本</a></li></ul></li><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.3%20%E5%AD%A6%E7%94%9F%E5%AE%9E%E8%AE%AD%E4%BB%BB%E5%8A%A1" level="2">3.3 学生实训任务</a></li></ul></li><li><a class="toc-level-1" href="https://edu.shuidilab.cn/console/expdesktop/5665#4%20%E5%B0%8F%E7%BB%93" level="1">4 小结</a><ul></ul></li></ul></div><h1 id="h1--python-mysql-kettle-r-"><a name="基于Python+MySQL+Kettle+R的某点评网数据采集分析" class="reference-link"></a><span class="header-link octicon octicon-link"></span>基于Python+MySQL+Kettle+R的某点评网数据采集分析</h1><p>本案例涉及数据采集、存储、查询、清洗和可视化分析等数据处理全流程所涉及的各种典型操作,涵盖Python、MySQL、Navicat、Kettle、R语言等系统和软件的使用方法。本案例适合高校大数据相关专业教学,可以作为学习大数据的综合实践案例。通过本案例,将有助于读者综合运用大数据知识以及各种工具软件,实现数据全流程操作。</p>
<h1 id="h1-0-"><a name="0. 案例简介" class="reference-link"></a><span class="header-link octicon octicon-link"></span>0. 案例简介</h1><p>本次实践的数据来源于国内某点评网站Top100餐厅网页,包括商户名称、所在商区、口味、环境、服务评分以及人均消费等数据,如图所示:<br><img src="./水滴实验营_files/00-案例大众点评top100页.png" alt=""></p>
<h2 id="h2-0-1-"><a name="0.1 案例目的" class="reference-link"></a><span class="header-link octicon octicon-link"></span>0.1 案例目的</h2><p>案例综合实践的目的就在于通过对该点评数据进行统计分析,评估各城市餐厅、菜品,帮助游客选择适合自己的餐厅就餐。</p>
<ul>
<li>编写Python爬虫程序采集数据</li><li>了解MySQL操作,存储数据</li><li>了解Kettle进行数据清洗、转换流程和操作</li><li>使用Python、R进行数据统计分析可视化展示</li></ul>
<h2 id="h2-0-2-"><a name="0.2 适用对象" class="reference-link"></a><span class="header-link octicon octicon-link"></span>0.2 适用对象</h2><ul>
<li>高校(高职)教师</li><li>高校(高职)学生</li><li>大数据学习者</li><li>数据处理分析者</li></ul>
<h2 id="h2-0-3-"><a name="0.3 时间安排" class="reference-link"></a><span class="header-link octicon octicon-link"></span>0.3 时间安排</h2><p>本案例可以作为大数据课程结束后的“大作业”,或者可以作为学生暑期或寒假大数据相关专业实习实践基础案例,建议2天。</p>
<h2 id="h2-0-4-"><a name="0.4 硬件要求" class="reference-link"></a><span class="header-link octicon octicon-link"></span>0.4 硬件要求</h2><p>本案例可以在单个虚机上完成,也可以在虚拟集群环境下完成。单虚机上完成本案例实验时,建议硬件配置为:8CPU,8GB以上内存,100GB以上硬盘。</p>
<h2 id="h2-0-5-"><a name="0.5 软件工具" class="reference-link"></a><span class="header-link octicon octicon-link"></span>0.5 软件工具</h2><p>本案例所涉及的系统及软件包括:Windows, Python, Pycharm, MySQL, Navicat, Kettle、R等。<br>相关软件的版本建议如下:</p>
<ul>
<li>Windows:Windows 7</li><li>Mcrosoft office: 2013</li><li>JDK:1.8</li><li>Python: 3.7(64位)</li><li>Pycharm: Community 2018.2</li><li>MySQL:5.7</li><li>Navicat for MySQL:11.1.13(64位)</li><li>Kettle:7.0</li><li>R:3.5.1</li><li>Rstudio:1.1<br><strong>*注意:</strong>案例实验操作,请使用chrome浏览器或者较新的360浏览器极速模式。<h2 id="h2-0-6-"><a name="0.6 案例任务" class="reference-link"></a><span class="header-link octicon octicon-link"></span>0.6 案例任务</h2>本案例需要完成以下实验任务:<br>(1) Python编程爬取点某评网数据,存储于MySQL数据库<br>(2) 利用Kettle对MySQL表数据进行清洗,一份存储到MySQL新表,一份转换成CSV文件<br>(3) 使用Python、R进行数据统计分析、可视化展示</li></ul>
<h2 id="h2-0-7-"><a name="0.7 实验步骤概述" class="reference-link"></a><span class="header-link octicon octicon-link"></span>0.7 实验步骤概述</h2><p>本案例共包含3个实验步骤:<br>1:Python编程爬取某点评网数据存储于MySQL数据库<br>2:利用Kettle进行数据转换、清洗<br>3:利用Python、R进行数据统计分析、可视化展示<br>下面表格分别给出了每个实验步骤所需的知识储备、训练技能和任务清单。</p>
<p>表1:Python编程爬取某点评网数据存储于MySQL数据库</p>
<table>
<thead>
<tr>
<th style="text-align:left">所需知识储备</th>
<th style="text-align:left">Python基础知识、使用、编程,MySQL数据库基础知识、Navicat使用</th>
</tr>
</thead>
<tbody>
<tr>
<td style="text-align:left">训练技能</td>
<td style="text-align:left">Python编程调试、Mysql、Navicat基本操作</td>
</tr>
<tr>
<td style="text-align:left">任务清单</td>
<td style="text-align:left">Python爬虫某点评网数据、解析数据、创建数据库存储数据</td>
</tr>
</tbody>
</table>
<p>表2:使用Kettle进行数据转换、清洗</p>
<table>
<thead>
<tr>
<th style="text-align:left">所需知识储备</th>
<th style="text-align:left">Kettle基本使用</th>
</tr>
</thead>
<tbody>
<tr>
<td style="text-align:left">训练技能</td>
<td style="text-align:left">Kettle基本使用、数据清洗、数据转换</td>
</tr>
<tr>
<td style="text-align:left">任务清单</td>
<td style="text-align:left">使用Kettle建立数据清洗、转换流程,一是完成MySQL数据表清洗存储到新表,二是MySQL表数据清洗后转换成CSV文件</td>
</tr>
</tbody>
</table>
<p>表3:利用Python进行数据统计分析、可视化展示</p>
<table>
<thead>
<tr>
<th style="text-align:left">所需知识储备</th>
<th style="text-align:left">Python、R工具选择其一,熟悉基本使用</th>
</tr>
</thead>
<tbody>
<tr>
<td style="text-align:left">训练技能</td>
<td style="text-align:left">编程调试能力、数据统计分析能力、数据可视化设计</td>
</tr>
<tr>
<td style="text-align:left">任务清单</td>
<td style="text-align:left">编程调试、数据统计分析、数据可视化</td>
</tr>
</tbody>
</table>
<h1 id="h1-1-python-"><a name="1. Python编程爬取某点评网数据" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1. Python编程爬取某点评网数据</h1><p>本节介绍什么是爬虫、爬虫基本原理、爬虫基本步骤、爬虫实战案例。</p>
<h2 id="h2-1-1-"><a name="1.1 什么是爬虫?" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.1 什么是爬虫?</h2><p>爬虫,即网络爬虫,可以理解为在网络上爬行的一只蜘蛛,在互联网这张大网上爬来爬去,抓取各种资源。</p>
<p><img src="./水滴实验营_files/01-爬虫页.png" alt=""></p>
<p>用户浏览网页,输入URL网址后,实际上经过DNS服务器解析找到网页服务器向其发送一个请求,服务器解析后发送响应消息,用户就看到HTML代码的网页了,包含文本、图片等内容。</p>
<p>爬虫爬取的就是这些内容,通过分析HTML代码,获取相应的图片和文本内容,例如:打开chrome浏览器,输入:<a href="http://www.163.com/">http://www.163.com</a>, 页面如下<br><img src="./水滴实验营_files/02-163网页.png" alt=""><br>右键点击“查看网页源代码”,我们可以看到HTML代码如下,里面包含标签、块等元素及属性信息。</p>
<p><img src="./水滴实验营_files/03-163网页源代码.png" alt=""></p>
<h2 id="h2-1-2-"><a name="1.2 爬虫基本原理" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.2 爬虫基本原理</h2><p><img src="./水滴实验营_files/04-爬虫基本原理.png" alt=""></p>
<p>通过程序模拟浏览器向服务器站点发起请求,将站点返回的响应,例如HTML代码/JSON数据/二进制数据(图片、视频)等爬取到本地,提取自己需要的数据存放起来使用。</p>
<h2 id="h2-1-3-"><a name="1.3 爬虫基本步骤" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.3 爬虫基本步骤</h2><p><img src="./水滴实验营_files/05-爬虫基本步骤.png" alt=""></p>
<p>(1)发起请求<br>使用HTTP协议向目标站点发起请求,也就是发送一个Request,Request包含请求头、请求体等, 等待服务器响应。<br>(2)获取响应内容<br>如果服务器能正常响应,会得到一个Response,Response的内容便是所要获取的页面内容,类型可能是HTML,Json字符串,二进制数据(图片或者视频)等类型。<br>(3)解析内容<br>得到的内容不同解析方式也不同。</p>
<ul>
<li>HTML页面:正则表达式解析,或者使用第三方解析库如Beautifulsoup,pyquery等进行解析。</li><li>Json数据:转换为Json对象解析。</li><li>二进制数据:可以做保存或者进一步的处理。</li></ul>
<p>(4)保存数据<br>保存形式多样,可以存为文本,也可以保存到数据库,或者保存特定格式的文件。</p>
<h2 id="h2-1-4-"><a name="1.4 爬虫实战案例——某点评网数据爬取" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4 爬虫实战案例——某点评网数据爬取</h2><p>本爬虫案例爬取某点评网站50个城市排名TOP100的餐厅网页,并分析这些餐厅名称、ID、地址、所属商区、菜品分类、口味得分、环境得分、服务得分、人均消费等存储到MySQL数据库。</p>
<h3 id="h3-1-4-1-"><a name="1.4.1 拟爬取网页初步分析" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4.1 拟爬取网页初步分析</h3><p>本案例拟爬取某点评网站50个城市排名TOP100的餐厅网页,每个城市对应一个网页地址,以北京为例,北京排名TOP100的餐厅网页(本文档中dp01.ds.quancube.cn为内部网址,请使用虚机内部的chrome浏览器打开,注意复制链接请使用自动弹出的复制按钮进行复制)<br><a href="http://dp01.ds.quancube.cn/?rankId=d5036cf54fcb57e9dceb9fefe3917fff71862f838d1255ea693b953b1d49c7c0">http://dp01.ds.quancube.cn/?rankId=d5036cf54fcb57e9dceb9fefe3917fff71862f838d1255ea693b953b1d49c7c0</a></p>
<p><img src="./水滴实验营_files/06-北京大众点评.png" alt=""></p>
<p>通过观察每个城市的链接主要区别在于rankId,每个城市有特定的ID,例如:<br> 北京排名TOP100的餐厅HTML网页的rankId: 652e242ca0227790d18acff20ae02272<br> 济南排名TOP100餐厅HTML网页的rankId: 4be7d758168826991eb4b3e779fdf41571862f838d1255ea693b953b1d49c7c0<br>济南HTML网页地址可通过修改rankId得到为:<br><a href="http://dp01.ds.quancube.cn/?rankId=4be7d758168826991eb4b3e779fdf41571862f838d1255ea693b953b1d49c7c0">http://dp01.ds.quancube.cn/?rankId=4be7d758168826991eb4b3e779fdf41571862f838d1255ea693b953b1d49c7c0</a><br>因此先获取到相应城市的ID,以便后续进行数据抓取。打开上面济南排名TOP100餐厅的HTML网页,显示如下:</p>
<p><img src="./水滴实验营_files/07-济南大众点评.png" alt=""></p>
<p>按住“Fn”键不动接着按下F12键,点击“Network”和“XHR”出现如下界面,出现如下界面。</p>
<p><img src="./水滴实验营_files/08-Fn+F12济南.png" alt=""></p>
<p>接着按下Ctrl+R键,出现如下界面</p>
<p><img src="./水滴实验营_files/09-Ctrl+R济南.png" alt=""></p>
<p>点击最下方红框处,出现如下界面,点击Headers看到general中Request URL:<br><a href="http://dp01.ds.quancube.cn/mylist/ajax/shoprank?rankId=4be7d758168826991eb4b3e779fdf41571862f838d1255ea693b953b1d49c7c0">http://dp01.ds.quancube.cn/mylist/ajax/shoprank?rankId=4be7d758168826991eb4b3e779fdf41571862f838d1255ea693b953b1d49c7c0</a><br>, 该网页所有数据通过Ajax接口获得,并且 在Response Headers中看到“Content-Type:application/json;charset=UTF-8”, 获得的响应是JSON数据,编码是UTF-8格式。</p>
<p><img src="./水滴实验营_files/10-header.png" alt=""></p>
<p>在浏览器中,输入进入该链接<br><a href="http://dp01.ds.quancube.cn/mylist/ajax/shoprank?rankId=4be7d758168826991eb4b3e779fdf41571862f838d1255ea693b953b1d49c7c0">http://dp01.ds.quancube.cn/mylist/ajax/shoprank?rankId=4be7d758168826991eb4b3e779fdf41571862f838d1255ea693b953b1d49c7c0</a><br> 得到JSON字符串,如下图所示,最终抓取的数据只需要解析json便可获得所需字段,主要包括城市、商铺名称、编号、星级、所在商区、菜品分类、口味评分、环境评分、服务评分、人均消费、网址、图片。同之前请求一样,只需要替换rankId便可进行多城市数据爬取。</p>
<p><img src="./水滴实验营_files/11-ajaxcode.png" alt=""></p>
<p>我们获得的50个城市rankId,首先对前30个城市进行数据爬取。</p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"上海"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"fce2e3a36450422b7fad3f2b90370efd71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L1"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"北京"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"d5036cf54fcb57e9dceb9fefe3917fff71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L2"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"广州"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"e749e3e04032ee6b165fbea6fe2dafab71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L3"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"深圳"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"e049aa251858f43d095fc4c61d62a9ec71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L4"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"天津"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"2e5d0080237ff3c8f5b5d3f315c7c4a508e25c702ab1b810071e8e2c39502be1"</span><span class="pun">],</span></code></li><li class="L5"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"杭州"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"91621282e559e9fc9c5b3e816cb1619c71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L6"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"南京"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"d6339a01dbd98141f8e684e1ad8af5c871862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L7"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"苏州"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"536e0e568df850d1e6ba74b0cf72e19771862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L8"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"成都"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"c950bc35ad04316c76e89bf2dc86bfe071862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L9"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"武汉"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"d96a24c312ed7b96fcc0cedd6c08f68c08e25c702ab1b810071e8e2c39502be1"</span><span class="pun">],</span></code></li><li class="L0"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"重庆"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"6229984ceb373efb8fd1beec7eb4dcfd71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L1"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"西安"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"ad66274c7f5f8d27ffd7f6b39ec447b608e25c702ab1b810071e8e2c39502be1"</span><span class="pun">],</span></code></li><li class="L2"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"青岛"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"9874ff91c8b50ad831ea897b0cd5315b71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L3"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"济南"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"4be7d758168826991eb4b3e779fdf41571862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L4"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"威海"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"fb5f85ca4c696d61c6e7a7e6c19b2f4530aacdebee4c4c9365dc18972daccaf7"</span><span class="pun">],</span></code></li><li class="L5"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"长春"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"261f655c7a9713696580432624721cd871862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L6"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"大连"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"21f56d987e57bd3e8df308351e3f4b9171862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L7"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"佛山"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"4e5d3a805c4f1ed6059c7074e5e86adf30aacdebee4c4c9365dc18972daccaf7"</span><span class="pun">],</span></code></li><li class="L8"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"贵阳"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"7ae3d5049e1c200ed6b2db0ac51604ba30aacdebee4c4c9365dc18972daccaf7"</span><span class="pun">],</span></code></li><li class="L9"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"合肥"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"94af1388885f0b7483fdf72cba086a6030aacdebee4c4c9365dc18972daccaf7"</span><span class="pun">],</span></code></li><li class="L0"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"呼和浩特"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"d64d5919fc236699402122b5770b4a6f71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L1"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"昆明"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"45159d85c35f742c1f2635d6ea6fa3ae30aacdebee4c4c9365dc18972daccaf7"</span><span class="pun">],</span></code></li><li class="L2"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"兰州"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"b201713f2fb5f7235b6e2a6de8faf6a630aacdebee4c4c9365dc18972daccaf7"</span><span class="pun">],</span></code></li><li class="L3"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"南宁"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"ff6f15347ab16f86ffb34e2700c4870930aacdebee4c4c9365dc18972daccaf7"</span><span class="pun">],</span></code></li><li class="L4"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"秦皇岛"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"ca02d37fa01b0f9613555237517e19ff71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L5"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"沈阳"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"eec1fb195780b91fc9dd065409c1d28471862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L6"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"太原"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"293e6160d971fe48ebbed1c8033bd9ed71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L7"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"唐山"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"2ae2237467ce4784b1705be502b0315271862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L8"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"无锡"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"51ff28145b3f63e2f9ec4b2a34e8850c71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L9"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"扬州"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"02eeb7a9021c215641f3ec0e3cede50b71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">]</span></code></li></ol></pre><p>有的网站会设置反爬机制,或者监听到外部爬取数据会封号,导致数据无法继续爬取。该点评网网页没有特别反爬限制,我们只需要不断轮换用户代理便可绕过反爬。请求头的设置如下:</p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="com">#用户代理,是一个特殊字符串头,使得服务器能够识别客户使用的操作系统及版本、浏览器及版本、浏览器渲染引擎、浏览器语言、浏览器插件等#</span></code></li><li class="L1"><code><span class="pln">USER_AGENT_LIST </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[</span></code></li><li class="L2"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"</span><span class="pun">,</span></code></li><li class="L3"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11"</span><span class="pun">,</span></code></li><li class="L4"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6"</span><span class="pun">,</span></code></li><li class="L5"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6"</span><span class="pun">,</span></code></li><li class="L6"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1"</span><span class="pun">,</span></code></li><li class="L7"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5"</span><span class="pun">]</span></code></li><li class="L8"><code><span class="pln"> </span><span class="com">#设置随机选择用户代理#</span></code></li><li class="L9"><code><span class="pln">head </span><span class="pun">=</span><span class="pln"> </span><span class="pun">{</span></code></li><li class="L0"><code><span class="str">'User-Agent'</span><span class="pun">:</span><span class="pln"> </span><span class="str">'{0}'</span><span class="pun">.</span><span class="pln">format</span><span class="pun">(</span><span class="pln">random</span><span class="pun">.</span><span class="pln">sample</span><span class="pun">(</span><span class="pln">USER_AGENT_LIST</span><span class="pun">,</span><span class="pln"> </span><span class="lit">1</span><span class="pun">)[</span><span class="lit">0</span><span class="pun">])</span><span class="pln"> </span><span class="com"># 随机获取</span></code></li><li class="L1"><code><span class="pun">}</span></code></li></ol></pre><p><strong>*小贴士:什么是JSON?</strong><br>JSON 指的是 JavaScript 对象表示法(JavaScript Object Notation)</p>
<ul>
<li>JSON 是轻量级的文本数据交换格式</li><li>JSON 独立于语言</li><li>JSON 具有自我描述性,更易理解</li><li>JSON 使用 JavaScript 语法来描述数据对象,但是 JSON 仍然独立于语言和平台。JSON 解析器和 JSON 库支持许多不同的编程语言。<br>例子:<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pun">{</span></code></li><li class="L1"><code><span class="str">"employees"</span><span class="pun">:</span><span class="pln"> </span><span class="pun">[</span></code></li><li class="L2"><code><span class="pun">{</span><span class="pln"> </span><span class="str">"firstName"</span><span class="pun">:</span><span class="str">"Bill"</span><span class="pln"> </span><span class="pun">,</span><span class="pln"> </span><span class="str">"lastName"</span><span class="pun">:</span><span class="str">"Gates"</span><span class="pln"> </span><span class="pun">},</span></code></li><li class="L3"><code><span class="pun">{</span><span class="pln"> </span><span class="str">"firstName"</span><span class="pun">:</span><span class="str">"George"</span><span class="pln"> </span><span class="pun">,</span><span class="pln"> </span><span class="str">"lastName"</span><span class="pun">:</span><span class="str">"Bush"</span><span class="pln"> </span><span class="pun">},</span></code></li><li class="L4"><code><span class="pun">{</span><span class="pln"> </span><span class="str">"firstName"</span><span class="pun">:</span><span class="str">"Thomas"</span><span class="pln"> </span><span class="pun">,</span><span class="pln"> </span><span class="str">"lastName"</span><span class="pun">:</span><span class="str">"Carter"</span><span class="pln"> </span><span class="pun">}</span></code></li><li class="L5"><code><span class="pun">]</span></code></li><li class="L6"><code><span class="pun">}</span></code></li></ol></pre>这个 employee 对象是包含 3 个员工记录(对象)的数组。<h3 id="h3-1-4-2-mysql-"><a name="1.4.2 创建MySQL数据库" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4.2 创建MySQL数据库</h3>为了存放爬取的数据,我们采用MySQL数据库。我们可以通过命令行创建并操作数据库,也可以利用GUI工具Navicat进行,本案例使用Navicat工具,并提供sql脚本文件。学生直接导入脚本文件即可使用数据库表。首先确认MySQL数据库是否已启动,在“计算机”上点击右键,点击“管理”,弹出“计算机管理”界面。</li></ul>
<p><img src="./水滴实验营_files/00-计算机.png" alt=""></p>
<p>点击“服务和应用程序”下的“服务”,查看MySQL服务状态,如下图所示,状态显示“已启动”,表明MySQL服务正常运行,若没有显示,则在该服务上点击右键手动启动。</p>
<p><img src="./水滴实验营_files/03-mysql服务启动.png" alt=""></p>
<p>双击桌面上的“Navicat for MySQL”图标,进入如下界面,点击文件-&gt;新建连接-&gt;MySQL</p>
<p><img src="./水滴实验营_files/04-新建链接.png" alt=""></p>
<p>出现如下界面,设置参数:</p>
<p><img src="./水滴实验营_files/05-连接参数配置.png" alt=""></p>
<ul>
<li>连接名:用户自定义</li><li>主机名或IP地址:localhost </li><li>端口:3306</li><li>用户名:root </li><li>密码:shuidilab<a href="https://github.com/123" title="@123" class="at-link">@123</a></li></ul>
<p>点击“测试连接”,弹出“连接成功”对话框,即连接成功。点击“确定”,显示新建连接“con1”,双击该连接,显示如下界面。</p>
<p><img src="./水滴实验营_files/06-显示新建连接.png" alt=""></p>
<p>在con1上点击右键,点击“新建数据库”,弹出界面,输入数据库名称“testdazhong”,选择字符集和排序规则,如下图所示。</p>
<p><img src="./水滴实验营_files/07-新建数据库.png" alt=""></p>
<p>点击“确定”进入如下页面,显示新创建数据库“testdazhong”</p>
<p><img src="./水滴实验营_files/08-显示数据库.png" alt=""></p>
<p>双击“testdazhong”,然后在“testdazhong”上点击右键,点击“运行SQL文件”,弹出如下界面,从C盘sqlfile目录下选择文件“fdazhongfood.sql”</p>
<p><img src="./水滴实验营_files/09-运行SQL.png" alt=""></p>
<p>点击“开始”,出现如下界面表示导入成功,关闭该界面。</p>
<p><img src="./水滴实验营_files/10-运行SQL成功.png" alt=""></p>
<p>在“testdazhong”下的“表”上点击右键,点击“刷新”,显示如下界面,可以看到数据表dazhonginfo已经导入。</p>
<p><img src="./水滴实验营_files/11-数据表显示.png" alt=""></p>
<p>选中数据表“dazhonginfo”,点击“设计表”,我们可以看到数据表各个字段的类型、长度的定义,如下图所示。</p>
<p><img src="./水滴实验营_files/12-设计表.png" alt=""></p>
<p>各字段含义如下:</p>
<table>
<thead>
<tr>
<th style="text-align:left">序号</th>
<th style="text-align:left">字段名称</th>
<th style="text-align:left">字段意义</th>
<th style="text-align:left">备注</th>
</tr>
</thead>
<tbody>
<tr>
<td style="text-align:left">1</td>
<td style="text-align:left">Id</td>
<td style="text-align:left">记录序号</td>
<td style="text-align:left"></td>
</tr>
<tr>
<td style="text-align:left">2</td>
<td style="text-align:left">city</td>
<td style="text-align:left">城市</td>
<td style="text-align:left"></td>
</tr>
<tr>
<td style="text-align:left">3</td>
<td style="text-align:left">shopName</td>
<td style="text-align:left">商铺名称</td>
<td style="text-align:left"></td>
</tr>
<tr>
<td style="text-align:left">4</td>
<td style="text-align:left">shopId</td>
<td style="text-align:left">商铺编号</td>
<td style="text-align:left"></td>
</tr>
<tr>
<td style="text-align:left">5</td>
<td style="text-align:left">shopPower</td>
<td style="text-align:left">商铺星级(大众点评自己的评星)</td>
<td style="text-align:left">30:三星 35:三星半 40:四星 45:四星半 50:五星</td>
</tr>
<tr>
<td style="text-align:left">6</td>
<td style="text-align:left">mainRegionName</td>
<td style="text-align:left">所在商区</td>
<td style="text-align:left"></td>
</tr>
<tr>
<td style="text-align:left">7</td>
<td style="text-align:left">mainCategoryName</td>
<td style="text-align:left">所属菜品分类</td>
<td style="text-align:left"></td>
</tr>
<tr>
<td style="text-align:left">8</td>
<td style="text-align:left">tasteScore</td>
<td style="text-align:left">口味评分</td>
<td style="text-align:left"></td>
</tr>
<tr>
<td style="text-align:left">9</td>
<td style="text-align:left">environmentScore</td>
<td style="text-align:left">环境评分</td>
<td style="text-align:left"></td>
</tr>
<tr>
<td style="text-align:left">10</td>
<td style="text-align:left">serviceScore</td>
<td style="text-align:left">服务评分</td>
<td style="text-align:left"></td>
</tr>
<tr>
<td style="text-align:left">11</td>
<td style="text-align:left">avgPrice</td>
<td style="text-align:left">人均消费</td>
<td style="text-align:left"></td>
</tr>
<tr>
<td style="text-align:left">12</td>
<td style="text-align:left">shopAddress</td>
<td style="text-align:left">商铺地址</td>
<td style="text-align:left"></td>
</tr>
<tr>
<td style="text-align:left">13</td>
<td style="text-align:left">shopUrl</td>
<td style="text-align:left">商铺网址</td>
<td style="text-align:left"></td>
</tr>
<tr>
<td style="text-align:left">14</td>
<td style="text-align:left">defaultPic</td>
<td style="text-align:left">商铺图片</td>
<td style="text-align:left"></td>
</tr>
</tbody>
</table>
<p>关闭,双击“dazhonginfo”表,我们可以看到表的内容,如图所示,该表是空表。</p>
<p><img src="./水滴实验营_files/13-打开空表.png" alt=""></p>
<h3 id="h3-1-4-3-python-"><a name="1.4.3 Python编程爬虫" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4.3 Python编程爬虫</h3><h4 id="h4-1-4-3-1-"><a name="1.4.3.1 新建项目" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4.3.1 新建项目</h4><p>打开桌面上的JetBrains PyCharm,点击“+创建新项目”创建一个新项目,如果之前已经使用过PyCharm,选择”文件”-&gt;“新建项目”创建新项目。</p>
<p><img src="./水滴实验营_files/1-首页.png" alt=""></p>
<p>新创建项目进入如下界面,在Location选择项目目录,或者直接输入新目录名称,例如C:\dazhongspider。然后选择第二个现存的解释器,默认:C:\Program Files\Python37\python.exe。</p>
<p><img src="./水滴实验营_files/2-选择编译环境.png" alt=""></p>
<p>然后单击New Project对话框底部的“ Create”按钮,启动过程中弹出如下对话框,将复选框“√”勾掉,关闭。</p>
<p><img src="./水滴实验营_files/3-弹出对话框关闭.png" alt=""></p>
<p>进入工程页面,点击“文件”-&gt;“设置”-&gt;“外观&amp;行为”,点击下方“外观”,在右边UI选项,主题下拉框中选择“IntelliJ”,如图所示,点击“确定”修改完成pycharm视图背景,返回工程页面,点击“视图”菜单勾选“工具栏”。</p>
<p>若没有看到具体的工程“dazhongspider”,则在工程页面中点击左侧的“Project”标签,看到创建的python项目,如下图所示。</p>
<p><img src="./水滴实验营_files/6-显示工程.png" alt=""></p>
<h4 id="h4-1-4-3-2-"><a name="1.4.3.2设置爬虫代码目录" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4.3.2设置爬虫代码目录</h4><p>为了便于项目管理,设置不同的目录编写程序,选中该项目“dazhongspider”,点击右键选择“新建”-&gt;“目录”,如图所示。</p>
<p><img src="./水滴实验营_files/7-新建目录.png" alt=""></p>
<p>在弹出的界面中(如下图所示),设置爬虫程序的目录名称为“spider”。</p>
<p><img src="./水滴实验营_files/8-输入新建目录名称.png" alt=""></p>
<h4 id="h4-1-4-3-3-python-"><a name="1.4.3.3 python编程数据存储" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4.3.3 python编程数据存储</h4><p>选中“dazhongspider”项目-&gt;“spider”目录,点击 “新建”-&gt;“Python File”,如下图所示。</p>
<p><img src="./水滴实验营_files/9-新建python程序文件.png" alt=""></p>
<p>在弹出的对话框中输入文件名“dbconnect”,点击“确定”进入下图所示的“dbconnect.py”文件编辑界面。</p>
<p><img src="./水滴实验营_files/11-dbconnect编辑界面.png" alt=""></p>
<p>输入下面数据库操作代码到“dbconnect.py”文件中并保存。</p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="kwd">import</span><span class="pln"> pymysql</span></code></li><li class="L1"><code><span class="com"># 数据库操作类,这个类用于连接数据库、插入数据、查询数据</span></code></li><li class="L2"><code><span class="kwd">class</span><span class="pln"> </span><span class="typ">DBHelper</span><span class="pun">():</span></code></li><li class="L3"><code><span class="pln"> </span><span class="com">#定义构造函数,self实例变量属性</span></code></li><li class="L4"><code><span class="pln"> </span><span class="kwd">def</span><span class="pln"> __init__</span><span class="pun">(</span><span class="kwd">self</span><span class="pun">):</span></code></li><li class="L5"><code><span class="pln"> </span><span class="kwd">self</span><span class="pun">.</span><span class="pln">host </span><span class="pun">=</span><span class="pln"> </span><span class="str">'localhost'</span><span class="pln"> </span><span class="com"># MySQL服务器地址</span></code></li><li class="L6"><code><span class="pln"> </span><span class="kwd">self</span><span class="pun">.</span><span class="pln">port </span><span class="pun">=</span><span class="pln"> </span><span class="lit">3306</span><span class="pln"> </span><span class="com"># MySQL服务器端口号</span></code></li><li class="L7"><code><span class="pln"> </span><span class="kwd">self</span><span class="pun">.</span><span class="pln">user </span><span class="pun">=</span><span class="pln"> </span><span class="str">'root'</span><span class="pln"> </span><span class="com">#用户名</span></code></li><li class="L8"><code><span class="pln"> </span><span class="kwd">self</span><span class="pun">.</span><span class="pln">passwd </span><span class="pun">=</span><span class="pln"> </span><span class="str">'shuidilab@123'</span><span class="pln"> </span><span class="com">#密码</span></code></li><li class="L9"><code><span class="pln"> </span><span class="kwd">self</span><span class="pun">.</span><span class="pln">db </span><span class="pun">=</span><span class="pln"> </span><span class="str">'testdazhong'</span><span class="pln"> </span><span class="com">#数据库名称</span></code></li><li class="L0"><code><span class="pln"> </span><span class="com"># 定义连接到数据库的方法</span></code></li><li class="L1"><code><span class="pln"> </span><span class="kwd">def</span><span class="pln"> connectDatabase</span><span class="pun">(</span><span class="kwd">self</span><span class="pun">):</span></code></li><li class="L2"><code><span class="pln"> </span><span class="com">#创建连接对象</span></code></li><li class="L3"><code><span class="pln"> conn </span><span class="pun">=</span><span class="pln"> pymysql</span><span class="pun">.</span><span class="pln">connect</span><span class="pun">(</span><span class="pln">host</span><span class="pun">=</span><span class="kwd">self</span><span class="pun">.</span><span class="pln">host</span><span class="pun">,</span></code></li><li class="L4"><code><span class="pln"> port</span><span class="pun">=</span><span class="kwd">self</span><span class="pun">.</span><span class="pln">port</span><span class="pun">,</span></code></li><li class="L5"><code><span class="pln"> user</span><span class="pun">=</span><span class="kwd">self</span><span class="pun">.</span><span class="pln">user</span><span class="pun">,</span></code></li><li class="L6"><code><span class="pln"> passwd</span><span class="pun">=</span><span class="kwd">self</span><span class="pun">.</span><span class="pln">passwd</span><span class="pun">,</span></code></li><li class="L7"><code><span class="pln"> db</span><span class="pun">=</span><span class="kwd">self</span><span class="pun">.</span><span class="pln">db</span><span class="pun">,</span></code></li><li class="L8"><code><span class="pln"> charset</span><span class="pun">=</span><span class="str">'utf8'</span><span class="pun">)</span><span class="pln"> </span><span class="com"># 要指定编码,否则中文可能乱码</span></code></li><li class="L9"><code><span class="pln"> </span><span class="kwd">return</span><span class="pln"> conn</span></code></li><li class="L0"><code><span class="pln"> </span><span class="com"># 插入数据</span></code></li><li class="L1"><code><span class="pln"> </span><span class="kwd">def</span><span class="pln"> insert</span><span class="pun">(</span><span class="kwd">self</span><span class="pun">,</span><span class="pln"> sql</span><span class="pun">,</span><span class="pln"> </span><span class="pun">*</span><span class="kwd">params</span><span class="pun">):</span><span class="pln"> </span><span class="com"># 注意这里params要加*,因为传递过来的是元组,*表示参数个数不定</span></code></li><li class="L2"><code><span class="pln"> conn </span><span class="pun">=</span><span class="pln"> </span><span class="kwd">self</span><span class="pun">.</span><span class="pln">connectDatabase</span><span class="pun">()</span></code></li><li class="L3"><code><span class="pln"> </span><span class="com">#创建游标对象</span></code></li><li class="L4"><code><span class="pln"> cur </span><span class="pun">=</span><span class="pln"> conn</span><span class="pun">.</span><span class="pln">cursor</span><span class="pun">()</span></code></li><li class="L5"><code><span class="pln"> </span><span class="com">#执行sql语句</span></code></li><li class="L6"><code><span class="pln"> cur</span><span class="pun">.</span><span class="pln">execute</span><span class="pun">(</span><span class="pln">sql</span><span class="pun">,</span><span class="pln"> </span><span class="kwd">params</span><span class="pun">)</span></code></li><li class="L7"><code><span class="pln"> </span><span class="com">#注意要commit,提交</span></code></li><li class="L8"><code><span class="pln"> conn</span><span class="pun">.</span><span class="pln">commit</span><span class="pun">()</span></code></li><li class="L9"><code><span class="pln"> cur</span><span class="pun">.</span><span class="pln">close</span><span class="pun">()</span></code></li><li class="L0"><code><span class="pln"> conn</span><span class="pun">.</span><span class="pln">close</span><span class="pun">()</span></code></li><li class="L1"><code><span class="pln"> </span><span class="com"># 查询数据</span></code></li><li class="L2"><code><span class="pln"> </span><span class="kwd">def</span><span class="pln"> </span><span class="kwd">select</span><span class="pun">(</span><span class="kwd">self</span><span class="pun">,</span><span class="pln"> sql</span><span class="pun">):</span></code></li><li class="L3"><code><span class="pln"> conn </span><span class="pun">=</span><span class="pln"> </span><span class="kwd">self</span><span class="pun">.</span><span class="pln">connectDatabase</span><span class="pun">()</span></code></li><li class="L4"><code><span class="pln"> cur </span><span class="pun">=</span><span class="pln"> conn</span><span class="pun">.</span><span class="pln">cursor</span><span class="pun">()</span></code></li><li class="L5"><code><span class="pln"> </span><span class="kwd">try</span><span class="pun">:</span></code></li><li class="L6"><code><span class="pln"> </span><span class="com"># 执行SQL语句</span></code></li><li class="L7"><code><span class="pln"> cur</span><span class="pun">.</span><span class="pln">execute</span><span class="pun">(</span><span class="pln">sql</span><span class="pun">)</span></code></li><li class="L8"><code><span class="pln"> conn</span><span class="pun">.</span><span class="pln">commit</span><span class="pun">()</span></code></li><li class="L9"><code><span class="pln"> </span><span class="com"># 获取所有记录列表</span></code></li><li class="L0"><code><span class="pln"> results </span><span class="pun">=</span><span class="pln"> cur</span><span class="pun">.</span><span class="pln">fetchall</span><span class="pun">()</span></code></li><li class="L1"><code><span class="pln"> </span><span class="kwd">return</span><span class="pln"> results</span></code></li><li class="L2"><code><span class="pln"> </span><span class="kwd">except</span><span class="pun">:</span></code></li><li class="L3"><code><span class="pln"> </span><span class="kwd">print</span><span class="pun">(</span><span class="str">"Error: unable to fecth data"</span><span class="pun">)</span></code></li><li class="L4"><code><span class="pln"> cur</span><span class="pun">.</span><span class="pln">close</span><span class="pun">()</span></code></li><li class="L5"><code><span class="pln"> conn</span><span class="pun">.</span><span class="pln">close</span><span class="pun">()</span></code></li></ol></pre><h4 id="h4-1-4-3-4-python-"><a name="1.4.3.4 Python爬虫编程" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4.3.4 Python爬虫编程</h4><p>如下图所示,选中“dazhongspider”项目-&gt;“spider”目录,点击“新建”-&gt;“Python File”。</p>
<p><img src="./水滴实验营_files/12-新建spider程序.png" alt=""></p>
<p>在弹出的对话框,输入文件名 “spider”,点击“确定”,进入“spider.py”的编辑界面。</p>
<p><img src="./水滴实验营_files/14-spider编辑页.png" alt=""></p>
<p>输入下面的爬虫代码到该文件中并保存。</p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="kwd">import</span><span class="pln"> json</span></code></li><li class="L1"><code><span class="kwd">import</span><span class="pln"> random</span></code></li><li class="L2"><code><span class="kwd">import</span><span class="pln"> requests</span></code></li><li class="L3"><code><span class="kwd">import</span><span class="pln"> dbconnect</span></code></li><li class="L4"><code><span class="com"># 城市列表</span></code></li><li class="L5"><code><span class="pln">listcity </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[[</span><span class="str">"上海"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"fce2e3a36450422b7fad3f2b90370efd71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L6"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"北京"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"d5036cf54fcb57e9dceb9fefe3917fff71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L7"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"广州"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"e749e3e04032ee6b165fbea6fe2dafab71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L8"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"深圳"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"e049aa251858f43d095fc4c61d62a9ec71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L9"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"天津"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"2e5d0080237ff3c8f5b5d3f315c7c4a508e25c702ab1b810071e8e2c39502be1"</span><span class="pun">],</span></code></li><li class="L0"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"杭州"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"91621282e559e9fc9c5b3e816cb1619c71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L1"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"南京"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"d6339a01dbd98141f8e684e1ad8af5c871862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L2"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"苏州"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"536e0e568df850d1e6ba74b0cf72e19771862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L3"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"成都"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"c950bc35ad04316c76e89bf2dc86bfe071862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L4"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"武汉"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"d96a24c312ed7b96fcc0cedd6c08f68c08e25c702ab1b810071e8e2c39502be1"</span><span class="pun">],</span></code></li><li class="L5"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"重庆"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"6229984ceb373efb8fd1beec7eb4dcfd71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L6"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"西安"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"ad66274c7f5f8d27ffd7f6b39ec447b608e25c702ab1b810071e8e2c39502be1"</span><span class="pun">],</span></code></li><li class="L7"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"青岛"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"9874ff91c8b50ad831ea897b0cd5315b71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L8"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"济南"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"4be7d758168826991eb4b3e779fdf41571862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L9"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"威海"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"fb5f85ca4c696d61c6e7a7e6c19b2f4530aacdebee4c4c9365dc18972daccaf7"</span><span class="pun">],</span></code></li><li class="L0"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"长春"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"261f655c7a9713696580432624721cd871862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L1"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"大连"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"21f56d987e57bd3e8df308351e3f4b9171862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L2"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"佛山"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"4e5d3a805c4f1ed6059c7074e5e86adf30aacdebee4c4c9365dc18972daccaf7"</span><span class="pun">],</span></code></li><li class="L3"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"贵阳"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"7ae3d5049e1c200ed6b2db0ac51604ba30aacdebee4c4c9365dc18972daccaf7"</span><span class="pun">],</span></code></li><li class="L4"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"合肥"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"94af1388885f0b7483fdf72cba086a6030aacdebee4c4c9365dc18972daccaf7"</span><span class="pun">],</span></code></li><li class="L5"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"呼和浩特"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"d64d5919fc236699402122b5770b4a6f71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L6"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"昆明"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"45159d85c35f742c1f2635d6ea6fa3ae30aacdebee4c4c9365dc18972daccaf7"</span><span class="pun">],</span></code></li><li class="L7"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"兰州"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"b201713f2fb5f7235b6e2a6de8faf6a630aacdebee4c4c9365dc18972daccaf7"</span><span class="pun">],</span></code></li><li class="L8"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"南宁"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"ff6f15347ab16f86ffb34e2700c4870930aacdebee4c4c9365dc18972daccaf7"</span><span class="pun">],</span></code></li><li class="L9"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"秦皇岛"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"ca02d37fa01b0f9613555237517e19ff71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L0"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"沈阳"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"eec1fb195780b91fc9dd065409c1d28471862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L1"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"太原"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"293e6160d971fe48ebbed1c8033bd9ed71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L2"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"唐山"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"2ae2237467ce4784b1705be502b0315271862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L3"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"无锡"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"51ff28145b3f63e2f9ec4b2a34e8850c71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L4"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"扬州"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"02eeb7a9021c215641f3ec0e3cede50b71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">]</span></code></li><li class="L5"><code><span class="pln"> </span><span class="pun">]</span></code></li><li class="L6"><code><span class="com">#用户代理,是一个特殊字符串头,使得服务器能够识别客户使用的操作系统及版本、浏览器及版本、浏览器渲染引擎、浏览器语言、浏览器插件等#</span></code></li><li class="L7"><code><span class="pln">USER_AGENT_LIST </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[</span></code></li><li class="L8"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"</span><span class="pun">,</span></code></li><li class="L9"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11"</span><span class="pun">,</span></code></li><li class="L0"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6"</span><span class="pun">,</span></code></li><li class="L1"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6"</span><span class="pun">,</span></code></li><li class="L2"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1"</span><span class="pun">,</span></code></li><li class="L3"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5"</span><span class="pun">]</span></code></li><li class="L4"><code><span class="com">#设置随机选择用户代理#</span></code></li><li class="L5"><code><span class="pln">head </span><span class="pun">=</span><span class="pln"> </span><span class="pun">{</span></code></li><li class="L6"><code><span class="str">'User-Agent'</span><span class="pun">:</span><span class="pln"> </span><span class="str">'{0}'</span><span class="pun">.</span><span class="pln">format</span><span class="pun">(</span><span class="pln">random</span><span class="pun">.</span><span class="pln">sample</span><span class="pun">(</span><span class="pln">USER_AGENT_LIST</span><span class="pun">,</span><span class="pln"> </span><span class="lit">1</span><span class="pun">)[</span><span class="lit">0</span><span class="pun">])</span><span class="pln"> </span><span class="com"># 从列表中随机获取1个</span></code></li><li class="L7"><code><span class="pun">}</span></code></li><li class="L8"><code><span class="pln">flag </span><span class="pun">=</span><span class="pln"> </span><span class="lit">0</span><span class="pln"> </span><span class="com">#总条数</span></code></li><li class="L9"><code><span class="pln">code </span><span class="pun">=</span><span class="pln"> </span><span class="lit">0</span><span class="pln"> </span><span class="com">#当前条数</span></code></li><li class="L0"><code></code></li><li class="L1"><code><span class="com"># 定义爬虫函数(调用解析函数)#</span></code></li><li class="L2"><code><span class="kwd">def</span><span class="pln"> cinfoSpider</span><span class="pun">(</span><span class="pln">citylist</span><span class="pun">):</span></code></li><li class="L3"><code><span class="pln"> city </span><span class="pun">=</span><span class="pln"> citylist</span><span class="pun">[</span><span class="lit">0</span><span class="pun">]</span></code></li><li class="L4"><code><span class="pln"> url </span><span class="pun">=</span><span class="pln"> citylist</span><span class="pun">[</span><span class="lit">1</span><span class="pun">]</span></code></li><li class="L5"><code><span class="pln"> </span><span class="com">#爬虫地址#</span></code></li><li class="L6"><code><span class="pln"> cbase_url </span><span class="pun">=</span><span class="pln"> </span><span class="str">"http://dp01.ds.quancube.cn/mylist/ajax/shoprank?rankId="</span><span class="pun">+</span><span class="pln">url</span></code></li><li class="L7"><code><span class="pln"> </span><span class="com">#get请求获取网页数据#</span></code></li><li class="L8"><code><span class="pln"> html </span><span class="pun">=</span><span class="pln"> requests</span><span class="pun">.</span><span class="kwd">get</span><span class="pun">(</span><span class="pln">cbase_url</span><span class="pun">,</span><span class="pln"> headers</span><span class="pun">=</span><span class="pln">head</span><span class="pun">)</span></code></li><li class="L9"><code><span class="pln"> </span><span class="com">#调用cfindFood函数解析#</span></code></li><li class="L0"><code><span class="pln"> cfindinfo</span><span class="pun">(</span><span class="pln">city</span><span class="pun">=</span><span class="pln">city</span><span class="pun">,</span><span class="pln"> data</span><span class="pun">=</span><span class="pln">str</span><span class="pun">(</span><span class="pln">html</span><span class="pun">.</span><span class="pln">text</span><span class="pun">))</span></code></li><li class="L1"><code></code></li><li class="L2"><code><span class="com"># 定义解析页面函数#</span></code></li><li class="L3"><code><span class="kwd">def</span><span class="pln"> cfindinfo</span><span class="pun">(</span><span class="pln">city</span><span class="pun">,</span><span class="pln">data</span><span class="pun">):</span></code></li><li class="L4"><code><span class="pln"> </span><span class="kwd">global</span><span class="pln"> flag</span><span class="pun">,</span><span class="pln">code</span></code></li><li class="L5"><code><span class="pln"> </span><span class="com">#连接mysql数据库#</span></code></li><li class="L6"><code><span class="pln"> mysql_db </span><span class="pun">=</span><span class="pln"> dbconnect</span><span class="pun">.</span><span class="typ">DBHelper</span><span class="pun">()</span></code></li><li class="L7"><code><span class="pln"> mysql_db</span><span class="pun">.</span><span class="pln">connectDatabase</span><span class="pun">();</span></code></li><li class="L8"><code><span class="pln"> </span><span class="com">#解析返回的JSON数据#</span></code></li><li class="L9"><code><span class="pln"> </span><span class="kwd">for</span><span class="pln"> data </span><span class="kwd">in</span><span class="pln"> json</span><span class="pun">.</span><span class="pln">loads</span><span class="pun">(</span><span class="pln">data</span><span class="pun">)[</span><span class="str">"shopBeans"</span><span class="pun">]:</span></code></li><li class="L0"><code><span class="pln"> flag </span><span class="pun">+=</span><span class="lit">1</span></code></li><li class="L1"><code><span class="pln"> </span><span class="com"># 商铺名称</span></code></li><li class="L2"><code><span class="pln"> shopName </span><span class="pun">=</span><span class="pln"> data</span><span class="pun">[</span><span class="str">"shopName"</span><span class="pun">]</span></code></li><li class="L3"><code><span class="pln"> </span><span class="com"># 商品编号</span></code></li><li class="L4"><code><span class="pln"> shopId </span><span class="pun">=</span><span class="pln"> data</span><span class="pun">[</span><span class="str">"shopId"</span><span class="pun">]</span></code></li><li class="L5"><code><span class="pln"> </span><span class="com"># 商铺星级</span></code></li><li class="L6"><code><span class="pln"> shopPower </span><span class="pun">=</span><span class="pln"> data</span><span class="pun">[</span><span class="str">"shopPower"</span><span class="pun">]</span></code></li><li class="L7"><code><span class="pln"> </span><span class="com"># 所在商区</span></code></li><li class="L8"><code><span class="pln"> mainRegionName </span><span class="pun">=</span><span class="pln"> data</span><span class="pun">[</span><span class="str">"mainRegionName"</span><span class="pun">]</span></code></li><li class="L9"><code><span class="pln"> </span><span class="com"># 分类名称</span></code></li><li class="L0"><code><span class="pln"> mainCategoryName </span><span class="pun">=</span><span class="pln"> data</span><span class="pun">[</span><span class="str">"mainCategoryName"</span><span class="pun">]</span></code></li><li class="L1"><code><span class="pln"> </span><span class="com"># 口味评分</span></code></li><li class="L2"><code><span class="pln"> tasteScore </span><span class="pun">=</span><span class="pln"> data</span><span class="pun">[</span><span class="str">"score1"</span><span class="pun">]</span></code></li><li class="L3"><code><span class="pln"> </span><span class="com"># 环境评分</span></code></li><li class="L4"><code><span class="pln"> environmentScore </span><span class="pun">=</span><span class="pln"> data</span><span class="pun">[</span><span class="str">"score2"</span><span class="pun">]</span></code></li><li class="L5"><code><span class="pln"> </span><span class="com"># 服务评分</span></code></li><li class="L6"><code><span class="pln"> serviceScore </span><span class="pun">=</span><span class="pln"> data</span><span class="pun">[</span><span class="str">"score3"</span><span class="pun">]</span></code></li><li class="L7"><code><span class="pln"> </span><span class="com"># 人均消费</span></code></li><li class="L8"><code><span class="pln"> avgPrice </span><span class="pun">=</span><span class="pln"> data</span><span class="pun">[</span><span class="str">"avgPrice"</span><span class="pun">]</span></code></li><li class="L9"><code><span class="pln"> </span><span class="com"># 详细地址</span></code></li><li class="L0"><code><span class="pln"> shopAddress </span><span class="pun">=</span><span class="pln"> data</span><span class="pun">[</span><span class="str">"address"</span><span class="pun">]</span></code></li><li class="L1"><code><span class="pln"> </span><span class="com"># 商铺网址</span></code></li><li class="L2"><code><span class="pln"> shopUrl </span><span class="pun">=</span><span class="pln"> </span><span class="str">"http://dp01.ds.quancube.cn/shop/"</span><span class="pun">+</span><span class="pln">shopId</span></code></li><li class="L3"><code><span class="pln"> </span><span class="com"># 商铺图片</span></code></li><li class="L4"><code><span class="pln"> defaultPic </span><span class="pun">=</span><span class="pln"> data</span><span class="pun">[</span><span class="str">"defaultPic"</span><span class="pun">]</span></code></li><li class="L5"><code><span class="pln"> </span><span class="com">#将解析数据插入数据库#</span></code></li><li class="L6"><code><span class="pln"> </span><span class="com">#定义sql语句#</span></code></li><li class="L7"><code><span class="pln"> sql </span><span class="pun">=</span><span class="pln"> </span><span class="str">'''insert into dazhonginfo(city, shopName, shopId, shopPower, mainRegionName, mainCategoryName, tasteScore, environmentScore, serviceScore, avgPrice, shopAddress, shopUrl, defaultPic) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'''</span></code></li><li class="L8"><code><span class="pln"> </span><span class="com">#插入这些参数的数据#</span></code></li><li class="L9"><code><span class="pln"> </span><span class="kwd">params</span><span class="pln"> </span><span class="pun">=</span><span class="pln"> </span><span class="pun">(</span><span class="pln">city</span><span class="pun">,</span><span class="pln"> shopName</span><span class="pun">,</span><span class="pln"> shopId</span><span class="pun">,</span><span class="pln"> shopPower</span><span class="pun">,</span><span class="pln"> mainRegionName</span><span class="pun">,</span><span class="pln"> mainCategoryName</span><span class="pun">,</span><span class="pln"> tasteScore</span><span class="pun">,</span><span class="pln"> environmentScore</span><span class="pun">,</span><span class="pln"> serviceScore</span><span class="pun">,</span><span class="pln"> avgPrice</span><span class="pun">,</span><span class="pln"> shopAddress</span><span class="pun">,</span><span class="pln"> shopUrl</span><span class="pun">,</span><span class="pln"> defaultPic</span><span class="pun">)</span></code></li><li class="L0"><code><span class="pln"> </span><span class="kwd">try</span><span class="pun">:</span></code></li><li class="L1"><code><span class="pln"> mysql_db</span><span class="pun">.</span><span class="pln">insert</span><span class="pun">(</span><span class="pln">sql</span><span class="pun">,*</span><span class="kwd">params</span><span class="pun">)</span></code></li><li class="L2"><code><span class="pln"> code </span><span class="pun">+=</span><span class="lit">1</span></code></li><li class="L3"><code><span class="pln"> </span><span class="kwd">print</span><span class="pun">(</span><span class="str">"----- 插入:"</span><span class="pun">,</span><span class="pln"> code</span><span class="pun">,</span><span class="pln"> </span><span class="str">"条------"</span><span class="pun">)</span></code></li><li class="L4"><code><span class="pln"> </span><span class="kwd">except</span><span class="pun">:</span></code></li><li class="L5"><code><span class="pln"> </span><span class="kwd">print</span><span class="pun">(</span><span class="str">"已存在不再重复插入!!"</span><span class="pun">)</span></code></li><li class="L6"><code><span class="pln"> </span><span class="kwd">print</span><span class="pun">(</span><span class="str">"总条数:"</span><span class="pun">,</span><span class="pln"> flag</span><span class="pun">)</span></code></li><li class="L7"><code></code></li><li class="L8"><code><span class="kwd">if</span><span class="pln"> __name__ </span><span class="pun">==</span><span class="pln"> </span><span class="str">'__main__'</span><span class="pun">:</span></code></li><li class="L9"><code><span class="pln"> </span><span class="com">#循环执行对每个城市网页的爬虫#</span></code></li><li class="L0"><code><span class="pln"> </span><span class="kwd">for</span><span class="pln"> city_data </span><span class="kwd">in</span><span class="pln"> listcity</span><span class="pun">:</span></code></li><li class="L1"><code><span class="pln"> cinfoSpider</span><span class="pun">(</span><span class="pln">city_data</span><span class="pun">)</span></code></li></ol></pre><h4 id="h4-1-4-3-5-"><a name="1.4.3.5 设置依赖包" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4.3.5 设置依赖包</h4><p>选中项目“dazhongspider”,点击“文件”-&gt;“设置”,查看项目解释器,点击“项目:dazhongspider”中的“Project Interpreter”,是否已安装requests、pymysql依赖包,若有,点击“确定”,若没有,则按如下步骤添加,以“requests”为列,首先点击右侧的“+”号</p>
<p><img src="./水滴实验营_files/15-选择项目依赖包.png" alt=""></p>
<p>进入如下界面</p>
<p><img src="./水滴实验营_files/30-安装依赖包首页.png" alt=""></p>
<p>点击下方“Manage repositories”,进入下载源管理界面,选择“quancube”下载源点击“-”并点击“确定”删除该下载源,只保留“tsinghua”清华下载源。</p>
<p><img src="./水滴实验营_files/31-删除清华下载源.png" alt=""></p>
<p>在查询框中输入“requests”,然后在列表中选择“requests”,点击“install package”</p>
<p><img src="./水滴实验营_files/16-检索requests依赖包.png" alt=""></p>
<p>安装过程可能需要等待一会儿,出现“Package ‘requests-**‘ installed successfully”表明依赖包安装成功,其他依赖包的安装过程类似,请按照该步骤继续安装“pymysql”。</p>
<p><img src="./水滴实验营_files/17-依赖包安装成功.png" alt=""></p>
<p>安装完成后关闭安装页面,进入“设置”页面后,点击“确定”,回到工程主页面。</p>
<p><strong>注意:</strong>若多次安装依赖包仍不成功,可选择其他国内下载源中的依赖包进行安装。</p>
<h4 id="h4-1-4-3-6-"><a name="1.4.3.6 执行爬虫爬取数据" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4.3.6 执行爬虫爬取数据</h4><p>在dazhongspider工程spider目录中选中“spider.py”文件,点击右键选择“运行spider”</p>
<p><img src="./水滴实验营_files/18-运行python程序.png" alt=""></p>
<p>执行爬虫程序开始爬取数据,在“运行”控制台打印程序输出的信息,如下图所示。</p>
<p><img src="./水滴实验营_files/19-运行控制台显示结果.png" alt=""></p>
<h4 id="h4-1-4-3-7-"><a name="1.4.3.7 查看爬取的数据" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4.3.7 查看爬取的数据</h4><p>点击Navicat软件下“testdazhong”数据库,如果数据表已打开,如图所示,请先关闭该文件。</p>
<p><img src="./水滴实验营_files/20-已打开数据空表.png" alt=""></p>
<p>显示数据表“dazhonginfo”</p>
<p><img src="./水滴实验营_files/21-显示数据库表.png" alt=""></p>
<p>双击该表,即可查看爬取的数据,如下图所示,点击右下角-&gt; ,可查看下一页数据。</p>
<p><img src="./水滴实验营_files/22-数据库表数据.png" alt=""></p>
<h4 id="h4-1-4-3-8-"><a name="1.4.3.8 学生实训任务" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4.3.8 学生实训任务</h4><h5 id="h5-1-4-3-8-1-20-"><a name="1.4.3.8.1 增加其他20个城市进行爬虫" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4.3.8.1 增加其他20个城市进行爬虫</h5><p>学生自行添加下表中的其他20个城市的rankId,进行爬虫实验。<br><strong>注意:程序中第30个城市rankId最后需要加逗号,最后第50个城市rankId无需逗号。</strong></p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pun">[</span><span class="str">"珠海"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"cf609e6234aca7061ab6a47e8d906e8c30aacdebee4c4c9365dc18972daccaf7"</span><span class="pun">],</span></code></li><li class="L1"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"安庆"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"35e4c8f1738ecc8ad87b613926a9d6dd1eebfd971c239635bc3b5e190325c01c"</span><span class="pun">],</span></code></li><li class="L2"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"安阳"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"d217beb464e7778289c854eafe2fdab21eebfd971c239635bc3b5e190325c01c"</span><span class="pun">],</span></code></li><li class="L3"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"保定"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"2dfe97d8557f1cccc3484afc2dc54e9c71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L4"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"蚌埠"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"1b3e2c870e22cb1c98035d64f71b82361eebfd971c239635bc3b5e190325c01c"</span><span class="pun">],</span></code></li><li class="L5"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"北海"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"d0490e0c48a4225ee85d2a0ec802ab131eebfd971c239635bc3b5e190325c01c"</span><span class="pun">],</span></code></li><li class="L6"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"滨州"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"7ed4b634645044dc1c746c8a1afb14481eebfd971c239635bc3b5e190325c01c"</span><span class="pun">],</span></code></li><li class="L7"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"保山"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"a589b29ec2067666fca30ea28844e70c1eebfd971c239635bc3b5e190325c01c"</span><span class="pun">],</span></code></li><li class="L8"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"宝鸡"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"701cef49e3c2f39394ce006c0b9945b21eebfd971c239635bc3b5e190325c01c"</span><span class="pun">],</span></code></li><li class="L9"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"北安"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"5ac6152df84456d942f1a69afe59c3fa1eebfd971c239635bc3b5e190325c01c"</span><span class="pun">],</span></code></li><li class="L0"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"承德"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"ad798bb841d96db01ce329855213a02ad97945289578b7557a191d6c125161e7"</span><span class="pun">],</span></code></li><li class="L1"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"沧州"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"76de7d4c29a0660101822920a658ecded97945289578b7557a191d6c125161e7"</span><span class="pun">],</span></code></li><li class="L2"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"大庆"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"a18c61815481274c318a383a902247ef71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L3"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"丹东"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"ad2b737489102d91905e887d5eb4d838d97945289578b7557a191d6c125161e7"</span><span class="pun">],</span></code></li><li class="L4"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"东营"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"cb6678ec79300273ed82ba7f7c09ccc11eebfd971c239635bc3b5e190325c01c"</span><span class="pun">],</span></code></li><li class="L5"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"德州"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"475860687e32c17b501eb86d7e8966d21eebfd971c239635bc3b5e190325c01c"</span><span class="pun">],</span></code></li><li class="L6"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"鄂尔多斯"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"8255b0e7540dbba8157071204385cd65d97945289578b7557a191d6c125161e7"</span><span class="pun">],</span></code></li><li class="L7"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"抚顺"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"4c4a091ef17e08597fd6a7c6bc0aab23d97945289578b7557a191d6c125161e7"</span><span class="pun">],</span></code></li><li class="L8"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"聊城"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"e34386f82b808950f4a1f859113352fd1eebfd971c239635bc3b5e190325c01c"</span><span class="pun">],</span></code></li><li class="L9"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"连云港"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"c2687c56fb887be66d324c728ab95c2571862f838d1255ea693b953b1d49c7c0"</span><span class="pun">]</span></code></li></ol></pre><p>执行爬虫程序继续爬取数据,运行界面开始显示“已存在不再重复插入”。</p>
<p><img src="./水滴实验营_files/23-重复执行部分城市输出信息.png" alt=""></p>
<p>这是因为前面已经爬取前30个城市数据是正常输出,继续执行即可,结束后,出现如下信息。</p>
<p><img src="./水滴实验营_files/24-50个城市执行完成信息.png" alt=""></p>
<h5 id="h5-1-4-3-8-2-phoneno-"><a name="1.4.3.8.2 爬取数据增加电话号码“phoneNo”字段" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4.3.8.2 爬取数据增加电话号码“phoneNo”字段</h5><p>学生可自行实践,提示:新建数据库、增加字段、爬虫程序修改。</p>
<h5 id="h5-1-4-3-8-3-"><a name="1.4.3.8.3 爬虫豆瓣电影正则表达式分析" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4.3.8.3 爬虫豆瓣电影正则表达式分析</h5><p>1.正则表达式介绍<br>(1) 正则表达式<br> 正则表达式是用来匹配与查找字符串的,从网上爬取数据自然或多或少会用到正则表达式,python的正则表达式要先引入re模块,正则表达式以r引导,例如<strong>r“\d+”</strong>正则表达式表示匹配连续的多个数值,search是re中的函数,从”abc123cd”字符串中搜索连续的数值,得到”123”。<br>(2) 字符串”\d”匹配0~9之间的一个数值<br>(3) 字符”+”重复前面一个匹配字符一次或者多次.<br> <strong>r”b\d+”</strong>第一个字符要匹配”b”,后面是连续的多个数字,因此”是b1233”,不是”a12”.<br>(4)字符”<em>“</em>重复前面一个匹配字符零次或者多次.<br> “” 与 “+”类似,但有区别,列如:可见 r”ab+“匹配的是”ab”,但是r”ab “匹配的是”a”,因为表示”b”可以重复零次,但是”+“却要求”b”重复一次以上.<br>(5)字符”?”重复前面一个匹配字符零次或者一次.<br> 匹配结果”ab”,重复b一次.<br>(6) 字符”.“代表任何一个字符,但是没有特别声明时不代表字符”\n”.<br>“.”代表了字符”x”.<br>(7) “|”代表把左右分成两个部分 .<br> 匹配”ab”或者”ba”都可以.<br><strong>(8)</strong> 特殊字符使用反斜杠”“引导,例如”\r”、”\n”、”\t”、”\”分别表示回车、换行、制表符号与反斜线自己本身.<br>(9) 字符”\b”表示单词结尾,单词结尾包括各种空白字符或者字符串结尾.<br>结果匹配”car”,因为”car”后面是一个空格.<br>(10) “[]中的字符是任选择一个,如果字符ASCll码中连续的一组,那么可以使用”-“字符连接,例如[0-9]表示0-9的其中一个数字,[A-Z]表示A-Z的其中一个大写字符,[0-9A-z]表示0-9的其中一个数字或者A-z的其中一个大写字符.<br>(11) “^”出现在[]的第一个字符位置,就代表<em>取</em>反,例如[ ^ab0-9]表示不是a、b,也不是0-9的数字.<br>(12) “\s”匹配任何空白字符,等价”[\r\n\x20\t\f\v]”<br>(13) “\w”匹配包括下划线子内的单词字符,等价于”[a-zA-Z0-9]”<br>(14) “$”字符比配字符串的结尾位置<br> 最后一个”ab”,而不是第一个”ab”<br>(15) 使用括号(…)可以把(…)看出一个整体,经常与”+”、”<em>“、”?”</em>的连续使用,对(…)部分进行重复.<br>匹配”abab”,”+“对”ab”进行了重复<br>(16) 查找匹配字符串<em>**</em><br> 正则表达式re库的search函数使用正则表达式对要匹配的字符串进行匹配,如果匹配不成功返回None,如果匹配成功返回一个匹配对象,匹配对象调用start()函数得到匹配字符的开始位置,匹配对象调用end()函数得到匹配字符串的结束位置,search虽然只返回匹配第一次匹配的结果,但是我们只要连续使用search函数就可以找到字符串全部匹配的字符串.匹配找出英文句子中所有单词,以使用正则表达式r”[A-Za-z]+\b”匹配单词,它表示匹配由大小写字母组成的连续多个字符,一般是一个单词,之后”\b”表示单词结尾.程序开始匹配到一个单词后m.start(),m.end()就是单词的起始位置,s[start:end]为截取的单词,之后程序再次匹配字符串s=s[end:],即字符串的后半段,一直到匹配完毕为止就找出每个单词.<br>2.正则表达式分析爬虫数据<br>爬取豆瓣电影TOP250影片并利用正则表达式进行信息分析,爬虫网址链接<a href="https://movie.douban.com/top250">https://movie.douban.com/top250</a> , 模拟浏览器部分请多设置几组随机选择一个,以防IP被封。</p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="kwd">import</span><span class="pln"> requests</span></code></li><li class="L1"><code><span class="kwd">import</span><span class="pln"> re</span></code></li><li class="L2"><code><span class="kwd">import</span><span class="pln"> random</span></code></li><li class="L3"><code></code></li><li class="L4"><code><span class="kwd">def</span><span class="pln"> get_douban_top</span><span class="pun">(</span><span class="pln">url</span><span class="pun">):</span></code></li><li class="L5"><code><span class="pln"> USER_AGENT_LIST </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[</span></code></li><li class="L6"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"</span><span class="pun">,</span></code></li><li class="L7"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11"</span><span class="pun">,</span></code></li><li class="L8"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6"</span><span class="pun">,</span></code></li><li class="L9"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6"</span><span class="pun">,</span></code></li><li class="L0"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1"</span><span class="pun">,</span></code></li><li class="L1"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5"</span><span class="pun">]</span></code></li><li class="L2"><code><span class="pln"> </span><span class="com"># 设置随机选择用户代理#</span></code></li><li class="L3"><code><span class="pln"> headers </span><span class="pun">=</span><span class="pln"> </span><span class="pun">{</span></code></li><li class="L4"><code><span class="pln"> </span><span class="str">'User-Agent'</span><span class="pun">:</span><span class="pln"> </span><span class="str">'{0}'</span><span class="pun">.</span><span class="pln">format</span><span class="pun">(</span><span class="pln">random</span><span class="pun">.</span><span class="pln">sample</span><span class="pun">(</span><span class="pln">USER_AGENT_LIST</span><span class="pun">,</span><span class="pln"> </span><span class="lit">1</span><span class="pun">)[</span><span class="lit">0</span><span class="pun">])</span><span class="pln"> </span><span class="com"># 从列表中随机获取1个</span></code></li><li class="L5"><code><span class="pln"> </span><span class="pun">}</span></code></li><li class="L6"><code></code></li><li class="L7"><code><span class="pln"> response </span><span class="pun">=</span><span class="pln"> requests</span><span class="pun">.</span><span class="kwd">get</span><span class="pun">(</span><span class="pln">url</span><span class="pun">,</span><span class="pln"> headers</span><span class="pun">=</span><span class="pln">headers</span><span class="pun">)</span></code></li><li class="L8"><code><span class="pln"> </span><span class="kwd">if</span><span class="pln"> response</span><span class="pun">.</span><span class="pln">status_code </span><span class="pun">==</span><span class="pln"> </span><span class="lit">200</span><span class="pun">:</span></code></li><li class="L9"><code><span class="pln"> </span><span class="kwd">return</span><span class="pln"> response</span><span class="pun">.</span><span class="pln">text</span></code></li><li class="L0"><code><span class="pln"> </span><span class="kwd">return</span><span class="pln"> </span><span class="kwd">None</span></code></li><li class="L1"><code></code></li><li class="L2"><code></code></li><li class="L3"><code><span class="kwd">def</span><span class="pln"> analyze</span><span class="pun">(</span><span class="pln">offset</span><span class="pun">):</span></code></li><li class="L4"><code><span class="pln"> url </span><span class="pun">=</span><span class="pln"> </span><span class="str">'https://movie.douban.com/top250/?start = offset &amp; filter ='</span><span class="pun">+</span><span class="pln">str</span><span class="pun">(</span><span class="pln">offset</span><span class="pun">)</span></code></li><li class="L5"><code><span class="pln"> html </span><span class="pun">=</span><span class="pln"> get_douban_top</span><span class="pun">(</span><span class="pln">url</span><span class="pun">)</span></code></li><li class="L6"><code></code></li><li class="L7"><code><span class="pln"> </span><span class="kwd">for</span><span class="pln"> item </span><span class="kwd">in</span><span class="pln"> parse_douban_top</span><span class="pun">(</span><span class="pln">html</span><span class="pun">):</span></code></li><li class="L8"><code><span class="pln"> </span><span class="kwd">print</span><span class="pun">(</span><span class="pln">item</span><span class="pun">)</span></code></li><li class="L9"><code></code></li><li class="L0"><code><span class="kwd">def</span><span class="pln"> parse_douban_top</span><span class="pun">(</span><span class="pln">html</span><span class="pun">):</span></code></li><li class="L1"><code><span class="pln"> regex </span><span class="pun">=</span><span class="pln"> </span><span class="str">'&lt;em class=""&gt;(\d+)&lt;/em&gt;.*?&lt;span class="title"&gt;(.*?)&lt;/span&gt;.*?&lt;p class=""&gt;(.*?)&lt;/p&gt;.*?&lt;span class="rating_num" property="v:average"&gt;(.*?)&lt;/span&gt;.*?&lt;span&gt;(.*?)&lt;/span&gt;.*?&lt;span class="inq"&gt;(.*?)&lt;/span&gt;'</span></code></li><li class="L2"><code></code></li><li class="L3"><code><span class="pln"> jsonResult </span><span class="pun">=</span><span class="pln"> re</span><span class="pun">.</span><span class="pln">compile</span><span class="pun">(</span></code></li><li class="L4"><code><span class="pln"> regex</span><span class="pun">,</span><span class="pln"> re</span><span class="pun">.</span><span class="pln">S</span></code></li><li class="L5"><code><span class="pln"> </span><span class="pun">)</span></code></li><li class="L6"><code></code></li><li class="L7"><code><span class="pln"> items </span><span class="pun">=</span><span class="pln"> re</span><span class="pun">.</span><span class="pln">findall</span><span class="pun">(</span><span class="pln">jsonResult</span><span class="pun">,</span><span class="pln"> html</span><span class="pun">)</span></code></li><li class="L8"><code></code></li><li class="L9"><code><span class="pln"> </span><span class="kwd">for</span><span class="pln"> item </span><span class="kwd">in</span><span class="pln"> items</span><span class="pun">:</span></code></li><li class="L0"><code></code></li><li class="L1"><code><span class="pln"> content </span><span class="pun">=</span><span class="pln"> </span><span class="str">""</span></code></li><li class="L2"><code></code></li><li class="L3"><code><span class="pln"> </span><span class="kwd">for</span><span class="pln"> every_list </span><span class="kwd">in</span><span class="pln"> item</span><span class="pun">[</span><span class="lit">2</span><span class="pun">].</span><span class="pln">split</span><span class="pun">():</span></code></li><li class="L4"><code><span class="pln"> </span><span class="com"># split()通过指定分隔符对字符串进行切片</span></code></li><li class="L5"><code><span class="pln"> content </span><span class="pun">=</span><span class="pln"> content </span><span class="pun">+</span><span class="pln"> </span><span class="str">""</span><span class="pun">.</span><span class="kwd">join</span><span class="pun">(</span><span class="pln">every_list</span><span class="pun">)</span></code></li><li class="L6"><code><span class="pln"> content </span><span class="pun">=</span><span class="pln"> re</span><span class="pun">.</span><span class="kwd">sub</span><span class="pun">(</span><span class="str">'&amp;nbsp;'</span><span class="pun">,</span><span class="pln"> </span><span class="str">' '</span><span class="pun">,</span><span class="pln"> content</span><span class="pun">)</span></code></li><li class="L7"><code><span class="pln"> content </span><span class="pun">=</span><span class="pln"> re</span><span class="pun">.</span><span class="kwd">sub</span><span class="pun">(</span><span class="str">'&lt;br&gt;'</span><span class="pun">,</span><span class="pln"> </span><span class="str">' '</span><span class="pun">,</span><span class="pln"> content</span><span class="pun">)</span></code></li><li class="L8"><code></code></li><li class="L9"><code><span class="pln"> </span><span class="kwd">yield</span><span class="pln"> </span><span class="pun">{</span></code></li><li class="L0"><code><span class="pln"> </span><span class="str">"index"</span><span class="pun">:</span><span class="pln"> item</span><span class="pun">[</span><span class="lit">0</span><span class="pun">],</span></code></li><li class="L1"><code><span class="pln"> </span><span class="str">"name"</span><span class="pun">:</span><span class="pln"> item</span><span class="pun">[</span><span class="lit">1</span><span class="pun">],</span></code></li><li class="L2"><code><span class="pln"> </span><span class="str">"describe"</span><span class="pun">:</span><span class="pln"> content</span><span class="pun">,</span></code></li><li class="L3"><code><span class="pln"> </span><span class="str">"star"</span><span class="pun">:</span><span class="pln"> item</span><span class="pun">[</span><span class="lit">3</span><span class="pun">],</span></code></li><li class="L4"><code><span class="pln"> </span><span class="str">"evaluate"</span><span class="pun">:</span><span class="pln"> item</span><span class="pun">[</span><span class="lit">4</span><span class="pun">],</span></code></li><li class="L5"><code><span class="pln"> </span><span class="str">"title"</span><span class="pun">:</span><span class="pln"> item</span><span class="pun">[</span><span class="lit">5</span><span class="pun">]</span></code></li><li class="L6"><code><span class="pln"> </span><span class="pun">}</span></code></li><li class="L7"><code><span class="kwd">if</span><span class="pln"> __name__ </span><span class="pun">==</span><span class="pln"> </span><span class="str">'__main__'</span><span class="pun">:</span></code></li><li class="L8"><code><span class="pln"> </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="lit">5</span><span class="pun">):</span></code></li><li class="L9"><code><span class="pln"> analyze</span><span class="pun">(</span><span class="pln">i</span><span class="pun">*</span><span class="lit">50</span><span class="pun">)</span></code></li></ol></pre><h5 id="h5-1-4-3-8-4-scrapy-"><a name="1.4.3.8.4 Scrapy爬虫框架" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4.3.8.4 Scrapy爬虫框架</h5><p>1.安装scrapy框架<br>更改pip源,C盘下搜索pip.in,完成下述修改</p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">index</span><span class="pun">-</span><span class="pln">url </span><span class="pun">=</span><span class="pln"> https</span><span class="pun">:</span><span class="com">//pypi.tuna.tsinghua.edu.cn/simple</span></code></li><li class="L1"><code></code></li><li class="L2"><code><span class="pln">trusted</span><span class="pun">-</span><span class="pln">host </span><span class="pun">=</span><span class="pln"> pypi</span><span class="pun">.</span><span class="pln">tuna</span><span class="pun">.</span><span class="pln">tsinghua</span><span class="pun">.</span><span class="pln">edu</span><span class="pun">.</span><span class="pln">cn</span></code></li></ol></pre><p>命令行下利用pip先后安装wheel、lxml、Pywin32,</p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">pip install wheel</span></code></li><li class="L1"><code><span class="pln">pip install lxml</span></code></li><li class="L2"><code><span class="pln">pip install </span><span class="typ">Pywin32</span></code></li></ol></pre><p>然后从辅材下载Twisted‑18.9.0‑cp37‑cp37m‑win_amd64.whl,通过文件菜单将该文件上传到实验环境c:/updownload目录下,解压后到解压文件目录下执行安装,注意这里务必写上绝对路径,不写将从互联网安装失败。</p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">pip install </span><span class="pun">/解压目录/</span><span class="typ">Twisted</span><span class="pun">‑</span><span class="lit">18.9</span><span class="pun">.</span><span class="lit">0</span><span class="pun">‑</span><span class="pln">cp37</span><span class="pun">‑</span><span class="pln">cp37m</span><span class="pun">‑</span><span class="pln">win_amd64</span><span class="pun">.</span><span class="pln">whl</span></code></li></ol></pre><p>最后安装scrapy.</p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">pip install scrapy</span></code></li></ol></pre><p>可能出现安装失败,执行下述操作,安装成功。</p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">easy_install </span><span class="pun">-</span><span class="pln">U pip</span></code></li><li class="L1"><code><span class="pln">pip install pyOpenSSL</span></code></li><li class="L2"><code><span class="pln">pip install scrapy</span></code></li></ol></pre><p>2.Scrapy介绍<br>(1)Scrapy框架简介<br>Scrapy是:由Python语言开发的一个快速、高层次的屏幕抓取和web抓取框架,用于抓取web站点并从页面中提取结构化的数据,只需要实现少量的代码,就能够快速的抓取。<br>(2)Scrapy框架运行原理<br><img src="./水滴实验营_files/b4c0be4d-1342-46f1-be06-99142a0ee31a_scrapy原理.png" alt=""><br>Scrapy主要包括了以下组件:<br>引擎(Scrapy Engine)<br>Item 项目<br>调度器(Scheduler)<br>下载器(Downloader)<br>爬虫(Spiders)<br>项目管道(Pipeline)<br>下载器中间件(Downloader Middlewares)<br>爬虫中间件(Spider Middlewares)<br>调度中间件(Scheduler Middewares)<br>(3)Scrapy框架入门<br>使用大概分为下面四步<br>[1]创建一个scrapy项目</p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">scrapy startproject mySpider</span></code></li></ol></pre><p>[2] 生成一个爬虫<br>scrapy genspider demo “demo.cn”<br>[3]提取数据<br>完善spider 使用xpath等<br>[4]pipeline中保存数据<br>[5]在命令中运行爬虫</p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">scrapy crawl qb</span></code></li></ol></pre><p>或在pycharm中运行爬虫</p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="kwd">from</span><span class="pln"> scrapy </span><span class="kwd">import</span><span class="pln"> cmdline</span></code></li><li class="L1"><code><span class="pln">cmdline</span><span class="pun">.</span><span class="pln">execute</span><span class="pun">(</span><span class="str">"scrapy crawl qb"</span><span class="pun">.</span><span class="pln">split</span><span class="pun">())</span></code></li></ol></pre><p>3.Scrapy框架来爬取腾讯招聘网<br>打开网站<a href="https://careers.tencent.com/search.html?pcid=40001">https://careers.tencent.com/search.html?pcid=40001</a><br>进入开发者模式,发现数据没有存放在源代码里面,那么我们查看Ajax请求,看看Ajax请求中有没有存放职位信息的数据,按CTR+R查看request URL,如下图所示:<br><img src="./水滴实验营_files/78deafce-c3e2-4e6e-9269-339baa462934_岗位代码截图.jpg" alt=""></p>
<p>Ajax请求中存放着职位数据,那可以通过Ajax请求来获取,URL链接如下:<br><a href="https://careers.tencent.com/tencentcareer/api/post/Query?timestamp=1629464904109&amp;countryId=&amp;cityId=&amp;bgIds=&amp;productId=&amp;categoryId=40001001,40001002,40001003,40001004,40001005,40001006&amp;parentCategoryId=&amp;attrId=&amp;keyword=&amp;pageIndex=2&amp;pageSize=10&amp;language=zh-cn&amp;area=cn">https://careers.tencent.com/tencentcareer/api/post/Query?timestamp=1629464904109&amp;countryId=&amp;cityId=&amp;bgIds=&amp;productId=&amp;categoryId=40001001,40001002,40001003,40001004,40001005,40001006&amp;parentCategoryId=&amp;attrId=&amp;keyword=&amp;pageIndex=2&amp;pageSize=10&amp;language=zh-cn&amp;area=cn</a><br>发现该请求的URL长,那么删减一下该URL的数据,<br>第一页:<br><a href="https://careers.tencent.com/tencentcareer/api/post/Query?categoryId=40001001,40001002,40001003,40001004,40001005,40001006&amp;pageIndex=1&amp;pageSize=10&amp;language=zh-cn&amp;area=cn">https://careers.tencent.com/tencentcareer/api/post/Query?categoryId=40001001,40001002,40001003,40001004,40001005,40001006&amp;pageIndex=1&amp;pageSize=10&amp;language=zh-cn&amp;area=cn</a><br>第二页:<br><a href="https://careers.tencent.com/tencentcareer/api/post/Query?categoryId=40001001,40001002,40001003,40001004,40001005,40001006&amp;pageIndex=2&amp;pageSize=10&amp;language=zh-cn&amp;area=cn">https://careers.tencent.com/tencentcareer/api/post/Query?categoryId=40001001,40001002,40001003,40001004,40001005,40001006&amp;pageIndex=2&amp;pageSize=10&amp;language=zh-cn&amp;area=cn</a><br>第三页:<br><a href="https://careers.tencent.com/tencentcareer/api/post/Query?categoryId=40001001,40001002,40001003,40001004,40001005,40001006&amp;pageIndex=3&amp;pageSize=10&amp;language=zh-cn&amp;area=cn">https://careers.tencent.com/tencentcareer/api/post/Query?categoryId=40001001,40001002,40001003,40001004,40001005,40001006&amp;pageIndex=3&amp;pageSize=10&amp;language=zh-cn&amp;area=cn</a><br>过简单删减可以得出该URL可以为上面的URL,而且可通过修改pageIndex的值爬取不同页面数据。</p>
<p>创建scrapy项目</p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">scrapy startproject </span><span class="typ">Tencent</span></code></li></ol></pre><p>可以pycharm的项目目录下会多了很多文件<br>进入Tencent目录,执行下述命令,创建spider爬虫</p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">scrapy genspider tencent careers</span><span class="pun">.</span><span class="pln">tencent</span><span class="pun">.</span><span class="pln">com</span></code></li></ol></pre><p>执行后,在spiders文件夹中会多了一个tencent.py文件,该文件就是刚才创建的spider爬虫,其文件内容下所示:</p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="kwd">import</span><span class="pln"> scrapy</span></code></li><li class="L1"><code><span class="kwd">class</span><span class="pln"> </span><span class="typ">Tencent1Spider</span><span class="pun">(</span><span class="pln">scrapy</span><span class="pun">.</span><span class="typ">Spider</span><span class="pun">):</span></code></li><li class="L2"><code><span class="pln"> name </span><span class="pun">=</span><span class="pln"> </span><span class="str">'tencent'</span></code></li><li class="L3"><code><span class="pln"> allowed_domains </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[</span><span class="str">'careers.tencent.com'</span><span class="pun">]</span></code></li><li class="L4"><code><span class="pln"> start_urls </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[</span><span class="str">'http://careers.tencent.com/'</span><span class="pun">]</span></code></li><li class="L5"><code></code></li><li class="L6"><code><span class="pln"> </span><span class="kwd">def</span><span class="pln"> parse</span><span class="pun">(</span><span class="kwd">self</span><span class="pun">,</span><span class="pln"> response</span><span class="pun">):</span></code></li><li class="L7"><code><span class="pln"> </span><span class="kwd">pass</span></code></li></ol></pre><p>其中</p>
<p>name是我们的爬虫名;<br>allowed_domains是域名,也就是爬虫爬取的范围;<br>start_urls是爬虫最开始爬取的URL链接;<br>parse()是用来解析响应、提取数据。</p>
<p>定义字段<br>在编写代码提取数据前,我们先来在items.py定义爬取的字段,字段类型为scrapy.Field,代码如下所示:</p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="kwd">import</span><span class="pln"> scrapy</span></code></li><li class="L1"><code></code></li><li class="L2"><code><span class="kwd">class</span><span class="pln"> </span><span class="typ">TencentItem</span><span class="pun">(</span><span class="pln">scrapy</span><span class="pun">.</span><span class="typ">Item</span><span class="pun">):</span></code></li><li class="L3"><code><span class="pln"> </span><span class="com"># define the fields for your item here like:</span></code></li><li class="L4"><code><span class="pln"> </span><span class="typ">RecruitPostName</span><span class="pln"> </span><span class="pun">=</span><span class="pln"> scrapy</span><span class="pun">.</span><span class="typ">Field</span><span class="pun">()</span><span class="pln"> </span><span class="com">#职位名</span></code></li><li class="L5"><code><span class="pln"> </span><span class="typ">LocationName</span><span class="pln"> </span><span class="pun">=</span><span class="pln"> scrapy</span><span class="pun">.</span><span class="typ">Field</span><span class="pun">()</span><span class="pln"> </span><span class="com">#地址</span></code></li><li class="L6"><code><span class="pln"> </span><span class="typ">Responsibility</span><span class="pln"> </span><span class="pun">=</span><span class="pln"> scrapy</span><span class="pun">.</span><span class="typ">Field</span><span class="pun">()</span><span class="pln"> </span><span class="com">#工作要求</span></code></li></ol></pre><p>定义好字段后,接下来开始在tencent.py中编写代码来提取数据,修改爬虫地址和解析函数,具体代码如下所示:</p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="kwd">import</span><span class="pln"> scrapy</span></code></li><li class="L1"><code><span class="kwd">from</span><span class="pln"> </span><span class="typ">Tencent</span><span class="pun">.</span><span class="pln">items </span><span class="kwd">import</span><span class="pln"> </span><span class="typ">TencentItem</span></code></li><li class="L2"><code><span class="kwd">class</span><span class="pln"> </span><span class="typ">TencentSpider</span><span class="pun">(</span><span class="pln">scrapy</span><span class="pun">.</span><span class="typ">Spider</span><span class="pun">):</span></code></li><li class="L3"><code><span class="pln"> name </span><span class="pun">=</span><span class="pln"> </span><span class="str">'tencent'</span></code></li><li class="L4"><code><span class="pln"> allowed_domains </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[</span><span class="str">'careers.tencent.com'</span><span class="pun">]</span></code></li><li class="L5"><code><span class="pln"> start_urls </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[</span><span class="str">'https://careers.tencent.com/tencentcareer/api/post/Query?categoryId=40001001,40001002,40001003,40001004,40001005,40001006&amp;pageIndex=1&amp;pageSize=10&amp;language=zh-cn&amp;area=cn'</span><span class="pun">]</span></code></li><li class="L6"><code></code></li><li class="L7"><code><span class="pln"> </span><span class="kwd">def</span><span class="pln"> parse</span><span class="pun">(</span><span class="kwd">self</span><span class="pun">,</span><span class="pln"> response</span><span class="pun">):</span></code></li><li class="L8"><code><span class="pln"> json</span><span class="pun">=</span><span class="pln">response</span><span class="pun">.</span><span class="pln">json</span><span class="pun">()</span></code></li><li class="L9"><code><span class="pln"> datas </span><span class="pun">=</span><span class="pln"> json</span><span class="pun">.</span><span class="kwd">get</span><span class="pun">(</span><span class="str">'Data'</span><span class="pun">).</span><span class="kwd">get</span><span class="pun">(</span><span class="str">'Posts'</span><span class="pun">)</span></code></li><li class="L0"><code><span class="pln"> </span><span class="kwd">for</span><span class="pln"> data </span><span class="kwd">in</span><span class="pln"> datas</span><span class="pun">:</span></code></li><li class="L1"><code><span class="pln"> item</span><span class="pun">=</span><span class="typ">TencentItem</span><span class="pun">()</span></code></li><li class="L2"><code><span class="pln"> item</span><span class="pun">[</span><span class="str">'RecruitPostName'</span><span class="pun">]=</span><span class="pln">data</span><span class="pun">.</span><span class="kwd">get</span><span class="pun">(</span><span class="str">'RecruitPostName'</span><span class="pun">),</span></code></li><li class="L3"><code><span class="pln"> item</span><span class="pun">[</span><span class="str">'LocationName'</span><span class="pun">]=</span><span class="pln">data</span><span class="pun">.</span><span class="kwd">get</span><span class="pun">(</span><span class="str">'LocationName'</span><span class="pun">),</span></code></li><li class="L4"><code><span class="pln"> item</span><span class="pun">[</span><span class="str">'Responsibility'</span><span class="pun">]=</span><span class="pln">data</span><span class="pun">.</span><span class="kwd">get</span><span class="pun">(</span><span class="str">'Responsibility'</span><span class="pun">).</span><span class="pln">replace</span><span class="pun">(</span><span class="str">'\n'</span><span class="pun">,</span><span class="str">''</span><span class="pun">)</span></code></li><li class="L5"><code><span class="pln"> </span><span class="kwd">yield</span><span class="pln"> item</span></code></li></ol></pre><p>首先导入items.py文件中的TencentItem,再修改start_urls的URL链接,定义一个json变量来接收网页响应的json()数据,通过for循环把每条职位信息循环遍历并提取我们想要的数据并放在item字典里面,其中item=TencentItem()相当于创建一个空字典item={}</p>
<p>创建一个for循环来进行翻页</p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="lit">2</span><span class="pun">,</span><span class="lit">4</span><span class="pun">):</span></code></li><li class="L1"><code><span class="pln"> next_url</span><span class="pun">=</span><span class="pln">f</span><span class="str">'https://careers.tencent.com/tencentcareer/api/post/Query?categoryId=40001001,40001002,40001003,40001004,40001005,40001006&amp;pageIndex={i}&amp;pageSize=10&amp;language=zh-cn&amp;area=cn'</span></code></li><li class="L2"><code><span class="pln"> </span><span class="kwd">yield</span><span class="pln"> scrapy</span><span class="pun">.</span><span class="typ">Request</span><span class="pun">(</span></code></li><li class="L3"><code><span class="pln"> next_url</span><span class="pun">,</span></code></li><li class="L4"><code><span class="pln"> callback</span><span class="pun">=</span><span class="kwd">self</span><span class="pun">.</span><span class="pln">parse</span></code></li><li class="L5"><code><span class="pln"> </span><span class="pun">)</span></code></li></ol></pre><p>调用yield生成器来返回数据给引擎,并调用scrapy.Request()方法,该方法能构建一个requests,同时指定提取数据的callback函数<br>在启动爬取前,我们先要在settings.py文件中编写一些代码,具体代码如下所示:</p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">LOG_LEVEL</span><span class="pun">=</span><span class="str">"WARNING"</span><span class="pln"> </span></code></li><li class="L1"><code><span class="pln">ITEM_PIPELINES </span><span class="pun">=</span><span class="pln"> </span><span class="pun">{</span><span class="pln"> </span></code></li><li class="L2"><code><span class="pln"> </span><span class="str">'Tencent.pipelines.TencentPipeline'</span><span class="pun">:</span><span class="pln"> </span><span class="lit">300</span><span class="pun">,</span></code></li><li class="L3"><code><span class="pun">}</span></code></li><li class="L4"><code><span class="pln">USER_AGENT </span><span class="pun">=</span><span class="pln"> </span><span class="str">'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36'</span></code></li></ol></pre><p>其中:<br>LOG_LEVEL的作用的屏蔽log日志的输出;<br>ITEM_PIPELINES的作用是开启引擎;</p>
<p>当我们要把数据保存成文件的时候,不需要任何额外的代码,只要执行如下代码即可</p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">scrapy crawl tencent </span><span class="pun">-</span><span class="pln">o tencent</span><span class="pun">.</span><span class="pln">csv</span></code></li></ol></pre><h1 id="h1-2-kettle-"><a name="2. 利用Kettle进行数据清洗转换" class="reference-link"></a><span class="header-link octicon octicon-link"></span>2. 利用Kettle进行数据清洗转换</h1><p>爬取的数据通常存在缺失值、无效值等问题,不能直接用于数据统计分析,因此利用kettle将数据进行清洗、转换;将清洗后的数据提供给学生,学生利用Python软件进行统计分析。本部分实验目标首先将MySQL数据库中的爬取数据进行清洗,然后以两种形式存储,一种存储到MySQL新表中,用于Python编程统计分析、可视化;一种转换成CSV文件。</p>
<h2 id="h2-2-1-kettle-"><a name="2.1 Kettle 基本使用" class="reference-link"></a><span class="header-link octicon octicon-link"></span>2.1 Kettle 基本使用</h2><p>(1)启动:桌面上双击“Spoon.bat”图标启动Kettle,显示如下界面。</p>
<p><img src="./水滴实验营_files/01-启动kettle.png" alt=""></p>
<p>(2)新建转换:选中左侧栏内的“转换”菜单,右键选择“新建”即可建立一个新的转换任务,如图所示。</p>
<p><img src="./水滴实验营_files/02-新建转换.png" alt=""></p>
<p>(3)新建组件:新建输入、流程、输出等组件,选择相应组件,直接拖到右边工作区,例如新建CSV文件输入,将核心对象-&gt;输入-&gt;CSV文件输入,直接拖到右边工作台或者通过双击添加该组件,同样可以将新对象-&gt;转换-&gt;排序记录,直接拖到右边工作台。</p>
<p><img src="./水滴实验营_files/03-新建组件.png" alt=""></p>
<p>(4)建立连接:按下“shift”键,在“CSV文件输入”和“排序记录”间用鼠标划线,或者按住鼠标中间滚轮移动鼠标即可连接两个节点。</p>
<p><img src="./水滴实验营_files/04-建立连接.png" alt=""></p>
<p>(5)确认连接:双击连接线,会弹出对话框,注意起始步骤和目标步骤如果有值为空,则代表连接不成功,若使连接生效未勾选,则连接不生效,请重新建立连接。</p>
<p><img src="./水滴实验营_files/05-确认连接.png" alt=""></p>
<p>(6)删除连接:选中拟删除的连接线,点击右键选择“删除节点连接”。<br>(7)连接分支选择:在组件连接过程中,会弹出对话框,一般主要流程会选择“主要输出步骤”,存在分支的时候选择“Result is TRUE”和“Result is FALSE”,例如选择核心对象-&gt;流程-&gt;过滤记录和空操作建立如下分支,如下图所示。</p>
<p><img src="./水滴实验营_files/06-分支选择设置1.png" alt=""></p>
<h2 id="h2-2-2-"><a name="2.2 数据清洗内容" class="reference-link"></a><span class="header-link octicon octicon-link"></span>2.2 数据清洗内容</h2><p>浏览“dazhonginfo”数据表的数据,发现爬取的数据存在以下问题:<br>(1) 存在缺失值,即空的字段,需要删除该条记录。<br>(2) 存在无效值,avgPrice人均消费字段的值出现0,代表人均消费是0,不合理,对于tasteScore,environmentScore、serviceScore、avgPrice数值型字段,如果出现0值,则删除该条记录。<br>(3) shopUrl和defaultPic列后续用不到,需要删除。</p>
<h2 id="h2-2-3-mysql-"><a name="2.3 建立MySQL新表存储清洗后数据" class="reference-link"></a><span class="header-link octicon octicon-link"></span>2.3 建立MySQL新表存储清洗后数据</h2><p>在Navicat软件中选中“testdazhong”数据库,点击右键选择“运行SQL文件”,选择“C:\sqlfile\dazhongoutput.sql”,点击“开始”导入dazhongoutput数据表</p>
<p><img src="./水滴实验营_files/07-输出表导入.png" alt=""></p>
<p>提示“*** executed successfully”信息表示导入成功。关闭该页面,选中testdazhong数据库下面的“表”,点击鼠标右键选择“刷新”,看到“dazhongoutput”表。</p>
<p><img src="./水滴实验营_files/09-刷新输出表.png" alt=""></p>
<p>双击“dazhongoutput”表可打开查看相关字段,各字段解释同数据表dazhonginfo.</p>
<p><img src="./水滴实验营_files/10-输出空表.png" alt=""></p>
<h2 id="h2-2-4-kettle-"><a name="2.4 Kettle数据清洗、转换" class="reference-link"></a><span class="header-link octicon octicon-link"></span>2.4 Kettle数据清洗、转换</h2><h3 id="h3-2-4-1-"><a name="2.4.1 创建转换任务设计输入组件" class="reference-link"></a><span class="header-link octicon octicon-link"></span>2.4.1 创建转换任务设计输入组件</h3><p>在“主对象树”标签下双击“转换”,新建一个转换任务</p>
<p><img src="./水滴实验营_files/11-新建转换.png" alt=""></p>
<p>打开“输入”-&gt;“表输入”,将“表输入”拖到右边工作台上。</p>
<p><img src="./水滴实验营_files/12-新建表输入.png" alt=""></p>
<p>双击“表输入”,出现如下界面,数据库连接处点击“新建”,在弹出对话框处设置连接名称,可任意定义,选择连接类型为“MySQL”,连接方式为“Native(JDBC)”,设置数据库连接信息,这里信息与dbconnect.py中数据库连接信息一致。<br>主机名或IP地址:localhost ##代表本机<br>端口:3306<br>用户名:root ##根用户权限<br>密码:shuidilab<a href="https://github.com/123" title="@123" class="at-link">@123</a><br>数据库设置为testdazhong,存储3000条爬取的数据。<br><img src="./水滴实验营_files/13-数据库连接.png" alt=""></p>
<p>点击“测试”,出现如下信息表示数据库连接成功,点击“确定”,返回数据库连接页,点击“确认”返回表输入页。</p>
<p><img src="./水滴实验营_files/14-数据库连接成功.png" alt=""></p>
<p>在下图中的表输入页,输入查询数据库的SQL查询语句,例如选择表中所有数据,</p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">SELECT </span><span class="pun">*</span><span class="pln"> FROM dazhonginfo</span></code></li></ol></pre><p>“dazhonginfo”是数据库表名称,然后点击“预览”,选择预览的记录条数,例如”3000”,配置完成。</p>
<p><img src="./水滴实验营_files/15-设置SQL查询语句.png" alt=""></p>
<p>点击“确定”进行数据库中表信息预览,如图所示。</p>
<p><img src="./水滴实验营_files/16-浏览数据表数据.png" alt=""></p>
<h3 id="h3-2-4-2-"><a name="2.4.2 设计数据清洗组件" class="reference-link"></a><span class="header-link octicon octicon-link"></span>2.4.2 设计数据清洗组件</h3><h4 id="h4-2-4-2-1-"><a name="2.4.2.1过滤空值" class="reference-link"></a><span class="header-link octicon octicon-link"></span>2.4.2.1过滤空值</h4><p>选中“流程”-&gt;“过滤记录”,将“过滤记录”拖到右边工作台上。</p>
<p><img src="./水滴实验营_files/17-1新建第1个过滤记录.png" alt=""></p>
<p>双击“过滤记录”,设置步骤名称为“过滤空值”。</p>
<p><img src="./水滴实验营_files/17-设置过滤空值名称.png" alt=""></p>
<p>点击“确定”,出现如下图所示。</p>
<p><img src="./水滴实验营_files/18-过滤空值设置完成.png" alt=""></p>
<p>选中“流程”-&gt; “空操作(什么也不做)”,将“空操作(什么也不做)”拖到右边工作台上。</p>
<p><img src="./水滴实验营_files/19-设置第1个空操作.png" alt=""></p>
<p>按下“shift”键,从“表输入”向“过滤空值”进行连线,如果弹出对话框,选择“主要输出步骤”, 线连接成功后,点击连线,显示如图。</p>
<p><img src="./水滴实验营_files/20-表输入-过滤空值连线.png" alt=""></p>
<p>然后同样操作从 “过滤空值”向“空操作(什么也不做)”进行连线,在此过程中会弹出对话框,选择“Result is FALSE”,建立如图所示连接图。</p>
<p><img src="./水滴实验营_files/21-过滤空值-空操作连线.png" alt=""> </p>
<p>点击“过滤空值”和“空操作(什么也不做)”两者间的连线,如图所示,点击“确定”关闭该页。</p>
<p><img src="./水滴实验营_files/21-双击连线查看连接属性.png" alt=""></p>
<p>双击“过滤空值”,添加条件,第一个红框中选择一个字段“Id”,如图所示,点击“确定”。</p>
<p><img src="./水滴实验营_files/22-过滤空值添加第1个条件.png" alt=""></p>
<p>在第二个红框中选择函数,选择“IS NOT NULL”,点击函数对话框的“确定”,如图所示,完成了第一个过滤条件的设置。</p>
<p><img src="./水滴实验营_files/23-过滤空值添加第1个条件2.png" alt=""></p>
<p>点击右侧“+”号,出现如图所示第二个条件。</p>
<p> <img src="./水滴实验营_files/24-添加第2个条件.png" alt=""></p>
<p>点击“null= []”,出现如下界面,按照配置“Id”字段的步骤配置第二个字段“city”,点击最下方的确定按钮。</p>
<p> <img src="./水滴实验营_files/25-设置第二个过滤空值条件.png" alt=""></p>
<p>重复上述步骤,依次添加“shopName”,“shopId”,“shopPower”,“mainRegionName”, “ mainCategoryName”,“tasteScore”,“environmentScore”,“serviceScore”,“avgPrice”,“shopAddress”,“shopUrl”、“defaultPic”这些字段均不为空的条件,如下图所示,点击“确定”,保存过滤条件,完成第一个组件“过滤空值”的条件设置。</p>
<p> <img src="./水滴实验营_files/26-第1个组件过滤空值条件完成.png" alt=""></p>
<h4 id="h4-2-4-2-2-0-"><a name="2.4.2.2过滤0值" class="reference-link"></a><span class="header-link octicon octicon-link"></span>2.4.2.2过滤0值</h4><p>继续,选中“流程”-&gt;“过滤记录”,将“过滤记录”拖到右边工作台上,双击“过滤记录”设置步骤名称为“过滤0值”,选中“流程”-&gt; “空操作(什么也不做)”,将“空操作(什么也不做)2”拖到右边工作台上,显示为“空操作(什么也不做)2”,如图所示。</p>
<p> <img src="./水滴实验营_files/27-第2个过滤组件设置.png" alt=""></p>
<p>按下“shift”键,从“过滤空值”向“过滤0值”进行连线,若弹出对话框选择“Result is TURE”连接即可,点击连线,显示如图。</p>
<p> <img src="./水滴实验营_files/28-第二个过滤0值连线.png" alt=""></p>
<p>同样操作连接“过滤0值” -&gt; “空操作(什么也不做)2”, 在此过程中会弹出对话框,选择“Result is FALSE”,建立如图所示连接图。</p>
<p> <img src="./水滴实验营_files/28-第二个过滤0值连线.png" alt=""></p>
<p>双击“过滤0值”,设置条件,如图所示,左侧第一个红框处选择字段avgPrice,第二个红框处选择函数不等于“&lt; &gt;”,第三个红框处设置为整数0,点击“确定”,这样建立如图所示的过滤0值的条件。</p>
<p> <img src="./水滴实验营_files/30-设置过滤0值条件.png" alt=""></p>
<p>按照同样步骤添加tasteScore,点击“+”号,出现如下界面,点击“null=[]”,进入配置页面进行相关配置,点击红框处“确定”。依次进行environmentScore、serviceScore字段的条件设置,最后如图所示。</p>
<p> <img src="./水滴实验营_files/31-第2个过滤条件完成.png" alt=""></p>
<h4 id="h4-2-4-2-3-"><a name="2.4.2.3排序" class="reference-link"></a><span class="header-link octicon octicon-link"></span>2.4.2.3排序</h4><p>打开“转换”-&gt;“排序记录”,将“排序记录”拖到右边工作台,如图所示。</p>
<p> <img src="./水滴实验营_files/32-新建排序记录组件.png" alt=""></p>
<p>连接“过滤0值”和“排序记录”,弹出对话框选择“Result is True”双击“排序记录”,如图所示,清洗后的数据按照Id升序进行排序,点击字段名称下方,选择字段“Id”,升序选择“是”,大小写敏感选择“否”,点击“确定”保存。</p>
<p> <img src="./水滴实验营_files/33-排序记录设置.png" alt=""></p>
<h3 id="h3-2-4-3-"><a name="2.4.3 设计输出组件" class="reference-link"></a><span class="header-link octicon octicon-link"></span>2.4.3 设计输出组件</h3><p>选中“输出”中的“文本文件输出”和“表输出”添加到右边工作台,如图所示</p>
<p> <img src="./水滴实验营_files/34-设计输出组件.png" alt=""></p>
<p>先连接“排序记录”与“文本文件输出”,然后连接“排序记录”与“表输出”,将弹出对话框,如图所示,选择“复制”,清洗后的所有数据将存储到文本文件和表中各一份,选择“分发”则数据循环向两个文件输入,我们选择“复制”。</p>
<p> <img src="./水滴实验营_files/35-连接时设置复制数据.png" alt=""></p>
<p>双击“文本文件输出”,配置信息,首先在文件标签页,扩展名改为“csv”,在文件名称处选择文件路径,文件名可以是已经创建的空csv文件,也可以直接输入一个新文件名,例如“dazhongdata”。</p>
<p><img src="./水滴实验营_files/36-csv文件内容设置.png" alt=""></p>
<p>点击“内容”标签页,分隔符设置为“,”号。</p>
<p><img src="./水滴实验营_files/37-csv文件内容设置.png" alt=""></p>
<p>点击“字段”标签页,点击“获取字段”按钮,获得输入数据的14个字段,删除最后2个字段,具体操作:选中待删除行,点击右键,选择“删除选中行”,点击“最小宽度”按钮,最后点击“确定”按钮关闭该页。</p>
<p><img src="./水滴实验营_files/38-csv文件输出设置.png" alt=""></p>
<p>双击“表输出”,选择数据库连接“dbcon”,点击“目标表”处的浏览,选择连接输出表“dazhongoutput”,点击“确定”。</p>
<p><img src="./水滴实验营_files/39-表输出设置.png" alt=""></p>
<p>在表输出页面,“提交记录数量”部分输入拟提取的数据记录行数,请输入已爬取的记录行数,如3000或5000,代表最多提取3000或5000行到输出表中,若实际数据记录行数小于该值,则按照实际数量输出到表中,勾选指定数据库字段,点击“数据库字段”标签页,点击“获取字段”,然后删除最后两个字段“shopUrl”和“defaultPic”所在的行,选中所在行点击右键选择“删除选中行”。</p>
<p><img src="./水滴实验营_files/40-表输出2.png" alt=""></p>
<p>点击“确定”,数据清洗转换流程设计完成,如下图所示,保存该ktr文件。</p>
<p><img src="./水滴实验营_files/41-1ktr流程.png" alt=""></p>
<h3 id="h3-2-4-4-"><a name="2.4.4 执行清洗和转换" class="reference-link"></a><span class="header-link octicon octicon-link"></span>2.4.4 执行清洗和转换</h3><p>点击红框处“执行这个转换”进行数据清洗和转换</p>
<p><img src="./水滴实验营_files/41-2运行ktr文件.png" alt=""></p>
<p>弹出“执行转换”页面,选择“本地执行”,点击“启动”按钮执行。</p>
<p><img src="./水滴实验营_files/41-3启动本地执行.png" alt=""></p>
<p>执行完成后,在执行结果“步骤度量”中可以看到数据清洗、转换的步骤。</p>
<p><img src="./水滴实验营_files/42-执行结果.png" alt=""></p>
<p>打开输出的CSV文件“C:\changedata\dazhongdata.csv”,可以看到数据如下。</p>
<p><img src="./水滴实验营_files/42-清洗后的CSV文件.png" alt=""></p>
<p>查看数据库表“dazhongoutput”,可以看到如下数据,数据清洗转换成功。</p>
<p><img src="./水滴实验营_files/43-清洗后的SQL文件.png" alt=""></p>
<h2 id="h2-2-5-"><a name="2.5 学生实训任务" class="reference-link"></a><span class="header-link octicon octicon-link"></span>2.5 学生实训任务</h2><p>自行设计多个清洗规则,利用kettle创建数据清洗流程,生成ktr文件,进行数据清洗转换。</p>
<h1 id="h1-3-"><a name="3 数据统计分析" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3 数据统计分析</h1><h2 id="h2-3-1-python-mysql-"><a name="3.1 Python+MySQL数据统计分析" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1 Python+MySQL数据统计分析</h2><h3 id="h3-3-1-1-sql-"><a name="3.1.1 基本SQL语句学习" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.1 基本SQL语句学习</h3><p>本节进行Python编程完成数据统计分析,数据通过调用MySQL数据库中表获取,因此需要学习基本SQL语句操作数据库。</p>
<p>启动Navicat,选中数据库“testdazhong”,点击下图工具栏“查询”-&gt;“新建查询”。</p>
<p><img src="./水滴实验营_files/1-新建查询.png" alt=""></p>
<p>显示查询编辑器,在查询编辑器中可编辑SQL语句,例如:查询“dazhongoutput”表中所有记录的语句:</p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">SELECT </span><span class="pun">*</span><span class="pln"> FROM dazhongoutput</span></code></li></ol></pre><p>点击“运行”,显示结果如下图所示。</p>
<p><img src="./水滴实验营_files/3-查询结果.png" alt=""></p>
<p>查询shopPower列的不同值(即商铺有哪些星级)的语句:</p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">SELECT DISTINCT</span><span class="pun">(</span><span class="pln">shopPower</span><span class="pun">)</span><span class="pln"> </span><span class="kwd">from</span><span class="pln"> dazhongoutput</span></code></li></ol></pre><p>点击“运行”,显示结果如下图所示。</p>
<p><img src="./水滴实验营_files/4-查询星级.png" alt=""></p>
<p>查询“济南5星级商铺的数量”,查询条件: “城市是济南并且shopPower=50”,统计shopId列的值的数量,SQL语句:</p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">SELECT COUNT</span><span class="pun">(</span><span class="pln">shopId</span><span class="pun">)</span><span class="pln"> </span><span class="kwd">from</span><span class="pln"> dazhongoutput </span><span class="kwd">where</span><span class="pln"> shopPower</span><span class="pun">=</span><span class="lit">50</span><span class="pln"> </span><span class="kwd">and</span><span class="pln"> city</span><span class="pun">=</span><span class="str">"济南"</span></code></li></ol></pre><p>Where后面子句用于规定选择的标准。点击“运行”,显示结果如下图所示。</p>
<p><img src="./水滴实验营_files/5-查询5星级商铺数量.png" alt=""></p>
<p>分类汇总济南市“菜品分类”数量、“菜品分类”名称,SQL语句:</p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">SELECT COUNT</span><span class="pun">(</span><span class="pln">mainCategoryName</span><span class="pun">),</span><span class="pln">mainCategoryName </span><span class="kwd">from</span><span class="pln"> dazhongoutput </span><span class="kwd">where</span><span class="pln"> city</span><span class="pun">=</span><span class="str">"济南"</span><span class="pln"> GROUP BY mainCategoryName</span></code></li></ol></pre><p>GROUP BY 后面的子句是根据该分组进行分类汇总。点击“运行”,显示结果如下图所示。</p>
<p><img src="./水滴实验营_files/6-分类查询.png" alt=""></p>
<h3 id="h3-3-1-2-"><a name="3.1.2 打开项目" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.2 打开项目</h3><p>启动Pycharm软件,系统默认打开“dazhongspider”项目。</p>
<h3 id="h3-3-1-3-"><a name="3.1.3 设置代码目录" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.3 设置代码目录</h3><p>设置数据分析程序目录,选中该项目“dazhongspider”,点击右键选择“新建”-&gt;“目录”,如图所示。</p>
<p><img src="./水滴实验营_files/1-新建目录.png" alt=""></p>
<p>在弹出的界面中(如图所示),设置数据统计分析程序的目录名称为“dataanalysis”。</p>
<p><img src="./水滴实验营_files/2-目录名称.png" alt=""></p>
<h3 id="h3-3-1-4-python-"><a name="3.1.4 Python编程进行数据统计可视化" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4 Python编程进行数据统计可视化</h3><h4 id="h4-3-1-4-1-20-"><a name="3.1.4.1 20个大中城市星级商铺统计分析" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.1 20个大中城市星级商铺统计分析</h4><h5 id="h5-3-1-4-1-1-"><a name="3.1.4.1.1 新建代码文件" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.1.1 新建代码文件</h5><p>在项目窗口内,在“dataanalysis”文件夹上单击鼠标右键,在弹出的菜单中点击“新建”,再选择“Python File”。</p>
<p><img src="./水滴实验营_files/3-新建py文件.png" alt=""></p>
<p>在弹出的对话框中(如下图所示),在“Name”中输入“starclass”,点击“确定”按钮,进入“starclass.py”文件编辑器。</p>
<p><img src="./水滴实验营_files/4-新建文件名称.png" alt=""></p>
<h5 id="h5-3-1-4-1-2-python-"><a name="3.1.4.1.2 编写python程序代码" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.1.2 编写python程序代码</h5><p>输入下面代码到新建的“starclass.py”文件中并保存。</p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="kwd">from</span><span class="pln"> pyecharts </span><span class="kwd">import</span><span class="pln"> </span><span class="typ">Bar</span></code></li><li class="L1"><code><span class="kwd">from</span><span class="pln"> spider </span><span class="kwd">import</span><span class="pln"> dbconnect</span></code></li><li class="L2"><code><span class="com"># 统计一个城市各星级饭店数量函数</span></code></li><li class="L3"><code><span class="kwd">def</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="typ">City</span><span class="pun">):</span></code></li><li class="L4"><code><span class="pln"> </span><span class="com">#连接数据库</span></code></li><li class="L5"><code><span class="pln"> mysql_db </span><span class="pun">=</span><span class="pln"> dbconnect</span><span class="pun">.</span><span class="typ">DBHelper</span><span class="pun">()</span></code></li><li class="L6"><code><span class="pln"> mysql_db</span><span class="pun">.</span><span class="pln">connectDatabase</span><span class="pun">();</span></code></li><li class="L7"><code><span class="pln"> </span><span class="com">#查询有哪些星级,返回唯一不同的星级</span></code></li><li class="L8"><code><span class="pln"> starsql </span><span class="pun">=</span><span class="pln"> </span><span class="str">"SELECT DISTINCT(shopPower) from dazhongoutput ;"</span></code></li><li class="L9"><code><span class="pln"> data </span><span class="pun">=</span><span class="pln"> mysql_db</span><span class="pun">.</span><span class="kwd">select</span><span class="pun">(</span><span class="pln">starsql</span><span class="pun">)</span><span class="pln"> </span><span class="com">#返回所有查询记录行,每一行是一个元组,所有行的元组再组装成一个元组data</span></code></li><li class="L0"><code><span class="pln"> list_code </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L1"><code><span class="pln"> </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> data</span><span class="pun">:</span></code></li><li class="L2"><code><span class="pln"> </span><span class="com">#选择该城市某个星级按照商铺ID统计商铺数量</span></code></li><li class="L3"><code><span class="pln"> countsql </span><span class="pun">=</span><span class="pln"> </span><span class="str">'''SELECT COUNT(shopId) from dazhongoutput where shopPower=%s and city="%s";'''</span><span class="pln"> </span><span class="pun">%</span><span class="pln"> </span><span class="pun">(</span><span class="pln">i</span><span class="pun">[</span><span class="lit">0</span><span class="pun">],</span><span class="pln"> </span><span class="typ">City</span><span class="pun">)</span></code></li><li class="L4"><code><span class="pln"> countstar_city </span><span class="pun">=</span><span class="pln"> mysql_db</span><span class="pun">.</span><span class="kwd">select</span><span class="pun">(</span><span class="pln">countsql</span><span class="pun">)</span></code></li><li class="L5"><code><span class="pln"> </span><span class="com">#将所有星级和饭店数量放于list_code列表</span></code></li><li class="L6"><code><span class="pln"> </span><span class="kwd">for</span><span class="pln"> m </span><span class="kwd">in</span><span class="pln"> countstar_city</span><span class="pun">:</span></code></li><li class="L7"><code><span class="pln"> list_code</span><span class="pun">.</span><span class="pln">append</span><span class="pun">([</span><span class="pln">i</span><span class="pun">[</span><span class="lit">0</span><span class="pun">],</span><span class="pln"> m</span><span class="pun">[</span><span class="lit">0</span><span class="pun">]])</span></code></li><li class="L8"><code><span class="pln"> </span><span class="kwd">print</span><span class="pun">(</span><span class="pln">list_code</span><span class="pun">)</span></code></li><li class="L9"><code><span class="pln"> </span><span class="com">#返回商铺星级、数量</span></code></li><li class="L0"><code><span class="pln"> </span><span class="kwd">return</span><span class="pln"> list_code</span></code></li><li class="L1"><code></code></li><li class="L2"><code><span class="com"># 所有城市各星级饭店统计函数</span></code></li><li class="L3"><code><span class="kwd">def</span><span class="pln"> mianStar</span><span class="pun">():</span></code></li><li class="L4"><code><span class="pln"> </span><span class="com">#调用starshopCount(City)统计出下述城市各星级饭店数量</span></code></li><li class="L5"><code><span class="pln"> shop1 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"广州"</span><span class="pun">)</span></code></li><li class="L6"><code><span class="pln"> shop2 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"杭州"</span><span class="pun">)</span></code></li><li class="L7"><code><span class="pln"> shop3 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"南京"</span><span class="pun">)</span></code></li><li class="L8"><code><span class="pln"> shop4 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"成都"</span><span class="pun">)</span></code></li><li class="L9"><code><span class="pln"> shop5 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"武汉"</span><span class="pun">)</span></code></li><li class="L0"><code><span class="pln"> shop6 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"济南"</span><span class="pun">)</span></code></li><li class="L1"><code><span class="pln"> shop7 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"南宁"</span><span class="pun">)</span></code></li><li class="L2"><code><span class="pln"> shop8 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"合肥"</span><span class="pun">)</span></code></li><li class="L3"><code><span class="pln"> shop9 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"昆明"</span><span class="pun">)</span></code></li><li class="L4"><code><span class="pln"> shop10 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"西安"</span><span class="pun">)</span></code></li><li class="L5"><code><span class="pln"> shop11 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"北京"</span><span class="pun">)</span></code></li><li class="L6"><code><span class="pln"> shop12 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"上海"</span><span class="pun">)</span></code></li><li class="L7"><code><span class="pln"> shop13 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"深圳"</span><span class="pun">)</span></code></li><li class="L8"><code><span class="pln"> shop14 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"天津"</span><span class="pun">)</span></code></li><li class="L9"><code><span class="pln"> shop15 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"重庆"</span><span class="pun">)</span></code></li><li class="L0"><code><span class="pln"> shop16 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"青岛"</span><span class="pun">)</span></code></li><li class="L1"><code><span class="pln"> shop17 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"威海"</span><span class="pun">)</span></code></li><li class="L2"><code><span class="pln"> shop18 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"长春"</span><span class="pun">)</span></code></li><li class="L3"><code><span class="pln"> shop19 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"大连"</span><span class="pun">)</span></code></li><li class="L4"><code><span class="pln"> shop20 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"贵阳"</span><span class="pun">)</span></code></li><li class="L5"><code></code></li><li class="L6"><code><span class="pln"> </span><span class="com">#定义X轴和Y轴显示数据列表,开始为NULL</span></code></li><li class="L7"><code><span class="pln"> list_x </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L8"><code><span class="pln"> list_y1</span><span class="pun">,</span><span class="pln"> list_y2</span><span class="pun">,</span><span class="pln"> list_y3</span><span class="pun">,</span><span class="pln"> list_y4</span><span class="pun">,</span><span class="pln"> list_y5</span><span class="pun">,</span><span class="pln"> list_y6</span><span class="pun">,</span><span class="pln"> list_y7</span><span class="pun">,</span><span class="pln"> list_y8</span><span class="pun">,</span><span class="pln"> list_y9</span><span class="pun">,</span><span class="pln"> list_y10</span><span class="pun">,</span><span class="pln">list_y11</span><span class="pun">,</span><span class="pln"> list_y12</span><span class="pun">,</span><span class="pln"> list_y13</span><span class="pun">,</span><span class="pln"> list_y14</span><span class="pun">,</span><span class="pln"> list_y15</span><span class="pun">,</span><span class="pln"> list_y16</span><span class="pun">,</span><span class="pln"> list_y17</span><span class="pun">,</span><span class="pln"> list_y18</span><span class="pun">,</span><span class="pln"> list_y19</span><span class="pun">,</span><span class="pln"> list_y20 </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L9"><code><span class="pln"> </span><span class="com">#数据列表赋值,zip将所有城市星级饭店列表,打包为一个包含多个元组的zip迭代对象,多变量循环</span></code></li><li class="L0"><code><span class="pln"> </span><span class="kwd">for</span><span class="pln"> data1</span><span class="pun">,</span><span class="pln"> data2</span><span class="pun">,</span><span class="pln"> data3</span><span class="pun">,</span><span class="pln"> data4</span><span class="pun">,</span><span class="pln"> data5</span><span class="pun">,</span><span class="pln"> data6</span><span class="pun">,</span><span class="pln"> data7</span><span class="pun">,</span><span class="pln"> data8</span><span class="pun">,</span><span class="pln"> data9</span><span class="pun">,</span><span class="pln"> data10</span><span class="pun">,</span><span class="pln"> data11</span><span class="pun">,</span><span class="pln"> data12</span><span class="pun">,</span><span class="pln"> data13</span><span class="pun">,</span><span class="pln"> data14</span><span class="pun">,</span><span class="pln"> data15</span><span class="pun">,</span><span class="pln"> data16</span><span class="pun">,</span><span class="pln"> data17</span><span class="pun">,</span><span class="pln"> data18</span><span class="pun">,</span><span class="pln"> data19</span><span class="pun">,</span><span class="pln"> data20 </span><span class="kwd">in</span><span class="pln"> zip</span><span class="pun">(</span></code></li><li class="L1"><code><span class="pln"> shop1</span><span class="pun">,</span><span class="pln"> shop2</span><span class="pun">,</span><span class="pln"> shop3</span><span class="pun">,</span><span class="pln"> shop4</span><span class="pun">,</span><span class="pln"> shop5</span><span class="pun">,</span><span class="pln"> shop6</span><span class="pun">,</span><span class="pln"> shop7</span><span class="pun">,</span><span class="pln"> shop8</span><span class="pun">,</span><span class="pln"> shop9</span><span class="pun">,</span><span class="pln"> shop10</span><span class="pun">,</span></code></li><li class="L2"><code><span class="pln"> shop11</span><span class="pun">,</span><span class="pln"> shop12</span><span class="pun">,</span><span class="pln"> shop13</span><span class="pun">,</span><span class="pln"> shop14</span><span class="pun">,</span><span class="pln"> shop15</span><span class="pun">,</span><span class="pln"> shop16</span><span class="pun">,</span><span class="pln"> shop17</span><span class="pun">,</span><span class="pln"> shop18</span><span class="pun">,</span><span class="pln"> shop19</span><span class="pun">,</span><span class="pln"> shop20</span></code></li><li class="L3"><code><span class="pln"> </span><span class="pun">):</span></code></li><li class="L4"><code><span class="pln"> </span><span class="com">#提取星级,建立X轴列表</span></code></li><li class="L5"><code><span class="pln"> </span><span class="kwd">if</span><span class="pln"> data1</span><span class="pun">[</span><span class="lit">0</span><span class="pun">]</span><span class="pln"> </span><span class="pun">==</span><span class="pln"> </span><span class="str">'30'</span><span class="pun">:</span></code></li><li class="L6"><code><span class="pln"> list_x</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="str">"三星级"</span><span class="pun">)</span></code></li><li class="L7"><code><span class="pln"> </span><span class="kwd">elif</span><span class="pln"> data1</span><span class="pun">[</span><span class="lit">0</span><span class="pun">]</span><span class="pln"> </span><span class="pun">==</span><span class="pln"> </span><span class="str">'35'</span><span class="pun">:</span></code></li><li class="L8"><code><span class="pln"> list_x</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="str">"三星半"</span><span class="pun">)</span></code></li><li class="L9"><code><span class="pln"> </span><span class="kwd">elif</span><span class="pln"> data1</span><span class="pun">[</span><span class="lit">0</span><span class="pun">]</span><span class="pln"> </span><span class="pun">==</span><span class="pln"> </span><span class="str">'40'</span><span class="pun">:</span></code></li><li class="L0"><code><span class="pln"> list_x</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="str">"四星级"</span><span class="pun">)</span></code></li><li class="L1"><code><span class="pln"> </span><span class="kwd">elif</span><span class="pln"> data1</span><span class="pun">[</span><span class="lit">0</span><span class="pun">]</span><span class="pln"> </span><span class="pun">==</span><span class="pln"> </span><span class="str">'45'</span><span class="pun">:</span></code></li><li class="L2"><code><span class="pln"> list_x</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="str">"四星半"</span><span class="pun">)</span></code></li><li class="L3"><code><span class="pln"> </span><span class="kwd">else</span><span class="pun">:</span></code></li><li class="L4"><code><span class="pln"> list_x</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="str">"五星级"</span><span class="pun">)</span></code></li><li class="L5"><code><span class="pln"> </span><span class="com">#20城市不同星级饭店数量</span></code></li><li class="L6"><code><span class="pln"> list_y1</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data1</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L7"><code><span class="pln"> list_y2</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data2</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L8"><code><span class="pln"> list_y3</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data3</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L9"><code><span class="pln"> list_y4</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data4</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L0"><code><span class="pln"> list_y5</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data5</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L1"><code><span class="pln"> list_y6</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data6</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L2"><code><span class="pln"> list_y7</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data7</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L3"><code><span class="pln"> list_y8</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data8</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L4"><code><span class="pln"> list_y9</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data9</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L5"><code><span class="pln"> list_y10</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data10</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L6"><code><span class="pln"> list_y11</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data11</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L7"><code><span class="pln"> list_y12</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data12</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L8"><code><span class="pln"> list_y13</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data13</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L9"><code><span class="pln"> list_y14</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data14</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L0"><code><span class="pln"> list_y15</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data15</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L1"><code><span class="pln"> list_y16</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data16</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L2"><code><span class="pln"> list_y17</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data17</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L3"><code><span class="pln"> list_y18</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data18</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L4"><code><span class="pln"> list_y19</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data19</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L5"><code><span class="pln"> list_y20</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data20</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L6"><code><span class="pln"> </span><span class="kwd">print</span><span class="pun">(</span><span class="pln">list_x</span><span class="pun">,</span><span class="pln"> list_y1</span><span class="pun">,</span><span class="pln"> list_y2</span><span class="pun">,</span><span class="pln"> list_y3</span><span class="pun">,</span><span class="pln"> list_y4</span><span class="pun">,</span><span class="pln"> list_y5</span><span class="pun">,</span><span class="pln"> list_y6</span><span class="pun">,</span><span class="pln"> list_y7</span><span class="pun">,</span><span class="pln"> list_y8</span><span class="pun">,</span><span class="pln"> list_y9</span><span class="pun">,</span><span class="pln"> list_y10</span><span class="pun">,</span></code></li><li class="L7"><code><span class="pln"> list_y11</span><span class="pun">,</span><span class="pln"> list_y12</span><span class="pun">,</span><span class="pln"> list_y13</span><span class="pun">,</span><span class="pln"> list_y14</span><span class="pun">,</span><span class="pln"> list_y15</span><span class="pun">,</span><span class="pln"> list_y16</span><span class="pun">,</span><span class="pln"> list_y17</span><span class="pun">,</span><span class="pln"> list_y18</span><span class="pun">,</span><span class="pln"> list_y19</span><span class="pun">,</span><span class="pln"> list_y20</span></code></li><li class="L8"><code><span class="pln"> </span><span class="pun">)</span></code></li><li class="L9"><code><span class="pln"> </span><span class="com"># 饭店星级排行画图</span></code></li><li class="L0"><code><span class="pln"> plotstar</span><span class="pun">(</span><span class="pln">list_x</span><span class="pun">,</span><span class="pln"> list_y1</span><span class="pun">,</span><span class="pln"> list_y2</span><span class="pun">,</span><span class="pln"> list_y3</span><span class="pun">,</span><span class="pln"> list_y4</span><span class="pun">,</span><span class="pln"> list_y5</span><span class="pun">,</span><span class="pln"> list_y6</span><span class="pun">,</span><span class="pln"> list_y7</span><span class="pun">,</span><span class="pln"> list_y8</span><span class="pun">,</span><span class="pln"> list_y9</span><span class="pun">,</span><span class="pln"> list_y10</span><span class="pun">,</span></code></li><li class="L1"><code><span class="pln"> list_y11</span><span class="pun">,</span><span class="pln"> list_y12</span><span class="pun">,</span><span class="pln"> list_y13</span><span class="pun">,</span><span class="pln"> list_y14</span><span class="pun">,</span><span class="pln"> list_y15</span><span class="pun">,</span><span class="pln"> list_y16</span><span class="pun">,</span><span class="pln"> list_y17</span><span class="pun">,</span><span class="pln"> list_y18</span><span class="pun">,</span><span class="pln"> list_y19</span><span class="pun">,</span><span class="pln"> list_y20</span></code></li><li class="L2"><code><span class="pln"> </span><span class="pun">)</span></code></li><li class="L3"><code></code></li><li class="L4"><code><span class="com"># 饭店星级排行画图函数</span></code></li><li class="L5"><code><span class="kwd">def</span><span class="pln"> plotstar</span><span class="pun">(</span><span class="pln">list_x</span><span class="pun">,</span><span class="pln"> list_y1</span><span class="pun">,</span><span class="pln"> list_y2</span><span class="pun">,</span><span class="pln"> list_y3</span><span class="pun">,</span><span class="pln"> list_y4</span><span class="pun">,</span><span class="pln"> list_y5</span><span class="pun">,</span><span class="pln"> list_y6</span><span class="pun">,</span><span class="pln"> list_y7</span><span class="pun">,</span><span class="pln"> list_y8</span><span class="pun">,</span><span class="pln"> list_y9</span><span class="pun">,</span><span class="pln"> list_y10</span><span class="pun">,</span></code></li><li class="L6"><code><span class="pln"> list_y11</span><span class="pun">,</span><span class="pln"> list_y12</span><span class="pun">,</span><span class="pln"> list_y13</span><span class="pun">,</span><span class="pln"> list_y14</span><span class="pun">,</span><span class="pln"> list_y15</span><span class="pun">,</span><span class="pln"> list_y16</span><span class="pun">,</span><span class="pln"> list_y17</span><span class="pun">,</span><span class="pln"> list_y18</span><span class="pun">,</span><span class="pln"> list_y19</span><span class="pun">,</span><span class="pln"> list_y20</span><span class="pun">):</span></code></li><li class="L7"><code><span class="pln"> attr </span><span class="pun">=</span><span class="pln"> list_x</span></code></li><li class="L8"><code><span class="pln"> </span><span class="com">#设置柱状图大小标题,可根据实际数据情况统计</span></code></li><li class="L9"><code><span class="pln"> starbar </span><span class="pun">=</span><span class="pln"> </span><span class="typ">Bar</span><span class="pun">(</span><span class="str">"20个大中城市饭店星级统计"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"数据来源于大众点评TOP100"</span><span class="pun">,</span><span class="pln">width</span><span class="pun">=</span><span class="lit">1300</span><span class="pun">,</span><span class="pln">height</span><span class="pun">=</span><span class="lit">500</span><span class="pun">)</span></code></li><li class="L0"><code><span class="pln"> </span><span class="com">##添加柱状图数据和配置项,标记最大值和最小值,最右侧设置工具栏,可对图进行更多操作,图例位置</span></code></li><li class="L1"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"广州"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y1</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L2"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"杭州"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y2</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L3"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"南京"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y3</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L4"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"成都"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y4</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L5"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"武汉"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y5</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L6"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"济南"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y6</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L7"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"南宁"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y7</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L8"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"合肥"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y8</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L9"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"昆明"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y9</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L0"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"西安"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y10</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L1"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"北京"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y11</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L2"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"上海"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y12</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L3"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"深圳"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y13</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L4"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"天津"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y14</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L5"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"重庆"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y15</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L6"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"青岛"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y16</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L7"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"威海"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y17</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L8"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"长春"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y18</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L9"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"大连"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y19</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L0"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"贵阳"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y20</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">],</span><span class="pln"> is_more_utils</span><span class="pun">=</span><span class="kwd">True</span><span class="pun">,</span><span class="pln"> legend_top</span><span class="pun">=</span><span class="str">'bottom'</span><span class="pun">)</span></code></li><li class="L1"><code><span class="com">#生成html文件数据可视化展示</span></code></li><li class="L2"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="pln">render</span><span class="pun">(</span><span class="str">"20个大中城市饭店星级排名.html"</span><span class="pun">)</span></code></li><li class="L3"><code></code></li><li class="L4"><code><span class="kwd">if</span><span class="pln"> __name__ </span><span class="pun">==</span><span class="pln"> </span><span class="str">'__main__'</span><span class="pun">:</span></code></li><li class="L5"><code><span class="pln"> mianStar</span><span class="pun">()</span></code></li></ol></pre><h5 id="h5-3-1-4-1-3-"><a name="3.1.4.1.3 设置依赖包" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.1.3 设置依赖包</h5><p>选中项目,点击“文件”-&gt;“设置”,查看项目下的“Project Interpreter”,点击右侧的“+”号</p>
<p><img src="./水滴实验营_files/5-添加依赖包1.png" alt=""></p>
<p>弹出如下界面后,在查询框中输入“pytest-runner”,然后在列表中选择“pytest-runner”,点击“install package”,如图所示,提示安装成功。</p>
<p><img src="./水滴实验营_files/6-依赖包安装成功.png" alt=""></p>
<p>然后按照上述操作继续安装“pyecharts-jupyter-installer”、“pyecharts”和“pyecharts-snapshot”库,安装完成后关闭安装页面,进入“Project Interpreter”设置页面后,点击“确定”,回到项目主页面。若出现安装不成功,可能是由于网络连接问题,请再次安装。</p>
<h5 id="h5-3-1-4-1-4-"><a name="3.1.4.1.4 执行程序" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.1.4 执行程序</h5><p>选中“starclass.py”,点击右键选择“运行starclass”,执行程序,运行窗口显示打印信息为商铺星级、数量,在项目“dataanalysis”目录下,可以看到生成的“20个大中城市饭店星级排名.html”,如下图所示。</p>
<p><img src="./水滴实验营_files/9-星级排名运行结果.png" alt=""></p>
<h5 id="h5-3-1-4-1-5-"><a name="3.1.4.1.5 查看可视化数据" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.1.5 查看可视化数据</h5><p>进入该程序文件所在文件夹C:\dazhongspider\dataanalysis,可以看到该HTML文件,打开后可看到统计图表(如下面链接展示)。</p>
<p><a href="https://jn1.is.shanhe.com/shuidilab/images/qilucasetrainning/analysis/20%E4%B8%AA%E5%A4%A7%E4%B8%AD%E5%9F%8E%E5%B8%82%E9%A5%AD%E5%BA%97%E6%98%9F%E7%BA%A7%E6%8E%92%E5%90%8D.html" title="20个大中城市饭店星级排名">https://jn1.is.shanhe.com/shuidilab/images/qilucasetrainning/analysis/20个大中城市饭店星级排名.html</a></p>
<p>点击最右侧各个图标,可以查看数据图表、折线图等。</p>
<h4 id="h4-3-1-4-2-3-"><a name="3.1.4.2 山东3大城市菜品分类统计" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.2 山东3大城市菜品分类统计</h4><p>统计山东省内济南、青岛、威海三个城市的菜品分类。</p>
<h5 id="h5-3-1-4-2-1-"><a name="3.1.4.2.1 新建代码" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.2.1 新建代码</h5><p>参照3.1.4.1.1步骤,在项目窗口内,在“dataanalysis”文件夹上单击鼠标右键,在弹出的菜单中点击“新建”,再选择“Python File”,在弹出的对话框中,在“Name”中输入“foodclass”,点击“确定”按钮,进入.py文件编辑器。</p>
<h5 id="h5-3-1-4-2-2-python-"><a name="3.1.4.2.2 编写python程序代码" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.2.2 编写python程序代码</h5><p>输入如下代码到新建的“foodclass.py”文件中并保存。</p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="kwd">from</span><span class="pln"> pyecharts </span><span class="kwd">import</span><span class="pln"> </span><span class="typ">Pie</span><span class="pun">,</span><span class="pln"> </span><span class="typ">Page</span><span class="pun">,</span><span class="typ">Bar</span></code></li><li class="L1"><code><span class="kwd">from</span><span class="pln"> spider </span><span class="kwd">import</span><span class="pln"> dbconnect</span></code></li><li class="L2"><code></code></li><li class="L3"><code><span class="com">#查询统计某城市菜品分类名</span></code></li><li class="L4"><code><span class="kwd">def</span><span class="pln"> foodClass</span><span class="pun">(</span><span class="pln">city</span><span class="pun">):</span></code></li><li class="L5"><code><span class="pln"> </span><span class="com">#连接数据库</span></code></li><li class="L6"><code><span class="pln"> mysql_db </span><span class="pun">=</span><span class="pln"> dbconnect</span><span class="pun">.</span><span class="typ">DBHelper</span><span class="pun">()</span></code></li><li class="L7"><code><span class="pln"> mysql_db</span><span class="pun">.</span><span class="pln">connectDatabase</span><span class="pun">();</span></code></li><li class="L8"><code><span class="pln"> fsql </span><span class="pun">=</span><span class="pln"> </span><span class="str">'''SELECT COUNT(mainCategoryName),mainCategoryName from dazhongoutput where city="%s" GROUP BY mainCategoryName;'''</span><span class="pun">%(</span><span class="pln">city</span><span class="pun">)</span></code></li><li class="L9"><code><span class="pln"> </span><span class="com">#查询获得菜品分类商店梳理、菜品分类名称数据返回</span></code></li><li class="L0"><code><span class="pln"> listData </span><span class="pun">=</span><span class="pln"> mysql_db</span><span class="pun">.</span><span class="kwd">select</span><span class="pun">(</span><span class="pln">fsql</span><span class="pun">)</span></code></li><li class="L1"><code><span class="pln"> </span><span class="kwd">return</span><span class="pln"> listData</span></code></li><li class="L2"><code></code></li><li class="L3"><code><span class="com">#菜品分类画图函数</span></code></li><li class="L4"><code><span class="kwd">def</span><span class="pln"> plotfoodclass</span><span class="pun">():</span></code></li><li class="L5"><code><span class="pln"> </span><span class="com">#查询统计济南的菜品分类以及数量</span></code></li><li class="L6"><code><span class="pln"> list_address </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L7"><code><span class="pln"> list_sum </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L8"><code><span class="pln"> </span><span class="pun">[</span><span class="pln">list_sum</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">i</span><span class="pun">[</span><span class="lit">0</span><span class="pun">])</span><span class="pln"> </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> foodClass</span><span class="pun">(</span><span class="str">"济南"</span><span class="pun">)]</span></code></li><li class="L9"><code><span class="pln"> </span><span class="pun">[</span><span class="pln">list_address</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">i</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span><span class="pln"> </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> foodClass</span><span class="pun">(</span><span class="str">"济南"</span><span class="pun">)]</span></code></li><li class="L0"><code><span class="pln"> </span><span class="kwd">print</span><span class="pun">(</span><span class="pln">list_address</span><span class="pun">)</span></code></li><li class="L1"><code><span class="pln"> </span><span class="kwd">print</span><span class="pun">(</span><span class="pln">list_sum</span><span class="pun">)</span></code></li><li class="L2"><code><span class="pln"> </span><span class="com">#定义页面</span></code></li><li class="L3"><code><span class="pln"> page </span><span class="pun">=</span><span class="pln"> </span><span class="typ">Page</span><span class="pun">()</span></code></li><li class="L4"><code><span class="pln"> </span><span class="com">#添加饼图的数据和配置项</span></code></li><li class="L5"><code><span class="pln"> fpie1 </span><span class="pun">=</span><span class="pln"> </span><span class="typ">Pie</span><span class="pun">(</span><span class="str">"济南菜品分类"</span><span class="pun">,</span><span class="pln"> title_pos</span><span class="pun">=</span><span class="str">'center'</span><span class="pun">)</span></code></li><li class="L6"><code><span class="pln"> fpie1</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">""</span><span class="pun">,</span><span class="pln"> list_address</span><span class="pun">,</span><span class="pln"> list_sum</span><span class="pun">,</span><span class="pln"> radius</span><span class="pun">=[</span><span class="lit">30</span><span class="pun">,</span><span class="pln"> </span><span class="lit">75</span><span class="pun">],</span><span class="pln"> label_text_color</span><span class="pun">=</span><span class="kwd">None</span><span class="pun">,</span></code></li><li class="L7"><code><span class="pln"> is_label_show</span><span class="pun">=</span><span class="kwd">True</span><span class="pun">,</span><span class="pln"> legend_orient</span><span class="pun">=</span><span class="str">'vertical'</span><span class="pun">,</span></code></li><li class="L8"><code><span class="pln"> legend_pos</span><span class="pun">=</span><span class="str">'auto'</span><span class="pun">,</span><span class="pln"> is_legend_show</span><span class="pun">=</span><span class="kwd">False</span><span class="pun">)</span></code></li><li class="L9"><code><span class="pln"> </span><span class="com">#添加柱状图的数据和配置型项</span></code></li><li class="L0"><code><span class="pln"> fbar1 </span><span class="pun">=</span><span class="pln"> </span><span class="typ">Bar</span><span class="pun">(</span><span class="str">"济南菜品分类"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"数据来源于大众点评TOP100"</span><span class="pun">,</span><span class="pln">title_pos </span><span class="pun">=</span><span class="str">'center'</span><span class="pun">)</span></code></li><li class="L1"><code><span class="pln"> fbar1</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"济南"</span><span class="pun">,</span><span class="pln"> list_address</span><span class="pun">,</span><span class="pln"> list_sum</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">],</span><span class="pln">legend_pos</span><span class="pun">=</span><span class="str">"right"</span><span class="pun">)</span></code></li><li class="L2"><code></code></li><li class="L3"><code><span class="pln"> </span><span class="com"># 查询统计青岛的菜品分类以及数量</span></code></li><li class="L4"><code><span class="pln"> list_address </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L5"><code><span class="pln"> list_sum </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L6"><code><span class="pln"> </span><span class="pun">[</span><span class="pln">list_sum</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">i</span><span class="pun">[</span><span class="lit">0</span><span class="pun">])</span><span class="pln"> </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> foodClass</span><span class="pun">(</span><span class="str">"青岛"</span><span class="pun">)]</span></code></li><li class="L7"><code><span class="pln"> </span><span class="pun">[</span><span class="pln">list_address</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">i</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span><span class="pln"> </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> foodClass</span><span class="pun">(</span><span class="str">"青岛"</span><span class="pun">)]</span></code></li><li class="L8"><code><span class="pln"> </span><span class="kwd">print</span><span class="pun">(</span><span class="pln">list_address</span><span class="pun">)</span></code></li><li class="L9"><code><span class="pln"> </span><span class="kwd">print</span><span class="pun">(</span><span class="pln">list_sum</span><span class="pun">)</span></code></li><li class="L0"><code><span class="pln"> </span><span class="com"># 添加饼图的数据和配置项</span></code></li><li class="L1"><code><span class="pln"> fpie2 </span><span class="pun">=</span><span class="pln"> </span><span class="typ">Pie</span><span class="pun">(</span><span class="str">"青岛菜品分类"</span><span class="pun">,</span><span class="pln"> title_pos</span><span class="pun">=</span><span class="str">'center'</span><span class="pun">)</span></code></li><li class="L2"><code><span class="pln"> fpie2</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">""</span><span class="pun">,</span><span class="pln"> list_address</span><span class="pun">,</span><span class="pln"> list_sum</span><span class="pun">,</span><span class="pln"> radius</span><span class="pun">=[</span><span class="lit">30</span><span class="pun">,</span><span class="pln"> </span><span class="lit">75</span><span class="pun">],</span><span class="pln"> label_text_color</span><span class="pun">=</span><span class="kwd">None</span><span class="pun">,</span></code></li><li class="L3"><code><span class="pln"> is_label_show</span><span class="pun">=</span><span class="kwd">True</span><span class="pun">,</span><span class="pln"> legend_orient</span><span class="pun">=</span><span class="str">'vertical'</span><span class="pun">,</span></code></li><li class="L4"><code><span class="pln"> legend_pos</span><span class="pun">=</span><span class="str">'auto'</span><span class="pun">,</span><span class="pln"> is_legend_show</span><span class="pun">=</span><span class="kwd">False</span><span class="pun">)</span></code></li><li class="L5"><code><span class="pln"> </span><span class="com"># 添加柱状图的数据和配置型项</span></code></li><li class="L6"><code><span class="pln"> fbar2 </span><span class="pun">=</span><span class="pln"> </span><span class="typ">Bar</span><span class="pun">(</span><span class="str">"青岛菜品分类"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"数据来源于大众点评TOP100"</span><span class="pun">,</span><span class="pln"> title_pos</span><span class="pun">=</span><span class="str">'center'</span><span class="pun">)</span></code></li><li class="L7"><code><span class="pln"> fbar2</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"青岛"</span><span class="pun">,</span><span class="pln"> list_address</span><span class="pun">,</span><span class="pln"> list_sum</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">],</span><span class="pln"> legend_pos</span><span class="pun">=</span><span class="str">"right"</span><span class="pun">)</span></code></li><li class="L8"><code></code></li><li class="L9"><code><span class="pln"> </span><span class="com"># 查询统计威海的菜品分类以及数量</span></code></li><li class="L0"><code><span class="pln"> list_address </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L1"><code><span class="pln"> list_sum </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L2"><code><span class="pln"> </span><span class="pun">[</span><span class="pln">list_sum</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">i</span><span class="pun">[</span><span class="lit">0</span><span class="pun">])</span><span class="pln"> </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> foodClass</span><span class="pun">(</span><span class="str">"威海"</span><span class="pun">)]</span></code></li><li class="L3"><code><span class="pln"> </span><span class="pun">[</span><span class="pln">list_address</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">i</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span><span class="pln"> </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> foodClass</span><span class="pun">(</span><span class="str">"威海"</span><span class="pun">)]</span></code></li><li class="L4"><code><span class="pln"> </span><span class="kwd">print</span><span class="pun">(</span><span class="pln">list_address</span><span class="pun">)</span></code></li><li class="L5"><code><span class="pln"> </span><span class="kwd">print</span><span class="pun">(</span><span class="pln">list_sum</span><span class="pun">)</span></code></li><li class="L6"><code><span class="pln"> </span><span class="com"># 添加饼图的数据和配置项</span></code></li><li class="L7"><code><span class="pln"> fpie3 </span><span class="pun">=</span><span class="pln"> </span><span class="typ">Pie</span><span class="pun">(</span><span class="str">"威海菜品分类"</span><span class="pun">,</span><span class="pln"> title_pos</span><span class="pun">=</span><span class="str">'center'</span><span class="pun">)</span></code></li><li class="L8"><code><span class="pln"> fpie3</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">""</span><span class="pun">,</span><span class="pln"> list_address</span><span class="pun">,</span><span class="pln"> list_sum</span><span class="pun">,</span><span class="pln"> radius</span><span class="pun">=[</span><span class="lit">30</span><span class="pun">,</span><span class="pln"> </span><span class="lit">75</span><span class="pun">],</span><span class="pln"> label_text_color</span><span class="pun">=</span><span class="kwd">None</span><span class="pun">,</span></code></li><li class="L9"><code><span class="pln"> is_label_show</span><span class="pun">=</span><span class="kwd">True</span><span class="pun">,</span><span class="pln"> legend_orient</span><span class="pun">=</span><span class="str">'vertical'</span><span class="pun">,</span></code></li><li class="L0"><code><span class="pln"> legend_pos</span><span class="pun">=</span><span class="str">'auto'</span><span class="pun">,</span><span class="pln"> is_legend_show</span><span class="pun">=</span><span class="kwd">False</span><span class="pun">)</span></code></li><li class="L1"><code><span class="pln"> </span><span class="com"># 添加柱状图的数据和配置型项</span></code></li><li class="L2"><code><span class="pln"> fbar3 </span><span class="pun">=</span><span class="pln"> </span><span class="typ">Bar</span><span class="pun">(</span><span class="str">"威海菜品分类"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"数据来源于大众点评TOP100"</span><span class="pun">,</span><span class="pln"> title_pos</span><span class="pun">=</span><span class="str">'center'</span><span class="pun">)</span></code></li><li class="L3"><code><span class="pln"> fbar3</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"威海"</span><span class="pun">,</span><span class="pln"> list_address</span><span class="pun">,</span><span class="pln"> list_sum</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">],</span><span class="pln"> legend_pos</span><span class="pun">=</span><span class="str">"right"</span><span class="pun">)</span></code></li><li class="L4"><code></code></li><li class="L5"><code><span class="pln"> </span><span class="com">#页面上添加图形</span></code></li><li class="L6"><code><span class="pln"> page</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="pln">fpie1</span><span class="pun">)</span></code></li><li class="L7"><code><span class="pln"> page</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="pln">fbar1</span><span class="pun">)</span></code></li><li class="L8"><code><span class="pln"> page</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="pln">fpie2</span><span class="pun">)</span></code></li><li class="L9"><code><span class="pln"> page</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="pln">fbar2</span><span class="pun">)</span></code></li><li class="L0"><code><span class="pln"> page</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="pln">fpie3</span><span class="pun">)</span></code></li><li class="L1"><code><span class="pln"> page</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="pln">fbar3</span><span class="pun">)</span></code></li><li class="L2"><code></code></li><li class="L3"><code><span class="pln"> page</span><span class="pun">.</span><span class="pln">render</span><span class="pun">(</span><span class="str">"山东3大城市菜品分类.html"</span><span class="pun">)</span></code></li><li class="L4"><code><span class="kwd">if</span><span class="pln"> __name__ </span><span class="pun">==</span><span class="pln"> </span><span class="str">'__main__'</span><span class="pun">:</span></code></li><li class="L5"><code><span class="pln"> plotfoodclass</span><span class="pun">()</span></code></li></ol></pre><h5 id="h5-3-1-4-2-3-"><a name="3.1.4.2.3 设置依赖包" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.2.3 设置依赖包</h5><p>选中项目,点击“文件”-&gt;“设置”,查看项目下“Project Interpreter”,是否存在“pytest-runner”、“pyecharts-jupyter-installer”、“pyecharts-snapshot”和“pyecharts”依赖包,若无,则参照3.1.4.1.3进行安装。</p>
<h5 id="h5-3-1-4-2-4-"><a name="3.1.4.2.4 执行程序" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.2.4 执行程序</h5><p>选中“foodclass.py”,点击右键选择“运行foodclass”,执行程序,运行窗口显示打印信息为菜品分类、每种分类商铺数,“dataanalysis”目录下显示生成的HTML页面。</p>
<p><img src="./水滴实验营_files/13-大城市菜品分类.png" alt=""></p>
<h5 id="h5-3-1-4-2-5-"><a name="3.1.4.2.5 查看可视化数据" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.2.5 查看可视化数据</h5><p>进入该项目所在文件夹C:\dazhongspider\dataanalysis,可以看到该HTML文件,双击打开,可看到统计图表(如下面链接展示)。</p>
<p> <a href="https://jn1.is.shanhe.com/shuidilab/images/qilucasetrainning/analysis/%E5%B1%B1%E4%B8%9C3%E5%A4%A7%E5%9F%8E%E5%B8%82%E8%8F%9C%E5%93%81%E5%88%86%E7%B1%BB.html" title="山东3大城市菜品分类">https://jn1.is.shanhe.com/shuidilab/images/qilucasetrainning/analysis/山东3大城市菜品分类.html</a></p>
<p> 点击最右侧各个图标,可以查看数据图表、折线图等。</p>
<h4 id="h4-3-1-4-3-3-top20-"><a name="3.1.4.3 山东3大城市TOP20饭店服务的综合评价" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.3 山东3大城市TOP20饭店服务的综合评价</h4><p>针对山东省内济南、青岛、威海三个城市,评价每个店铺的“口味”、“环境”、“服务”,根据这些指标求平均得到综合评价指标,并根据综合评价指标对商铺进行排名,最终得到山东3大城市TOP20商铺。</p>
<h5 id="h5-3-1-4-3-1-"><a name="3.1.4.3.1 新建代码" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.3.1 新建代码</h5><p>参照3.1.4.1.1步骤,在项目窗口内,在“dataanalysis”文件夹上单击鼠标右键,在弹出的菜单中点击“新建”,再选择“Python File”,在弹出的对话框中(如图所示),在“Name”中输入“evaluatetop20”,点击“确定”按钮。</p>
<h5 id="h5-3-1-4-3-2-python-"><a name="3.1.4.3.2 编写python程序代码" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.3.2 编写python程序代码</h5><p>输入如下代码到新建的“evaluatetop20.py”文件中并保存。</p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="kwd">from</span><span class="pln"> pyecharts </span><span class="kwd">import</span><span class="pln"> </span><span class="typ">Line</span><span class="pun">,</span><span class="pln"> </span><span class="typ">Bar</span><span class="pun">,</span><span class="pln"> </span><span class="typ">Page</span><span class="pun">,</span><span class="pln"> </span><span class="typ">WordCloud</span></code></li><li class="L1"><code><span class="kwd">from</span><span class="pln"> spider </span><span class="kwd">import</span><span class="pln"> dbconnect</span></code></li><li class="L2"><code></code></li><li class="L3"><code><span class="com">#查询商铺名称、计算综合评价分数=(口味+环境+服务)/3的函数</span></code></li><li class="L4"><code><span class="kwd">def</span><span class="pln"> evaluateshop</span><span class="pun">(</span><span class="pln">city</span><span class="pun">):</span></code></li><li class="L5"><code><span class="pln"> </span><span class="com">#连接数据库</span></code></li><li class="L6"><code><span class="pln"> mysql_db </span><span class="pun">=</span><span class="pln"> dbconnect</span><span class="pun">.</span><span class="typ">DBHelper</span><span class="pun">()</span></code></li><li class="L7"><code><span class="pln"> mysql_db</span><span class="pun">.</span><span class="pln">connectDatabase</span><span class="pun">();</span></code></li><li class="L8"><code><span class="pln"> </span><span class="com">#定义查询语句,查询商铺名称、计算综合评价分数</span></code></li><li class="L9"><code><span class="pln"> nsql </span><span class="pun">=</span><span class="pln"> </span><span class="str">'''SELECT shopName,(tasteScore+environmentScore+serviceScore)/3.0 as avg from dazhongoutput WHERE city="%s" ORDER BY (tasteScore+environmentScore+serviceScore)/3.0 DESC;'''</span><span class="pun">%(</span><span class="pln">city</span><span class="pun">)</span></code></li><li class="L0"><code><span class="pln"> listData </span><span class="pun">=</span><span class="pln">mysql_db</span><span class="pun">.</span><span class="kwd">select</span><span class="pun">(</span><span class="pln">nsql</span><span class="pun">)</span></code></li><li class="L1"><code><span class="pln"> listShopName </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L2"><code><span class="pln"> listAvg </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L3"><code><span class="pln"> </span><span class="com">#返回商铺名称、综合评价分数</span></code></li><li class="L4"><code><span class="pln"> </span><span class="kwd">for</span><span class="pln"> data </span><span class="kwd">in</span><span class="pln"> listData</span><span class="pun">:</span></code></li><li class="L5"><code><span class="pln"> listShopName</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data</span><span class="pun">[</span><span class="lit">0</span><span class="pun">])</span></code></li><li class="L6"><code><span class="pln"> listAvg</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">round</span><span class="pun">(</span><span class="pln">data</span><span class="pun">[</span><span class="lit">1</span><span class="pun">],</span><span class="pln"> </span><span class="lit">2</span><span class="pun">))</span></code></li><li class="L7"><code><span class="pln"> </span><span class="kwd">return</span><span class="pln"> </span><span class="pun">[</span><span class="pln">listShopName</span><span class="pun">,</span><span class="pln">listAvg</span><span class="pun">]</span></code></li><li class="L8"><code></code></li><li class="L9"><code><span class="com">#TOP20商铺综合分数画图函数</span></code></li><li class="L0"><code><span class="kwd">def</span><span class="pln"> plottop20</span><span class="pun">(</span><span class="pln">list_y1</span><span class="pun">,</span><span class="pln"> list_y2</span><span class="pun">,</span><span class="pln"> list_y3</span><span class="pun">):</span></code></li><li class="L1"><code><span class="pln"> </span><span class="com">#定义页面</span></code></li><li class="L2"><code><span class="pln"> page </span><span class="pun">=</span><span class="pln"> </span><span class="typ">Page</span><span class="pun">()</span></code></li><li class="L3"><code></code></li><li class="L4"><code><span class="pln"> </span><span class="com">#折线图定义、添加数据和配置项</span></code></li><li class="L5"><code><span class="pln"> line1 </span><span class="pun">=</span><span class="pln"> </span><span class="typ">Line</span><span class="pun">(</span><span class="str">"济南饭店评分统计"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"数据来源于大众点评TOP100"</span><span class="pun">,</span><span class="pln"> width</span><span class="pun">=</span><span class="lit">1200</span><span class="pun">)</span></code></li><li class="L6"><code><span class="pln"> line1</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"综合"</span><span class="pun">,</span><span class="pln"> list_y1</span><span class="pun">[</span><span class="lit">0</span><span class="pun">][:</span><span class="lit">20</span><span class="pun">],</span><span class="pln"> list_y1</span><span class="pun">[</span><span class="lit">1</span><span class="pun">][:</span><span class="lit">20</span><span class="pun">],</span><span class="pln"> is_label_show</span><span class="pun">=</span><span class="kwd">True</span><span class="pun">,</span><span class="pln"> is_more_utils</span><span class="pun">=</span><span class="kwd">True</span><span class="pun">)</span></code></li><li class="L7"><code></code></li><li class="L8"><code><span class="pln"> line2 </span><span class="pun">=</span><span class="pln"> </span><span class="typ">Line</span><span class="pun">(</span><span class="str">"青岛饭店评分统计"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"数据来源于大众点评TOP100"</span><span class="pun">,</span><span class="pln"> width</span><span class="pun">=</span><span class="lit">1200</span><span class="pun">)</span></code></li><li class="L9"><code><span class="pln"> line2</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"综合"</span><span class="pun">,</span><span class="pln"> list_y2</span><span class="pun">[</span><span class="lit">0</span><span class="pun">][:</span><span class="lit">20</span><span class="pun">],</span><span class="pln"> list_y2</span><span class="pun">[</span><span class="lit">1</span><span class="pun">][:</span><span class="lit">20</span><span class="pun">],</span><span class="pln"> is_label_show</span><span class="pun">=</span><span class="kwd">True</span><span class="pun">,</span><span class="pln"> is_more_utils</span><span class="pun">=</span><span class="kwd">True</span><span class="pun">)</span></code></li><li class="L0"><code></code></li><li class="L1"><code><span class="pln"> line3 </span><span class="pun">=</span><span class="pln"> </span><span class="typ">Line</span><span class="pun">(</span><span class="str">"威海饭店评分统计"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"数据来源于大众点评TOP100"</span><span class="pun">,</span><span class="pln"> width</span><span class="pun">=</span><span class="lit">1200</span><span class="pun">)</span></code></li><li class="L2"><code><span class="pln"> line3</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"综合"</span><span class="pun">,</span><span class="pln"> list_y3</span><span class="pun">[</span><span class="lit">0</span><span class="pun">][:</span><span class="lit">20</span><span class="pun">],</span><span class="pln"> list_y3</span><span class="pun">[</span><span class="lit">1</span><span class="pun">][:</span><span class="lit">20</span><span class="pun">],</span><span class="pln"> is_label_show</span><span class="pun">=</span><span class="kwd">True</span><span class="pun">,</span><span class="pln"> is_more_utils</span><span class="pun">=</span><span class="kwd">True</span><span class="pun">)</span></code></li><li class="L3"><code><span class="pln"> </span><span class="com">#页面添加图形</span></code></li><li class="L4"><code><span class="pln"> page</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="pln">line1</span><span class="pun">)</span></code></li><li class="L5"><code><span class="pln"> page</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="pln">line2</span><span class="pun">)</span></code></li><li class="L6"><code><span class="pln"> page</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="pln">line3</span><span class="pun">)</span></code></li><li class="L7"><code><span class="pln"> </span><span class="com">#生成HTML,数据可视化展示</span></code></li><li class="L8"><code><span class="pln"> page</span><span class="pun">.</span><span class="pln">render</span><span class="pun">(</span><span class="str">"山东3大城市TOP20商铺综合服务评分.html"</span><span class="pun">)</span></code></li><li class="L9"><code></code></li><li class="L0"><code><span class="kwd">if</span><span class="pln"> __name__ </span><span class="pun">==</span><span class="pln"> </span><span class="str">'__main__'</span><span class="pun">:</span></code></li><li class="L1"><code><span class="pln"> </span><span class="com">#评估3大城市商铺综合分数</span></code></li><li class="L2"><code><span class="pln"> shop1 </span><span class="pun">=</span><span class="pln"> evaluateshop</span><span class="pun">(</span><span class="str">"济南"</span><span class="pun">)</span></code></li><li class="L3"><code><span class="pln"> shop2 </span><span class="pun">=</span><span class="pln"> evaluateshop</span><span class="pun">(</span><span class="str">"青岛"</span><span class="pun">)</span></code></li><li class="L4"><code><span class="pln"> shop3 </span><span class="pun">=</span><span class="pln"> evaluateshop</span><span class="pun">(</span><span class="str">"威海"</span><span class="pun">)</span></code></li><li class="L5"><code></code></li><li class="L6"><code><span class="pln"> </span><span class="com">#建立数据列表画图,生成HTML</span></code></li><li class="L7"><code><span class="pln"> list_x </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L8"><code><span class="pln"> list_y1</span><span class="pun">,</span><span class="pln"> list_y2</span><span class="pun">,</span><span class="pln"> list_y3 </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L9"><code><span class="pln"> </span><span class="kwd">for</span><span class="pln"> data</span><span class="pun">,</span><span class="pln"> data2</span><span class="pun">,</span><span class="pln"> data3 </span><span class="kwd">in</span><span class="pln"> zip</span><span class="pun">(</span><span class="pln">shop1</span><span class="pun">,</span><span class="pln"> shop2</span><span class="pun">,</span><span class="pln"> shop3</span><span class="pun">):</span></code></li><li class="L0"><code><span class="pln"> list_y1</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data</span><span class="pun">)</span></code></li><li class="L1"><code><span class="pln"> list_y2</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data2</span><span class="pun">)</span></code></li><li class="L2"><code><span class="pln"> list_y3</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data3</span><span class="pun">)</span></code></li><li class="L3"><code><span class="pln"> plottop20</span><span class="pun">(</span><span class="pln">list_y1</span><span class="pun">,</span><span class="pln"> list_y2</span><span class="pun">,</span><span class="pln"> list_y3</span><span class="pun">)</span></code></li></ol></pre><h5 id="h5-3-1-4-3-3-"><a name="3.1.4.3.3 设置依赖包" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.3.3 设置依赖包</h5><p>选中项目,点击“文件”-&gt;“设置”,查看“Project Interpreter”,是否存在“pytest-runner”、“pyecharts-jupyter-installer”、“pyecharts”和“pyecharts-snapshot”依赖包,若无,则参照3.1.4.1.3进行安装。</p>
<h5 id="h5-3-1-4-3-4-"><a name="3.1.4.3.4 执行程序" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.3.4 执行程序</h5><p>选中“evaluatetop20.py”,点击右键选择“evaluatetop20”,执行程序,运行窗口显示打印信息为菜品分类、每种分类商铺数,“dataanalysis”目录下显示生成的HTML页面</p>
<p><img src="./水滴实验营_files/14-大城市top20商铺服务评分.png" alt=""></p>
<h5 id="h5-3-1-4-3-5-"><a name="3.1.4.3.5 查看可视化数据" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.3.5 查看可视化数据</h5><p>进入该程序文件所在文件夹D:\dazhongspider\dataanalysis,可以看到该HTML文件,双击打开,看到统计图表(如下述链接展示),点击右侧图表可以查看柱状图、数据视图等。</p>
<p><a href="https://jn1.is.shanhe.com/shuidilab/images/qilucasetrainning/analysis/%E5%B1%B1%E4%B8%9C3%E5%A4%A7%E5%9F%8E%E5%B8%82TOP20%E5%95%86%E9%93%BA%E7%BB%BC%E5%90%88%E6%9C%8D%E5%8A%A1%E8%AF%84%E5%88%86.html" title="山东3大城市TOP20商铺综合服务评分">https://jn1.is.shanhe.com/shuidilab/images/qilucasetrainning/analysis/山东3大城市TOP20商铺综合服务评分.html</a></p>
<p>点击最右侧各个图标,可以查看数据图表、折线图等。</p>
<h2 id="h2-3-2-r-csv-"><a name="3.2 R编程+CSV文件数据统计分析可视化" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.2 R编程+CSV文件数据统计分析可视化</h2><p>本部分利用R统计分析CSV文件中20大城市的星级商铺统计分析,统计指标同3.1.3.3,其他统计分析学生自行实践.</p>
<h3 id="h3-3-2-1-"><a name="3.2.1 新建代码" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.2.1 新建代码</h3><p>双击“RStudio”图标启动软件,在窗口内,点击“File”-&gt;“New File”-&gt;“R Script”。</p>
<p><img src="./水滴实验营_files/1-新建R脚本.png" alt=""></p>
<p>进入R脚本编辑界面,如图所示,输入如下代码到脚本文件中,请注意代码中的”C:/changedata/dazhongdata.csv”指的是清洗后的数据文件,请根据前面自己的设置进行路径及文件名修改。</p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">data</span><span class="pun">&lt;-</span><span class="pln">read</span><span class="pun">.</span><span class="pln">csv</span><span class="pun">(</span><span class="pln">file</span><span class="pun">=</span><span class="str">"C:/changedata/dazhongdata.csv"</span><span class="pun">,</span><span class="pln">sep</span><span class="pun">=</span><span class="str">","</span><span class="pun">,</span><span class="pln">header</span><span class="pun">=</span><span class="pln">TRUE</span><span class="pun">,</span><span class="pln">encoding </span><span class="pun">=</span><span class="pln"> </span><span class="str">"utf-8"</span><span class="pun">)</span></code></li><li class="L1"><code><span class="pln">selectscore1</span><span class="pun">&lt;-</span><span class="pln">subset</span><span class="pun">(</span><span class="pln">data</span><span class="pun">,</span><span class="pln">city</span><span class="pun">==</span><span class="str">"济南"</span><span class="pun">,</span><span class="kwd">select</span><span class="pun">=</span><span class="pln">c</span><span class="pun">(</span><span class="pln">shopName</span><span class="pun">,</span><span class="pln">tasteScore</span><span class="pun">,</span><span class="pln">environmentScore</span><span class="pun">,</span><span class="pln">serviceScore</span><span class="pun">))</span></code></li><li class="L2"><code><span class="pln">meanscore1</span><span class="pun">&lt;-</span><span class="pln">rowMeans</span><span class="pun">(</span><span class="pln">selectscore1</span><span class="pun">[</span><span class="lit">2</span><span class="pun">:</span><span class="lit">4</span><span class="pun">])</span></code></li><li class="L3"><code><span class="pln">cdata1</span><span class="pun">&lt;-</span><span class="pln">cbind</span><span class="pun">(</span><span class="pln">selectscore1</span><span class="pun">[</span><span class="lit">1</span><span class="pun">],</span><span class="pln">meanscore1</span><span class="pun">)</span></code></li><li class="L4"><code><span class="pln">top20_jinan</span><span class="pun">&lt;-</span><span class="pln">cdata1</span><span class="pun">[</span><span class="pln">order</span><span class="pun">(</span><span class="pln">cdata1</span><span class="pun">[</span><span class="lit">2</span><span class="pun">],</span><span class="pln">decreasing </span><span class="pun">=</span><span class="pln"> TRUE</span><span class="pun">)[</span><span class="lit">1</span><span class="pun">:</span><span class="lit">20</span><span class="pun">],]</span></code></li><li class="L5"><code></code></li><li class="L6"><code><span class="kwd">print</span><span class="pun">(</span><span class="pln">top20_jinan</span><span class="pun">)</span></code></li><li class="L7"><code></code></li><li class="L8"><code><span class="pln">par</span><span class="pun">(</span><span class="pln">mfrow</span><span class="pun">=</span><span class="pln">c</span><span class="pun">(</span><span class="lit">3</span><span class="pun">,</span><span class="lit">1</span><span class="pun">))</span></code></li><li class="L9"><code><span class="pln">plot</span><span class="pun">(</span><span class="pln">factor</span><span class="pun">(</span><span class="pln">top20_jinan</span><span class="pun">[,</span><span class="lit">1</span><span class="pun">]),</span><span class="pln">top20_jinan</span><span class="pun">[,</span><span class="lit">2</span><span class="pun">],</span><span class="pln">xlable</span><span class="pun">=</span><span class="str">"济南TOP20商铺"</span><span class="pun">,</span><span class="pln">ylab</span><span class="pun">=</span><span class="str">"综合评分"</span><span class="pun">,</span><span class="pln">main</span><span class="pun">=</span><span class="str">"山东3大城市TOP20商铺综合评分"</span><span class="pun">)</span></code></li><li class="L0"><code></code></li><li class="L1"><code></code></li><li class="L2"><code><span class="pln">selectscore2</span><span class="pun">&lt;-</span><span class="pln">subset</span><span class="pun">(</span><span class="pln">data</span><span class="pun">,</span><span class="pln">city</span><span class="pun">==</span><span class="str">"青岛"</span><span class="pun">,</span><span class="kwd">select</span><span class="pun">=</span><span class="pln">c</span><span class="pun">(</span><span class="pln">shopName</span><span class="pun">,</span><span class="pln">tasteScore</span><span class="pun">,</span><span class="pln">environmentScore</span><span class="pun">,</span><span class="pln">serviceScore</span><span class="pun">))</span></code></li><li class="L3"><code><span class="pln">meanscore2</span><span class="pun">&lt;-</span><span class="pln">rowMeans</span><span class="pun">(</span><span class="pln">selectscore2</span><span class="pun">[</span><span class="lit">2</span><span class="pun">:</span><span class="lit">4</span><span class="pun">])</span></code></li><li class="L4"><code><span class="pln">cdata2</span><span class="pun">&lt;-</span><span class="pln">cbind</span><span class="pun">(</span><span class="pln">selectscore2</span><span class="pun">[</span><span class="lit">1</span><span class="pun">],</span><span class="pln">meanscore2</span><span class="pun">)</span></code></li><li class="L5"><code><span class="pln">top20_qingdao</span><span class="pun">&lt;-</span><span class="pln">cdata2</span><span class="pun">[</span><span class="pln">order</span><span class="pun">(</span><span class="pln">cdata2</span><span class="pun">[</span><span class="lit">2</span><span class="pun">],</span><span class="pln">decreasing </span><span class="pun">=</span><span class="pln"> TRUE</span><span class="pun">)[</span><span class="lit">1</span><span class="pun">:</span><span class="lit">20</span><span class="pun">],]</span></code></li><li class="L6"><code><span class="kwd">print</span><span class="pun">(</span><span class="pln">top20_qingdao</span><span class="pun">)</span></code></li><li class="L7"><code><span class="pln">plot</span><span class="pun">(</span><span class="pln">factor</span><span class="pun">(</span><span class="pln">top20_qingdao</span><span class="pun">[,</span><span class="lit">1</span><span class="pun">]),</span><span class="pln">top20_qingdao</span><span class="pun">[,</span><span class="lit">2</span><span class="pun">],</span><span class="pln">xlab</span><span class="pun">=</span><span class="str">"青岛TOP20商铺"</span><span class="pun">,</span><span class="pln">ylab</span><span class="pun">=</span><span class="str">"综合评分"</span><span class="pun">)</span></code></li><li class="L8"><code></code></li><li class="L9"><code><span class="pln">selectscore3</span><span class="pun">&lt;-</span><span class="pln">subset</span><span class="pun">(</span><span class="pln">data</span><span class="pun">,</span><span class="pln">city</span><span class="pun">==</span><span class="str">"威海"</span><span class="pun">,</span><span class="kwd">select</span><span class="pun">=</span><span class="pln">c</span><span class="pun">(</span><span class="pln">shopName</span><span class="pun">,</span><span class="pln">tasteScore</span><span class="pun">,</span><span class="pln">environmentScore</span><span class="pun">,</span><span class="pln">serviceScore</span><span class="pun">))</span></code></li><li class="L0"><code><span class="pln">meanscore3</span><span class="pun">&lt;-</span><span class="pln">rowMeans</span><span class="pun">(</span><span class="pln">selectscore3</span><span class="pun">[</span><span class="lit">2</span><span class="pun">:</span><span class="lit">4</span><span class="pun">])</span></code></li><li class="L1"><code><span class="pln">cdata3</span><span class="pun">&lt;-</span><span class="pln">cbind</span><span class="pun">(</span><span class="pln">selectscore3</span><span class="pun">[</span><span class="lit">1</span><span class="pun">],</span><span class="pln">meanscore3</span><span class="pun">)</span></code></li><li class="L2"><code><span class="pln">top20_weihai</span><span class="pun">&lt;-</span><span class="pln">cdata3</span><span class="pun">[</span><span class="pln">order</span><span class="pun">(</span><span class="pln">cdata3</span><span class="pun">[</span><span class="lit">2</span><span class="pun">],</span><span class="pln">decreasing </span><span class="pun">=</span><span class="pln"> TRUE</span><span class="pun">)[</span><span class="lit">1</span><span class="pun">:</span><span class="lit">20</span><span class="pun">],]</span></code></li><li class="L3"><code><span class="kwd">print</span><span class="pun">(</span><span class="pln">top20_weihai</span><span class="pun">)</span></code></li><li class="L4"><code><span class="pln">plot</span><span class="pun">(</span><span class="pln">factor</span><span class="pun">(</span><span class="pln">top20_weihai</span><span class="pun">[,</span><span class="lit">1</span><span class="pun">]),</span><span class="pln">top20_weihai</span><span class="pun">[,</span><span class="lit">2</span><span class="pun">],</span><span class="pln">xlab</span><span class="pun">=</span><span class="str">"威海TOP20商铺"</span><span class="pun">,</span><span class="pln">ylab</span><span class="pun">=</span><span class="str">"综合评分"</span><span class="pun">)</span></code></li></ol></pre><p>点击“保存”,弹出对话框,选择“UTF-8”编码,点击“确定”,弹出对话框,输入保存R脚本文件名称“top20”.</p>
<p><img src="./水滴实验营_files/2-编辑代码保存.png" alt=""></p>
<h3 id="h3-3-2-2-r-"><a name="3.2.2 执行R脚本" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.2.2 执行R脚本</h3><p>点击“source”按钮执行R脚本文件,在左下方控制台可以看到脚本中打印信息已输出,在右上方环境中可以看到脚本运行过程中各变量的值,在右下方显示执行结果,在plots标签页显示绘制的图形。</p>
<p><img src="./水滴实验营_files/3-运行结果.png" alt=""></p>
<p>在右下方工作区,点击“Zoom”可以放大到新窗口查看,点击“Export”可以导出图片,如图所示。</p>
<p><img src="./水滴实验营_files/4-运行结果图.png" alt=""></p>
<p><font color="red"> <strong>提示:</strong></font></p>
<ul>
<li>* 若执行脚本过程中出现“error in read.table(…….):more columns than column names”,表示某列未定义列名但有数据,请查看dazhongdata.csv数据表中最后一列后面的某列是不是没有列名但有一两行有数据,删除该列即可。</li><li>* 若执行脚本过程中出现“plot.new() : figure margins too large”错误,则将RStudio里的【Plots】窗口适当调大一点,或者把RStudio设置为全屏。如果仍然有该错误,则按下“Fn”不动后接着按下“F11”,将浏览器全屏。</li><li>* 若出现如下错误,则表示数据集没有相应城市的数据.</li></ul>
<p><img src="./水滴实验营_files/6-数据没有对应城市.png" alt=""></p>
<p>注释掉那部分代码,修改一页多图部分的代码,例如缺少一个城市的数据,则修改为</p>
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">par</span><span class="pun">(</span><span class="pln">mfrow</span><span class="pun">=</span><span class="pln">c</span><span class="pun">(</span><span class="lit">2</span><span class="pun">,</span><span class="lit">1</span><span class="pun">))</span><span class="pln"> </span><span class="com"># 将一列3个图修改为1列2个图</span></code></li></ol></pre><h2 id="h2-3-3-"><a name="3.3 学生实训任务" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.3 学生实训任务</h2><p>自行设计新的数据统计分析内容,基于上述某个python程序进行修改,进行可视化展示,可以是柱状图、饼图等图形的任何一个或多个图形均可。</p>
<h1 id="h1-4-"><a name="4 小结" class="reference-link"></a><span class="header-link octicon octicon-link"></span>4 小结</h1><p>本章的综合实验案例涵盖了Python、MySQL、Navicat、Kettle、R等系统和软件的使用,技术上涉及数据采集、存储、查询、清洗和可视化分析等数据处理全流程,通过本章实验案例,学生可以更加熟悉软件使用,掌握数据处理的流程以及编程技术,可以帮助学生加深对各种技术的理解,体会项目开发的流程,提高各项技能。</p>
</div></div></div></div><!--for03356147647--><div id="evaluation" style="display: none;"><!--if--><div class="qc-desktop-com"><div class="comment"><div style="border: 1px solid rgb(204, 204, 204); padding-left: 10px; border-radius: 4px; margin-bottom: 10px; height: 45px; box-shadow: rgb(221, 221, 221) 0px 0px 2px 2px;"><p style="font-size: 12px;">给实验评分</p><span>:</span><span id="exp_course_score"></span></div><!--if--><div class="commentBtn"><p id="evaluate_tip"></p><input type="button" value="发表" name="send_comment" avalon-events="click:eclick_0_events46send95comment"></div></div><div class="avatarListWrap" style="height: auto; overflow-y: auto;"><div class="avatarList"><p class="avatarList-nocomment">暂无评论</p></div><!--ms-for:c in course_evaluate.comment_list | limitBy(100)--><!--ms-for-end:--><p style="margin-right: 20px;" class="qc-paging" id="expdesltop_view_data_page"></p></div></div><!--if--><!-- 随材管理 --><div style="display: none;"><div style="padding: 20px 10px 10px; text-align: center; color: rgb(102, 102, 102); font-size: 14px;">暂无实验随材</div><!--if--></div><!-- 视频管理 --><div style="display: none;"><!--if--></div><!-- 代码管理 --><div style="display: none; padding: 10px;"><input type="file" id="guac_upload_file" style="opacity: 0; width: 0px;"><!--if--><!--
<button ms-if="show_vm.code_data" style="margin:10px;width:100px;background-color:#0dc49b"
ms-on-click="events.code_upload()" class="layui-btn">
上传文件
</button>
--><!--if--><!--if--><div style="padding: 20px 10px 10px; text-align: center; color: rgb(102, 102, 102); font-size: 14px;">实验环境尚未启动,无法上传/下载文件</div><!--if--><!--
<div ms-if="show_vm.code_data&&show_vm.show" class="trying-to-load">
<div class="load-container load1">
<div class="loader">Loading...</div>
</div>
</div>
<div style="padding:20px 10px 10px;text-align: center;color:#666;font-size: 14px;" ms-if="show_vm.title"
ms-html="show_vm.title"></div>
<div ms-if="d.type=='code'&&!show_vm.show" style="padding: 5px">
<table style="margin:0" class="create-materials-table" ms-if="step_code_list.length !=0">
<tr>
<th>文件名称</th>
<th>大小(KB)</th>
</tr>
<tr ms-for="(index, val) in @step_code_list">
<td>
<a ms-if = "val.filesize >0" style="color: #00a5c1;cursor:pointer" ms-attr="{href:val.href,download:val.filename}"
ms-html="val.filename"></a>
<a ms-if = "val.filesize ==0 || val.filesize <0" style="color: #00a5c1;cursor:pointer"
ms-html="val.filename"></a>
</td>
<td ms-html="val.filesize"></td>
</tr>
</table>
</div>
--></div></div><!--for03356147647--><div id="report" style="display: none;"><!--if--><!--if--><div id="report_data_form" class="qc-desktop-com"><div id="course"><!--ms-for:field in fields_report--><div class="icode-form-group clearfix"><p style="font-size: 14px;"><label>完成情况</label><!--ms-for:val in field.data--><label><input type="radio" checked="" name="radio" value="0" avalon-events="change:echange_0_events46completionFun40val46value41"><span>全部完成</span></label><!--for157171563893--><label><input type="radio" name="radio" value="1" avalon-events="change:echange_0_events46completionFun40val46value41"><span>部分完成</span></label><!--for157171563893--><label><input type="radio" name="radio" value="2" avalon-events="change:echange_0_events46completionFun40val46value41"><span>未完成</span></label><!--for157171563893--><!--ms-for-end:--></p><!--if--></div><!--for696174317876--><div class="icode-form-group clearfix"><!--if--><div style="margin: 0px 28px 0px 20px;" class="ueditor row" change-class="row "><div class="layui-upload"><button type="button" class="layui-btn layui-btn-normal" style="padding: 2px; width: 64px; height: 24px; line-height: 24px; margin: 0px;" id="testList" change-class="">附件上传</button><div class="layui-upload-list"><table class="layui-table" id="report_attachment" style="overflow: auto; max-height: 200px;"><thead><tr><th>文件名</th><th>大小</th><th>状态</th><th>操作</th></tr></thead><tbody id="demoList"><!--ms-for:val in report_data.attach_list--><!--ms-for-end:--></tbody></table></div><div id="testListAction"></div></div><label class="checkLabel" style="font-size: 14px; margin-left: 0px; margin-bottom: 10px;">报告内容</label><!--if--><script type="text/plain" style="width: 0px; height: 500px;" id="content"></script><!--if--></div></div><!--for696174317876--><!--ms-for-end:--><div style="height: 60px; display: flex; justify-content: center; align-items: center;"><!--if--><button class="layui-btn layui-btn-primary" change-class="layui-btn layui-btn-primary " avalon-events="click:eclick_0_events46save95report4041">保存</button><button class="layui-btn" change-class="layui-btn " avalon-events="click:eclick_0_events46submit95report4041">提交</button></div><!-- style ="background-color:ccc;color:#fff" --></div></div><!-- 随材管理 --><div style="display: none;"><div style="padding: 20px 10px 10px; text-align: center; color: rgb(102, 102, 102); font-size: 14px;">暂无实验随材</div><!--if--></div><!-- 视频管理 --><div style="display: none;"><!--if--></div><!-- 代码管理 --><div style="display: none; padding: 10px;"><input type="file" id="guac_upload_file" style="opacity: 0; width: 0px;"><!--if--><!--
<button ms-if="show_vm.code_data" style="margin:10px;width:100px;background-color:#0dc49b"
ms-on-click="events.code_upload()" class="layui-btn">
上传文件
</button>
--><!--if--><!--if--><div style="padding: 20px 10px 10px; text-align: center; color: rgb(102, 102, 102); font-size: 14px;">实验环境尚未启动,无法上传/下载文件</div><!--if--><!--
<div ms-if="show_vm.code_data&&show_vm.show" class="trying-to-load">
<div class="load-container load1">
<div class="loader">Loading...</div>
</div>
</div>
<div style="padding:20px 10px 10px;text-align: center;color:#666;font-size: 14px;" ms-if="show_vm.title"
ms-html="show_vm.title"></div>
<div ms-if="d.type=='code'&&!show_vm.show" style="padding: 5px">
<table style="margin:0" class="create-materials-table" ms-if="step_code_list.length !=0">
<tr>
<th>文件名称</th>
<th>大小(KB)</th>
</tr>
<tr ms-for="(index, val) in @step_code_list">
<td>
<a ms-if = "val.filesize >0" style="color: #00a5c1;cursor:pointer" ms-attr="{href:val.href,download:val.filename}"
ms-html="val.filename"></a>
<a ms-if = "val.filesize ==0 || val.filesize <0" style="color: #00a5c1;cursor:pointer"
ms-html="val.filename"></a>
</td>
<td ms-html="val.filesize"></td>
</tr>
</table>
</div>
--></div></div><!--for03356147647--><div id="material" style="display: none;"><!--if--><!--if--><!--if--><!-- 随材管理 --><div style="display: none;"><div style="padding: 20px 10px 10px; text-align: center; color: rgb(102, 102, 102); font-size: 14px;">暂无实验随材</div><div style="padding: 5px;"><!--if--></div></div><!-- 视频管理 --><div style="display: none;"><!--if--></div><!-- 代码管理 --><div style="display: none; padding: 10px;"><input type="file" id="guac_upload_file" style="opacity: 0; width: 0px;"><!--if--><!--
<button ms-if="show_vm.code_data" style="margin:10px;width:100px;background-color:#0dc49b"
ms-on-click="events.code_upload()" class="layui-btn">
上传文件
</button>
--><!--if--><!--if--><div style="padding: 20px 10px 10px; text-align: center; color: rgb(102, 102, 102); font-size: 14px;">实验环境尚未启动,无法上传/下载文件</div><!--if--><!--
<div ms-if="show_vm.code_data&&show_vm.show" class="trying-to-load">
<div class="load-container load1">
<div class="loader">Loading...</div>
</div>
</div>
<div style="padding:20px 10px 10px;text-align: center;color:#666;font-size: 14px;" ms-if="show_vm.title"
ms-html="show_vm.title"></div>
<div ms-if="d.type=='code'&&!show_vm.show" style="padding: 5px">
<table style="margin:0" class="create-materials-table" ms-if="step_code_list.length !=0">
<tr>
<th>文件名称</th>
<th>大小(KB)</th>
</tr>
<tr ms-for="(index, val) in @step_code_list">
<td>
<a ms-if = "val.filesize >0" style="color: #00a5c1;cursor:pointer" ms-attr="{href:val.href,download:val.filename}"
ms-html="val.filename"></a>
<a ms-if = "val.filesize ==0 || val.filesize <0" style="color: #00a5c1;cursor:pointer"
ms-html="val.filename"></a>
</td>
<td ms-html="val.filesize"></td>
</tr>
</table>
</div>
--></div></div><!--for03356147647--><div id="video" style="display: none;"><!--if--><!--if--><!--if--><!-- 随材管理 --><div style="display: none;"><div style="padding: 20px 10px 10px; text-align: center; color: rgb(102, 102, 102); font-size: 14px;">暂无实验随材</div><!--if--></div><!-- 视频管理 --><div style="display: none;"><div class="video_manage" style="padding: 5px;"><!--if--><div class="notdata"><img src="./水滴实验营_files/notdata.png">
暂无视频
</div></div></div><!-- 代码管理 --><div style="display: none; padding: 10px;"><input type="file" id="guac_upload_file" style="opacity: 0; width: 0px;"><!--if--><!--
<button ms-if="show_vm.code_data" style="margin:10px;width:100px;background-color:#0dc49b"
ms-on-click="events.code_upload()" class="layui-btn">
上传文件
</button>
--><!--if--><!--if--><div style="padding: 20px 10px 10px; text-align: center; color: rgb(102, 102, 102); font-size: 14px;">实验环境尚未启动,无法上传/下载文件</div><!--if--><!--
<div ms-if="show_vm.code_data&&show_vm.show" class="trying-to-load">
<div class="load-container load1">
<div class="loader">Loading...</div>
</div>
</div>
<div style="padding:20px 10px 10px;text-align: center;color:#666;font-size: 14px;" ms-if="show_vm.title"
ms-html="show_vm.title"></div>
<div ms-if="d.type=='code'&&!show_vm.show" style="padding: 5px">
<table style="margin:0" class="create-materials-table" ms-if="step_code_list.length !=0">
<tr>
<th>文件名称</th>
<th>大小(KB)</th>
</tr>
<tr ms-for="(index, val) in @step_code_list">
<td>
<a ms-if = "val.filesize >0" style="color: #00a5c1;cursor:pointer" ms-attr="{href:val.href,download:val.filename}"
ms-html="val.filename"></a>
<a ms-if = "val.filesize ==0 || val.filesize <0" style="color: #00a5c1;cursor:pointer"
ms-html="val.filename"></a>
</td>
<td ms-html="val.filesize"></td>
</tr>
</table>
</div>
--></div></div><!--for03356147647--><div id="file" style="display: none;"><!--if--><!--if--><!--if--><!-- 随材管理 --><div style="display: none;"><div style="padding: 20px 10px 10px; text-align: center; color: rgb(102, 102, 102); font-size: 14px;">暂无实验随材</div><!--if--></div><!-- 视频管理 --><div style="display: none;"><!--if--></div><!-- 代码管理 --><div style="display: none; padding: 10px;"><input type="file" id="guac_upload_file" style="opacity: 0; width: 0px;"><!--if--><!--
<button ms-if="show_vm.code_data" style="margin:10px;width:100px;background-color:#0dc49b"
ms-on-click="events.code_upload()" class="layui-btn">
上传文件
</button>
--><!--if--><!--if--><div style="padding: 20px 10px 10px; text-align: center; color: rgb(102, 102, 102); font-size: 14px;">实验环境尚未启动,无法上传/下载文件</div><!--if--><!--
<div ms-if="show_vm.code_data&&show_vm.show" class="trying-to-load">
<div class="load-container load1">
<div class="loader">Loading...</div>
</div>
</div>
<div style="padding:20px 10px 10px;text-align: center;color:#666;font-size: 14px;" ms-if="show_vm.title"
ms-html="show_vm.title"></div>
<div ms-if="d.type=='code'&&!show_vm.show" style="padding: 5px">
<table style="margin:0" class="create-materials-table" ms-if="step_code_list.length !=0">
<tr>
<th>文件名称</th>
<th>大小(KB)</th>
</tr>
<tr ms-for="(index, val) in @step_code_list">
<td>
<a ms-if = "val.filesize >0" style="color: #00a5c1;cursor:pointer" ms-attr="{href:val.href,download:val.filename}"
ms-html="val.filename"></a>
<a ms-if = "val.filesize ==0 || val.filesize <0" style="color: #00a5c1;cursor:pointer"
ms-html="val.filename"></a>
</td>
<td ms-html="val.filesize"></td>
</tr>
</table>
</div>
--></div></div><!--for03356147647--><!--ms-for-end:--></div></div></div><div class="expDesktop_con"><div id="vm_progress" class="hide" change-class="hide"><canvas style="width:90px;height:90px;" id="c"></canvas><input class="hide" type="range" id="r" min="0" max="100" value="0"></div><div class="console_bg_img" change-class=""><img src="./水滴实验营_files/console_bg_img.png"></div><div class="expDesktop_con_wrap clearfix"><div class="fl videoCon" id="displayvm" style="position: relative; height: 956px;"><div id="display" class="ui-layout-center" style="width: 100%;"></div><div class="container"><!-- HTML5 Canvas --><div id="noVNC_screen" style="display:none"><canvas id="noVNC_canvas" class="mid-canvas" width="0" height="0">
Canvas not supported.
</canvas></div><div class="window-overlay" style="display:none"><div class="modal"><div class="modal-header"><h4>
复制粘贴命令
<a class="close"><span class="icon-close icon-Large"></span></a></h4></div><div class="modal-content"><form class="form form-horizontal"><fieldset><div class="modal-content alert alert-danger"><h6>将内容粘贴至文本框中,最大支持2000个字符,暂不支持中文等非标准键盘值特殊字符。</h6></div><div class="item"><div class="control-label">文本内容</div><div class="controls"><textarea class="copyText" rows="10" cols="50" maxlength="2000" name="copyText"></textarea></div></div><div class="form-actions"><input class="btn btn-primary" type="submit" value="提交"><input class="btn btn-cancel" type="button" value="取消"></div></fieldset></form></div></div></div></div><div id="displaynotebook" class="ui-layout-center"><iframe class="interact-notebook" id="interact-notebook" src="./水滴实验营_files/saved_resource.html"></iframe></div></div></div></div></div><div id="toolbar-options" class="hidden"><a title="复制" href="https://edu.shuidilab.cn/console/expdesktop/5665#" name="textcopy"><img style="width: 22px;height: 20px;margin-top: 0px;margin-left: -1px;" src="./水滴实验营_files/copytext.png" alt="复制"></a></div><button id="btntoolbar" style="position: absolute; left: -99999px; top: 0px;"></button></div><script src="./水滴实验营_files/require.min.js.下载"></script><script async="" src="./水滴实验营_files/main-console.min.98fbde9e.js.下载"></script><div class="tool-container tool-bottom toolbar-primary" style="opacity: 0; display: none; top: -120px;"><div class="tool-items"><a title="复制" href="https://edu.shuidilab.cn/console/expdesktop/5665#" name="textcopy" class="tool-item"><img style="width: 22px;height: 20px;margin-top: 0px;margin-left: -1px;" src="./水滴实验营_files/copytext.png" alt="复制"></a></div><div class="arrow"></div></div><div class="layui-layer-move"></div><div class="qc-modal fadeOut" style=""></div></body></html>