mirror of
https://github.com/This-weiyuqing/ECGTest.git
synced 2026-10-18 03:04:11 +08:00
805 lines
280 KiB
HTML
805 lines
280 KiB
HTML
<!DOCTYPE html>
|
||
<!-- saved from url=(0048)https://edu.shuidilab.cn/console/expdesktop/5665 -->
|
||
<html lang="zh-cn" delegate-events="click,change"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><style class="vjs-styles-defaults">
|
||
.video-js {
|
||
width: 300px;
|
||
height: 150px;
|
||
}
|
||
|
||
.vjs-fluid {
|
||
padding-top: 56.25%
|
||
}
|
||
</style>
|
||
<meta http-equiv="Cache-Control" content="max-age=7200">
|
||
|
||
<meta name="viewport" content="width=device-width, user-scalable=no, initial-scale=1.0, maximum-scale=1.0, minimum-scale=1.0">
|
||
<meta http-equiv="X-UA-Compatible" content="ie=edge">
|
||
<meta name="description" content="“水滴实验营”是一个专门为高校师生、科研人员及IT从业人员提供计算机技术相关实验课程的在线教学实训服务平台。平台支持按需定制的实验环境、秒级就绪的实验桌面、互动互助的实验教学模式,并专门想高校教师提供实验监控、作业考试、学情分析等精准教学服务,同时也可为高等院校、科研机构以及IT培训机构定制专属的在线实验教学门户。">
|
||
<meta name="keywords" content="水滴实验营,水滴,实验,实验营,教学实训,大数据,云计算,人工智能,高性能计算,信息技术,实验室,大数据实验室,云计算实验室,IT在线教育,编程,在线编程,Linux,Python,在线实验,软件开发">
|
||
<link href="https://edu.shuidilab.cn/static/public/img/shuidi_favicon.ico" rel="shortcut icon" type="image/x-icon">
|
||
<title>水滴实验营</title>
|
||
|
||
<link rel="stylesheet" href="./水滴实验营_files/font_877519_0ywdpr3rijo.css">
|
||
<link href="./水滴实验营_files/toastr.css" rel="stylesheet">
|
||
|
||
<link rel="stylesheet" href="./水滴实验营_files/desktop.19d8939c.css">
|
||
|
||
|
||
|
||
|
||
|
||
<script>
|
||
var ua = navigator.userAgent.toLowerCase();
|
||
var s;
|
||
(s = ua.match(/rv:([\d.]+)\) like gecko/)) ? s = parseFloat(s[1]) >= 11:
|
||
(s = ua.match(/msie ([\d.]+)/)) ? s = false :
|
||
(s = ua.match(/firefox\/([\d.]+)/)) ? s = parseFloat(s[1]) >= 30 :
|
||
(s = ua.match(/chrome\/([\d.]+)/)) ? s = parseFloat(s[1]) >= 28 :
|
||
(s = ua.match(/version\/([\d.]+).*safari/)) ? s = parseFloat(s[1]) >= 7 : false;
|
||
if (!s) {
|
||
window.location.href = '/msg/update-browser';
|
||
}
|
||
</script>
|
||
|
||
<script>
|
||
window.SDC = {"cas_url": "https://sso.shuidilab.cn/login?service=https://edu.shuidilab.cn/auth/?next=/slab/userpanel/", "cas_server_url": "https://sso.shuidilab.cn/", "slab_url": "https://edu.shuidilab.cn/", "tlab_url": "/", "mlab_url": "/", "realtime_num": 20, "proxyBase": "/api/proxy", "menuspread": true, "t": {"LANG": "zh-hans"}, "env": {"showSdcLink": true}, "websockets": ["RegionOne"], "msgsocket": "", "SOCKJS_URL": "wsconsole.shuidilab.cn/mcenter", "DATASET_SOCK_URL": "/ws", "vm_id": null, "guacHost": "wsconsole.shuidilab.cn/gws/", "sock_type": "wss", "domain": {"user_domain": "https://jn1.is.shanhe.com/shuidilab/", "pcourse_domain": "https://jn1.is.shanhe.com/shuidilab/", "vcourse_domain": "https://jn1.is.shanhe.com/shuidilab/", "icourse_domain": "https://jn1.is.shanhe.com/shuidilab/", "report_domain": "https://jn1.is.shanhe.com/shuidilab/", "step_domain": "https://jn1.is.shanhe.com/shuidilab/", "step_att_domain": "https://jn1.is.shanhe.com/shuidilab/", "idauth_att_domain": "https://jn1.is.shanhe.com/shuidilab/", "invite_domain": "https://jn1.is.shanhe.com/shuidilab/"}, "user": {"project_id": "", "name": "18247649623", "user_pic_path": "file/defaultavatar.png ", "email": "This_weiyuqing@qq.com", "nickname": "student20221107012836", "user_id": "805f8fa8-5e3b-11ed-aaad-0242ac110002", "mobile": "18247649623", "address": "", "industry": "", "manage_spec": {"allow_custom_template": 0}, "id": "9732", "auth_token": "df3fb84d-cd76-4b9c-9b13-20ce42f1a38e", "roles": ["student"], "isAdmin": true, "user_type": 10, "userquota": {"lab_env_count": {"value": 20, "type": "", "number": "", "enabled": true, "quota_name": "\u53ef\u521b\u5efa\u5b9e\u9a8c\u73af\u5883\u6570\u91cf", "unit": "count"}, "vclass_count": {"value": 20, "type": "", "number": "", "enabled": true, "quota_name": "\u53ef\u521b\u5efa\u865a\u62df\u8bfe\u5802\u6570\u91cf", "unit": "count"}, "stu_class_student_count": {"value": 200, "type": "", "number": "", "enabled": true, "quota_name": "\u73ed\u7ea7\u5b66\u751f\u6570\u91cf", "unit": "count"}, "vclass_student_count": {"value": 50, "type": "", "number": "", "enabled": true, "quota_name": "\u865a\u62df\u8bfe\u5802\u5b66\u751f\u6570\u91cf", "unit": "count"}, "study_spec_count": {"value": 1, "type": "", "number": "", "enabled": true, "quota_name": "\u53ef\u521b\u5efa\u4e3b\u673a\u6570", "unit": "count"}, "software_info_count": {"value": 10, "type": "", "number": "", "enabled": true, "quota_name": "\u53ef\u5b89\u88c5\u8f6f\u4ef6\u6570\u91cf", "unit": "count"}, "lab_template_count": {"value": 20, "type": "", "number": "", "enabled": true, "quota_name": "\u53ef\u521b\u5efa\u5b9e\u9a8c\u6a21\u677f\u6570\u91cf", "unit": "count"}}}}
|
||
t = SDC.t;
|
||
</script>
|
||
|
||
|
||
<link id="layuicss-layer" rel="stylesheet" href="./水滴实验营_files/layer.css" media="all"><script type="text/javascript" charset="utf-8" async="" data-requirecontext="_" data-requiremodule="layermsg" src="./水滴实验营_files/layer.js.下载"></script><link type="text/css" rel="stylesheet" href="./水滴实验营_files/katex.min.css"></head>
|
||
|
||
<body style="background: rgb(0, 0, 0);"><input type="hidden" name="csrfmiddlewaretoken" value="ztx4Z2bG9Qv1B981SDZFa8hzxKS59fGJFkwhCBBAoHO0gKNXvbWO0YsiDodoCc4u"><style>
|
||
body{
|
||
overflow: hidden;
|
||
font-size: 12px;
|
||
}
|
||
body {
|
||
background: black
|
||
}
|
||
|
||
.topbar {
|
||
background: #222;
|
||
height: 22px;
|
||
position: relative;
|
||
overflow: hidden
|
||
}
|
||
|
||
.tip {
|
||
position: absolute;
|
||
top: 0;
|
||
width: 100%;
|
||
z-index: 2
|
||
}
|
||
|
||
.tip p {
|
||
padding: 3px 15px;
|
||
background: #658BA0;
|
||
color: white;
|
||
text-align: center;
|
||
width: 370px;
|
||
margin: 0 auto;
|
||
}
|
||
|
||
.pipe {
|
||
margin: 3px 0;
|
||
height: 1px;
|
||
background: #666
|
||
}
|
||
|
||
.dropdown {
|
||
width: 140px;
|
||
height: 22px;
|
||
z-index: 3
|
||
}
|
||
|
||
.message {
|
||
position: fixed;
|
||
left: 0;
|
||
top: 0;
|
||
width: 100%;
|
||
height: 22px;
|
||
color: #CEC17E;
|
||
line-height: 22px;
|
||
vertical-align: middle;
|
||
text-align: center;
|
||
display: none;
|
||
z-index: 0
|
||
}
|
||
|
||
.fullscreen {
|
||
position: fixed;
|
||
top: 2px;
|
||
right: 220px;
|
||
cursor: pointer;
|
||
display: block
|
||
}
|
||
|
||
.modal {
|
||
transform: translate(-50%, -50%);
|
||
}
|
||
|
||
.modal-header {
|
||
color: #000000;
|
||
background: #ebecef;
|
||
}
|
||
|
||
.modal-header .close {
|
||
color: #000000;
|
||
}
|
||
|
||
.form-horizontal .controls {
|
||
margin-right: 30px;
|
||
}
|
||
|
||
.mid-canvas {
|
||
display: block;
|
||
margin: 0 auto
|
||
}
|
||
|
||
input.dropdown-toggle {
|
||
height: 22px
|
||
}
|
||
|
||
.dropdown-text {
|
||
line-height: 22px;
|
||
border: 0 none
|
||
}
|
||
|
||
.dropdown-text::after {
|
||
top: 10px
|
||
}
|
||
|
||
.dropdown-content:hover, .dropdown-toggle:focus ~ .dropdown-content {
|
||
top: 24px
|
||
}
|
||
|
||
.dropdown-content {
|
||
top: 24px;
|
||
width: 140px;
|
||
}
|
||
#noVNC_screen {
|
||
position: absolute;
|
||
margin: 0px;
|
||
padding: 0px;
|
||
bottom: 0px;
|
||
top: 36px; /* the height of the control bar */
|
||
left: 0px;
|
||
right: 0px;
|
||
width: auto;
|
||
height: auto;
|
||
display: flex;
|
||
display: -webkit-flex;
|
||
align-items: center;
|
||
justify-content: center;
|
||
|
||
}
|
||
|
||
|
||
</style><div style="height: 100%;"><div class="console_residue_time hide" change-class="console_residue_time hide" style="display: none;"><div><span>00</span>:
|
||
<span>00</span>:
|
||
<span>00</span></div></div><div class="expDesktop clearfix"><div class="expDesktop_tool" style="right: 0px;"><div class="tools-toggle show_menu"><span title="隐藏工具" class="show-tools" avalon-events="click:eclick_0_events46desktop95tools95hide4041" style="display: block;"><img src="./水滴实验营_files/tools.png"></span><span title="显示工具" class="hide-tools" avalon-events="click:eclick_0_events46desktop95tools95show4041" style="display: none;"><img src="./水滴实验营_files/tools.png"></span></div><!--if--><div class="right-toolbar os-host os-theme-dark os-host-resize-disabled os-host-scrollbar-horizontal-hidden os-host-scrollbar-vertical-hidden os-host-transition"><div class="os-resize-observer-host"><div class="os-resize-observer observed" style="left: 0px; right: auto;"></div></div><div class="os-size-auto-observer" style="height: calc(100% + 1px); float: left;"><div class="os-resize-observer observed"></div></div><div class="os-content-glue" style="width: 139px; margin: 0px; height: 881px;"></div><div class="os-padding"><div class="os-viewport os-viewport-native-scrollbars-invisible" style="right: 0px; bottom: 0px;"><div class="os-content" style="padding: 0px; height: 100%; width: 100%;"><ul class="layui-nav layui-nav-tree tool_button" lay-filter="desktop"><li class="layui-nav-item layui-nav-itemed" style="display: none;"><a href="javascript:;">主机名<span class="layui-nav-more"></span></a><dl class="layui-nav-child"><!-- 二级菜单 --><!--ms-for:option in step.vm_name.data--><!--ms-for-end:--><dd><!--if--><!--if--></dd></dl></li><li class="layui-nav-item layui-nav-itemed" style="display: none;"><a href="javascript:;">实验<span class="layui-nav-more"></span></a><dl class="layui-nav-child"><!-- 二级菜单 --><!--ms-for:option in step.interacts.data--><!--ms-for-end:--></dl></li><li class="layui-nav-item layui-nav-itemed"><a href="javascript:;">操作<span class="layui-nav-more"></span></a><dl class="layui-nav-child"><!--ms-for:field in step.fields--><dd><button name="forced_close" class="hide consoleRipple" change-class="hide consoleRipple"><i class="fa fa-play-circle" change-class="fa fa-play-circle"></i><span>强制关机</span></button></dd><!--for0069487838--><dd><button name="create" class="consoleRipple" change-class="consoleRipple"><i class="fa fa-play-circle" change-class="fa fa-play-circle"></i><span>创建</span></button></dd><!--for0069487838--><dd><button name="delay" class="hide consoleRipple" change-class="hide consoleRipple"><i class="fa fa-clock-o" change-class="fa fa-clock-o"></i><span>延时</span></button></dd><!--for0069487838--><dd><button name="saveexp" class="hide consoleRipple" change-class="hide consoleRipple"><i class="fa fa-save" change-class="fa fa-save"></i><span>保存实验</span></button></dd><!--for0069487838--><dd><button name="lookrecover" class="consoleRipple" change-class="consoleRipple"><i class="fa fa-cloud" change-class="fa fa-cloud"></i><span>恢复实验</span></button></dd><!--for0069487838--><dd><button name="help" class="hide consoleRipple" change-class="hide consoleRipple"><i class="fa fa-share-alt" change-class="fa fa-share-alt"></i><span>请求协助</span></button></dd><!--for0069487838--><dd><button name="unhelp" class="hide consoleRipple" change-class="hide consoleRipple"><i class="fa fa-share-square" change-class="fa fa-share-square"></i><span>取消协助</span></button></dd><!--for0069487838--><dd><button name="viewtype" class="hide consoleRipple" change-class="hide consoleRipple"><i class="fa fa-columns" change-class="fa fa-columns"></i><span>显示方式</span></button></dd><!--for0069487838--><dd><button name="copytext" class="hide consoleRipple" change-class="hide consoleRipple"><i class="fa fa-copy" change-class="fa fa-copy"></i><span></span></button></dd><!--for0069487838--><dd><button name="scrapbooking" class="hide consoleRipple" change-class="hide consoleRipple"><i class="fa fa-scissors" change-class="fa fa-scissors"></i><span>剪贴板</span></button></dd><!--for0069487838--><dd><button name="fullscreen" class="hide consoleRipple" change-class="hide consoleRipple"><i class="fa fa-arrows-alt" change-class="fa fa-arrows-alt"></i><span>全屏</span></button></dd><!--for0069487838--><dd><button name="refresh" class="hide consoleRipple" change-class="hide consoleRipple"><i class="fa fa-refresh" change-class="fa fa-refresh"></i><span>刷新</span></button></dd><!--for0069487838--><dd><button name="close" class="hide consoleRipple" change-class="hide consoleRipple"><i class="fa fa-close" change-class="fa fa-close"></i><span>关闭</span></button></dd><!--for0069487838--><dd><button name="exception" class="consoleRipple" change-class="consoleRipple"><i class="fa fa-bug" change-class="fa fa-bug"></i><span>报告异常</span></button></dd><!--for0069487838--><!--ms-for-end:--></dl></li><span class="layui-nav-bar" style="top: 18px; height: 0px; opacity: 0;"></span></ul><ul class="layui-nav layui-nav-tree tool_button"><li class="layui-nav-item layui-nav-itemed" style="display: none;"><a href="javascript:;">扩展功能<span class="layui-nav-more"></span></a><dl class="layui-nav-child"><!-- 二级菜单 --><!--ms-for:option in step.extension.data--><dd change-class=""><!--if--></dd><!--for92837868529--><!--ms-for-end:--><!--if--><dd><!--if--></dd></dl></li><span class="layui-nav-bar"></span></ul><ul class="layui-nav layui-nav-tree tool_button" style="display: none"><li class="layui-nav-item"><a href="javascript:;">按键操作<span class="layui-nav-more"></span></a><dl class="layui-nav-child"><!-- 二级菜单 --><dd class="" id="sendCtrlAltDelButton"><button>Ctrl-Alt-Del</button></dd><dd class="" id="sendCtrlAltBackspaceButton"><button>Ctrl-Alt-Backspace</button></dd><dd class="" id="sendMemoryDumpButton"><button>RCtrl-Scroll-Scroll</button></dd><dd class="sendCtrlAltFunctionButton" data-n="1"><button>Ctrl-Alt-F1</button></dd><dd class="sendCtrlAltFunctionButton" data-n="2"><button>Ctrl-Alt-F2</button></dd><dd class="sendCtrlAltFunctionButton" data-n="3"><button>Ctrl-Alt-F3</button></dd><dd class="sendCtrlAltFunctionButton" data-n="4"><button>Ctrl-Alt-F4</button></dd><dd class="sendCtrlAltFunctionButton" data-n="5"><button>Ctrl-Alt-F5</button></dd><dd class="sendCtrlAltFunctionButton" data-n="6"><button>Ctrl-Alt-F6</button></dd><dd class="sendCtrlAltFunctionButton" data-n="7"><button>Ctrl-Alt-F7</button></dd><dd class="sendCtrlAltFunctionButton" data-n="8"><button>Ctrl-Alt-F8</button></dd><dd class="sendCtrlAltFunctionButton" data-n="9"><button>Ctrl-Alt-F9</button></dd><dd class="sendCtrlAltFunctionButton" data-n="10"><button>Ctrl-Alt-F10</button></dd><dd class="sendCtrlAltFunctionButton" data-n="11"><button>Ctrl-Alt-F11</button></dd></dl></li><span class="layui-nav-bar"></span></ul></div></div></div><div class="os-scrollbar os-scrollbar-horizontal os-scrollbar-unusable os-scrollbar-auto-hidden"><div class="os-scrollbar-track os-scrollbar-track-off"><div class="os-scrollbar-handle" style="width: 100%; transform: translate(0px, 0px);"></div></div></div><div class="os-scrollbar os-scrollbar-vertical os-scrollbar-unusable os-scrollbar-auto-hidden"><div class="os-scrollbar-track os-scrollbar-track-off"><div class="os-scrollbar-handle" style="height: 100%; transform: translate(0px, 0px);"></div></div></div><div class="os-scrollbar-corner"></div></div><div id="realtime_msg" data-placement="top-left"><span>消息</span><span style="margin-left: 6px;" id="msg_num" class="layui-badge">0</span></div></div><div id="editormd_view" class="expDesktop_editormd_box" style="width: 360px;"><div style="left: 334px; top: 182px;" class="editormd-toggle show_menu"><span title="移动图标" class="editormd-toggle-move"><img src="./水滴实验营_files/arrows_v.png"></span><span title="隐藏文档" class="layui-icon layui-icon-shrink-right show-editormd sd-font-awefontt" avalon-events="click:eclick_0_events46desktop95editormd95hide4041" style="display: block;"><!--
|
||
<img src="/static/public/img/desktop/edit.png" />
|
||
--></span><span class="layui-icon layui-icon-spread-left hide-editormd sd-font-awefontt" title="显示文档" change-class="layui-icon layui-icon-spread-left hide-editormd sd-font-awefontt" avalon-events="click:eclick_0_events46desktop95editormd95show4041" style="display: none;"><img style="display: none;" src="./水滴实验营_files/arrows_all.png" class="hide" change-class="hide"></span><span title="最大化" class="layui-icon layui-icon-layer maximize-editormd sd-font-awefontt sd-font-awefontt" avalon-events="click:eclick_0_events46desktop95editormd95maximize4041" style="display: block;"><!--
|
||
<img src="/static/public/img/desktop/arrows_all.png" />
|
||
--></span></div><div class="expDesktop_editormd" style="width: 360px; left: 0px;"><div class="shuidi-left-drag"></div><ul class="layui-nav layui-bg-cyan desktop_menu"><!--ms-for:(index,i) in doctabs--><li class="layui-nav-item layui-this" change-class="layui-this" avalon-events="click:eclick_0_events46changetabs40i44index41">文档</li><!--for303271319807--><li class="layui-nav-item" change-class="" avalon-events="click:eclick_0_events46changetabs40i44index41">评价</li><!--for303271319807--><li class="layui-nav-item" change-class="" avalon-events="click:eclick_0_events46changetabs40i44index41">我的报告</li><!--for303271319807--><li class="layui-nav-item" change-class="" avalon-events="click:eclick_0_events46changetabs40i44index41">随材</li><!--for303271319807--><li class="layui-nav-item" change-class="" avalon-events="click:eclick_0_events46changetabs40i44index41">视频</li><!--for303271319807--><li class="layui-nav-item" change-class="" avalon-events="click:eclick_0_events46changetabs40i44index41">文件</li><!--for303271319807--><!--ms-for-end:--><span class="layui-nav-bar"></span></ul><h4 class="expDesktop_title"><span title="案例实训-基于Python+MySQL+Kettle+R的某点评网数据采集分析">案例实训-基于Python+MySQL+Kettle+R的某点评网数据采集分析</span></h4><div class="expDesktop_editormd_console"><!--ms-for:(index,d) in doctabs--><div id="docmarkdown" style="height: 100%;"><div style="width: 100%;height: 100%;"><div style="width: 100%;height: 100%;overflow-y: auto"><div id="docmarkdown_doc" class="markdown-body editormd-html-preview"><div class="markdown-toc editormd-markdown-toc"><ul class="markdown-toc-list"><li><a class="toc-level-1" href="https://edu.shuidilab.cn/console/expdesktop/5665#%E5%9F%BA%E4%BA%8EPython+MySQL+Kettle+R%E7%9A%84%E6%9F%90%E7%82%B9%E8%AF%84%E7%BD%91%E6%95%B0%E6%8D%AE%E9%87%87%E9%9B%86%E5%88%86%E6%9E%90" level="1">基于Python+MySQL+Kettle+R的某点评网数据采集分析</a></li><li><a class="toc-level-1" href="https://edu.shuidilab.cn/console/expdesktop/5665#0.%20%E6%A1%88%E4%BE%8B%E7%AE%80%E4%BB%8B" level="1">0. 案例简介</a><ul><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#0.1%20%E6%A1%88%E4%BE%8B%E7%9B%AE%E7%9A%84" level="2">0.1 案例目的</a></li><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#0.2%20%E9%80%82%E7%94%A8%E5%AF%B9%E8%B1%A1" level="2">0.2 适用对象</a></li><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#0.3%20%E6%97%B6%E9%97%B4%E5%AE%89%E6%8E%92" level="2">0.3 时间安排</a></li><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#0.4%20%E7%A1%AC%E4%BB%B6%E8%A6%81%E6%B1%82" level="2">0.4 硬件要求</a></li><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#0.5%20%E8%BD%AF%E4%BB%B6%E5%B7%A5%E5%85%B7" level="2">0.5 软件工具</a></li><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#0.6%20%E6%A1%88%E4%BE%8B%E4%BB%BB%E5%8A%A1" level="2">0.6 案例任务</a></li><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#0.7%20%E5%AE%9E%E9%AA%8C%E6%AD%A5%E9%AA%A4%E6%A6%82%E8%BF%B0" level="2">0.7 实验步骤概述</a></li></ul></li><li><a class="toc-level-1" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.%20Python%E7%BC%96%E7%A8%8B%E7%88%AC%E5%8F%96%E6%9F%90%E7%82%B9%E8%AF%84%E7%BD%91%E6%95%B0%E6%8D%AE" level="1">1. Python编程爬取某点评网数据</a><ul><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.1%20%E4%BB%80%E4%B9%88%E6%98%AF%E7%88%AC%E8%99%AB%EF%BC%9F" level="2">1.1 什么是爬虫?</a></li><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.2%20%E7%88%AC%E8%99%AB%E5%9F%BA%E6%9C%AC%E5%8E%9F%E7%90%86" level="2">1.2 爬虫基本原理</a></li><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.3%20%E7%88%AC%E8%99%AB%E5%9F%BA%E6%9C%AC%E6%AD%A5%E9%AA%A4" level="2">1.3 爬虫基本步骤</a></li><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4%20%E7%88%AC%E8%99%AB%E5%AE%9E%E6%88%98%E6%A1%88%E4%BE%8B%E2%80%94%E2%80%94%E6%9F%90%E7%82%B9%E8%AF%84%E7%BD%91%E6%95%B0%E6%8D%AE%E7%88%AC%E5%8F%96" level="2">1.4 爬虫实战案例——某点评网数据爬取</a><ul><li><a class="toc-level-3" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4.1%20%E6%8B%9F%E7%88%AC%E5%8F%96%E7%BD%91%E9%A1%B5%E5%88%9D%E6%AD%A5%E5%88%86%E6%9E%90" level="3">1.4.1 拟爬取网页初步分析</a></li><li><a class="toc-level-3" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4.2%20%E5%88%9B%E5%BB%BAMySQL%E6%95%B0%E6%8D%AE%E5%BA%93" level="3">1.4.2 创建MySQL数据库</a></li><li><a class="toc-level-3" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4.3%20Python%E7%BC%96%E7%A8%8B%E7%88%AC%E8%99%AB" level="3">1.4.3 Python编程爬虫</a><ul><li><a class="toc-level-4" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4.3.1%20%E6%96%B0%E5%BB%BA%E9%A1%B9%E7%9B%AE" level="4">1.4.3.1 新建项目</a></li><li><a class="toc-level-4" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4.3.2%E8%AE%BE%E7%BD%AE%E7%88%AC%E8%99%AB%E4%BB%A3%E7%A0%81%E7%9B%AE%E5%BD%95" level="4">1.4.3.2设置爬虫代码目录</a></li><li><a class="toc-level-4" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4.3.3%20python%E7%BC%96%E7%A8%8B%E6%95%B0%E6%8D%AE%E5%AD%98%E5%82%A8" level="4">1.4.3.3 python编程数据存储</a></li><li><a class="toc-level-4" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4.3.4%20Python%E7%88%AC%E8%99%AB%E7%BC%96%E7%A8%8B" level="4">1.4.3.4 Python爬虫编程</a></li><li><a class="toc-level-4" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4.3.5%20%E8%AE%BE%E7%BD%AE%E4%BE%9D%E8%B5%96%E5%8C%85" level="4">1.4.3.5 设置依赖包</a></li><li><a class="toc-level-4" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4.3.6%20%E6%89%A7%E8%A1%8C%E7%88%AC%E8%99%AB%E7%88%AC%E5%8F%96%E6%95%B0%E6%8D%AE" level="4">1.4.3.6 执行爬虫爬取数据</a></li><li><a class="toc-level-4" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4.3.7%20%E6%9F%A5%E7%9C%8B%E7%88%AC%E5%8F%96%E7%9A%84%E6%95%B0%E6%8D%AE" level="4">1.4.3.7 查看爬取的数据</a></li><li><a class="toc-level-4" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4.3.8%20%E5%AD%A6%E7%94%9F%E5%AE%9E%E8%AE%AD%E4%BB%BB%E5%8A%A1" level="4">1.4.3.8 学生实训任务</a><ul><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4.3.8.1%20%E5%A2%9E%E5%8A%A0%E5%85%B6%E4%BB%9620%E4%B8%AA%E5%9F%8E%E5%B8%82%E8%BF%9B%E8%A1%8C%E7%88%AC%E8%99%AB" level="5">1.4.3.8.1 增加其他20个城市进行爬虫</a></li><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4.3.8.2%20%E7%88%AC%E5%8F%96%E6%95%B0%E6%8D%AE%E5%A2%9E%E5%8A%A0%E7%94%B5%E8%AF%9D%E5%8F%B7%E7%A0%81%E2%80%9CphoneNo%E2%80%9D%E5%AD%97%E6%AE%B5" level="5">1.4.3.8.2 爬取数据增加电话号码“phoneNo”字段</a></li><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4.3.8.3%20%E7%88%AC%E8%99%AB%E8%B1%86%E7%93%A3%E7%94%B5%E5%BD%B1%E6%AD%A3%E5%88%99%E8%A1%A8%E8%BE%BE%E5%BC%8F%E5%88%86%E6%9E%90" level="5">1.4.3.8.3 爬虫豆瓣电影正则表达式分析</a></li><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#1.4.3.8.4%20Scrapy%E7%88%AC%E8%99%AB%E6%A1%86%E6%9E%B6" level="5">1.4.3.8.4 Scrapy爬虫框架</a></li></ul></li></ul></li></ul></li></ul></li><li><a class="toc-level-1" href="https://edu.shuidilab.cn/console/expdesktop/5665#2.%20%E5%88%A9%E7%94%A8Kettle%E8%BF%9B%E8%A1%8C%E6%95%B0%E6%8D%AE%E6%B8%85%E6%B4%97%E8%BD%AC%E6%8D%A2" level="1">2. 利用Kettle进行数据清洗转换</a><ul><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#2.1%20Kettle%20%E5%9F%BA%E6%9C%AC%E4%BD%BF%E7%94%A8" level="2">2.1 Kettle 基本使用</a></li><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#2.2%20%E6%95%B0%E6%8D%AE%E6%B8%85%E6%B4%97%E5%86%85%E5%AE%B9" level="2">2.2 数据清洗内容</a></li><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#2.3%20%E5%BB%BA%E7%AB%8BMySQL%E6%96%B0%E8%A1%A8%E5%AD%98%E5%82%A8%E6%B8%85%E6%B4%97%E5%90%8E%E6%95%B0%E6%8D%AE" level="2">2.3 建立MySQL新表存储清洗后数据</a></li><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#2.4%20Kettle%E6%95%B0%E6%8D%AE%E6%B8%85%E6%B4%97%E3%80%81%E8%BD%AC%E6%8D%A2" level="2">2.4 Kettle数据清洗、转换</a><ul><li><a class="toc-level-3" href="https://edu.shuidilab.cn/console/expdesktop/5665#2.4.1%20%E5%88%9B%E5%BB%BA%E8%BD%AC%E6%8D%A2%E4%BB%BB%E5%8A%A1%E8%AE%BE%E8%AE%A1%E8%BE%93%E5%85%A5%E7%BB%84%E4%BB%B6" level="3">2.4.1 创建转换任务设计输入组件</a></li><li><a class="toc-level-3" href="https://edu.shuidilab.cn/console/expdesktop/5665#2.4.2%20%E8%AE%BE%E8%AE%A1%E6%95%B0%E6%8D%AE%E6%B8%85%E6%B4%97%E7%BB%84%E4%BB%B6" level="3">2.4.2 设计数据清洗组件</a><ul><li><a class="toc-level-4" href="https://edu.shuidilab.cn/console/expdesktop/5665#2.4.2.1%E8%BF%87%E6%BB%A4%E7%A9%BA%E5%80%BC" level="4">2.4.2.1过滤空值</a></li><li><a class="toc-level-4" href="https://edu.shuidilab.cn/console/expdesktop/5665#2.4.2.2%E8%BF%87%E6%BB%A40%E5%80%BC" level="4">2.4.2.2过滤0值</a></li><li><a class="toc-level-4" href="https://edu.shuidilab.cn/console/expdesktop/5665#2.4.2.3%E6%8E%92%E5%BA%8F" level="4">2.4.2.3排序</a></li></ul></li><li><a class="toc-level-3" href="https://edu.shuidilab.cn/console/expdesktop/5665#2.4.3%20%E8%AE%BE%E8%AE%A1%E8%BE%93%E5%87%BA%E7%BB%84%E4%BB%B6" level="3">2.4.3 设计输出组件</a></li><li><a class="toc-level-3" href="https://edu.shuidilab.cn/console/expdesktop/5665#2.4.4%20%E6%89%A7%E8%A1%8C%E6%B8%85%E6%B4%97%E5%92%8C%E8%BD%AC%E6%8D%A2" level="3">2.4.4 执行清洗和转换</a></li></ul></li><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#2.5%20%E5%AD%A6%E7%94%9F%E5%AE%9E%E8%AE%AD%E4%BB%BB%E5%8A%A1" level="2">2.5 学生实训任务</a></li></ul></li><li><a class="toc-level-1" href="https://edu.shuidilab.cn/console/expdesktop/5665#3%20%E6%95%B0%E6%8D%AE%E7%BB%9F%E8%AE%A1%E5%88%86%E6%9E%90" level="1">3 数据统计分析</a><ul><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1%20Python+MySQL%E6%95%B0%E6%8D%AE%E7%BB%9F%E8%AE%A1%E5%88%86%E6%9E%90" level="2">3.1 Python+MySQL数据统计分析</a><ul><li><a class="toc-level-3" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.1%20%E5%9F%BA%E6%9C%ACSQL%E8%AF%AD%E5%8F%A5%E5%AD%A6%E4%B9%A0" level="3">3.1.1 基本SQL语句学习</a></li><li><a class="toc-level-3" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.2%20%E6%89%93%E5%BC%80%E9%A1%B9%E7%9B%AE" level="3">3.1.2 打开项目</a></li><li><a class="toc-level-3" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.3%20%E8%AE%BE%E7%BD%AE%E4%BB%A3%E7%A0%81%E7%9B%AE%E5%BD%95" level="3">3.1.3 设置代码目录</a></li><li><a class="toc-level-3" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4%20Python%E7%BC%96%E7%A8%8B%E8%BF%9B%E8%A1%8C%E6%95%B0%E6%8D%AE%E7%BB%9F%E8%AE%A1%E5%8F%AF%E8%A7%86%E5%8C%96" level="3">3.1.4 Python编程进行数据统计可视化</a><ul><li><a class="toc-level-4" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.1%2020%E4%B8%AA%E5%A4%A7%E4%B8%AD%E5%9F%8E%E5%B8%82%E6%98%9F%E7%BA%A7%E5%95%86%E9%93%BA%E7%BB%9F%E8%AE%A1%E5%88%86%E6%9E%90" level="4">3.1.4.1 20个大中城市星级商铺统计分析</a><ul><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.1.1%20%E6%96%B0%E5%BB%BA%E4%BB%A3%E7%A0%81%E6%96%87%E4%BB%B6" level="5">3.1.4.1.1 新建代码文件</a></li><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.1.2%20%E7%BC%96%E5%86%99python%E7%A8%8B%E5%BA%8F%E4%BB%A3%E7%A0%81" level="5">3.1.4.1.2 编写python程序代码</a></li><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.1.3%20%E8%AE%BE%E7%BD%AE%E4%BE%9D%E8%B5%96%E5%8C%85" level="5">3.1.4.1.3 设置依赖包</a></li><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.1.4%20%E6%89%A7%E8%A1%8C%E7%A8%8B%E5%BA%8F" level="5">3.1.4.1.4 执行程序</a></li><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.1.5%20%E6%9F%A5%E7%9C%8B%E5%8F%AF%E8%A7%86%E5%8C%96%E6%95%B0%E6%8D%AE" level="5">3.1.4.1.5 查看可视化数据</a></li></ul></li><li><a class="toc-level-4" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.2%20%E5%B1%B1%E4%B8%9C3%E5%A4%A7%E5%9F%8E%E5%B8%82%E8%8F%9C%E5%93%81%E5%88%86%E7%B1%BB%E7%BB%9F%E8%AE%A1" level="4">3.1.4.2 山东3大城市菜品分类统计</a><ul><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.2.1%20%E6%96%B0%E5%BB%BA%E4%BB%A3%E7%A0%81" level="5">3.1.4.2.1 新建代码</a></li><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.2.2%20%E7%BC%96%E5%86%99python%E7%A8%8B%E5%BA%8F%E4%BB%A3%E7%A0%81" level="5">3.1.4.2.2 编写python程序代码</a></li><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.2.3%20%E8%AE%BE%E7%BD%AE%E4%BE%9D%E8%B5%96%E5%8C%85" level="5">3.1.4.2.3 设置依赖包</a></li><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.2.4%20%E6%89%A7%E8%A1%8C%E7%A8%8B%E5%BA%8F" level="5">3.1.4.2.4 执行程序</a></li><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.2.5%20%E6%9F%A5%E7%9C%8B%E5%8F%AF%E8%A7%86%E5%8C%96%E6%95%B0%E6%8D%AE" level="5">3.1.4.2.5 查看可视化数据</a></li></ul></li><li><a class="toc-level-4" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.3%20%E5%B1%B1%E4%B8%9C3%E5%A4%A7%E5%9F%8E%E5%B8%82TOP20%E9%A5%AD%E5%BA%97%E6%9C%8D%E5%8A%A1%E7%9A%84%E7%BB%BC%E5%90%88%E8%AF%84%E4%BB%B7" level="4">3.1.4.3 山东3大城市TOP20饭店服务的综合评价</a><ul><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.3.1%20%E6%96%B0%E5%BB%BA%E4%BB%A3%E7%A0%81" level="5">3.1.4.3.1 新建代码</a></li><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.3.2%20%E7%BC%96%E5%86%99python%E7%A8%8B%E5%BA%8F%E4%BB%A3%E7%A0%81" level="5">3.1.4.3.2 编写python程序代码</a></li><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.3.3%20%E8%AE%BE%E7%BD%AE%E4%BE%9D%E8%B5%96%E5%8C%85" level="5">3.1.4.3.3 设置依赖包</a></li><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.3.4%20%E6%89%A7%E8%A1%8C%E7%A8%8B%E5%BA%8F" level="5">3.1.4.3.4 执行程序</a></li><li><a class="toc-level-5" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.1.4.3.5%20%E6%9F%A5%E7%9C%8B%E5%8F%AF%E8%A7%86%E5%8C%96%E6%95%B0%E6%8D%AE" level="5">3.1.4.3.5 查看可视化数据</a></li></ul></li></ul></li></ul></li><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.2%20%20R%E7%BC%96%E7%A8%8B+CSV%E6%96%87%E4%BB%B6%E6%95%B0%E6%8D%AE%E7%BB%9F%E8%AE%A1%E5%88%86%E6%9E%90%E5%8F%AF%E8%A7%86%E5%8C%96" level="2">3.2 R编程+CSV文件数据统计分析可视化</a><ul><li><a class="toc-level-3" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.2.1%20%E6%96%B0%E5%BB%BA%E4%BB%A3%E7%A0%81" level="3">3.2.1 新建代码</a></li><li><a class="toc-level-3" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.2.2%20%E6%89%A7%E8%A1%8CR%E8%84%9A%E6%9C%AC" level="3">3.2.2 执行R脚本</a></li></ul></li><li><a class="toc-level-2" href="https://edu.shuidilab.cn/console/expdesktop/5665#3.3%20%E5%AD%A6%E7%94%9F%E5%AE%9E%E8%AE%AD%E4%BB%BB%E5%8A%A1" level="2">3.3 学生实训任务</a></li></ul></li><li><a class="toc-level-1" href="https://edu.shuidilab.cn/console/expdesktop/5665#4%20%E5%B0%8F%E7%BB%93" level="1">4 小结</a><ul></ul></li></ul></div><h1 id="h1--python-mysql-kettle-r-"><a name="基于Python+MySQL+Kettle+R的某点评网数据采集分析" class="reference-link"></a><span class="header-link octicon octicon-link"></span>基于Python+MySQL+Kettle+R的某点评网数据采集分析</h1><p>本案例涉及数据采集、存储、查询、清洗和可视化分析等数据处理全流程所涉及的各种典型操作,涵盖Python、MySQL、Navicat、Kettle、R语言等系统和软件的使用方法。本案例适合高校大数据相关专业教学,可以作为学习大数据的综合实践案例。通过本案例,将有助于读者综合运用大数据知识以及各种工具软件,实现数据全流程操作。</p>
|
||
<h1 id="h1-0-"><a name="0. 案例简介" class="reference-link"></a><span class="header-link octicon octicon-link"></span>0. 案例简介</h1><p>本次实践的数据来源于国内某点评网站Top100餐厅网页,包括商户名称、所在商区、口味、环境、服务评分以及人均消费等数据,如图所示:<br><img src="./水滴实验营_files/00-案例大众点评top100页.png" alt=""></p>
|
||
<h2 id="h2-0-1-"><a name="0.1 案例目的" class="reference-link"></a><span class="header-link octicon octicon-link"></span>0.1 案例目的</h2><p>案例综合实践的目的就在于通过对该点评数据进行统计分析,评估各城市餐厅、菜品,帮助游客选择适合自己的餐厅就餐。</p>
|
||
<ul>
|
||
<li>编写Python爬虫程序采集数据</li><li>了解MySQL操作,存储数据</li><li>了解Kettle进行数据清洗、转换流程和操作</li><li>使用Python、R进行数据统计分析可视化展示</li></ul>
|
||
<h2 id="h2-0-2-"><a name="0.2 适用对象" class="reference-link"></a><span class="header-link octicon octicon-link"></span>0.2 适用对象</h2><ul>
|
||
<li>高校(高职)教师</li><li>高校(高职)学生</li><li>大数据学习者</li><li>数据处理分析者</li></ul>
|
||
<h2 id="h2-0-3-"><a name="0.3 时间安排" class="reference-link"></a><span class="header-link octicon octicon-link"></span>0.3 时间安排</h2><p>本案例可以作为大数据课程结束后的“大作业”,或者可以作为学生暑期或寒假大数据相关专业实习实践基础案例,建议2天。</p>
|
||
<h2 id="h2-0-4-"><a name="0.4 硬件要求" class="reference-link"></a><span class="header-link octicon octicon-link"></span>0.4 硬件要求</h2><p>本案例可以在单个虚机上完成,也可以在虚拟集群环境下完成。单虚机上完成本案例实验时,建议硬件配置为:8CPU,8GB以上内存,100GB以上硬盘。</p>
|
||
<h2 id="h2-0-5-"><a name="0.5 软件工具" class="reference-link"></a><span class="header-link octicon octicon-link"></span>0.5 软件工具</h2><p>本案例所涉及的系统及软件包括:Windows, Python, Pycharm, MySQL, Navicat, Kettle、R等。<br>相关软件的版本建议如下:</p>
|
||
<ul>
|
||
<li>Windows:Windows 7</li><li>Mcrosoft office: 2013</li><li>JDK:1.8</li><li>Python: 3.7(64位)</li><li>Pycharm: Community 2018.2</li><li>MySQL:5.7</li><li>Navicat for MySQL:11.1.13(64位)</li><li>Kettle:7.0</li><li>R:3.5.1</li><li>Rstudio:1.1<br><strong>*注意:</strong>案例实验操作,请使用chrome浏览器或者较新的360浏览器极速模式。<h2 id="h2-0-6-"><a name="0.6 案例任务" class="reference-link"></a><span class="header-link octicon octicon-link"></span>0.6 案例任务</h2>本案例需要完成以下实验任务:<br>(1) Python编程爬取点某评网数据,存储于MySQL数据库<br>(2) 利用Kettle对MySQL表数据进行清洗,一份存储到MySQL新表,一份转换成CSV文件<br>(3) 使用Python、R进行数据统计分析、可视化展示</li></ul>
|
||
<h2 id="h2-0-7-"><a name="0.7 实验步骤概述" class="reference-link"></a><span class="header-link octicon octicon-link"></span>0.7 实验步骤概述</h2><p>本案例共包含3个实验步骤:<br>1:Python编程爬取某点评网数据存储于MySQL数据库<br>2:利用Kettle进行数据转换、清洗<br>3:利用Python、R进行数据统计分析、可视化展示<br>下面表格分别给出了每个实验步骤所需的知识储备、训练技能和任务清单。</p>
|
||
<p>表1:Python编程爬取某点评网数据存储于MySQL数据库</p>
|
||
<table>
|
||
<thead>
|
||
<tr>
|
||
<th style="text-align:left">所需知识储备</th>
|
||
<th style="text-align:left">Python基础知识、使用、编程,MySQL数据库基础知识、Navicat使用</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<td style="text-align:left">训练技能</td>
|
||
<td style="text-align:left">Python编程调试、Mysql、Navicat基本操作</td>
|
||
</tr>
|
||
<tr>
|
||
<td style="text-align:left">任务清单</td>
|
||
<td style="text-align:left">Python爬虫某点评网数据、解析数据、创建数据库存储数据</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
<p>表2:使用Kettle进行数据转换、清洗</p>
|
||
<table>
|
||
<thead>
|
||
<tr>
|
||
<th style="text-align:left">所需知识储备</th>
|
||
<th style="text-align:left">Kettle基本使用</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<td style="text-align:left">训练技能</td>
|
||
<td style="text-align:left">Kettle基本使用、数据清洗、数据转换</td>
|
||
</tr>
|
||
<tr>
|
||
<td style="text-align:left">任务清单</td>
|
||
<td style="text-align:left">使用Kettle建立数据清洗、转换流程,一是完成MySQL数据表清洗存储到新表,二是MySQL表数据清洗后转换成CSV文件</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
<p>表3:利用Python进行数据统计分析、可视化展示</p>
|
||
<table>
|
||
<thead>
|
||
<tr>
|
||
<th style="text-align:left">所需知识储备</th>
|
||
<th style="text-align:left">Python、R工具选择其一,熟悉基本使用</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<td style="text-align:left">训练技能</td>
|
||
<td style="text-align:left">编程调试能力、数据统计分析能力、数据可视化设计</td>
|
||
</tr>
|
||
<tr>
|
||
<td style="text-align:left">任务清单</td>
|
||
<td style="text-align:left">编程调试、数据统计分析、数据可视化</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
<h1 id="h1-1-python-"><a name="1. Python编程爬取某点评网数据" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1. Python编程爬取某点评网数据</h1><p>本节介绍什么是爬虫、爬虫基本原理、爬虫基本步骤、爬虫实战案例。</p>
|
||
<h2 id="h2-1-1-"><a name="1.1 什么是爬虫?" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.1 什么是爬虫?</h2><p>爬虫,即网络爬虫,可以理解为在网络上爬行的一只蜘蛛,在互联网这张大网上爬来爬去,抓取各种资源。</p>
|
||
<p><img src="./水滴实验营_files/01-爬虫页.png" alt=""></p>
|
||
<p>用户浏览网页,输入URL网址后,实际上经过DNS服务器解析找到网页服务器向其发送一个请求,服务器解析后发送响应消息,用户就看到HTML代码的网页了,包含文本、图片等内容。</p>
|
||
<p>爬虫爬取的就是这些内容,通过分析HTML代码,获取相应的图片和文本内容,例如:打开chrome浏览器,输入:<a href="http://www.163.com/">http://www.163.com</a>, 页面如下<br><img src="./水滴实验营_files/02-163网页.png" alt=""><br>右键点击“查看网页源代码”,我们可以看到HTML代码如下,里面包含标签、块等元素及属性信息。</p>
|
||
<p><img src="./水滴实验营_files/03-163网页源代码.png" alt=""></p>
|
||
<h2 id="h2-1-2-"><a name="1.2 爬虫基本原理" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.2 爬虫基本原理</h2><p><img src="./水滴实验营_files/04-爬虫基本原理.png" alt=""></p>
|
||
<p>通过程序模拟浏览器向服务器站点发起请求,将站点返回的响应,例如HTML代码/JSON数据/二进制数据(图片、视频)等爬取到本地,提取自己需要的数据存放起来使用。</p>
|
||
<h2 id="h2-1-3-"><a name="1.3 爬虫基本步骤" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.3 爬虫基本步骤</h2><p><img src="./水滴实验营_files/05-爬虫基本步骤.png" alt=""></p>
|
||
<p>(1)发起请求<br>使用HTTP协议向目标站点发起请求,也就是发送一个Request,Request包含请求头、请求体等, 等待服务器响应。<br>(2)获取响应内容<br>如果服务器能正常响应,会得到一个Response,Response的内容便是所要获取的页面内容,类型可能是HTML,Json字符串,二进制数据(图片或者视频)等类型。<br>(3)解析内容<br>得到的内容不同解析方式也不同。</p>
|
||
<ul>
|
||
<li>HTML页面:正则表达式解析,或者使用第三方解析库如Beautifulsoup,pyquery等进行解析。</li><li>Json数据:转换为Json对象解析。</li><li>二进制数据:可以做保存或者进一步的处理。</li></ul>
|
||
<p>(4)保存数据<br>保存形式多样,可以存为文本,也可以保存到数据库,或者保存特定格式的文件。</p>
|
||
<h2 id="h2-1-4-"><a name="1.4 爬虫实战案例——某点评网数据爬取" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4 爬虫实战案例——某点评网数据爬取</h2><p>本爬虫案例爬取某点评网站50个城市排名TOP100的餐厅网页,并分析这些餐厅名称、ID、地址、所属商区、菜品分类、口味得分、环境得分、服务得分、人均消费等存储到MySQL数据库。</p>
|
||
<h3 id="h3-1-4-1-"><a name="1.4.1 拟爬取网页初步分析" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4.1 拟爬取网页初步分析</h3><p>本案例拟爬取某点评网站50个城市排名TOP100的餐厅网页,每个城市对应一个网页地址,以北京为例,北京排名TOP100的餐厅网页(本文档中dp01.ds.quancube.cn为内部网址,请使用虚机内部的chrome浏览器打开,注意复制链接请使用自动弹出的复制按钮进行复制)<br><a href="http://dp01.ds.quancube.cn/?rankId=d5036cf54fcb57e9dceb9fefe3917fff71862f838d1255ea693b953b1d49c7c0">http://dp01.ds.quancube.cn/?rankId=d5036cf54fcb57e9dceb9fefe3917fff71862f838d1255ea693b953b1d49c7c0</a></p>
|
||
<p><img src="./水滴实验营_files/06-北京大众点评.png" alt=""></p>
|
||
<p>通过观察每个城市的链接主要区别在于rankId,每个城市有特定的ID,例如:<br> 北京排名TOP100的餐厅HTML网页的rankId: 652e242ca0227790d18acff20ae02272<br> 济南排名TOP100餐厅HTML网页的rankId: 4be7d758168826991eb4b3e779fdf41571862f838d1255ea693b953b1d49c7c0<br>济南HTML网页地址可通过修改rankId得到为:<br><a href="http://dp01.ds.quancube.cn/?rankId=4be7d758168826991eb4b3e779fdf41571862f838d1255ea693b953b1d49c7c0">http://dp01.ds.quancube.cn/?rankId=4be7d758168826991eb4b3e779fdf41571862f838d1255ea693b953b1d49c7c0</a><br>因此先获取到相应城市的ID,以便后续进行数据抓取。打开上面济南排名TOP100餐厅的HTML网页,显示如下:</p>
|
||
<p><img src="./水滴实验营_files/07-济南大众点评.png" alt=""></p>
|
||
<p>按住“Fn”键不动接着按下F12键,点击“Network”和“XHR”出现如下界面,出现如下界面。</p>
|
||
<p><img src="./水滴实验营_files/08-Fn+F12济南.png" alt=""></p>
|
||
<p>接着按下Ctrl+R键,出现如下界面</p>
|
||
<p><img src="./水滴实验营_files/09-Ctrl+R济南.png" alt=""></p>
|
||
<p>点击最下方红框处,出现如下界面,点击Headers看到general中Request URL:<br><a href="http://dp01.ds.quancube.cn/mylist/ajax/shoprank?rankId=4be7d758168826991eb4b3e779fdf41571862f838d1255ea693b953b1d49c7c0">http://dp01.ds.quancube.cn/mylist/ajax/shoprank?rankId=4be7d758168826991eb4b3e779fdf41571862f838d1255ea693b953b1d49c7c0</a><br>, 该网页所有数据通过Ajax接口获得,并且 在Response Headers中看到“Content-Type:application/json;charset=UTF-8”, 获得的响应是JSON数据,编码是UTF-8格式。</p>
|
||
<p><img src="./水滴实验营_files/10-header.png" alt=""></p>
|
||
<p>在浏览器中,输入进入该链接<br><a href="http://dp01.ds.quancube.cn/mylist/ajax/shoprank?rankId=4be7d758168826991eb4b3e779fdf41571862f838d1255ea693b953b1d49c7c0">http://dp01.ds.quancube.cn/mylist/ajax/shoprank?rankId=4be7d758168826991eb4b3e779fdf41571862f838d1255ea693b953b1d49c7c0</a><br> 得到JSON字符串,如下图所示,最终抓取的数据只需要解析json便可获得所需字段,主要包括城市、商铺名称、编号、星级、所在商区、菜品分类、口味评分、环境评分、服务评分、人均消费、网址、图片。同之前请求一样,只需要替换rankId便可进行多城市数据爬取。</p>
|
||
<p><img src="./水滴实验营_files/11-ajaxcode.png" alt=""></p>
|
||
<p>我们获得的50个城市rankId,首先对前30个城市进行数据爬取。</p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"上海"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"fce2e3a36450422b7fad3f2b90370efd71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L1"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"北京"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"d5036cf54fcb57e9dceb9fefe3917fff71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L2"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"广州"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"e749e3e04032ee6b165fbea6fe2dafab71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L3"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"深圳"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"e049aa251858f43d095fc4c61d62a9ec71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L4"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"天津"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"2e5d0080237ff3c8f5b5d3f315c7c4a508e25c702ab1b810071e8e2c39502be1"</span><span class="pun">],</span></code></li><li class="L5"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"杭州"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"91621282e559e9fc9c5b3e816cb1619c71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L6"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"南京"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"d6339a01dbd98141f8e684e1ad8af5c871862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L7"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"苏州"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"536e0e568df850d1e6ba74b0cf72e19771862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L8"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"成都"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"c950bc35ad04316c76e89bf2dc86bfe071862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L9"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"武汉"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"d96a24c312ed7b96fcc0cedd6c08f68c08e25c702ab1b810071e8e2c39502be1"</span><span class="pun">],</span></code></li><li class="L0"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"重庆"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"6229984ceb373efb8fd1beec7eb4dcfd71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L1"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"西安"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"ad66274c7f5f8d27ffd7f6b39ec447b608e25c702ab1b810071e8e2c39502be1"</span><span class="pun">],</span></code></li><li class="L2"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"青岛"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"9874ff91c8b50ad831ea897b0cd5315b71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L3"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"济南"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"4be7d758168826991eb4b3e779fdf41571862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L4"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"威海"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"fb5f85ca4c696d61c6e7a7e6c19b2f4530aacdebee4c4c9365dc18972daccaf7"</span><span class="pun">],</span></code></li><li class="L5"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"长春"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"261f655c7a9713696580432624721cd871862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L6"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"大连"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"21f56d987e57bd3e8df308351e3f4b9171862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L7"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"佛山"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"4e5d3a805c4f1ed6059c7074e5e86adf30aacdebee4c4c9365dc18972daccaf7"</span><span class="pun">],</span></code></li><li class="L8"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"贵阳"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"7ae3d5049e1c200ed6b2db0ac51604ba30aacdebee4c4c9365dc18972daccaf7"</span><span class="pun">],</span></code></li><li class="L9"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"合肥"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"94af1388885f0b7483fdf72cba086a6030aacdebee4c4c9365dc18972daccaf7"</span><span class="pun">],</span></code></li><li class="L0"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"呼和浩特"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"d64d5919fc236699402122b5770b4a6f71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L1"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"昆明"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"45159d85c35f742c1f2635d6ea6fa3ae30aacdebee4c4c9365dc18972daccaf7"</span><span class="pun">],</span></code></li><li class="L2"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"兰州"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"b201713f2fb5f7235b6e2a6de8faf6a630aacdebee4c4c9365dc18972daccaf7"</span><span class="pun">],</span></code></li><li class="L3"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"南宁"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"ff6f15347ab16f86ffb34e2700c4870930aacdebee4c4c9365dc18972daccaf7"</span><span class="pun">],</span></code></li><li class="L4"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"秦皇岛"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"ca02d37fa01b0f9613555237517e19ff71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L5"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"沈阳"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"eec1fb195780b91fc9dd065409c1d28471862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L6"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"太原"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"293e6160d971fe48ebbed1c8033bd9ed71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L7"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"唐山"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"2ae2237467ce4784b1705be502b0315271862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L8"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"无锡"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"51ff28145b3f63e2f9ec4b2a34e8850c71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L9"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"扬州"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"02eeb7a9021c215641f3ec0e3cede50b71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">]</span></code></li></ol></pre><p>有的网站会设置反爬机制,或者监听到外部爬取数据会封号,导致数据无法继续爬取。该点评网网页没有特别反爬限制,我们只需要不断轮换用户代理便可绕过反爬。请求头的设置如下:</p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="com">#用户代理,是一个特殊字符串头,使得服务器能够识别客户使用的操作系统及版本、浏览器及版本、浏览器渲染引擎、浏览器语言、浏览器插件等#</span></code></li><li class="L1"><code><span class="pln">USER_AGENT_LIST </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[</span></code></li><li class="L2"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"</span><span class="pun">,</span></code></li><li class="L3"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11"</span><span class="pun">,</span></code></li><li class="L4"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6"</span><span class="pun">,</span></code></li><li class="L5"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6"</span><span class="pun">,</span></code></li><li class="L6"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1"</span><span class="pun">,</span></code></li><li class="L7"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5"</span><span class="pun">]</span></code></li><li class="L8"><code><span class="pln"> </span><span class="com">#设置随机选择用户代理#</span></code></li><li class="L9"><code><span class="pln">head </span><span class="pun">=</span><span class="pln"> </span><span class="pun">{</span></code></li><li class="L0"><code><span class="str">'User-Agent'</span><span class="pun">:</span><span class="pln"> </span><span class="str">'{0}'</span><span class="pun">.</span><span class="pln">format</span><span class="pun">(</span><span class="pln">random</span><span class="pun">.</span><span class="pln">sample</span><span class="pun">(</span><span class="pln">USER_AGENT_LIST</span><span class="pun">,</span><span class="pln"> </span><span class="lit">1</span><span class="pun">)[</span><span class="lit">0</span><span class="pun">])</span><span class="pln"> </span><span class="com"># 随机获取</span></code></li><li class="L1"><code><span class="pun">}</span></code></li></ol></pre><p><strong>*小贴士:什么是JSON?</strong><br>JSON 指的是 JavaScript 对象表示法(JavaScript Object Notation)</p>
|
||
<ul>
|
||
<li>JSON 是轻量级的文本数据交换格式</li><li>JSON 独立于语言</li><li>JSON 具有自我描述性,更易理解</li><li>JSON 使用 JavaScript 语法来描述数据对象,但是 JSON 仍然独立于语言和平台。JSON 解析器和 JSON 库支持许多不同的编程语言。<br>例子:<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pun">{</span></code></li><li class="L1"><code><span class="str">"employees"</span><span class="pun">:</span><span class="pln"> </span><span class="pun">[</span></code></li><li class="L2"><code><span class="pun">{</span><span class="pln"> </span><span class="str">"firstName"</span><span class="pun">:</span><span class="str">"Bill"</span><span class="pln"> </span><span class="pun">,</span><span class="pln"> </span><span class="str">"lastName"</span><span class="pun">:</span><span class="str">"Gates"</span><span class="pln"> </span><span class="pun">},</span></code></li><li class="L3"><code><span class="pun">{</span><span class="pln"> </span><span class="str">"firstName"</span><span class="pun">:</span><span class="str">"George"</span><span class="pln"> </span><span class="pun">,</span><span class="pln"> </span><span class="str">"lastName"</span><span class="pun">:</span><span class="str">"Bush"</span><span class="pln"> </span><span class="pun">},</span></code></li><li class="L4"><code><span class="pun">{</span><span class="pln"> </span><span class="str">"firstName"</span><span class="pun">:</span><span class="str">"Thomas"</span><span class="pln"> </span><span class="pun">,</span><span class="pln"> </span><span class="str">"lastName"</span><span class="pun">:</span><span class="str">"Carter"</span><span class="pln"> </span><span class="pun">}</span></code></li><li class="L5"><code><span class="pun">]</span></code></li><li class="L6"><code><span class="pun">}</span></code></li></ol></pre>这个 employee 对象是包含 3 个员工记录(对象)的数组。<h3 id="h3-1-4-2-mysql-"><a name="1.4.2 创建MySQL数据库" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4.2 创建MySQL数据库</h3>为了存放爬取的数据,我们采用MySQL数据库。我们可以通过命令行创建并操作数据库,也可以利用GUI工具Navicat进行,本案例使用Navicat工具,并提供sql脚本文件。学生直接导入脚本文件即可使用数据库表。首先确认MySQL数据库是否已启动,在“计算机”上点击右键,点击“管理”,弹出“计算机管理”界面。</li></ul>
|
||
<p><img src="./水滴实验营_files/00-计算机.png" alt=""></p>
|
||
<p>点击“服务和应用程序”下的“服务”,查看MySQL服务状态,如下图所示,状态显示“已启动”,表明MySQL服务正常运行,若没有显示,则在该服务上点击右键手动启动。</p>
|
||
<p><img src="./水滴实验营_files/03-mysql服务启动.png" alt=""></p>
|
||
<p>双击桌面上的“Navicat for MySQL”图标,进入如下界面,点击文件->新建连接->MySQL</p>
|
||
<p><img src="./水滴实验营_files/04-新建链接.png" alt=""></p>
|
||
<p>出现如下界面,设置参数:</p>
|
||
<p><img src="./水滴实验营_files/05-连接参数配置.png" alt=""></p>
|
||
<ul>
|
||
<li>连接名:用户自定义</li><li>主机名或IP地址:localhost </li><li>端口:3306</li><li>用户名:root </li><li>密码:shuidilab<a href="https://github.com/123" title="@123" class="at-link">@123</a></li></ul>
|
||
<p>点击“测试连接”,弹出“连接成功”对话框,即连接成功。点击“确定”,显示新建连接“con1”,双击该连接,显示如下界面。</p>
|
||
<p><img src="./水滴实验营_files/06-显示新建连接.png" alt=""></p>
|
||
<p>在con1上点击右键,点击“新建数据库”,弹出界面,输入数据库名称“testdazhong”,选择字符集和排序规则,如下图所示。</p>
|
||
<p><img src="./水滴实验营_files/07-新建数据库.png" alt=""></p>
|
||
<p>点击“确定”进入如下页面,显示新创建数据库“testdazhong”</p>
|
||
<p><img src="./水滴实验营_files/08-显示数据库.png" alt=""></p>
|
||
<p>双击“testdazhong”,然后在“testdazhong”上点击右键,点击“运行SQL文件”,弹出如下界面,从C盘sqlfile目录下选择文件“fdazhongfood.sql”</p>
|
||
<p><img src="./水滴实验营_files/09-运行SQL.png" alt=""></p>
|
||
<p>点击“开始”,出现如下界面表示导入成功,关闭该界面。</p>
|
||
<p><img src="./水滴实验营_files/10-运行SQL成功.png" alt=""></p>
|
||
<p>在“testdazhong”下的“表”上点击右键,点击“刷新”,显示如下界面,可以看到数据表dazhonginfo已经导入。</p>
|
||
<p><img src="./水滴实验营_files/11-数据表显示.png" alt=""></p>
|
||
<p>选中数据表“dazhonginfo”,点击“设计表”,我们可以看到数据表各个字段的类型、长度的定义,如下图所示。</p>
|
||
<p><img src="./水滴实验营_files/12-设计表.png" alt=""></p>
|
||
<p>各字段含义如下:</p>
|
||
<table>
|
||
<thead>
|
||
<tr>
|
||
<th style="text-align:left">序号</th>
|
||
<th style="text-align:left">字段名称</th>
|
||
<th style="text-align:left">字段意义</th>
|
||
<th style="text-align:left">备注</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<td style="text-align:left">1</td>
|
||
<td style="text-align:left">Id</td>
|
||
<td style="text-align:left">记录序号</td>
|
||
<td style="text-align:left"></td>
|
||
</tr>
|
||
<tr>
|
||
<td style="text-align:left">2</td>
|
||
<td style="text-align:left">city</td>
|
||
<td style="text-align:left">城市</td>
|
||
<td style="text-align:left"></td>
|
||
</tr>
|
||
<tr>
|
||
<td style="text-align:left">3</td>
|
||
<td style="text-align:left">shopName</td>
|
||
<td style="text-align:left">商铺名称</td>
|
||
<td style="text-align:left"></td>
|
||
</tr>
|
||
<tr>
|
||
<td style="text-align:left">4</td>
|
||
<td style="text-align:left">shopId</td>
|
||
<td style="text-align:left">商铺编号</td>
|
||
<td style="text-align:left"></td>
|
||
</tr>
|
||
<tr>
|
||
<td style="text-align:left">5</td>
|
||
<td style="text-align:left">shopPower</td>
|
||
<td style="text-align:left">商铺星级(大众点评自己的评星)</td>
|
||
<td style="text-align:left">30:三星 35:三星半 40:四星 45:四星半 50:五星</td>
|
||
</tr>
|
||
<tr>
|
||
<td style="text-align:left">6</td>
|
||
<td style="text-align:left">mainRegionName</td>
|
||
<td style="text-align:left">所在商区</td>
|
||
<td style="text-align:left"></td>
|
||
</tr>
|
||
<tr>
|
||
<td style="text-align:left">7</td>
|
||
<td style="text-align:left">mainCategoryName</td>
|
||
<td style="text-align:left">所属菜品分类</td>
|
||
<td style="text-align:left"></td>
|
||
</tr>
|
||
<tr>
|
||
<td style="text-align:left">8</td>
|
||
<td style="text-align:left">tasteScore</td>
|
||
<td style="text-align:left">口味评分</td>
|
||
<td style="text-align:left"></td>
|
||
</tr>
|
||
<tr>
|
||
<td style="text-align:left">9</td>
|
||
<td style="text-align:left">environmentScore</td>
|
||
<td style="text-align:left">环境评分</td>
|
||
<td style="text-align:left"></td>
|
||
</tr>
|
||
<tr>
|
||
<td style="text-align:left">10</td>
|
||
<td style="text-align:left">serviceScore</td>
|
||
<td style="text-align:left">服务评分</td>
|
||
<td style="text-align:left"></td>
|
||
</tr>
|
||
<tr>
|
||
<td style="text-align:left">11</td>
|
||
<td style="text-align:left">avgPrice</td>
|
||
<td style="text-align:left">人均消费</td>
|
||
<td style="text-align:left"></td>
|
||
</tr>
|
||
<tr>
|
||
<td style="text-align:left">12</td>
|
||
<td style="text-align:left">shopAddress</td>
|
||
<td style="text-align:left">商铺地址</td>
|
||
<td style="text-align:left"></td>
|
||
</tr>
|
||
<tr>
|
||
<td style="text-align:left">13</td>
|
||
<td style="text-align:left">shopUrl</td>
|
||
<td style="text-align:left">商铺网址</td>
|
||
<td style="text-align:left"></td>
|
||
</tr>
|
||
<tr>
|
||
<td style="text-align:left">14</td>
|
||
<td style="text-align:left">defaultPic</td>
|
||
<td style="text-align:left">商铺图片</td>
|
||
<td style="text-align:left"></td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
<p>关闭,双击“dazhonginfo”表,我们可以看到表的内容,如图所示,该表是空表。</p>
|
||
<p><img src="./水滴实验营_files/13-打开空表.png" alt=""></p>
|
||
<h3 id="h3-1-4-3-python-"><a name="1.4.3 Python编程爬虫" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4.3 Python编程爬虫</h3><h4 id="h4-1-4-3-1-"><a name="1.4.3.1 新建项目" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4.3.1 新建项目</h4><p>打开桌面上的JetBrains PyCharm,点击“+创建新项目”创建一个新项目,如果之前已经使用过PyCharm,选择”文件”->“新建项目”创建新项目。</p>
|
||
<p><img src="./水滴实验营_files/1-首页.png" alt=""></p>
|
||
<p>新创建项目进入如下界面,在Location选择项目目录,或者直接输入新目录名称,例如C:\dazhongspider。然后选择第二个现存的解释器,默认:C:\Program Files\Python37\python.exe。</p>
|
||
<p><img src="./水滴实验营_files/2-选择编译环境.png" alt=""></p>
|
||
<p>然后单击New Project对话框底部的“ Create”按钮,启动过程中弹出如下对话框,将复选框“√”勾掉,关闭。</p>
|
||
<p><img src="./水滴实验营_files/3-弹出对话框关闭.png" alt=""></p>
|
||
<p>进入工程页面,点击“文件”->“设置”->“外观&行为”,点击下方“外观”,在右边UI选项,主题下拉框中选择“IntelliJ”,如图所示,点击“确定”修改完成pycharm视图背景,返回工程页面,点击“视图”菜单勾选“工具栏”。</p>
|
||
<p>若没有看到具体的工程“dazhongspider”,则在工程页面中点击左侧的“Project”标签,看到创建的python项目,如下图所示。</p>
|
||
<p><img src="./水滴实验营_files/6-显示工程.png" alt=""></p>
|
||
<h4 id="h4-1-4-3-2-"><a name="1.4.3.2设置爬虫代码目录" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4.3.2设置爬虫代码目录</h4><p>为了便于项目管理,设置不同的目录编写程序,选中该项目“dazhongspider”,点击右键选择“新建”->“目录”,如图所示。</p>
|
||
<p><img src="./水滴实验营_files/7-新建目录.png" alt=""></p>
|
||
<p>在弹出的界面中(如下图所示),设置爬虫程序的目录名称为“spider”。</p>
|
||
<p><img src="./水滴实验营_files/8-输入新建目录名称.png" alt=""></p>
|
||
<h4 id="h4-1-4-3-3-python-"><a name="1.4.3.3 python编程数据存储" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4.3.3 python编程数据存储</h4><p>选中“dazhongspider”项目->“spider”目录,点击 “新建”->“Python File”,如下图所示。</p>
|
||
<p><img src="./水滴实验营_files/9-新建python程序文件.png" alt=""></p>
|
||
<p>在弹出的对话框中输入文件名“dbconnect”,点击“确定”进入下图所示的“dbconnect.py”文件编辑界面。</p>
|
||
<p><img src="./水滴实验营_files/11-dbconnect编辑界面.png" alt=""></p>
|
||
<p>输入下面数据库操作代码到“dbconnect.py”文件中并保存。</p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="kwd">import</span><span class="pln"> pymysql</span></code></li><li class="L1"><code><span class="com"># 数据库操作类,这个类用于连接数据库、插入数据、查询数据</span></code></li><li class="L2"><code><span class="kwd">class</span><span class="pln"> </span><span class="typ">DBHelper</span><span class="pun">():</span></code></li><li class="L3"><code><span class="pln"> </span><span class="com">#定义构造函数,self实例变量属性</span></code></li><li class="L4"><code><span class="pln"> </span><span class="kwd">def</span><span class="pln"> __init__</span><span class="pun">(</span><span class="kwd">self</span><span class="pun">):</span></code></li><li class="L5"><code><span class="pln"> </span><span class="kwd">self</span><span class="pun">.</span><span class="pln">host </span><span class="pun">=</span><span class="pln"> </span><span class="str">'localhost'</span><span class="pln"> </span><span class="com"># MySQL服务器地址</span></code></li><li class="L6"><code><span class="pln"> </span><span class="kwd">self</span><span class="pun">.</span><span class="pln">port </span><span class="pun">=</span><span class="pln"> </span><span class="lit">3306</span><span class="pln"> </span><span class="com"># MySQL服务器端口号</span></code></li><li class="L7"><code><span class="pln"> </span><span class="kwd">self</span><span class="pun">.</span><span class="pln">user </span><span class="pun">=</span><span class="pln"> </span><span class="str">'root'</span><span class="pln"> </span><span class="com">#用户名</span></code></li><li class="L8"><code><span class="pln"> </span><span class="kwd">self</span><span class="pun">.</span><span class="pln">passwd </span><span class="pun">=</span><span class="pln"> </span><span class="str">'shuidilab@123'</span><span class="pln"> </span><span class="com">#密码</span></code></li><li class="L9"><code><span class="pln"> </span><span class="kwd">self</span><span class="pun">.</span><span class="pln">db </span><span class="pun">=</span><span class="pln"> </span><span class="str">'testdazhong'</span><span class="pln"> </span><span class="com">#数据库名称</span></code></li><li class="L0"><code><span class="pln"> </span><span class="com"># 定义连接到数据库的方法</span></code></li><li class="L1"><code><span class="pln"> </span><span class="kwd">def</span><span class="pln"> connectDatabase</span><span class="pun">(</span><span class="kwd">self</span><span class="pun">):</span></code></li><li class="L2"><code><span class="pln"> </span><span class="com">#创建连接对象</span></code></li><li class="L3"><code><span class="pln"> conn </span><span class="pun">=</span><span class="pln"> pymysql</span><span class="pun">.</span><span class="pln">connect</span><span class="pun">(</span><span class="pln">host</span><span class="pun">=</span><span class="kwd">self</span><span class="pun">.</span><span class="pln">host</span><span class="pun">,</span></code></li><li class="L4"><code><span class="pln"> port</span><span class="pun">=</span><span class="kwd">self</span><span class="pun">.</span><span class="pln">port</span><span class="pun">,</span></code></li><li class="L5"><code><span class="pln"> user</span><span class="pun">=</span><span class="kwd">self</span><span class="pun">.</span><span class="pln">user</span><span class="pun">,</span></code></li><li class="L6"><code><span class="pln"> passwd</span><span class="pun">=</span><span class="kwd">self</span><span class="pun">.</span><span class="pln">passwd</span><span class="pun">,</span></code></li><li class="L7"><code><span class="pln"> db</span><span class="pun">=</span><span class="kwd">self</span><span class="pun">.</span><span class="pln">db</span><span class="pun">,</span></code></li><li class="L8"><code><span class="pln"> charset</span><span class="pun">=</span><span class="str">'utf8'</span><span class="pun">)</span><span class="pln"> </span><span class="com"># 要指定编码,否则中文可能乱码</span></code></li><li class="L9"><code><span class="pln"> </span><span class="kwd">return</span><span class="pln"> conn</span></code></li><li class="L0"><code><span class="pln"> </span><span class="com"># 插入数据</span></code></li><li class="L1"><code><span class="pln"> </span><span class="kwd">def</span><span class="pln"> insert</span><span class="pun">(</span><span class="kwd">self</span><span class="pun">,</span><span class="pln"> sql</span><span class="pun">,</span><span class="pln"> </span><span class="pun">*</span><span class="kwd">params</span><span class="pun">):</span><span class="pln"> </span><span class="com"># 注意这里params要加*,因为传递过来的是元组,*表示参数个数不定</span></code></li><li class="L2"><code><span class="pln"> conn </span><span class="pun">=</span><span class="pln"> </span><span class="kwd">self</span><span class="pun">.</span><span class="pln">connectDatabase</span><span class="pun">()</span></code></li><li class="L3"><code><span class="pln"> </span><span class="com">#创建游标对象</span></code></li><li class="L4"><code><span class="pln"> cur </span><span class="pun">=</span><span class="pln"> conn</span><span class="pun">.</span><span class="pln">cursor</span><span class="pun">()</span></code></li><li class="L5"><code><span class="pln"> </span><span class="com">#执行sql语句</span></code></li><li class="L6"><code><span class="pln"> cur</span><span class="pun">.</span><span class="pln">execute</span><span class="pun">(</span><span class="pln">sql</span><span class="pun">,</span><span class="pln"> </span><span class="kwd">params</span><span class="pun">)</span></code></li><li class="L7"><code><span class="pln"> </span><span class="com">#注意要commit,提交</span></code></li><li class="L8"><code><span class="pln"> conn</span><span class="pun">.</span><span class="pln">commit</span><span class="pun">()</span></code></li><li class="L9"><code><span class="pln"> cur</span><span class="pun">.</span><span class="pln">close</span><span class="pun">()</span></code></li><li class="L0"><code><span class="pln"> conn</span><span class="pun">.</span><span class="pln">close</span><span class="pun">()</span></code></li><li class="L1"><code><span class="pln"> </span><span class="com"># 查询数据</span></code></li><li class="L2"><code><span class="pln"> </span><span class="kwd">def</span><span class="pln"> </span><span class="kwd">select</span><span class="pun">(</span><span class="kwd">self</span><span class="pun">,</span><span class="pln"> sql</span><span class="pun">):</span></code></li><li class="L3"><code><span class="pln"> conn </span><span class="pun">=</span><span class="pln"> </span><span class="kwd">self</span><span class="pun">.</span><span class="pln">connectDatabase</span><span class="pun">()</span></code></li><li class="L4"><code><span class="pln"> cur </span><span class="pun">=</span><span class="pln"> conn</span><span class="pun">.</span><span class="pln">cursor</span><span class="pun">()</span></code></li><li class="L5"><code><span class="pln"> </span><span class="kwd">try</span><span class="pun">:</span></code></li><li class="L6"><code><span class="pln"> </span><span class="com"># 执行SQL语句</span></code></li><li class="L7"><code><span class="pln"> cur</span><span class="pun">.</span><span class="pln">execute</span><span class="pun">(</span><span class="pln">sql</span><span class="pun">)</span></code></li><li class="L8"><code><span class="pln"> conn</span><span class="pun">.</span><span class="pln">commit</span><span class="pun">()</span></code></li><li class="L9"><code><span class="pln"> </span><span class="com"># 获取所有记录列表</span></code></li><li class="L0"><code><span class="pln"> results </span><span class="pun">=</span><span class="pln"> cur</span><span class="pun">.</span><span class="pln">fetchall</span><span class="pun">()</span></code></li><li class="L1"><code><span class="pln"> </span><span class="kwd">return</span><span class="pln"> results</span></code></li><li class="L2"><code><span class="pln"> </span><span class="kwd">except</span><span class="pun">:</span></code></li><li class="L3"><code><span class="pln"> </span><span class="kwd">print</span><span class="pun">(</span><span class="str">"Error: unable to fecth data"</span><span class="pun">)</span></code></li><li class="L4"><code><span class="pln"> cur</span><span class="pun">.</span><span class="pln">close</span><span class="pun">()</span></code></li><li class="L5"><code><span class="pln"> conn</span><span class="pun">.</span><span class="pln">close</span><span class="pun">()</span></code></li></ol></pre><h4 id="h4-1-4-3-4-python-"><a name="1.4.3.4 Python爬虫编程" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4.3.4 Python爬虫编程</h4><p>如下图所示,选中“dazhongspider”项目->“spider”目录,点击“新建”->“Python File”。</p>
|
||
<p><img src="./水滴实验营_files/12-新建spider程序.png" alt=""></p>
|
||
<p>在弹出的对话框,输入文件名 “spider”,点击“确定”,进入“spider.py”的编辑界面。</p>
|
||
<p><img src="./水滴实验营_files/14-spider编辑页.png" alt=""></p>
|
||
<p>输入下面的爬虫代码到该文件中并保存。</p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="kwd">import</span><span class="pln"> json</span></code></li><li class="L1"><code><span class="kwd">import</span><span class="pln"> random</span></code></li><li class="L2"><code><span class="kwd">import</span><span class="pln"> requests</span></code></li><li class="L3"><code><span class="kwd">import</span><span class="pln"> dbconnect</span></code></li><li class="L4"><code><span class="com"># 城市列表</span></code></li><li class="L5"><code><span class="pln">listcity </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[[</span><span class="str">"上海"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"fce2e3a36450422b7fad3f2b90370efd71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L6"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"北京"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"d5036cf54fcb57e9dceb9fefe3917fff71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L7"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"广州"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"e749e3e04032ee6b165fbea6fe2dafab71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L8"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"深圳"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"e049aa251858f43d095fc4c61d62a9ec71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L9"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"天津"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"2e5d0080237ff3c8f5b5d3f315c7c4a508e25c702ab1b810071e8e2c39502be1"</span><span class="pun">],</span></code></li><li class="L0"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"杭州"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"91621282e559e9fc9c5b3e816cb1619c71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L1"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"南京"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"d6339a01dbd98141f8e684e1ad8af5c871862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L2"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"苏州"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"536e0e568df850d1e6ba74b0cf72e19771862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L3"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"成都"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"c950bc35ad04316c76e89bf2dc86bfe071862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L4"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"武汉"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"d96a24c312ed7b96fcc0cedd6c08f68c08e25c702ab1b810071e8e2c39502be1"</span><span class="pun">],</span></code></li><li class="L5"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"重庆"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"6229984ceb373efb8fd1beec7eb4dcfd71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L6"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"西安"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"ad66274c7f5f8d27ffd7f6b39ec447b608e25c702ab1b810071e8e2c39502be1"</span><span class="pun">],</span></code></li><li class="L7"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"青岛"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"9874ff91c8b50ad831ea897b0cd5315b71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L8"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"济南"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"4be7d758168826991eb4b3e779fdf41571862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L9"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"威海"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"fb5f85ca4c696d61c6e7a7e6c19b2f4530aacdebee4c4c9365dc18972daccaf7"</span><span class="pun">],</span></code></li><li class="L0"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"长春"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"261f655c7a9713696580432624721cd871862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L1"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"大连"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"21f56d987e57bd3e8df308351e3f4b9171862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L2"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"佛山"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"4e5d3a805c4f1ed6059c7074e5e86adf30aacdebee4c4c9365dc18972daccaf7"</span><span class="pun">],</span></code></li><li class="L3"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"贵阳"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"7ae3d5049e1c200ed6b2db0ac51604ba30aacdebee4c4c9365dc18972daccaf7"</span><span class="pun">],</span></code></li><li class="L4"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"合肥"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"94af1388885f0b7483fdf72cba086a6030aacdebee4c4c9365dc18972daccaf7"</span><span class="pun">],</span></code></li><li class="L5"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"呼和浩特"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"d64d5919fc236699402122b5770b4a6f71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L6"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"昆明"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"45159d85c35f742c1f2635d6ea6fa3ae30aacdebee4c4c9365dc18972daccaf7"</span><span class="pun">],</span></code></li><li class="L7"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"兰州"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"b201713f2fb5f7235b6e2a6de8faf6a630aacdebee4c4c9365dc18972daccaf7"</span><span class="pun">],</span></code></li><li class="L8"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"南宁"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"ff6f15347ab16f86ffb34e2700c4870930aacdebee4c4c9365dc18972daccaf7"</span><span class="pun">],</span></code></li><li class="L9"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"秦皇岛"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"ca02d37fa01b0f9613555237517e19ff71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L0"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"沈阳"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"eec1fb195780b91fc9dd065409c1d28471862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L1"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"太原"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"293e6160d971fe48ebbed1c8033bd9ed71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L2"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"唐山"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"2ae2237467ce4784b1705be502b0315271862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L3"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"无锡"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"51ff28145b3f63e2f9ec4b2a34e8850c71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L4"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"扬州"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"02eeb7a9021c215641f3ec0e3cede50b71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">]</span></code></li><li class="L5"><code><span class="pln"> </span><span class="pun">]</span></code></li><li class="L6"><code><span class="com">#用户代理,是一个特殊字符串头,使得服务器能够识别客户使用的操作系统及版本、浏览器及版本、浏览器渲染引擎、浏览器语言、浏览器插件等#</span></code></li><li class="L7"><code><span class="pln">USER_AGENT_LIST </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[</span></code></li><li class="L8"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"</span><span class="pun">,</span></code></li><li class="L9"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11"</span><span class="pun">,</span></code></li><li class="L0"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6"</span><span class="pun">,</span></code></li><li class="L1"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6"</span><span class="pun">,</span></code></li><li class="L2"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1"</span><span class="pun">,</span></code></li><li class="L3"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5"</span><span class="pun">]</span></code></li><li class="L4"><code><span class="com">#设置随机选择用户代理#</span></code></li><li class="L5"><code><span class="pln">head </span><span class="pun">=</span><span class="pln"> </span><span class="pun">{</span></code></li><li class="L6"><code><span class="str">'User-Agent'</span><span class="pun">:</span><span class="pln"> </span><span class="str">'{0}'</span><span class="pun">.</span><span class="pln">format</span><span class="pun">(</span><span class="pln">random</span><span class="pun">.</span><span class="pln">sample</span><span class="pun">(</span><span class="pln">USER_AGENT_LIST</span><span class="pun">,</span><span class="pln"> </span><span class="lit">1</span><span class="pun">)[</span><span class="lit">0</span><span class="pun">])</span><span class="pln"> </span><span class="com"># 从列表中随机获取1个</span></code></li><li class="L7"><code><span class="pun">}</span></code></li><li class="L8"><code><span class="pln">flag </span><span class="pun">=</span><span class="pln"> </span><span class="lit">0</span><span class="pln"> </span><span class="com">#总条数</span></code></li><li class="L9"><code><span class="pln">code </span><span class="pun">=</span><span class="pln"> </span><span class="lit">0</span><span class="pln"> </span><span class="com">#当前条数</span></code></li><li class="L0"><code></code></li><li class="L1"><code><span class="com"># 定义爬虫函数(调用解析函数)#</span></code></li><li class="L2"><code><span class="kwd">def</span><span class="pln"> cinfoSpider</span><span class="pun">(</span><span class="pln">citylist</span><span class="pun">):</span></code></li><li class="L3"><code><span class="pln"> city </span><span class="pun">=</span><span class="pln"> citylist</span><span class="pun">[</span><span class="lit">0</span><span class="pun">]</span></code></li><li class="L4"><code><span class="pln"> url </span><span class="pun">=</span><span class="pln"> citylist</span><span class="pun">[</span><span class="lit">1</span><span class="pun">]</span></code></li><li class="L5"><code><span class="pln"> </span><span class="com">#爬虫地址#</span></code></li><li class="L6"><code><span class="pln"> cbase_url </span><span class="pun">=</span><span class="pln"> </span><span class="str">"http://dp01.ds.quancube.cn/mylist/ajax/shoprank?rankId="</span><span class="pun">+</span><span class="pln">url</span></code></li><li class="L7"><code><span class="pln"> </span><span class="com">#get请求获取网页数据#</span></code></li><li class="L8"><code><span class="pln"> html </span><span class="pun">=</span><span class="pln"> requests</span><span class="pun">.</span><span class="kwd">get</span><span class="pun">(</span><span class="pln">cbase_url</span><span class="pun">,</span><span class="pln"> headers</span><span class="pun">=</span><span class="pln">head</span><span class="pun">)</span></code></li><li class="L9"><code><span class="pln"> </span><span class="com">#调用cfindFood函数解析#</span></code></li><li class="L0"><code><span class="pln"> cfindinfo</span><span class="pun">(</span><span class="pln">city</span><span class="pun">=</span><span class="pln">city</span><span class="pun">,</span><span class="pln"> data</span><span class="pun">=</span><span class="pln">str</span><span class="pun">(</span><span class="pln">html</span><span class="pun">.</span><span class="pln">text</span><span class="pun">))</span></code></li><li class="L1"><code></code></li><li class="L2"><code><span class="com"># 定义解析页面函数#</span></code></li><li class="L3"><code><span class="kwd">def</span><span class="pln"> cfindinfo</span><span class="pun">(</span><span class="pln">city</span><span class="pun">,</span><span class="pln">data</span><span class="pun">):</span></code></li><li class="L4"><code><span class="pln"> </span><span class="kwd">global</span><span class="pln"> flag</span><span class="pun">,</span><span class="pln">code</span></code></li><li class="L5"><code><span class="pln"> </span><span class="com">#连接mysql数据库#</span></code></li><li class="L6"><code><span class="pln"> mysql_db </span><span class="pun">=</span><span class="pln"> dbconnect</span><span class="pun">.</span><span class="typ">DBHelper</span><span class="pun">()</span></code></li><li class="L7"><code><span class="pln"> mysql_db</span><span class="pun">.</span><span class="pln">connectDatabase</span><span class="pun">();</span></code></li><li class="L8"><code><span class="pln"> </span><span class="com">#解析返回的JSON数据#</span></code></li><li class="L9"><code><span class="pln"> </span><span class="kwd">for</span><span class="pln"> data </span><span class="kwd">in</span><span class="pln"> json</span><span class="pun">.</span><span class="pln">loads</span><span class="pun">(</span><span class="pln">data</span><span class="pun">)[</span><span class="str">"shopBeans"</span><span class="pun">]:</span></code></li><li class="L0"><code><span class="pln"> flag </span><span class="pun">+=</span><span class="lit">1</span></code></li><li class="L1"><code><span class="pln"> </span><span class="com"># 商铺名称</span></code></li><li class="L2"><code><span class="pln"> shopName </span><span class="pun">=</span><span class="pln"> data</span><span class="pun">[</span><span class="str">"shopName"</span><span class="pun">]</span></code></li><li class="L3"><code><span class="pln"> </span><span class="com"># 商品编号</span></code></li><li class="L4"><code><span class="pln"> shopId </span><span class="pun">=</span><span class="pln"> data</span><span class="pun">[</span><span class="str">"shopId"</span><span class="pun">]</span></code></li><li class="L5"><code><span class="pln"> </span><span class="com"># 商铺星级</span></code></li><li class="L6"><code><span class="pln"> shopPower </span><span class="pun">=</span><span class="pln"> data</span><span class="pun">[</span><span class="str">"shopPower"</span><span class="pun">]</span></code></li><li class="L7"><code><span class="pln"> </span><span class="com"># 所在商区</span></code></li><li class="L8"><code><span class="pln"> mainRegionName </span><span class="pun">=</span><span class="pln"> data</span><span class="pun">[</span><span class="str">"mainRegionName"</span><span class="pun">]</span></code></li><li class="L9"><code><span class="pln"> </span><span class="com"># 分类名称</span></code></li><li class="L0"><code><span class="pln"> mainCategoryName </span><span class="pun">=</span><span class="pln"> data</span><span class="pun">[</span><span class="str">"mainCategoryName"</span><span class="pun">]</span></code></li><li class="L1"><code><span class="pln"> </span><span class="com"># 口味评分</span></code></li><li class="L2"><code><span class="pln"> tasteScore </span><span class="pun">=</span><span class="pln"> data</span><span class="pun">[</span><span class="str">"score1"</span><span class="pun">]</span></code></li><li class="L3"><code><span class="pln"> </span><span class="com"># 环境评分</span></code></li><li class="L4"><code><span class="pln"> environmentScore </span><span class="pun">=</span><span class="pln"> data</span><span class="pun">[</span><span class="str">"score2"</span><span class="pun">]</span></code></li><li class="L5"><code><span class="pln"> </span><span class="com"># 服务评分</span></code></li><li class="L6"><code><span class="pln"> serviceScore </span><span class="pun">=</span><span class="pln"> data</span><span class="pun">[</span><span class="str">"score3"</span><span class="pun">]</span></code></li><li class="L7"><code><span class="pln"> </span><span class="com"># 人均消费</span></code></li><li class="L8"><code><span class="pln"> avgPrice </span><span class="pun">=</span><span class="pln"> data</span><span class="pun">[</span><span class="str">"avgPrice"</span><span class="pun">]</span></code></li><li class="L9"><code><span class="pln"> </span><span class="com"># 详细地址</span></code></li><li class="L0"><code><span class="pln"> shopAddress </span><span class="pun">=</span><span class="pln"> data</span><span class="pun">[</span><span class="str">"address"</span><span class="pun">]</span></code></li><li class="L1"><code><span class="pln"> </span><span class="com"># 商铺网址</span></code></li><li class="L2"><code><span class="pln"> shopUrl </span><span class="pun">=</span><span class="pln"> </span><span class="str">"http://dp01.ds.quancube.cn/shop/"</span><span class="pun">+</span><span class="pln">shopId</span></code></li><li class="L3"><code><span class="pln"> </span><span class="com"># 商铺图片</span></code></li><li class="L4"><code><span class="pln"> defaultPic </span><span class="pun">=</span><span class="pln"> data</span><span class="pun">[</span><span class="str">"defaultPic"</span><span class="pun">]</span></code></li><li class="L5"><code><span class="pln"> </span><span class="com">#将解析数据插入数据库#</span></code></li><li class="L6"><code><span class="pln"> </span><span class="com">#定义sql语句#</span></code></li><li class="L7"><code><span class="pln"> sql </span><span class="pun">=</span><span class="pln"> </span><span class="str">'''insert into dazhonginfo(city, shopName, shopId, shopPower, mainRegionName, mainCategoryName, tasteScore, environmentScore, serviceScore, avgPrice, shopAddress, shopUrl, defaultPic) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'''</span></code></li><li class="L8"><code><span class="pln"> </span><span class="com">#插入这些参数的数据#</span></code></li><li class="L9"><code><span class="pln"> </span><span class="kwd">params</span><span class="pln"> </span><span class="pun">=</span><span class="pln"> </span><span class="pun">(</span><span class="pln">city</span><span class="pun">,</span><span class="pln"> shopName</span><span class="pun">,</span><span class="pln"> shopId</span><span class="pun">,</span><span class="pln"> shopPower</span><span class="pun">,</span><span class="pln"> mainRegionName</span><span class="pun">,</span><span class="pln"> mainCategoryName</span><span class="pun">,</span><span class="pln"> tasteScore</span><span class="pun">,</span><span class="pln"> environmentScore</span><span class="pun">,</span><span class="pln"> serviceScore</span><span class="pun">,</span><span class="pln"> avgPrice</span><span class="pun">,</span><span class="pln"> shopAddress</span><span class="pun">,</span><span class="pln"> shopUrl</span><span class="pun">,</span><span class="pln"> defaultPic</span><span class="pun">)</span></code></li><li class="L0"><code><span class="pln"> </span><span class="kwd">try</span><span class="pun">:</span></code></li><li class="L1"><code><span class="pln"> mysql_db</span><span class="pun">.</span><span class="pln">insert</span><span class="pun">(</span><span class="pln">sql</span><span class="pun">,*</span><span class="kwd">params</span><span class="pun">)</span></code></li><li class="L2"><code><span class="pln"> code </span><span class="pun">+=</span><span class="lit">1</span></code></li><li class="L3"><code><span class="pln"> </span><span class="kwd">print</span><span class="pun">(</span><span class="str">"----- 插入:"</span><span class="pun">,</span><span class="pln"> code</span><span class="pun">,</span><span class="pln"> </span><span class="str">"条------"</span><span class="pun">)</span></code></li><li class="L4"><code><span class="pln"> </span><span class="kwd">except</span><span class="pun">:</span></code></li><li class="L5"><code><span class="pln"> </span><span class="kwd">print</span><span class="pun">(</span><span class="str">"已存在不再重复插入!!"</span><span class="pun">)</span></code></li><li class="L6"><code><span class="pln"> </span><span class="kwd">print</span><span class="pun">(</span><span class="str">"总条数:"</span><span class="pun">,</span><span class="pln"> flag</span><span class="pun">)</span></code></li><li class="L7"><code></code></li><li class="L8"><code><span class="kwd">if</span><span class="pln"> __name__ </span><span class="pun">==</span><span class="pln"> </span><span class="str">'__main__'</span><span class="pun">:</span></code></li><li class="L9"><code><span class="pln"> </span><span class="com">#循环执行对每个城市网页的爬虫#</span></code></li><li class="L0"><code><span class="pln"> </span><span class="kwd">for</span><span class="pln"> city_data </span><span class="kwd">in</span><span class="pln"> listcity</span><span class="pun">:</span></code></li><li class="L1"><code><span class="pln"> cinfoSpider</span><span class="pun">(</span><span class="pln">city_data</span><span class="pun">)</span></code></li></ol></pre><h4 id="h4-1-4-3-5-"><a name="1.4.3.5 设置依赖包" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4.3.5 设置依赖包</h4><p>选中项目“dazhongspider”,点击“文件”->“设置”,查看项目解释器,点击“项目:dazhongspider”中的“Project Interpreter”,是否已安装requests、pymysql依赖包,若有,点击“确定”,若没有,则按如下步骤添加,以“requests”为列,首先点击右侧的“+”号</p>
|
||
<p><img src="./水滴实验营_files/15-选择项目依赖包.png" alt=""></p>
|
||
<p>进入如下界面</p>
|
||
<p><img src="./水滴实验营_files/30-安装依赖包首页.png" alt=""></p>
|
||
<p>点击下方“Manage repositories”,进入下载源管理界面,选择“quancube”下载源点击“-”并点击“确定”删除该下载源,只保留“tsinghua”清华下载源。</p>
|
||
<p><img src="./水滴实验营_files/31-删除清华下载源.png" alt=""></p>
|
||
<p>在查询框中输入“requests”,然后在列表中选择“requests”,点击“install package”</p>
|
||
<p><img src="./水滴实验营_files/16-检索requests依赖包.png" alt=""></p>
|
||
<p>安装过程可能需要等待一会儿,出现“Package ‘requests-**‘ installed successfully”表明依赖包安装成功,其他依赖包的安装过程类似,请按照该步骤继续安装“pymysql”。</p>
|
||
<p><img src="./水滴实验营_files/17-依赖包安装成功.png" alt=""></p>
|
||
<p>安装完成后关闭安装页面,进入“设置”页面后,点击“确定”,回到工程主页面。</p>
|
||
<p><strong>注意:</strong>若多次安装依赖包仍不成功,可选择其他国内下载源中的依赖包进行安装。</p>
|
||
<h4 id="h4-1-4-3-6-"><a name="1.4.3.6 执行爬虫爬取数据" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4.3.6 执行爬虫爬取数据</h4><p>在dazhongspider工程spider目录中选中“spider.py”文件,点击右键选择“运行spider”</p>
|
||
<p><img src="./水滴实验营_files/18-运行python程序.png" alt=""></p>
|
||
<p>执行爬虫程序开始爬取数据,在“运行”控制台打印程序输出的信息,如下图所示。</p>
|
||
<p><img src="./水滴实验营_files/19-运行控制台显示结果.png" alt=""></p>
|
||
<h4 id="h4-1-4-3-7-"><a name="1.4.3.7 查看爬取的数据" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4.3.7 查看爬取的数据</h4><p>点击Navicat软件下“testdazhong”数据库,如果数据表已打开,如图所示,请先关闭该文件。</p>
|
||
<p><img src="./水滴实验营_files/20-已打开数据空表.png" alt=""></p>
|
||
<p>显示数据表“dazhonginfo”</p>
|
||
<p><img src="./水滴实验营_files/21-显示数据库表.png" alt=""></p>
|
||
<p>双击该表,即可查看爬取的数据,如下图所示,点击右下角-> ,可查看下一页数据。</p>
|
||
<p><img src="./水滴实验营_files/22-数据库表数据.png" alt=""></p>
|
||
<h4 id="h4-1-4-3-8-"><a name="1.4.3.8 学生实训任务" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4.3.8 学生实训任务</h4><h5 id="h5-1-4-3-8-1-20-"><a name="1.4.3.8.1 增加其他20个城市进行爬虫" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4.3.8.1 增加其他20个城市进行爬虫</h5><p>学生自行添加下表中的其他20个城市的rankId,进行爬虫实验。<br><strong>注意:程序中第30个城市rankId最后需要加逗号,最后第50个城市rankId无需逗号。</strong></p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pun">[</span><span class="str">"珠海"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"cf609e6234aca7061ab6a47e8d906e8c30aacdebee4c4c9365dc18972daccaf7"</span><span class="pun">],</span></code></li><li class="L1"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"安庆"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"35e4c8f1738ecc8ad87b613926a9d6dd1eebfd971c239635bc3b5e190325c01c"</span><span class="pun">],</span></code></li><li class="L2"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"安阳"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"d217beb464e7778289c854eafe2fdab21eebfd971c239635bc3b5e190325c01c"</span><span class="pun">],</span></code></li><li class="L3"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"保定"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"2dfe97d8557f1cccc3484afc2dc54e9c71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L4"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"蚌埠"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"1b3e2c870e22cb1c98035d64f71b82361eebfd971c239635bc3b5e190325c01c"</span><span class="pun">],</span></code></li><li class="L5"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"北海"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"d0490e0c48a4225ee85d2a0ec802ab131eebfd971c239635bc3b5e190325c01c"</span><span class="pun">],</span></code></li><li class="L6"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"滨州"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"7ed4b634645044dc1c746c8a1afb14481eebfd971c239635bc3b5e190325c01c"</span><span class="pun">],</span></code></li><li class="L7"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"保山"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"a589b29ec2067666fca30ea28844e70c1eebfd971c239635bc3b5e190325c01c"</span><span class="pun">],</span></code></li><li class="L8"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"宝鸡"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"701cef49e3c2f39394ce006c0b9945b21eebfd971c239635bc3b5e190325c01c"</span><span class="pun">],</span></code></li><li class="L9"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"北安"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"5ac6152df84456d942f1a69afe59c3fa1eebfd971c239635bc3b5e190325c01c"</span><span class="pun">],</span></code></li><li class="L0"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"承德"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"ad798bb841d96db01ce329855213a02ad97945289578b7557a191d6c125161e7"</span><span class="pun">],</span></code></li><li class="L1"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"沧州"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"76de7d4c29a0660101822920a658ecded97945289578b7557a191d6c125161e7"</span><span class="pun">],</span></code></li><li class="L2"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"大庆"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"a18c61815481274c318a383a902247ef71862f838d1255ea693b953b1d49c7c0"</span><span class="pun">],</span></code></li><li class="L3"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"丹东"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"ad2b737489102d91905e887d5eb4d838d97945289578b7557a191d6c125161e7"</span><span class="pun">],</span></code></li><li class="L4"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"东营"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"cb6678ec79300273ed82ba7f7c09ccc11eebfd971c239635bc3b5e190325c01c"</span><span class="pun">],</span></code></li><li class="L5"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"德州"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"475860687e32c17b501eb86d7e8966d21eebfd971c239635bc3b5e190325c01c"</span><span class="pun">],</span></code></li><li class="L6"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"鄂尔多斯"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"8255b0e7540dbba8157071204385cd65d97945289578b7557a191d6c125161e7"</span><span class="pun">],</span></code></li><li class="L7"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"抚顺"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"4c4a091ef17e08597fd6a7c6bc0aab23d97945289578b7557a191d6c125161e7"</span><span class="pun">],</span></code></li><li class="L8"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"聊城"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"e34386f82b808950f4a1f859113352fd1eebfd971c239635bc3b5e190325c01c"</span><span class="pun">],</span></code></li><li class="L9"><code><span class="pln"> </span><span class="pun">[</span><span class="str">"连云港"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"c2687c56fb887be66d324c728ab95c2571862f838d1255ea693b953b1d49c7c0"</span><span class="pun">]</span></code></li></ol></pre><p>执行爬虫程序继续爬取数据,运行界面开始显示“已存在不再重复插入”。</p>
|
||
<p><img src="./水滴实验营_files/23-重复执行部分城市输出信息.png" alt=""></p>
|
||
<p>这是因为前面已经爬取前30个城市数据是正常输出,继续执行即可,结束后,出现如下信息。</p>
|
||
<p><img src="./水滴实验营_files/24-50个城市执行完成信息.png" alt=""></p>
|
||
<h5 id="h5-1-4-3-8-2-phoneno-"><a name="1.4.3.8.2 爬取数据增加电话号码“phoneNo”字段" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4.3.8.2 爬取数据增加电话号码“phoneNo”字段</h5><p>学生可自行实践,提示:新建数据库、增加字段、爬虫程序修改。</p>
|
||
<h5 id="h5-1-4-3-8-3-"><a name="1.4.3.8.3 爬虫豆瓣电影正则表达式分析" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4.3.8.3 爬虫豆瓣电影正则表达式分析</h5><p>1.正则表达式介绍<br>(1) 正则表达式<br> 正则表达式是用来匹配与查找字符串的,从网上爬取数据自然或多或少会用到正则表达式,python的正则表达式要先引入re模块,正则表达式以r引导,例如<strong>r“\d+”</strong>正则表达式表示匹配连续的多个数值,search是re中的函数,从”abc123cd”字符串中搜索连续的数值,得到”123”。<br>(2) 字符串”\d”匹配0~9之间的一个数值<br>(3) 字符”+”重复前面一个匹配字符一次或者多次.<br> <strong>r”b\d+”</strong>第一个字符要匹配”b”,后面是连续的多个数字,因此”是b1233”,不是”a12”.<br>(4)字符”<em>“</em>重复前面一个匹配字符零次或者多次.<br> “” 与 “+”类似,但有区别,列如:可见 r”ab+“匹配的是”ab”,但是r”ab “匹配的是”a”,因为表示”b”可以重复零次,但是”+“却要求”b”重复一次以上.<br>(5)字符”?”重复前面一个匹配字符零次或者一次.<br> 匹配结果”ab”,重复b一次.<br>(6) 字符”.“代表任何一个字符,但是没有特别声明时不代表字符”\n”.<br>“.”代表了字符”x”.<br>(7) “|”代表把左右分成两个部分 .<br> 匹配”ab”或者”ba”都可以.<br><strong>(8)</strong> 特殊字符使用反斜杠”“引导,例如”\r”、”\n”、”\t”、”\”分别表示回车、换行、制表符号与反斜线自己本身.<br>(9) 字符”\b”表示单词结尾,单词结尾包括各种空白字符或者字符串结尾.<br>结果匹配”car”,因为”car”后面是一个空格.<br>(10) “[]中的字符是任选择一个,如果字符ASCll码中连续的一组,那么可以使用”-“字符连接,例如[0-9]表示0-9的其中一个数字,[A-Z]表示A-Z的其中一个大写字符,[0-9A-z]表示0-9的其中一个数字或者A-z的其中一个大写字符.<br>(11) “^”出现在[]的第一个字符位置,就代表<em>取</em>反,例如[ ^ab0-9]表示不是a、b,也不是0-9的数字.<br>(12) “\s”匹配任何空白字符,等价”[\r\n\x20\t\f\v]”<br>(13) “\w”匹配包括下划线子内的单词字符,等价于”[a-zA-Z0-9]”<br>(14) “$”字符比配字符串的结尾位置<br> 最后一个”ab”,而不是第一个”ab”<br>(15) 使用括号(…)可以把(…)看出一个整体,经常与”+”、”<em>“、”?”</em>的连续使用,对(…)部分进行重复.<br>匹配”abab”,”+“对”ab”进行了重复<br>(16) 查找匹配字符串<em>**</em><br> 正则表达式re库的search函数使用正则表达式对要匹配的字符串进行匹配,如果匹配不成功返回None,如果匹配成功返回一个匹配对象,匹配对象调用start()函数得到匹配字符的开始位置,匹配对象调用end()函数得到匹配字符串的结束位置,search虽然只返回匹配第一次匹配的结果,但是我们只要连续使用search函数就可以找到字符串全部匹配的字符串.匹配找出英文句子中所有单词,以使用正则表达式r”[A-Za-z]+\b”匹配单词,它表示匹配由大小写字母组成的连续多个字符,一般是一个单词,之后”\b”表示单词结尾.程序开始匹配到一个单词后m.start(),m.end()就是单词的起始位置,s[start:end]为截取的单词,之后程序再次匹配字符串s=s[end:],即字符串的后半段,一直到匹配完毕为止就找出每个单词.<br>2.正则表达式分析爬虫数据<br>爬取豆瓣电影TOP250影片并利用正则表达式进行信息分析,爬虫网址链接<a href="https://movie.douban.com/top250">https://movie.douban.com/top250</a> , 模拟浏览器部分请多设置几组随机选择一个,以防IP被封。</p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="kwd">import</span><span class="pln"> requests</span></code></li><li class="L1"><code><span class="kwd">import</span><span class="pln"> re</span></code></li><li class="L2"><code><span class="kwd">import</span><span class="pln"> random</span></code></li><li class="L3"><code></code></li><li class="L4"><code><span class="kwd">def</span><span class="pln"> get_douban_top</span><span class="pun">(</span><span class="pln">url</span><span class="pun">):</span></code></li><li class="L5"><code><span class="pln"> USER_AGENT_LIST </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[</span></code></li><li class="L6"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"</span><span class="pun">,</span></code></li><li class="L7"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11"</span><span class="pun">,</span></code></li><li class="L8"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6"</span><span class="pun">,</span></code></li><li class="L9"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6"</span><span class="pun">,</span></code></li><li class="L0"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1"</span><span class="pun">,</span></code></li><li class="L1"><code><span class="pln"> </span><span class="str">"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5"</span><span class="pun">]</span></code></li><li class="L2"><code><span class="pln"> </span><span class="com"># 设置随机选择用户代理#</span></code></li><li class="L3"><code><span class="pln"> headers </span><span class="pun">=</span><span class="pln"> </span><span class="pun">{</span></code></li><li class="L4"><code><span class="pln"> </span><span class="str">'User-Agent'</span><span class="pun">:</span><span class="pln"> </span><span class="str">'{0}'</span><span class="pun">.</span><span class="pln">format</span><span class="pun">(</span><span class="pln">random</span><span class="pun">.</span><span class="pln">sample</span><span class="pun">(</span><span class="pln">USER_AGENT_LIST</span><span class="pun">,</span><span class="pln"> </span><span class="lit">1</span><span class="pun">)[</span><span class="lit">0</span><span class="pun">])</span><span class="pln"> </span><span class="com"># 从列表中随机获取1个</span></code></li><li class="L5"><code><span class="pln"> </span><span class="pun">}</span></code></li><li class="L6"><code></code></li><li class="L7"><code><span class="pln"> response </span><span class="pun">=</span><span class="pln"> requests</span><span class="pun">.</span><span class="kwd">get</span><span class="pun">(</span><span class="pln">url</span><span class="pun">,</span><span class="pln"> headers</span><span class="pun">=</span><span class="pln">headers</span><span class="pun">)</span></code></li><li class="L8"><code><span class="pln"> </span><span class="kwd">if</span><span class="pln"> response</span><span class="pun">.</span><span class="pln">status_code </span><span class="pun">==</span><span class="pln"> </span><span class="lit">200</span><span class="pun">:</span></code></li><li class="L9"><code><span class="pln"> </span><span class="kwd">return</span><span class="pln"> response</span><span class="pun">.</span><span class="pln">text</span></code></li><li class="L0"><code><span class="pln"> </span><span class="kwd">return</span><span class="pln"> </span><span class="kwd">None</span></code></li><li class="L1"><code></code></li><li class="L2"><code></code></li><li class="L3"><code><span class="kwd">def</span><span class="pln"> analyze</span><span class="pun">(</span><span class="pln">offset</span><span class="pun">):</span></code></li><li class="L4"><code><span class="pln"> url </span><span class="pun">=</span><span class="pln"> </span><span class="str">'https://movie.douban.com/top250/?start = offset & filter ='</span><span class="pun">+</span><span class="pln">str</span><span class="pun">(</span><span class="pln">offset</span><span class="pun">)</span></code></li><li class="L5"><code><span class="pln"> html </span><span class="pun">=</span><span class="pln"> get_douban_top</span><span class="pun">(</span><span class="pln">url</span><span class="pun">)</span></code></li><li class="L6"><code></code></li><li class="L7"><code><span class="pln"> </span><span class="kwd">for</span><span class="pln"> item </span><span class="kwd">in</span><span class="pln"> parse_douban_top</span><span class="pun">(</span><span class="pln">html</span><span class="pun">):</span></code></li><li class="L8"><code><span class="pln"> </span><span class="kwd">print</span><span class="pun">(</span><span class="pln">item</span><span class="pun">)</span></code></li><li class="L9"><code></code></li><li class="L0"><code><span class="kwd">def</span><span class="pln"> parse_douban_top</span><span class="pun">(</span><span class="pln">html</span><span class="pun">):</span></code></li><li class="L1"><code><span class="pln"> regex </span><span class="pun">=</span><span class="pln"> </span><span class="str">'<em class="">(\d+)</em>.*?<span class="title">(.*?)</span>.*?<p class="">(.*?)</p>.*?<span class="rating_num" property="v:average">(.*?)</span>.*?<span>(.*?)</span>.*?<span class="inq">(.*?)</span>'</span></code></li><li class="L2"><code></code></li><li class="L3"><code><span class="pln"> jsonResult </span><span class="pun">=</span><span class="pln"> re</span><span class="pun">.</span><span class="pln">compile</span><span class="pun">(</span></code></li><li class="L4"><code><span class="pln"> regex</span><span class="pun">,</span><span class="pln"> re</span><span class="pun">.</span><span class="pln">S</span></code></li><li class="L5"><code><span class="pln"> </span><span class="pun">)</span></code></li><li class="L6"><code></code></li><li class="L7"><code><span class="pln"> items </span><span class="pun">=</span><span class="pln"> re</span><span class="pun">.</span><span class="pln">findall</span><span class="pun">(</span><span class="pln">jsonResult</span><span class="pun">,</span><span class="pln"> html</span><span class="pun">)</span></code></li><li class="L8"><code></code></li><li class="L9"><code><span class="pln"> </span><span class="kwd">for</span><span class="pln"> item </span><span class="kwd">in</span><span class="pln"> items</span><span class="pun">:</span></code></li><li class="L0"><code></code></li><li class="L1"><code><span class="pln"> content </span><span class="pun">=</span><span class="pln"> </span><span class="str">""</span></code></li><li class="L2"><code></code></li><li class="L3"><code><span class="pln"> </span><span class="kwd">for</span><span class="pln"> every_list </span><span class="kwd">in</span><span class="pln"> item</span><span class="pun">[</span><span class="lit">2</span><span class="pun">].</span><span class="pln">split</span><span class="pun">():</span></code></li><li class="L4"><code><span class="pln"> </span><span class="com"># split()通过指定分隔符对字符串进行切片</span></code></li><li class="L5"><code><span class="pln"> content </span><span class="pun">=</span><span class="pln"> content </span><span class="pun">+</span><span class="pln"> </span><span class="str">""</span><span class="pun">.</span><span class="kwd">join</span><span class="pun">(</span><span class="pln">every_list</span><span class="pun">)</span></code></li><li class="L6"><code><span class="pln"> content </span><span class="pun">=</span><span class="pln"> re</span><span class="pun">.</span><span class="kwd">sub</span><span class="pun">(</span><span class="str">'&nbsp;'</span><span class="pun">,</span><span class="pln"> </span><span class="str">' '</span><span class="pun">,</span><span class="pln"> content</span><span class="pun">)</span></code></li><li class="L7"><code><span class="pln"> content </span><span class="pun">=</span><span class="pln"> re</span><span class="pun">.</span><span class="kwd">sub</span><span class="pun">(</span><span class="str">'<br>'</span><span class="pun">,</span><span class="pln"> </span><span class="str">' '</span><span class="pun">,</span><span class="pln"> content</span><span class="pun">)</span></code></li><li class="L8"><code></code></li><li class="L9"><code><span class="pln"> </span><span class="kwd">yield</span><span class="pln"> </span><span class="pun">{</span></code></li><li class="L0"><code><span class="pln"> </span><span class="str">"index"</span><span class="pun">:</span><span class="pln"> item</span><span class="pun">[</span><span class="lit">0</span><span class="pun">],</span></code></li><li class="L1"><code><span class="pln"> </span><span class="str">"name"</span><span class="pun">:</span><span class="pln"> item</span><span class="pun">[</span><span class="lit">1</span><span class="pun">],</span></code></li><li class="L2"><code><span class="pln"> </span><span class="str">"describe"</span><span class="pun">:</span><span class="pln"> content</span><span class="pun">,</span></code></li><li class="L3"><code><span class="pln"> </span><span class="str">"star"</span><span class="pun">:</span><span class="pln"> item</span><span class="pun">[</span><span class="lit">3</span><span class="pun">],</span></code></li><li class="L4"><code><span class="pln"> </span><span class="str">"evaluate"</span><span class="pun">:</span><span class="pln"> item</span><span class="pun">[</span><span class="lit">4</span><span class="pun">],</span></code></li><li class="L5"><code><span class="pln"> </span><span class="str">"title"</span><span class="pun">:</span><span class="pln"> item</span><span class="pun">[</span><span class="lit">5</span><span class="pun">]</span></code></li><li class="L6"><code><span class="pln"> </span><span class="pun">}</span></code></li><li class="L7"><code><span class="kwd">if</span><span class="pln"> __name__ </span><span class="pun">==</span><span class="pln"> </span><span class="str">'__main__'</span><span class="pun">:</span></code></li><li class="L8"><code><span class="pln"> </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="lit">5</span><span class="pun">):</span></code></li><li class="L9"><code><span class="pln"> analyze</span><span class="pun">(</span><span class="pln">i</span><span class="pun">*</span><span class="lit">50</span><span class="pun">)</span></code></li></ol></pre><h5 id="h5-1-4-3-8-4-scrapy-"><a name="1.4.3.8.4 Scrapy爬虫框架" class="reference-link"></a><span class="header-link octicon octicon-link"></span>1.4.3.8.4 Scrapy爬虫框架</h5><p>1.安装scrapy框架<br>更改pip源,C盘下搜索pip.in,完成下述修改</p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">index</span><span class="pun">-</span><span class="pln">url </span><span class="pun">=</span><span class="pln"> https</span><span class="pun">:</span><span class="com">//pypi.tuna.tsinghua.edu.cn/simple</span></code></li><li class="L1"><code></code></li><li class="L2"><code><span class="pln">trusted</span><span class="pun">-</span><span class="pln">host </span><span class="pun">=</span><span class="pln"> pypi</span><span class="pun">.</span><span class="pln">tuna</span><span class="pun">.</span><span class="pln">tsinghua</span><span class="pun">.</span><span class="pln">edu</span><span class="pun">.</span><span class="pln">cn</span></code></li></ol></pre><p>命令行下利用pip先后安装wheel、lxml、Pywin32,</p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">pip install wheel</span></code></li><li class="L1"><code><span class="pln">pip install lxml</span></code></li><li class="L2"><code><span class="pln">pip install </span><span class="typ">Pywin32</span></code></li></ol></pre><p>然后从辅材下载Twisted‑18.9.0‑cp37‑cp37m‑win_amd64.whl,通过文件菜单将该文件上传到实验环境c:/updownload目录下,解压后到解压文件目录下执行安装,注意这里务必写上绝对路径,不写将从互联网安装失败。</p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">pip install </span><span class="pun">/解压目录/</span><span class="typ">Twisted</span><span class="pun">‑</span><span class="lit">18.9</span><span class="pun">.</span><span class="lit">0</span><span class="pun">‑</span><span class="pln">cp37</span><span class="pun">‑</span><span class="pln">cp37m</span><span class="pun">‑</span><span class="pln">win_amd64</span><span class="pun">.</span><span class="pln">whl</span></code></li></ol></pre><p>最后安装scrapy.</p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">pip install scrapy</span></code></li></ol></pre><p>可能出现安装失败,执行下述操作,安装成功。</p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">easy_install </span><span class="pun">-</span><span class="pln">U pip</span></code></li><li class="L1"><code><span class="pln">pip install pyOpenSSL</span></code></li><li class="L2"><code><span class="pln">pip install scrapy</span></code></li></ol></pre><p>2.Scrapy介绍<br>(1)Scrapy框架简介<br>Scrapy是:由Python语言开发的一个快速、高层次的屏幕抓取和web抓取框架,用于抓取web站点并从页面中提取结构化的数据,只需要实现少量的代码,就能够快速的抓取。<br>(2)Scrapy框架运行原理<br><img src="./水滴实验营_files/b4c0be4d-1342-46f1-be06-99142a0ee31a_scrapy原理.png" alt=""><br>Scrapy主要包括了以下组件:<br>引擎(Scrapy Engine)<br>Item 项目<br>调度器(Scheduler)<br>下载器(Downloader)<br>爬虫(Spiders)<br>项目管道(Pipeline)<br>下载器中间件(Downloader Middlewares)<br>爬虫中间件(Spider Middlewares)<br>调度中间件(Scheduler Middewares)<br>(3)Scrapy框架入门<br>使用大概分为下面四步<br>[1]创建一个scrapy项目</p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">scrapy startproject mySpider</span></code></li></ol></pre><p>[2] 生成一个爬虫<br>scrapy genspider demo “demo.cn”<br>[3]提取数据<br>完善spider 使用xpath等<br>[4]pipeline中保存数据<br>[5]在命令中运行爬虫</p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">scrapy crawl qb</span></code></li></ol></pre><p>或在pycharm中运行爬虫</p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="kwd">from</span><span class="pln"> scrapy </span><span class="kwd">import</span><span class="pln"> cmdline</span></code></li><li class="L1"><code><span class="pln">cmdline</span><span class="pun">.</span><span class="pln">execute</span><span class="pun">(</span><span class="str">"scrapy crawl qb"</span><span class="pun">.</span><span class="pln">split</span><span class="pun">())</span></code></li></ol></pre><p>3.Scrapy框架来爬取腾讯招聘网<br>打开网站<a href="https://careers.tencent.com/search.html?pcid=40001">https://careers.tencent.com/search.html?pcid=40001</a><br>进入开发者模式,发现数据没有存放在源代码里面,那么我们查看Ajax请求,看看Ajax请求中有没有存放职位信息的数据,按CTR+R查看request URL,如下图所示:<br><img src="./水滴实验营_files/78deafce-c3e2-4e6e-9269-339baa462934_岗位代码截图.jpg" alt=""></p>
|
||
<p>Ajax请求中存放着职位数据,那可以通过Ajax请求来获取,URL链接如下:<br><a href="https://careers.tencent.com/tencentcareer/api/post/Query?timestamp=1629464904109&countryId=&cityId=&bgIds=&productId=&categoryId=40001001,40001002,40001003,40001004,40001005,40001006&parentCategoryId=&attrId=&keyword=&pageIndex=2&pageSize=10&language=zh-cn&area=cn">https://careers.tencent.com/tencentcareer/api/post/Query?timestamp=1629464904109&countryId=&cityId=&bgIds=&productId=&categoryId=40001001,40001002,40001003,40001004,40001005,40001006&parentCategoryId=&attrId=&keyword=&pageIndex=2&pageSize=10&language=zh-cn&area=cn</a><br>发现该请求的URL长,那么删减一下该URL的数据,<br>第一页:<br><a href="https://careers.tencent.com/tencentcareer/api/post/Query?categoryId=40001001,40001002,40001003,40001004,40001005,40001006&pageIndex=1&pageSize=10&language=zh-cn&area=cn">https://careers.tencent.com/tencentcareer/api/post/Query?categoryId=40001001,40001002,40001003,40001004,40001005,40001006&pageIndex=1&pageSize=10&language=zh-cn&area=cn</a><br>第二页:<br><a href="https://careers.tencent.com/tencentcareer/api/post/Query?categoryId=40001001,40001002,40001003,40001004,40001005,40001006&pageIndex=2&pageSize=10&language=zh-cn&area=cn">https://careers.tencent.com/tencentcareer/api/post/Query?categoryId=40001001,40001002,40001003,40001004,40001005,40001006&pageIndex=2&pageSize=10&language=zh-cn&area=cn</a><br>第三页:<br><a href="https://careers.tencent.com/tencentcareer/api/post/Query?categoryId=40001001,40001002,40001003,40001004,40001005,40001006&pageIndex=3&pageSize=10&language=zh-cn&area=cn">https://careers.tencent.com/tencentcareer/api/post/Query?categoryId=40001001,40001002,40001003,40001004,40001005,40001006&pageIndex=3&pageSize=10&language=zh-cn&area=cn</a><br>过简单删减可以得出该URL可以为上面的URL,而且可通过修改pageIndex的值爬取不同页面数据。</p>
|
||
<p>创建scrapy项目</p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">scrapy startproject </span><span class="typ">Tencent</span></code></li></ol></pre><p>可以pycharm的项目目录下会多了很多文件<br>进入Tencent目录,执行下述命令,创建spider爬虫</p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">scrapy genspider tencent careers</span><span class="pun">.</span><span class="pln">tencent</span><span class="pun">.</span><span class="pln">com</span></code></li></ol></pre><p>执行后,在spiders文件夹中会多了一个tencent.py文件,该文件就是刚才创建的spider爬虫,其文件内容下所示:</p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="kwd">import</span><span class="pln"> scrapy</span></code></li><li class="L1"><code><span class="kwd">class</span><span class="pln"> </span><span class="typ">Tencent1Spider</span><span class="pun">(</span><span class="pln">scrapy</span><span class="pun">.</span><span class="typ">Spider</span><span class="pun">):</span></code></li><li class="L2"><code><span class="pln"> name </span><span class="pun">=</span><span class="pln"> </span><span class="str">'tencent'</span></code></li><li class="L3"><code><span class="pln"> allowed_domains </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[</span><span class="str">'careers.tencent.com'</span><span class="pun">]</span></code></li><li class="L4"><code><span class="pln"> start_urls </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[</span><span class="str">'http://careers.tencent.com/'</span><span class="pun">]</span></code></li><li class="L5"><code></code></li><li class="L6"><code><span class="pln"> </span><span class="kwd">def</span><span class="pln"> parse</span><span class="pun">(</span><span class="kwd">self</span><span class="pun">,</span><span class="pln"> response</span><span class="pun">):</span></code></li><li class="L7"><code><span class="pln"> </span><span class="kwd">pass</span></code></li></ol></pre><p>其中</p>
|
||
<p>name是我们的爬虫名;<br>allowed_domains是域名,也就是爬虫爬取的范围;<br>start_urls是爬虫最开始爬取的URL链接;<br>parse()是用来解析响应、提取数据。</p>
|
||
<p>定义字段<br>在编写代码提取数据前,我们先来在items.py定义爬取的字段,字段类型为scrapy.Field,代码如下所示:</p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="kwd">import</span><span class="pln"> scrapy</span></code></li><li class="L1"><code></code></li><li class="L2"><code><span class="kwd">class</span><span class="pln"> </span><span class="typ">TencentItem</span><span class="pun">(</span><span class="pln">scrapy</span><span class="pun">.</span><span class="typ">Item</span><span class="pun">):</span></code></li><li class="L3"><code><span class="pln"> </span><span class="com"># define the fields for your item here like:</span></code></li><li class="L4"><code><span class="pln"> </span><span class="typ">RecruitPostName</span><span class="pln"> </span><span class="pun">=</span><span class="pln"> scrapy</span><span class="pun">.</span><span class="typ">Field</span><span class="pun">()</span><span class="pln"> </span><span class="com">#职位名</span></code></li><li class="L5"><code><span class="pln"> </span><span class="typ">LocationName</span><span class="pln"> </span><span class="pun">=</span><span class="pln"> scrapy</span><span class="pun">.</span><span class="typ">Field</span><span class="pun">()</span><span class="pln"> </span><span class="com">#地址</span></code></li><li class="L6"><code><span class="pln"> </span><span class="typ">Responsibility</span><span class="pln"> </span><span class="pun">=</span><span class="pln"> scrapy</span><span class="pun">.</span><span class="typ">Field</span><span class="pun">()</span><span class="pln"> </span><span class="com">#工作要求</span></code></li></ol></pre><p>定义好字段后,接下来开始在tencent.py中编写代码来提取数据,修改爬虫地址和解析函数,具体代码如下所示:</p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="kwd">import</span><span class="pln"> scrapy</span></code></li><li class="L1"><code><span class="kwd">from</span><span class="pln"> </span><span class="typ">Tencent</span><span class="pun">.</span><span class="pln">items </span><span class="kwd">import</span><span class="pln"> </span><span class="typ">TencentItem</span></code></li><li class="L2"><code><span class="kwd">class</span><span class="pln"> </span><span class="typ">TencentSpider</span><span class="pun">(</span><span class="pln">scrapy</span><span class="pun">.</span><span class="typ">Spider</span><span class="pun">):</span></code></li><li class="L3"><code><span class="pln"> name </span><span class="pun">=</span><span class="pln"> </span><span class="str">'tencent'</span></code></li><li class="L4"><code><span class="pln"> allowed_domains </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[</span><span class="str">'careers.tencent.com'</span><span class="pun">]</span></code></li><li class="L5"><code><span class="pln"> start_urls </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[</span><span class="str">'https://careers.tencent.com/tencentcareer/api/post/Query?categoryId=40001001,40001002,40001003,40001004,40001005,40001006&pageIndex=1&pageSize=10&language=zh-cn&area=cn'</span><span class="pun">]</span></code></li><li class="L6"><code></code></li><li class="L7"><code><span class="pln"> </span><span class="kwd">def</span><span class="pln"> parse</span><span class="pun">(</span><span class="kwd">self</span><span class="pun">,</span><span class="pln"> response</span><span class="pun">):</span></code></li><li class="L8"><code><span class="pln"> json</span><span class="pun">=</span><span class="pln">response</span><span class="pun">.</span><span class="pln">json</span><span class="pun">()</span></code></li><li class="L9"><code><span class="pln"> datas </span><span class="pun">=</span><span class="pln"> json</span><span class="pun">.</span><span class="kwd">get</span><span class="pun">(</span><span class="str">'Data'</span><span class="pun">).</span><span class="kwd">get</span><span class="pun">(</span><span class="str">'Posts'</span><span class="pun">)</span></code></li><li class="L0"><code><span class="pln"> </span><span class="kwd">for</span><span class="pln"> data </span><span class="kwd">in</span><span class="pln"> datas</span><span class="pun">:</span></code></li><li class="L1"><code><span class="pln"> item</span><span class="pun">=</span><span class="typ">TencentItem</span><span class="pun">()</span></code></li><li class="L2"><code><span class="pln"> item</span><span class="pun">[</span><span class="str">'RecruitPostName'</span><span class="pun">]=</span><span class="pln">data</span><span class="pun">.</span><span class="kwd">get</span><span class="pun">(</span><span class="str">'RecruitPostName'</span><span class="pun">),</span></code></li><li class="L3"><code><span class="pln"> item</span><span class="pun">[</span><span class="str">'LocationName'</span><span class="pun">]=</span><span class="pln">data</span><span class="pun">.</span><span class="kwd">get</span><span class="pun">(</span><span class="str">'LocationName'</span><span class="pun">),</span></code></li><li class="L4"><code><span class="pln"> item</span><span class="pun">[</span><span class="str">'Responsibility'</span><span class="pun">]=</span><span class="pln">data</span><span class="pun">.</span><span class="kwd">get</span><span class="pun">(</span><span class="str">'Responsibility'</span><span class="pun">).</span><span class="pln">replace</span><span class="pun">(</span><span class="str">'\n'</span><span class="pun">,</span><span class="str">''</span><span class="pun">)</span></code></li><li class="L5"><code><span class="pln"> </span><span class="kwd">yield</span><span class="pln"> item</span></code></li></ol></pre><p>首先导入items.py文件中的TencentItem,再修改start_urls的URL链接,定义一个json变量来接收网页响应的json()数据,通过for循环把每条职位信息循环遍历并提取我们想要的数据并放在item字典里面,其中item=TencentItem()相当于创建一个空字典item={}</p>
|
||
<p>创建一个for循环来进行翻页</p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> range</span><span class="pun">(</span><span class="lit">2</span><span class="pun">,</span><span class="lit">4</span><span class="pun">):</span></code></li><li class="L1"><code><span class="pln"> next_url</span><span class="pun">=</span><span class="pln">f</span><span class="str">'https://careers.tencent.com/tencentcareer/api/post/Query?categoryId=40001001,40001002,40001003,40001004,40001005,40001006&pageIndex={i}&pageSize=10&language=zh-cn&area=cn'</span></code></li><li class="L2"><code><span class="pln"> </span><span class="kwd">yield</span><span class="pln"> scrapy</span><span class="pun">.</span><span class="typ">Request</span><span class="pun">(</span></code></li><li class="L3"><code><span class="pln"> next_url</span><span class="pun">,</span></code></li><li class="L4"><code><span class="pln"> callback</span><span class="pun">=</span><span class="kwd">self</span><span class="pun">.</span><span class="pln">parse</span></code></li><li class="L5"><code><span class="pln"> </span><span class="pun">)</span></code></li></ol></pre><p>调用yield生成器来返回数据给引擎,并调用scrapy.Request()方法,该方法能构建一个requests,同时指定提取数据的callback函数<br>在启动爬取前,我们先要在settings.py文件中编写一些代码,具体代码如下所示:</p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">LOG_LEVEL</span><span class="pun">=</span><span class="str">"WARNING"</span><span class="pln"> </span></code></li><li class="L1"><code><span class="pln">ITEM_PIPELINES </span><span class="pun">=</span><span class="pln"> </span><span class="pun">{</span><span class="pln"> </span></code></li><li class="L2"><code><span class="pln"> </span><span class="str">'Tencent.pipelines.TencentPipeline'</span><span class="pun">:</span><span class="pln"> </span><span class="lit">300</span><span class="pun">,</span></code></li><li class="L3"><code><span class="pun">}</span></code></li><li class="L4"><code><span class="pln">USER_AGENT </span><span class="pun">=</span><span class="pln"> </span><span class="str">'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36'</span></code></li></ol></pre><p>其中:<br>LOG_LEVEL的作用的屏蔽log日志的输出;<br>ITEM_PIPELINES的作用是开启引擎;</p>
|
||
<p>当我们要把数据保存成文件的时候,不需要任何额外的代码,只要执行如下代码即可</p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">scrapy crawl tencent </span><span class="pun">-</span><span class="pln">o tencent</span><span class="pun">.</span><span class="pln">csv</span></code></li></ol></pre><h1 id="h1-2-kettle-"><a name="2. 利用Kettle进行数据清洗转换" class="reference-link"></a><span class="header-link octicon octicon-link"></span>2. 利用Kettle进行数据清洗转换</h1><p>爬取的数据通常存在缺失值、无效值等问题,不能直接用于数据统计分析,因此利用kettle将数据进行清洗、转换;将清洗后的数据提供给学生,学生利用Python软件进行统计分析。本部分实验目标首先将MySQL数据库中的爬取数据进行清洗,然后以两种形式存储,一种存储到MySQL新表中,用于Python编程统计分析、可视化;一种转换成CSV文件。</p>
|
||
<h2 id="h2-2-1-kettle-"><a name="2.1 Kettle 基本使用" class="reference-link"></a><span class="header-link octicon octicon-link"></span>2.1 Kettle 基本使用</h2><p>(1)启动:桌面上双击“Spoon.bat”图标启动Kettle,显示如下界面。</p>
|
||
<p><img src="./水滴实验营_files/01-启动kettle.png" alt=""></p>
|
||
<p>(2)新建转换:选中左侧栏内的“转换”菜单,右键选择“新建”即可建立一个新的转换任务,如图所示。</p>
|
||
<p><img src="./水滴实验营_files/02-新建转换.png" alt=""></p>
|
||
<p>(3)新建组件:新建输入、流程、输出等组件,选择相应组件,直接拖到右边工作区,例如新建CSV文件输入,将核心对象->输入->CSV文件输入,直接拖到右边工作台或者通过双击添加该组件,同样可以将新对象->转换->排序记录,直接拖到右边工作台。</p>
|
||
<p><img src="./水滴实验营_files/03-新建组件.png" alt=""></p>
|
||
<p>(4)建立连接:按下“shift”键,在“CSV文件输入”和“排序记录”间用鼠标划线,或者按住鼠标中间滚轮移动鼠标即可连接两个节点。</p>
|
||
<p><img src="./水滴实验营_files/04-建立连接.png" alt=""></p>
|
||
<p>(5)确认连接:双击连接线,会弹出对话框,注意起始步骤和目标步骤如果有值为空,则代表连接不成功,若使连接生效未勾选,则连接不生效,请重新建立连接。</p>
|
||
<p><img src="./水滴实验营_files/05-确认连接.png" alt=""></p>
|
||
<p>(6)删除连接:选中拟删除的连接线,点击右键选择“删除节点连接”。<br>(7)连接分支选择:在组件连接过程中,会弹出对话框,一般主要流程会选择“主要输出步骤”,存在分支的时候选择“Result is TRUE”和“Result is FALSE”,例如选择核心对象->流程->过滤记录和空操作建立如下分支,如下图所示。</p>
|
||
<p><img src="./水滴实验营_files/06-分支选择设置1.png" alt=""></p>
|
||
<h2 id="h2-2-2-"><a name="2.2 数据清洗内容" class="reference-link"></a><span class="header-link octicon octicon-link"></span>2.2 数据清洗内容</h2><p>浏览“dazhonginfo”数据表的数据,发现爬取的数据存在以下问题:<br>(1) 存在缺失值,即空的字段,需要删除该条记录。<br>(2) 存在无效值,avgPrice人均消费字段的值出现0,代表人均消费是0,不合理,对于tasteScore,environmentScore、serviceScore、avgPrice数值型字段,如果出现0值,则删除该条记录。<br>(3) shopUrl和defaultPic列后续用不到,需要删除。</p>
|
||
<h2 id="h2-2-3-mysql-"><a name="2.3 建立MySQL新表存储清洗后数据" class="reference-link"></a><span class="header-link octicon octicon-link"></span>2.3 建立MySQL新表存储清洗后数据</h2><p>在Navicat软件中选中“testdazhong”数据库,点击右键选择“运行SQL文件”,选择“C:\sqlfile\dazhongoutput.sql”,点击“开始”导入dazhongoutput数据表</p>
|
||
<p><img src="./水滴实验营_files/07-输出表导入.png" alt=""></p>
|
||
<p>提示“*** executed successfully”信息表示导入成功。关闭该页面,选中testdazhong数据库下面的“表”,点击鼠标右键选择“刷新”,看到“dazhongoutput”表。</p>
|
||
<p><img src="./水滴实验营_files/09-刷新输出表.png" alt=""></p>
|
||
<p>双击“dazhongoutput”表可打开查看相关字段,各字段解释同数据表dazhonginfo.</p>
|
||
<p><img src="./水滴实验营_files/10-输出空表.png" alt=""></p>
|
||
<h2 id="h2-2-4-kettle-"><a name="2.4 Kettle数据清洗、转换" class="reference-link"></a><span class="header-link octicon octicon-link"></span>2.4 Kettle数据清洗、转换</h2><h3 id="h3-2-4-1-"><a name="2.4.1 创建转换任务设计输入组件" class="reference-link"></a><span class="header-link octicon octicon-link"></span>2.4.1 创建转换任务设计输入组件</h3><p>在“主对象树”标签下双击“转换”,新建一个转换任务</p>
|
||
<p><img src="./水滴实验营_files/11-新建转换.png" alt=""></p>
|
||
<p>打开“输入”->“表输入”,将“表输入”拖到右边工作台上。</p>
|
||
<p><img src="./水滴实验营_files/12-新建表输入.png" alt=""></p>
|
||
<p>双击“表输入”,出现如下界面,数据库连接处点击“新建”,在弹出对话框处设置连接名称,可任意定义,选择连接类型为“MySQL”,连接方式为“Native(JDBC)”,设置数据库连接信息,这里信息与dbconnect.py中数据库连接信息一致。<br>主机名或IP地址:localhost ##代表本机<br>端口:3306<br>用户名:root ##根用户权限<br>密码:shuidilab<a href="https://github.com/123" title="@123" class="at-link">@123</a><br>数据库设置为testdazhong,存储3000条爬取的数据。<br><img src="./水滴实验营_files/13-数据库连接.png" alt=""></p>
|
||
<p>点击“测试”,出现如下信息表示数据库连接成功,点击“确定”,返回数据库连接页,点击“确认”返回表输入页。</p>
|
||
<p><img src="./水滴实验营_files/14-数据库连接成功.png" alt=""></p>
|
||
<p>在下图中的表输入页,输入查询数据库的SQL查询语句,例如选择表中所有数据,</p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">SELECT </span><span class="pun">*</span><span class="pln"> FROM dazhonginfo</span></code></li></ol></pre><p>“dazhonginfo”是数据库表名称,然后点击“预览”,选择预览的记录条数,例如”3000”,配置完成。</p>
|
||
<p><img src="./水滴实验营_files/15-设置SQL查询语句.png" alt=""></p>
|
||
<p>点击“确定”进行数据库中表信息预览,如图所示。</p>
|
||
<p><img src="./水滴实验营_files/16-浏览数据表数据.png" alt=""></p>
|
||
<h3 id="h3-2-4-2-"><a name="2.4.2 设计数据清洗组件" class="reference-link"></a><span class="header-link octicon octicon-link"></span>2.4.2 设计数据清洗组件</h3><h4 id="h4-2-4-2-1-"><a name="2.4.2.1过滤空值" class="reference-link"></a><span class="header-link octicon octicon-link"></span>2.4.2.1过滤空值</h4><p>选中“流程”->“过滤记录”,将“过滤记录”拖到右边工作台上。</p>
|
||
<p><img src="./水滴实验营_files/17-1新建第1个过滤记录.png" alt=""></p>
|
||
<p>双击“过滤记录”,设置步骤名称为“过滤空值”。</p>
|
||
<p><img src="./水滴实验营_files/17-设置过滤空值名称.png" alt=""></p>
|
||
<p>点击“确定”,出现如下图所示。</p>
|
||
<p><img src="./水滴实验营_files/18-过滤空值设置完成.png" alt=""></p>
|
||
<p>选中“流程”-> “空操作(什么也不做)”,将“空操作(什么也不做)”拖到右边工作台上。</p>
|
||
<p><img src="./水滴实验营_files/19-设置第1个空操作.png" alt=""></p>
|
||
<p>按下“shift”键,从“表输入”向“过滤空值”进行连线,如果弹出对话框,选择“主要输出步骤”, 线连接成功后,点击连线,显示如图。</p>
|
||
<p><img src="./水滴实验营_files/20-表输入-过滤空值连线.png" alt=""></p>
|
||
<p>然后同样操作从 “过滤空值”向“空操作(什么也不做)”进行连线,在此过程中会弹出对话框,选择“Result is FALSE”,建立如图所示连接图。</p>
|
||
<p><img src="./水滴实验营_files/21-过滤空值-空操作连线.png" alt=""> </p>
|
||
<p>点击“过滤空值”和“空操作(什么也不做)”两者间的连线,如图所示,点击“确定”关闭该页。</p>
|
||
<p><img src="./水滴实验营_files/21-双击连线查看连接属性.png" alt=""></p>
|
||
<p>双击“过滤空值”,添加条件,第一个红框中选择一个字段“Id”,如图所示,点击“确定”。</p>
|
||
<p><img src="./水滴实验营_files/22-过滤空值添加第1个条件.png" alt=""></p>
|
||
<p>在第二个红框中选择函数,选择“IS NOT NULL”,点击函数对话框的“确定”,如图所示,完成了第一个过滤条件的设置。</p>
|
||
<p><img src="./水滴实验营_files/23-过滤空值添加第1个条件2.png" alt=""></p>
|
||
<p>点击右侧“+”号,出现如图所示第二个条件。</p>
|
||
<p> <img src="./水滴实验营_files/24-添加第2个条件.png" alt=""></p>
|
||
<p>点击“null= []”,出现如下界面,按照配置“Id”字段的步骤配置第二个字段“city”,点击最下方的确定按钮。</p>
|
||
<p> <img src="./水滴实验营_files/25-设置第二个过滤空值条件.png" alt=""></p>
|
||
<p>重复上述步骤,依次添加“shopName”,“shopId”,“shopPower”,“mainRegionName”, “ mainCategoryName”,“tasteScore”,“environmentScore”,“serviceScore”,“avgPrice”,“shopAddress”,“shopUrl”、“defaultPic”这些字段均不为空的条件,如下图所示,点击“确定”,保存过滤条件,完成第一个组件“过滤空值”的条件设置。</p>
|
||
<p> <img src="./水滴实验营_files/26-第1个组件过滤空值条件完成.png" alt=""></p>
|
||
<h4 id="h4-2-4-2-2-0-"><a name="2.4.2.2过滤0值" class="reference-link"></a><span class="header-link octicon octicon-link"></span>2.4.2.2过滤0值</h4><p>继续,选中“流程”->“过滤记录”,将“过滤记录”拖到右边工作台上,双击“过滤记录”设置步骤名称为“过滤0值”,选中“流程”-> “空操作(什么也不做)”,将“空操作(什么也不做)2”拖到右边工作台上,显示为“空操作(什么也不做)2”,如图所示。</p>
|
||
<p> <img src="./水滴实验营_files/27-第2个过滤组件设置.png" alt=""></p>
|
||
<p>按下“shift”键,从“过滤空值”向“过滤0值”进行连线,若弹出对话框选择“Result is TURE”连接即可,点击连线,显示如图。</p>
|
||
<p> <img src="./水滴实验营_files/28-第二个过滤0值连线.png" alt=""></p>
|
||
<p>同样操作连接“过滤0值” -> “空操作(什么也不做)2”, 在此过程中会弹出对话框,选择“Result is FALSE”,建立如图所示连接图。</p>
|
||
<p> <img src="./水滴实验营_files/28-第二个过滤0值连线.png" alt=""></p>
|
||
<p>双击“过滤0值”,设置条件,如图所示,左侧第一个红框处选择字段avgPrice,第二个红框处选择函数不等于“< >”,第三个红框处设置为整数0,点击“确定”,这样建立如图所示的过滤0值的条件。</p>
|
||
<p> <img src="./水滴实验营_files/30-设置过滤0值条件.png" alt=""></p>
|
||
<p>按照同样步骤添加tasteScore,点击“+”号,出现如下界面,点击“null=[]”,进入配置页面进行相关配置,点击红框处“确定”。依次进行environmentScore、serviceScore字段的条件设置,最后如图所示。</p>
|
||
<p> <img src="./水滴实验营_files/31-第2个过滤条件完成.png" alt=""></p>
|
||
<h4 id="h4-2-4-2-3-"><a name="2.4.2.3排序" class="reference-link"></a><span class="header-link octicon octicon-link"></span>2.4.2.3排序</h4><p>打开“转换”->“排序记录”,将“排序记录”拖到右边工作台,如图所示。</p>
|
||
<p> <img src="./水滴实验营_files/32-新建排序记录组件.png" alt=""></p>
|
||
<p>连接“过滤0值”和“排序记录”,弹出对话框选择“Result is True”双击“排序记录”,如图所示,清洗后的数据按照Id升序进行排序,点击字段名称下方,选择字段“Id”,升序选择“是”,大小写敏感选择“否”,点击“确定”保存。</p>
|
||
<p> <img src="./水滴实验营_files/33-排序记录设置.png" alt=""></p>
|
||
<h3 id="h3-2-4-3-"><a name="2.4.3 设计输出组件" class="reference-link"></a><span class="header-link octicon octicon-link"></span>2.4.3 设计输出组件</h3><p>选中“输出”中的“文本文件输出”和“表输出”添加到右边工作台,如图所示</p>
|
||
<p> <img src="./水滴实验营_files/34-设计输出组件.png" alt=""></p>
|
||
<p>先连接“排序记录”与“文本文件输出”,然后连接“排序记录”与“表输出”,将弹出对话框,如图所示,选择“复制”,清洗后的所有数据将存储到文本文件和表中各一份,选择“分发”则数据循环向两个文件输入,我们选择“复制”。</p>
|
||
<p> <img src="./水滴实验营_files/35-连接时设置复制数据.png" alt=""></p>
|
||
<p>双击“文本文件输出”,配置信息,首先在文件标签页,扩展名改为“csv”,在文件名称处选择文件路径,文件名可以是已经创建的空csv文件,也可以直接输入一个新文件名,例如“dazhongdata”。</p>
|
||
<p><img src="./水滴实验营_files/36-csv文件内容设置.png" alt=""></p>
|
||
<p>点击“内容”标签页,分隔符设置为“,”号。</p>
|
||
<p><img src="./水滴实验营_files/37-csv文件内容设置.png" alt=""></p>
|
||
<p>点击“字段”标签页,点击“获取字段”按钮,获得输入数据的14个字段,删除最后2个字段,具体操作:选中待删除行,点击右键,选择“删除选中行”,点击“最小宽度”按钮,最后点击“确定”按钮关闭该页。</p>
|
||
<p><img src="./水滴实验营_files/38-csv文件输出设置.png" alt=""></p>
|
||
<p>双击“表输出”,选择数据库连接“dbcon”,点击“目标表”处的浏览,选择连接输出表“dazhongoutput”,点击“确定”。</p>
|
||
<p><img src="./水滴实验营_files/39-表输出设置.png" alt=""></p>
|
||
<p>在表输出页面,“提交记录数量”部分输入拟提取的数据记录行数,请输入已爬取的记录行数,如3000或5000,代表最多提取3000或5000行到输出表中,若实际数据记录行数小于该值,则按照实际数量输出到表中,勾选指定数据库字段,点击“数据库字段”标签页,点击“获取字段”,然后删除最后两个字段“shopUrl”和“defaultPic”所在的行,选中所在行点击右键选择“删除选中行”。</p>
|
||
<p><img src="./水滴实验营_files/40-表输出2.png" alt=""></p>
|
||
<p>点击“确定”,数据清洗转换流程设计完成,如下图所示,保存该ktr文件。</p>
|
||
<p><img src="./水滴实验营_files/41-1ktr流程.png" alt=""></p>
|
||
<h3 id="h3-2-4-4-"><a name="2.4.4 执行清洗和转换" class="reference-link"></a><span class="header-link octicon octicon-link"></span>2.4.4 执行清洗和转换</h3><p>点击红框处“执行这个转换”进行数据清洗和转换</p>
|
||
<p><img src="./水滴实验营_files/41-2运行ktr文件.png" alt=""></p>
|
||
<p>弹出“执行转换”页面,选择“本地执行”,点击“启动”按钮执行。</p>
|
||
<p><img src="./水滴实验营_files/41-3启动本地执行.png" alt=""></p>
|
||
<p>执行完成后,在执行结果“步骤度量”中可以看到数据清洗、转换的步骤。</p>
|
||
<p><img src="./水滴实验营_files/42-执行结果.png" alt=""></p>
|
||
<p>打开输出的CSV文件“C:\changedata\dazhongdata.csv”,可以看到数据如下。</p>
|
||
<p><img src="./水滴实验营_files/42-清洗后的CSV文件.png" alt=""></p>
|
||
<p>查看数据库表“dazhongoutput”,可以看到如下数据,数据清洗转换成功。</p>
|
||
<p><img src="./水滴实验营_files/43-清洗后的SQL文件.png" alt=""></p>
|
||
<h2 id="h2-2-5-"><a name="2.5 学生实训任务" class="reference-link"></a><span class="header-link octicon octicon-link"></span>2.5 学生实训任务</h2><p>自行设计多个清洗规则,利用kettle创建数据清洗流程,生成ktr文件,进行数据清洗转换。</p>
|
||
<h1 id="h1-3-"><a name="3 数据统计分析" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3 数据统计分析</h1><h2 id="h2-3-1-python-mysql-"><a name="3.1 Python+MySQL数据统计分析" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1 Python+MySQL数据统计分析</h2><h3 id="h3-3-1-1-sql-"><a name="3.1.1 基本SQL语句学习" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.1 基本SQL语句学习</h3><p>本节进行Python编程完成数据统计分析,数据通过调用MySQL数据库中表获取,因此需要学习基本SQL语句操作数据库。</p>
|
||
<p>启动Navicat,选中数据库“testdazhong”,点击下图工具栏“查询”->“新建查询”。</p>
|
||
<p><img src="./水滴实验营_files/1-新建查询.png" alt=""></p>
|
||
<p>显示查询编辑器,在查询编辑器中可编辑SQL语句,例如:查询“dazhongoutput”表中所有记录的语句:</p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">SELECT </span><span class="pun">*</span><span class="pln"> FROM dazhongoutput</span></code></li></ol></pre><p>点击“运行”,显示结果如下图所示。</p>
|
||
<p><img src="./水滴实验营_files/3-查询结果.png" alt=""></p>
|
||
<p>查询shopPower列的不同值(即商铺有哪些星级)的语句:</p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">SELECT DISTINCT</span><span class="pun">(</span><span class="pln">shopPower</span><span class="pun">)</span><span class="pln"> </span><span class="kwd">from</span><span class="pln"> dazhongoutput</span></code></li></ol></pre><p>点击“运行”,显示结果如下图所示。</p>
|
||
<p><img src="./水滴实验营_files/4-查询星级.png" alt=""></p>
|
||
<p>查询“济南5星级商铺的数量”,查询条件: “城市是济南并且shopPower=50”,统计shopId列的值的数量,SQL语句:</p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">SELECT COUNT</span><span class="pun">(</span><span class="pln">shopId</span><span class="pun">)</span><span class="pln"> </span><span class="kwd">from</span><span class="pln"> dazhongoutput </span><span class="kwd">where</span><span class="pln"> shopPower</span><span class="pun">=</span><span class="lit">50</span><span class="pln"> </span><span class="kwd">and</span><span class="pln"> city</span><span class="pun">=</span><span class="str">"济南"</span></code></li></ol></pre><p>Where后面子句用于规定选择的标准。点击“运行”,显示结果如下图所示。</p>
|
||
<p><img src="./水滴实验营_files/5-查询5星级商铺数量.png" alt=""></p>
|
||
<p>分类汇总济南市“菜品分类”数量、“菜品分类”名称,SQL语句:</p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">SELECT COUNT</span><span class="pun">(</span><span class="pln">mainCategoryName</span><span class="pun">),</span><span class="pln">mainCategoryName </span><span class="kwd">from</span><span class="pln"> dazhongoutput </span><span class="kwd">where</span><span class="pln"> city</span><span class="pun">=</span><span class="str">"济南"</span><span class="pln"> GROUP BY mainCategoryName</span></code></li></ol></pre><p>GROUP BY 后面的子句是根据该分组进行分类汇总。点击“运行”,显示结果如下图所示。</p>
|
||
<p><img src="./水滴实验营_files/6-分类查询.png" alt=""></p>
|
||
<h3 id="h3-3-1-2-"><a name="3.1.2 打开项目" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.2 打开项目</h3><p>启动Pycharm软件,系统默认打开“dazhongspider”项目。</p>
|
||
<h3 id="h3-3-1-3-"><a name="3.1.3 设置代码目录" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.3 设置代码目录</h3><p>设置数据分析程序目录,选中该项目“dazhongspider”,点击右键选择“新建”->“目录”,如图所示。</p>
|
||
<p><img src="./水滴实验营_files/1-新建目录.png" alt=""></p>
|
||
<p>在弹出的界面中(如图所示),设置数据统计分析程序的目录名称为“dataanalysis”。</p>
|
||
<p><img src="./水滴实验营_files/2-目录名称.png" alt=""></p>
|
||
<h3 id="h3-3-1-4-python-"><a name="3.1.4 Python编程进行数据统计可视化" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4 Python编程进行数据统计可视化</h3><h4 id="h4-3-1-4-1-20-"><a name="3.1.4.1 20个大中城市星级商铺统计分析" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.1 20个大中城市星级商铺统计分析</h4><h5 id="h5-3-1-4-1-1-"><a name="3.1.4.1.1 新建代码文件" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.1.1 新建代码文件</h5><p>在项目窗口内,在“dataanalysis”文件夹上单击鼠标右键,在弹出的菜单中点击“新建”,再选择“Python File”。</p>
|
||
<p><img src="./水滴实验营_files/3-新建py文件.png" alt=""></p>
|
||
<p>在弹出的对话框中(如下图所示),在“Name”中输入“starclass”,点击“确定”按钮,进入“starclass.py”文件编辑器。</p>
|
||
<p><img src="./水滴实验营_files/4-新建文件名称.png" alt=""></p>
|
||
<h5 id="h5-3-1-4-1-2-python-"><a name="3.1.4.1.2 编写python程序代码" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.1.2 编写python程序代码</h5><p>输入下面代码到新建的“starclass.py”文件中并保存。</p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="kwd">from</span><span class="pln"> pyecharts </span><span class="kwd">import</span><span class="pln"> </span><span class="typ">Bar</span></code></li><li class="L1"><code><span class="kwd">from</span><span class="pln"> spider </span><span class="kwd">import</span><span class="pln"> dbconnect</span></code></li><li class="L2"><code><span class="com"># 统计一个城市各星级饭店数量函数</span></code></li><li class="L3"><code><span class="kwd">def</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="typ">City</span><span class="pun">):</span></code></li><li class="L4"><code><span class="pln"> </span><span class="com">#连接数据库</span></code></li><li class="L5"><code><span class="pln"> mysql_db </span><span class="pun">=</span><span class="pln"> dbconnect</span><span class="pun">.</span><span class="typ">DBHelper</span><span class="pun">()</span></code></li><li class="L6"><code><span class="pln"> mysql_db</span><span class="pun">.</span><span class="pln">connectDatabase</span><span class="pun">();</span></code></li><li class="L7"><code><span class="pln"> </span><span class="com">#查询有哪些星级,返回唯一不同的星级</span></code></li><li class="L8"><code><span class="pln"> starsql </span><span class="pun">=</span><span class="pln"> </span><span class="str">"SELECT DISTINCT(shopPower) from dazhongoutput ;"</span></code></li><li class="L9"><code><span class="pln"> data </span><span class="pun">=</span><span class="pln"> mysql_db</span><span class="pun">.</span><span class="kwd">select</span><span class="pun">(</span><span class="pln">starsql</span><span class="pun">)</span><span class="pln"> </span><span class="com">#返回所有查询记录行,每一行是一个元组,所有行的元组再组装成一个元组data</span></code></li><li class="L0"><code><span class="pln"> list_code </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L1"><code><span class="pln"> </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> data</span><span class="pun">:</span></code></li><li class="L2"><code><span class="pln"> </span><span class="com">#选择该城市某个星级按照商铺ID统计商铺数量</span></code></li><li class="L3"><code><span class="pln"> countsql </span><span class="pun">=</span><span class="pln"> </span><span class="str">'''SELECT COUNT(shopId) from dazhongoutput where shopPower=%s and city="%s";'''</span><span class="pln"> </span><span class="pun">%</span><span class="pln"> </span><span class="pun">(</span><span class="pln">i</span><span class="pun">[</span><span class="lit">0</span><span class="pun">],</span><span class="pln"> </span><span class="typ">City</span><span class="pun">)</span></code></li><li class="L4"><code><span class="pln"> countstar_city </span><span class="pun">=</span><span class="pln"> mysql_db</span><span class="pun">.</span><span class="kwd">select</span><span class="pun">(</span><span class="pln">countsql</span><span class="pun">)</span></code></li><li class="L5"><code><span class="pln"> </span><span class="com">#将所有星级和饭店数量放于list_code列表</span></code></li><li class="L6"><code><span class="pln"> </span><span class="kwd">for</span><span class="pln"> m </span><span class="kwd">in</span><span class="pln"> countstar_city</span><span class="pun">:</span></code></li><li class="L7"><code><span class="pln"> list_code</span><span class="pun">.</span><span class="pln">append</span><span class="pun">([</span><span class="pln">i</span><span class="pun">[</span><span class="lit">0</span><span class="pun">],</span><span class="pln"> m</span><span class="pun">[</span><span class="lit">0</span><span class="pun">]])</span></code></li><li class="L8"><code><span class="pln"> </span><span class="kwd">print</span><span class="pun">(</span><span class="pln">list_code</span><span class="pun">)</span></code></li><li class="L9"><code><span class="pln"> </span><span class="com">#返回商铺星级、数量</span></code></li><li class="L0"><code><span class="pln"> </span><span class="kwd">return</span><span class="pln"> list_code</span></code></li><li class="L1"><code></code></li><li class="L2"><code><span class="com"># 所有城市各星级饭店统计函数</span></code></li><li class="L3"><code><span class="kwd">def</span><span class="pln"> mianStar</span><span class="pun">():</span></code></li><li class="L4"><code><span class="pln"> </span><span class="com">#调用starshopCount(City)统计出下述城市各星级饭店数量</span></code></li><li class="L5"><code><span class="pln"> shop1 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"广州"</span><span class="pun">)</span></code></li><li class="L6"><code><span class="pln"> shop2 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"杭州"</span><span class="pun">)</span></code></li><li class="L7"><code><span class="pln"> shop3 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"南京"</span><span class="pun">)</span></code></li><li class="L8"><code><span class="pln"> shop4 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"成都"</span><span class="pun">)</span></code></li><li class="L9"><code><span class="pln"> shop5 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"武汉"</span><span class="pun">)</span></code></li><li class="L0"><code><span class="pln"> shop6 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"济南"</span><span class="pun">)</span></code></li><li class="L1"><code><span class="pln"> shop7 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"南宁"</span><span class="pun">)</span></code></li><li class="L2"><code><span class="pln"> shop8 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"合肥"</span><span class="pun">)</span></code></li><li class="L3"><code><span class="pln"> shop9 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"昆明"</span><span class="pun">)</span></code></li><li class="L4"><code><span class="pln"> shop10 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"西安"</span><span class="pun">)</span></code></li><li class="L5"><code><span class="pln"> shop11 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"北京"</span><span class="pun">)</span></code></li><li class="L6"><code><span class="pln"> shop12 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"上海"</span><span class="pun">)</span></code></li><li class="L7"><code><span class="pln"> shop13 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"深圳"</span><span class="pun">)</span></code></li><li class="L8"><code><span class="pln"> shop14 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"天津"</span><span class="pun">)</span></code></li><li class="L9"><code><span class="pln"> shop15 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"重庆"</span><span class="pun">)</span></code></li><li class="L0"><code><span class="pln"> shop16 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"青岛"</span><span class="pun">)</span></code></li><li class="L1"><code><span class="pln"> shop17 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"威海"</span><span class="pun">)</span></code></li><li class="L2"><code><span class="pln"> shop18 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"长春"</span><span class="pun">)</span></code></li><li class="L3"><code><span class="pln"> shop19 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"大连"</span><span class="pun">)</span></code></li><li class="L4"><code><span class="pln"> shop20 </span><span class="pun">=</span><span class="pln"> starshopCount</span><span class="pun">(</span><span class="str">"贵阳"</span><span class="pun">)</span></code></li><li class="L5"><code></code></li><li class="L6"><code><span class="pln"> </span><span class="com">#定义X轴和Y轴显示数据列表,开始为NULL</span></code></li><li class="L7"><code><span class="pln"> list_x </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L8"><code><span class="pln"> list_y1</span><span class="pun">,</span><span class="pln"> list_y2</span><span class="pun">,</span><span class="pln"> list_y3</span><span class="pun">,</span><span class="pln"> list_y4</span><span class="pun">,</span><span class="pln"> list_y5</span><span class="pun">,</span><span class="pln"> list_y6</span><span class="pun">,</span><span class="pln"> list_y7</span><span class="pun">,</span><span class="pln"> list_y8</span><span class="pun">,</span><span class="pln"> list_y9</span><span class="pun">,</span><span class="pln"> list_y10</span><span class="pun">,</span><span class="pln">list_y11</span><span class="pun">,</span><span class="pln"> list_y12</span><span class="pun">,</span><span class="pln"> list_y13</span><span class="pun">,</span><span class="pln"> list_y14</span><span class="pun">,</span><span class="pln"> list_y15</span><span class="pun">,</span><span class="pln"> list_y16</span><span class="pun">,</span><span class="pln"> list_y17</span><span class="pun">,</span><span class="pln"> list_y18</span><span class="pun">,</span><span class="pln"> list_y19</span><span class="pun">,</span><span class="pln"> list_y20 </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L9"><code><span class="pln"> </span><span class="com">#数据列表赋值,zip将所有城市星级饭店列表,打包为一个包含多个元组的zip迭代对象,多变量循环</span></code></li><li class="L0"><code><span class="pln"> </span><span class="kwd">for</span><span class="pln"> data1</span><span class="pun">,</span><span class="pln"> data2</span><span class="pun">,</span><span class="pln"> data3</span><span class="pun">,</span><span class="pln"> data4</span><span class="pun">,</span><span class="pln"> data5</span><span class="pun">,</span><span class="pln"> data6</span><span class="pun">,</span><span class="pln"> data7</span><span class="pun">,</span><span class="pln"> data8</span><span class="pun">,</span><span class="pln"> data9</span><span class="pun">,</span><span class="pln"> data10</span><span class="pun">,</span><span class="pln"> data11</span><span class="pun">,</span><span class="pln"> data12</span><span class="pun">,</span><span class="pln"> data13</span><span class="pun">,</span><span class="pln"> data14</span><span class="pun">,</span><span class="pln"> data15</span><span class="pun">,</span><span class="pln"> data16</span><span class="pun">,</span><span class="pln"> data17</span><span class="pun">,</span><span class="pln"> data18</span><span class="pun">,</span><span class="pln"> data19</span><span class="pun">,</span><span class="pln"> data20 </span><span class="kwd">in</span><span class="pln"> zip</span><span class="pun">(</span></code></li><li class="L1"><code><span class="pln"> shop1</span><span class="pun">,</span><span class="pln"> shop2</span><span class="pun">,</span><span class="pln"> shop3</span><span class="pun">,</span><span class="pln"> shop4</span><span class="pun">,</span><span class="pln"> shop5</span><span class="pun">,</span><span class="pln"> shop6</span><span class="pun">,</span><span class="pln"> shop7</span><span class="pun">,</span><span class="pln"> shop8</span><span class="pun">,</span><span class="pln"> shop9</span><span class="pun">,</span><span class="pln"> shop10</span><span class="pun">,</span></code></li><li class="L2"><code><span class="pln"> shop11</span><span class="pun">,</span><span class="pln"> shop12</span><span class="pun">,</span><span class="pln"> shop13</span><span class="pun">,</span><span class="pln"> shop14</span><span class="pun">,</span><span class="pln"> shop15</span><span class="pun">,</span><span class="pln"> shop16</span><span class="pun">,</span><span class="pln"> shop17</span><span class="pun">,</span><span class="pln"> shop18</span><span class="pun">,</span><span class="pln"> shop19</span><span class="pun">,</span><span class="pln"> shop20</span></code></li><li class="L3"><code><span class="pln"> </span><span class="pun">):</span></code></li><li class="L4"><code><span class="pln"> </span><span class="com">#提取星级,建立X轴列表</span></code></li><li class="L5"><code><span class="pln"> </span><span class="kwd">if</span><span class="pln"> data1</span><span class="pun">[</span><span class="lit">0</span><span class="pun">]</span><span class="pln"> </span><span class="pun">==</span><span class="pln"> </span><span class="str">'30'</span><span class="pun">:</span></code></li><li class="L6"><code><span class="pln"> list_x</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="str">"三星级"</span><span class="pun">)</span></code></li><li class="L7"><code><span class="pln"> </span><span class="kwd">elif</span><span class="pln"> data1</span><span class="pun">[</span><span class="lit">0</span><span class="pun">]</span><span class="pln"> </span><span class="pun">==</span><span class="pln"> </span><span class="str">'35'</span><span class="pun">:</span></code></li><li class="L8"><code><span class="pln"> list_x</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="str">"三星半"</span><span class="pun">)</span></code></li><li class="L9"><code><span class="pln"> </span><span class="kwd">elif</span><span class="pln"> data1</span><span class="pun">[</span><span class="lit">0</span><span class="pun">]</span><span class="pln"> </span><span class="pun">==</span><span class="pln"> </span><span class="str">'40'</span><span class="pun">:</span></code></li><li class="L0"><code><span class="pln"> list_x</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="str">"四星级"</span><span class="pun">)</span></code></li><li class="L1"><code><span class="pln"> </span><span class="kwd">elif</span><span class="pln"> data1</span><span class="pun">[</span><span class="lit">0</span><span class="pun">]</span><span class="pln"> </span><span class="pun">==</span><span class="pln"> </span><span class="str">'45'</span><span class="pun">:</span></code></li><li class="L2"><code><span class="pln"> list_x</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="str">"四星半"</span><span class="pun">)</span></code></li><li class="L3"><code><span class="pln"> </span><span class="kwd">else</span><span class="pun">:</span></code></li><li class="L4"><code><span class="pln"> list_x</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="str">"五星级"</span><span class="pun">)</span></code></li><li class="L5"><code><span class="pln"> </span><span class="com">#20城市不同星级饭店数量</span></code></li><li class="L6"><code><span class="pln"> list_y1</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data1</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L7"><code><span class="pln"> list_y2</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data2</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L8"><code><span class="pln"> list_y3</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data3</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L9"><code><span class="pln"> list_y4</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data4</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L0"><code><span class="pln"> list_y5</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data5</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L1"><code><span class="pln"> list_y6</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data6</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L2"><code><span class="pln"> list_y7</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data7</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L3"><code><span class="pln"> list_y8</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data8</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L4"><code><span class="pln"> list_y9</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data9</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L5"><code><span class="pln"> list_y10</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data10</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L6"><code><span class="pln"> list_y11</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data11</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L7"><code><span class="pln"> list_y12</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data12</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L8"><code><span class="pln"> list_y13</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data13</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L9"><code><span class="pln"> list_y14</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data14</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L0"><code><span class="pln"> list_y15</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data15</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L1"><code><span class="pln"> list_y16</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data16</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L2"><code><span class="pln"> list_y17</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data17</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L3"><code><span class="pln"> list_y18</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data18</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L4"><code><span class="pln"> list_y19</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data19</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L5"><code><span class="pln"> list_y20</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data20</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span></code></li><li class="L6"><code><span class="pln"> </span><span class="kwd">print</span><span class="pun">(</span><span class="pln">list_x</span><span class="pun">,</span><span class="pln"> list_y1</span><span class="pun">,</span><span class="pln"> list_y2</span><span class="pun">,</span><span class="pln"> list_y3</span><span class="pun">,</span><span class="pln"> list_y4</span><span class="pun">,</span><span class="pln"> list_y5</span><span class="pun">,</span><span class="pln"> list_y6</span><span class="pun">,</span><span class="pln"> list_y7</span><span class="pun">,</span><span class="pln"> list_y8</span><span class="pun">,</span><span class="pln"> list_y9</span><span class="pun">,</span><span class="pln"> list_y10</span><span class="pun">,</span></code></li><li class="L7"><code><span class="pln"> list_y11</span><span class="pun">,</span><span class="pln"> list_y12</span><span class="pun">,</span><span class="pln"> list_y13</span><span class="pun">,</span><span class="pln"> list_y14</span><span class="pun">,</span><span class="pln"> list_y15</span><span class="pun">,</span><span class="pln"> list_y16</span><span class="pun">,</span><span class="pln"> list_y17</span><span class="pun">,</span><span class="pln"> list_y18</span><span class="pun">,</span><span class="pln"> list_y19</span><span class="pun">,</span><span class="pln"> list_y20</span></code></li><li class="L8"><code><span class="pln"> </span><span class="pun">)</span></code></li><li class="L9"><code><span class="pln"> </span><span class="com"># 饭店星级排行画图</span></code></li><li class="L0"><code><span class="pln"> plotstar</span><span class="pun">(</span><span class="pln">list_x</span><span class="pun">,</span><span class="pln"> list_y1</span><span class="pun">,</span><span class="pln"> list_y2</span><span class="pun">,</span><span class="pln"> list_y3</span><span class="pun">,</span><span class="pln"> list_y4</span><span class="pun">,</span><span class="pln"> list_y5</span><span class="pun">,</span><span class="pln"> list_y6</span><span class="pun">,</span><span class="pln"> list_y7</span><span class="pun">,</span><span class="pln"> list_y8</span><span class="pun">,</span><span class="pln"> list_y9</span><span class="pun">,</span><span class="pln"> list_y10</span><span class="pun">,</span></code></li><li class="L1"><code><span class="pln"> list_y11</span><span class="pun">,</span><span class="pln"> list_y12</span><span class="pun">,</span><span class="pln"> list_y13</span><span class="pun">,</span><span class="pln"> list_y14</span><span class="pun">,</span><span class="pln"> list_y15</span><span class="pun">,</span><span class="pln"> list_y16</span><span class="pun">,</span><span class="pln"> list_y17</span><span class="pun">,</span><span class="pln"> list_y18</span><span class="pun">,</span><span class="pln"> list_y19</span><span class="pun">,</span><span class="pln"> list_y20</span></code></li><li class="L2"><code><span class="pln"> </span><span class="pun">)</span></code></li><li class="L3"><code></code></li><li class="L4"><code><span class="com"># 饭店星级排行画图函数</span></code></li><li class="L5"><code><span class="kwd">def</span><span class="pln"> plotstar</span><span class="pun">(</span><span class="pln">list_x</span><span class="pun">,</span><span class="pln"> list_y1</span><span class="pun">,</span><span class="pln"> list_y2</span><span class="pun">,</span><span class="pln"> list_y3</span><span class="pun">,</span><span class="pln"> list_y4</span><span class="pun">,</span><span class="pln"> list_y5</span><span class="pun">,</span><span class="pln"> list_y6</span><span class="pun">,</span><span class="pln"> list_y7</span><span class="pun">,</span><span class="pln"> list_y8</span><span class="pun">,</span><span class="pln"> list_y9</span><span class="pun">,</span><span class="pln"> list_y10</span><span class="pun">,</span></code></li><li class="L6"><code><span class="pln"> list_y11</span><span class="pun">,</span><span class="pln"> list_y12</span><span class="pun">,</span><span class="pln"> list_y13</span><span class="pun">,</span><span class="pln"> list_y14</span><span class="pun">,</span><span class="pln"> list_y15</span><span class="pun">,</span><span class="pln"> list_y16</span><span class="pun">,</span><span class="pln"> list_y17</span><span class="pun">,</span><span class="pln"> list_y18</span><span class="pun">,</span><span class="pln"> list_y19</span><span class="pun">,</span><span class="pln"> list_y20</span><span class="pun">):</span></code></li><li class="L7"><code><span class="pln"> attr </span><span class="pun">=</span><span class="pln"> list_x</span></code></li><li class="L8"><code><span class="pln"> </span><span class="com">#设置柱状图大小标题,可根据实际数据情况统计</span></code></li><li class="L9"><code><span class="pln"> starbar </span><span class="pun">=</span><span class="pln"> </span><span class="typ">Bar</span><span class="pun">(</span><span class="str">"20个大中城市饭店星级统计"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"数据来源于大众点评TOP100"</span><span class="pun">,</span><span class="pln">width</span><span class="pun">=</span><span class="lit">1300</span><span class="pun">,</span><span class="pln">height</span><span class="pun">=</span><span class="lit">500</span><span class="pun">)</span></code></li><li class="L0"><code><span class="pln"> </span><span class="com">##添加柱状图数据和配置项,标记最大值和最小值,最右侧设置工具栏,可对图进行更多操作,图例位置</span></code></li><li class="L1"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"广州"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y1</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L2"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"杭州"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y2</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L3"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"南京"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y3</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L4"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"成都"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y4</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L5"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"武汉"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y5</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L6"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"济南"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y6</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L7"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"南宁"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y7</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L8"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"合肥"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y8</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L9"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"昆明"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y9</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L0"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"西安"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y10</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L1"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"北京"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y11</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L2"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"上海"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y12</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L3"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"深圳"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y13</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L4"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"天津"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y14</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L5"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"重庆"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y15</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L6"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"青岛"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y16</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L7"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"威海"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y17</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L8"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"长春"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y18</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L9"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"大连"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y19</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">])</span></code></li><li class="L0"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"贵阳"</span><span class="pun">,</span><span class="pln"> attr</span><span class="pun">,</span><span class="pln"> list_y20</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">],</span><span class="pln"> is_more_utils</span><span class="pun">=</span><span class="kwd">True</span><span class="pun">,</span><span class="pln"> legend_top</span><span class="pun">=</span><span class="str">'bottom'</span><span class="pun">)</span></code></li><li class="L1"><code><span class="com">#生成html文件数据可视化展示</span></code></li><li class="L2"><code><span class="pln"> starbar</span><span class="pun">.</span><span class="pln">render</span><span class="pun">(</span><span class="str">"20个大中城市饭店星级排名.html"</span><span class="pun">)</span></code></li><li class="L3"><code></code></li><li class="L4"><code><span class="kwd">if</span><span class="pln"> __name__ </span><span class="pun">==</span><span class="pln"> </span><span class="str">'__main__'</span><span class="pun">:</span></code></li><li class="L5"><code><span class="pln"> mianStar</span><span class="pun">()</span></code></li></ol></pre><h5 id="h5-3-1-4-1-3-"><a name="3.1.4.1.3 设置依赖包" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.1.3 设置依赖包</h5><p>选中项目,点击“文件”->“设置”,查看项目下的“Project Interpreter”,点击右侧的“+”号</p>
|
||
<p><img src="./水滴实验营_files/5-添加依赖包1.png" alt=""></p>
|
||
<p>弹出如下界面后,在查询框中输入“pytest-runner”,然后在列表中选择“pytest-runner”,点击“install package”,如图所示,提示安装成功。</p>
|
||
<p><img src="./水滴实验营_files/6-依赖包安装成功.png" alt=""></p>
|
||
<p>然后按照上述操作继续安装“pyecharts-jupyter-installer”、“pyecharts”和“pyecharts-snapshot”库,安装完成后关闭安装页面,进入“Project Interpreter”设置页面后,点击“确定”,回到项目主页面。若出现安装不成功,可能是由于网络连接问题,请再次安装。</p>
|
||
<h5 id="h5-3-1-4-1-4-"><a name="3.1.4.1.4 执行程序" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.1.4 执行程序</h5><p>选中“starclass.py”,点击右键选择“运行starclass”,执行程序,运行窗口显示打印信息为商铺星级、数量,在项目“dataanalysis”目录下,可以看到生成的“20个大中城市饭店星级排名.html”,如下图所示。</p>
|
||
<p><img src="./水滴实验营_files/9-星级排名运行结果.png" alt=""></p>
|
||
<h5 id="h5-3-1-4-1-5-"><a name="3.1.4.1.5 查看可视化数据" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.1.5 查看可视化数据</h5><p>进入该程序文件所在文件夹C:\dazhongspider\dataanalysis,可以看到该HTML文件,打开后可看到统计图表(如下面链接展示)。</p>
|
||
<p><a href="https://jn1.is.shanhe.com/shuidilab/images/qilucasetrainning/analysis/20%E4%B8%AA%E5%A4%A7%E4%B8%AD%E5%9F%8E%E5%B8%82%E9%A5%AD%E5%BA%97%E6%98%9F%E7%BA%A7%E6%8E%92%E5%90%8D.html" title="20个大中城市饭店星级排名">https://jn1.is.shanhe.com/shuidilab/images/qilucasetrainning/analysis/20个大中城市饭店星级排名.html</a></p>
|
||
<p>点击最右侧各个图标,可以查看数据图表、折线图等。</p>
|
||
<h4 id="h4-3-1-4-2-3-"><a name="3.1.4.2 山东3大城市菜品分类统计" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.2 山东3大城市菜品分类统计</h4><p>统计山东省内济南、青岛、威海三个城市的菜品分类。</p>
|
||
<h5 id="h5-3-1-4-2-1-"><a name="3.1.4.2.1 新建代码" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.2.1 新建代码</h5><p>参照3.1.4.1.1步骤,在项目窗口内,在“dataanalysis”文件夹上单击鼠标右键,在弹出的菜单中点击“新建”,再选择“Python File”,在弹出的对话框中,在“Name”中输入“foodclass”,点击“确定”按钮,进入.py文件编辑器。</p>
|
||
<h5 id="h5-3-1-4-2-2-python-"><a name="3.1.4.2.2 编写python程序代码" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.2.2 编写python程序代码</h5><p>输入如下代码到新建的“foodclass.py”文件中并保存。</p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="kwd">from</span><span class="pln"> pyecharts </span><span class="kwd">import</span><span class="pln"> </span><span class="typ">Pie</span><span class="pun">,</span><span class="pln"> </span><span class="typ">Page</span><span class="pun">,</span><span class="typ">Bar</span></code></li><li class="L1"><code><span class="kwd">from</span><span class="pln"> spider </span><span class="kwd">import</span><span class="pln"> dbconnect</span></code></li><li class="L2"><code></code></li><li class="L3"><code><span class="com">#查询统计某城市菜品分类名</span></code></li><li class="L4"><code><span class="kwd">def</span><span class="pln"> foodClass</span><span class="pun">(</span><span class="pln">city</span><span class="pun">):</span></code></li><li class="L5"><code><span class="pln"> </span><span class="com">#连接数据库</span></code></li><li class="L6"><code><span class="pln"> mysql_db </span><span class="pun">=</span><span class="pln"> dbconnect</span><span class="pun">.</span><span class="typ">DBHelper</span><span class="pun">()</span></code></li><li class="L7"><code><span class="pln"> mysql_db</span><span class="pun">.</span><span class="pln">connectDatabase</span><span class="pun">();</span></code></li><li class="L8"><code><span class="pln"> fsql </span><span class="pun">=</span><span class="pln"> </span><span class="str">'''SELECT COUNT(mainCategoryName),mainCategoryName from dazhongoutput where city="%s" GROUP BY mainCategoryName;'''</span><span class="pun">%(</span><span class="pln">city</span><span class="pun">)</span></code></li><li class="L9"><code><span class="pln"> </span><span class="com">#查询获得菜品分类商店梳理、菜品分类名称数据返回</span></code></li><li class="L0"><code><span class="pln"> listData </span><span class="pun">=</span><span class="pln"> mysql_db</span><span class="pun">.</span><span class="kwd">select</span><span class="pun">(</span><span class="pln">fsql</span><span class="pun">)</span></code></li><li class="L1"><code><span class="pln"> </span><span class="kwd">return</span><span class="pln"> listData</span></code></li><li class="L2"><code></code></li><li class="L3"><code><span class="com">#菜品分类画图函数</span></code></li><li class="L4"><code><span class="kwd">def</span><span class="pln"> plotfoodclass</span><span class="pun">():</span></code></li><li class="L5"><code><span class="pln"> </span><span class="com">#查询统计济南的菜品分类以及数量</span></code></li><li class="L6"><code><span class="pln"> list_address </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L7"><code><span class="pln"> list_sum </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L8"><code><span class="pln"> </span><span class="pun">[</span><span class="pln">list_sum</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">i</span><span class="pun">[</span><span class="lit">0</span><span class="pun">])</span><span class="pln"> </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> foodClass</span><span class="pun">(</span><span class="str">"济南"</span><span class="pun">)]</span></code></li><li class="L9"><code><span class="pln"> </span><span class="pun">[</span><span class="pln">list_address</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">i</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span><span class="pln"> </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> foodClass</span><span class="pun">(</span><span class="str">"济南"</span><span class="pun">)]</span></code></li><li class="L0"><code><span class="pln"> </span><span class="kwd">print</span><span class="pun">(</span><span class="pln">list_address</span><span class="pun">)</span></code></li><li class="L1"><code><span class="pln"> </span><span class="kwd">print</span><span class="pun">(</span><span class="pln">list_sum</span><span class="pun">)</span></code></li><li class="L2"><code><span class="pln"> </span><span class="com">#定义页面</span></code></li><li class="L3"><code><span class="pln"> page </span><span class="pun">=</span><span class="pln"> </span><span class="typ">Page</span><span class="pun">()</span></code></li><li class="L4"><code><span class="pln"> </span><span class="com">#添加饼图的数据和配置项</span></code></li><li class="L5"><code><span class="pln"> fpie1 </span><span class="pun">=</span><span class="pln"> </span><span class="typ">Pie</span><span class="pun">(</span><span class="str">"济南菜品分类"</span><span class="pun">,</span><span class="pln"> title_pos</span><span class="pun">=</span><span class="str">'center'</span><span class="pun">)</span></code></li><li class="L6"><code><span class="pln"> fpie1</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">""</span><span class="pun">,</span><span class="pln"> list_address</span><span class="pun">,</span><span class="pln"> list_sum</span><span class="pun">,</span><span class="pln"> radius</span><span class="pun">=[</span><span class="lit">30</span><span class="pun">,</span><span class="pln"> </span><span class="lit">75</span><span class="pun">],</span><span class="pln"> label_text_color</span><span class="pun">=</span><span class="kwd">None</span><span class="pun">,</span></code></li><li class="L7"><code><span class="pln"> is_label_show</span><span class="pun">=</span><span class="kwd">True</span><span class="pun">,</span><span class="pln"> legend_orient</span><span class="pun">=</span><span class="str">'vertical'</span><span class="pun">,</span></code></li><li class="L8"><code><span class="pln"> legend_pos</span><span class="pun">=</span><span class="str">'auto'</span><span class="pun">,</span><span class="pln"> is_legend_show</span><span class="pun">=</span><span class="kwd">False</span><span class="pun">)</span></code></li><li class="L9"><code><span class="pln"> </span><span class="com">#添加柱状图的数据和配置型项</span></code></li><li class="L0"><code><span class="pln"> fbar1 </span><span class="pun">=</span><span class="pln"> </span><span class="typ">Bar</span><span class="pun">(</span><span class="str">"济南菜品分类"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"数据来源于大众点评TOP100"</span><span class="pun">,</span><span class="pln">title_pos </span><span class="pun">=</span><span class="str">'center'</span><span class="pun">)</span></code></li><li class="L1"><code><span class="pln"> fbar1</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"济南"</span><span class="pun">,</span><span class="pln"> list_address</span><span class="pun">,</span><span class="pln"> list_sum</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">],</span><span class="pln">legend_pos</span><span class="pun">=</span><span class="str">"right"</span><span class="pun">)</span></code></li><li class="L2"><code></code></li><li class="L3"><code><span class="pln"> </span><span class="com"># 查询统计青岛的菜品分类以及数量</span></code></li><li class="L4"><code><span class="pln"> list_address </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L5"><code><span class="pln"> list_sum </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L6"><code><span class="pln"> </span><span class="pun">[</span><span class="pln">list_sum</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">i</span><span class="pun">[</span><span class="lit">0</span><span class="pun">])</span><span class="pln"> </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> foodClass</span><span class="pun">(</span><span class="str">"青岛"</span><span class="pun">)]</span></code></li><li class="L7"><code><span class="pln"> </span><span class="pun">[</span><span class="pln">list_address</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">i</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span><span class="pln"> </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> foodClass</span><span class="pun">(</span><span class="str">"青岛"</span><span class="pun">)]</span></code></li><li class="L8"><code><span class="pln"> </span><span class="kwd">print</span><span class="pun">(</span><span class="pln">list_address</span><span class="pun">)</span></code></li><li class="L9"><code><span class="pln"> </span><span class="kwd">print</span><span class="pun">(</span><span class="pln">list_sum</span><span class="pun">)</span></code></li><li class="L0"><code><span class="pln"> </span><span class="com"># 添加饼图的数据和配置项</span></code></li><li class="L1"><code><span class="pln"> fpie2 </span><span class="pun">=</span><span class="pln"> </span><span class="typ">Pie</span><span class="pun">(</span><span class="str">"青岛菜品分类"</span><span class="pun">,</span><span class="pln"> title_pos</span><span class="pun">=</span><span class="str">'center'</span><span class="pun">)</span></code></li><li class="L2"><code><span class="pln"> fpie2</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">""</span><span class="pun">,</span><span class="pln"> list_address</span><span class="pun">,</span><span class="pln"> list_sum</span><span class="pun">,</span><span class="pln"> radius</span><span class="pun">=[</span><span class="lit">30</span><span class="pun">,</span><span class="pln"> </span><span class="lit">75</span><span class="pun">],</span><span class="pln"> label_text_color</span><span class="pun">=</span><span class="kwd">None</span><span class="pun">,</span></code></li><li class="L3"><code><span class="pln"> is_label_show</span><span class="pun">=</span><span class="kwd">True</span><span class="pun">,</span><span class="pln"> legend_orient</span><span class="pun">=</span><span class="str">'vertical'</span><span class="pun">,</span></code></li><li class="L4"><code><span class="pln"> legend_pos</span><span class="pun">=</span><span class="str">'auto'</span><span class="pun">,</span><span class="pln"> is_legend_show</span><span class="pun">=</span><span class="kwd">False</span><span class="pun">)</span></code></li><li class="L5"><code><span class="pln"> </span><span class="com"># 添加柱状图的数据和配置型项</span></code></li><li class="L6"><code><span class="pln"> fbar2 </span><span class="pun">=</span><span class="pln"> </span><span class="typ">Bar</span><span class="pun">(</span><span class="str">"青岛菜品分类"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"数据来源于大众点评TOP100"</span><span class="pun">,</span><span class="pln"> title_pos</span><span class="pun">=</span><span class="str">'center'</span><span class="pun">)</span></code></li><li class="L7"><code><span class="pln"> fbar2</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"青岛"</span><span class="pun">,</span><span class="pln"> list_address</span><span class="pun">,</span><span class="pln"> list_sum</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">],</span><span class="pln"> legend_pos</span><span class="pun">=</span><span class="str">"right"</span><span class="pun">)</span></code></li><li class="L8"><code></code></li><li class="L9"><code><span class="pln"> </span><span class="com"># 查询统计威海的菜品分类以及数量</span></code></li><li class="L0"><code><span class="pln"> list_address </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L1"><code><span class="pln"> list_sum </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L2"><code><span class="pln"> </span><span class="pun">[</span><span class="pln">list_sum</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">i</span><span class="pun">[</span><span class="lit">0</span><span class="pun">])</span><span class="pln"> </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> foodClass</span><span class="pun">(</span><span class="str">"威海"</span><span class="pun">)]</span></code></li><li class="L3"><code><span class="pln"> </span><span class="pun">[</span><span class="pln">list_address</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">i</span><span class="pun">[</span><span class="lit">1</span><span class="pun">])</span><span class="pln"> </span><span class="kwd">for</span><span class="pln"> i </span><span class="kwd">in</span><span class="pln"> foodClass</span><span class="pun">(</span><span class="str">"威海"</span><span class="pun">)]</span></code></li><li class="L4"><code><span class="pln"> </span><span class="kwd">print</span><span class="pun">(</span><span class="pln">list_address</span><span class="pun">)</span></code></li><li class="L5"><code><span class="pln"> </span><span class="kwd">print</span><span class="pun">(</span><span class="pln">list_sum</span><span class="pun">)</span></code></li><li class="L6"><code><span class="pln"> </span><span class="com"># 添加饼图的数据和配置项</span></code></li><li class="L7"><code><span class="pln"> fpie3 </span><span class="pun">=</span><span class="pln"> </span><span class="typ">Pie</span><span class="pun">(</span><span class="str">"威海菜品分类"</span><span class="pun">,</span><span class="pln"> title_pos</span><span class="pun">=</span><span class="str">'center'</span><span class="pun">)</span></code></li><li class="L8"><code><span class="pln"> fpie3</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">""</span><span class="pun">,</span><span class="pln"> list_address</span><span class="pun">,</span><span class="pln"> list_sum</span><span class="pun">,</span><span class="pln"> radius</span><span class="pun">=[</span><span class="lit">30</span><span class="pun">,</span><span class="pln"> </span><span class="lit">75</span><span class="pun">],</span><span class="pln"> label_text_color</span><span class="pun">=</span><span class="kwd">None</span><span class="pun">,</span></code></li><li class="L9"><code><span class="pln"> is_label_show</span><span class="pun">=</span><span class="kwd">True</span><span class="pun">,</span><span class="pln"> legend_orient</span><span class="pun">=</span><span class="str">'vertical'</span><span class="pun">,</span></code></li><li class="L0"><code><span class="pln"> legend_pos</span><span class="pun">=</span><span class="str">'auto'</span><span class="pun">,</span><span class="pln"> is_legend_show</span><span class="pun">=</span><span class="kwd">False</span><span class="pun">)</span></code></li><li class="L1"><code><span class="pln"> </span><span class="com"># 添加柱状图的数据和配置型项</span></code></li><li class="L2"><code><span class="pln"> fbar3 </span><span class="pun">=</span><span class="pln"> </span><span class="typ">Bar</span><span class="pun">(</span><span class="str">"威海菜品分类"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"数据来源于大众点评TOP100"</span><span class="pun">,</span><span class="pln"> title_pos</span><span class="pun">=</span><span class="str">'center'</span><span class="pun">)</span></code></li><li class="L3"><code><span class="pln"> fbar3</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"威海"</span><span class="pun">,</span><span class="pln"> list_address</span><span class="pun">,</span><span class="pln"> list_sum</span><span class="pun">,</span><span class="pln"> mark_point</span><span class="pun">=[</span><span class="str">"max"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"min"</span><span class="pun">],</span><span class="pln"> legend_pos</span><span class="pun">=</span><span class="str">"right"</span><span class="pun">)</span></code></li><li class="L4"><code></code></li><li class="L5"><code><span class="pln"> </span><span class="com">#页面上添加图形</span></code></li><li class="L6"><code><span class="pln"> page</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="pln">fpie1</span><span class="pun">)</span></code></li><li class="L7"><code><span class="pln"> page</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="pln">fbar1</span><span class="pun">)</span></code></li><li class="L8"><code><span class="pln"> page</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="pln">fpie2</span><span class="pun">)</span></code></li><li class="L9"><code><span class="pln"> page</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="pln">fbar2</span><span class="pun">)</span></code></li><li class="L0"><code><span class="pln"> page</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="pln">fpie3</span><span class="pun">)</span></code></li><li class="L1"><code><span class="pln"> page</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="pln">fbar3</span><span class="pun">)</span></code></li><li class="L2"><code></code></li><li class="L3"><code><span class="pln"> page</span><span class="pun">.</span><span class="pln">render</span><span class="pun">(</span><span class="str">"山东3大城市菜品分类.html"</span><span class="pun">)</span></code></li><li class="L4"><code><span class="kwd">if</span><span class="pln"> __name__ </span><span class="pun">==</span><span class="pln"> </span><span class="str">'__main__'</span><span class="pun">:</span></code></li><li class="L5"><code><span class="pln"> plotfoodclass</span><span class="pun">()</span></code></li></ol></pre><h5 id="h5-3-1-4-2-3-"><a name="3.1.4.2.3 设置依赖包" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.2.3 设置依赖包</h5><p>选中项目,点击“文件”->“设置”,查看项目下“Project Interpreter”,是否存在“pytest-runner”、“pyecharts-jupyter-installer”、“pyecharts-snapshot”和“pyecharts”依赖包,若无,则参照3.1.4.1.3进行安装。</p>
|
||
<h5 id="h5-3-1-4-2-4-"><a name="3.1.4.2.4 执行程序" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.2.4 执行程序</h5><p>选中“foodclass.py”,点击右键选择“运行foodclass”,执行程序,运行窗口显示打印信息为菜品分类、每种分类商铺数,“dataanalysis”目录下显示生成的HTML页面。</p>
|
||
<p><img src="./水滴实验营_files/13-大城市菜品分类.png" alt=""></p>
|
||
<h5 id="h5-3-1-4-2-5-"><a name="3.1.4.2.5 查看可视化数据" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.2.5 查看可视化数据</h5><p>进入该项目所在文件夹C:\dazhongspider\dataanalysis,可以看到该HTML文件,双击打开,可看到统计图表(如下面链接展示)。</p>
|
||
<p> <a href="https://jn1.is.shanhe.com/shuidilab/images/qilucasetrainning/analysis/%E5%B1%B1%E4%B8%9C3%E5%A4%A7%E5%9F%8E%E5%B8%82%E8%8F%9C%E5%93%81%E5%88%86%E7%B1%BB.html" title="山东3大城市菜品分类">https://jn1.is.shanhe.com/shuidilab/images/qilucasetrainning/analysis/山东3大城市菜品分类.html</a></p>
|
||
<p> 点击最右侧各个图标,可以查看数据图表、折线图等。</p>
|
||
<h4 id="h4-3-1-4-3-3-top20-"><a name="3.1.4.3 山东3大城市TOP20饭店服务的综合评价" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.3 山东3大城市TOP20饭店服务的综合评价</h4><p>针对山东省内济南、青岛、威海三个城市,评价每个店铺的“口味”、“环境”、“服务”,根据这些指标求平均得到综合评价指标,并根据综合评价指标对商铺进行排名,最终得到山东3大城市TOP20商铺。</p>
|
||
<h5 id="h5-3-1-4-3-1-"><a name="3.1.4.3.1 新建代码" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.3.1 新建代码</h5><p>参照3.1.4.1.1步骤,在项目窗口内,在“dataanalysis”文件夹上单击鼠标右键,在弹出的菜单中点击“新建”,再选择“Python File”,在弹出的对话框中(如图所示),在“Name”中输入“evaluatetop20”,点击“确定”按钮。</p>
|
||
<h5 id="h5-3-1-4-3-2-python-"><a name="3.1.4.3.2 编写python程序代码" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.3.2 编写python程序代码</h5><p>输入如下代码到新建的“evaluatetop20.py”文件中并保存。</p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="kwd">from</span><span class="pln"> pyecharts </span><span class="kwd">import</span><span class="pln"> </span><span class="typ">Line</span><span class="pun">,</span><span class="pln"> </span><span class="typ">Bar</span><span class="pun">,</span><span class="pln"> </span><span class="typ">Page</span><span class="pun">,</span><span class="pln"> </span><span class="typ">WordCloud</span></code></li><li class="L1"><code><span class="kwd">from</span><span class="pln"> spider </span><span class="kwd">import</span><span class="pln"> dbconnect</span></code></li><li class="L2"><code></code></li><li class="L3"><code><span class="com">#查询商铺名称、计算综合评价分数=(口味+环境+服务)/3的函数</span></code></li><li class="L4"><code><span class="kwd">def</span><span class="pln"> evaluateshop</span><span class="pun">(</span><span class="pln">city</span><span class="pun">):</span></code></li><li class="L5"><code><span class="pln"> </span><span class="com">#连接数据库</span></code></li><li class="L6"><code><span class="pln"> mysql_db </span><span class="pun">=</span><span class="pln"> dbconnect</span><span class="pun">.</span><span class="typ">DBHelper</span><span class="pun">()</span></code></li><li class="L7"><code><span class="pln"> mysql_db</span><span class="pun">.</span><span class="pln">connectDatabase</span><span class="pun">();</span></code></li><li class="L8"><code><span class="pln"> </span><span class="com">#定义查询语句,查询商铺名称、计算综合评价分数</span></code></li><li class="L9"><code><span class="pln"> nsql </span><span class="pun">=</span><span class="pln"> </span><span class="str">'''SELECT shopName,(tasteScore+environmentScore+serviceScore)/3.0 as avg from dazhongoutput WHERE city="%s" ORDER BY (tasteScore+environmentScore+serviceScore)/3.0 DESC;'''</span><span class="pun">%(</span><span class="pln">city</span><span class="pun">)</span></code></li><li class="L0"><code><span class="pln"> listData </span><span class="pun">=</span><span class="pln">mysql_db</span><span class="pun">.</span><span class="kwd">select</span><span class="pun">(</span><span class="pln">nsql</span><span class="pun">)</span></code></li><li class="L1"><code><span class="pln"> listShopName </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L2"><code><span class="pln"> listAvg </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L3"><code><span class="pln"> </span><span class="com">#返回商铺名称、综合评价分数</span></code></li><li class="L4"><code><span class="pln"> </span><span class="kwd">for</span><span class="pln"> data </span><span class="kwd">in</span><span class="pln"> listData</span><span class="pun">:</span></code></li><li class="L5"><code><span class="pln"> listShopName</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data</span><span class="pun">[</span><span class="lit">0</span><span class="pun">])</span></code></li><li class="L6"><code><span class="pln"> listAvg</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">round</span><span class="pun">(</span><span class="pln">data</span><span class="pun">[</span><span class="lit">1</span><span class="pun">],</span><span class="pln"> </span><span class="lit">2</span><span class="pun">))</span></code></li><li class="L7"><code><span class="pln"> </span><span class="kwd">return</span><span class="pln"> </span><span class="pun">[</span><span class="pln">listShopName</span><span class="pun">,</span><span class="pln">listAvg</span><span class="pun">]</span></code></li><li class="L8"><code></code></li><li class="L9"><code><span class="com">#TOP20商铺综合分数画图函数</span></code></li><li class="L0"><code><span class="kwd">def</span><span class="pln"> plottop20</span><span class="pun">(</span><span class="pln">list_y1</span><span class="pun">,</span><span class="pln"> list_y2</span><span class="pun">,</span><span class="pln"> list_y3</span><span class="pun">):</span></code></li><li class="L1"><code><span class="pln"> </span><span class="com">#定义页面</span></code></li><li class="L2"><code><span class="pln"> page </span><span class="pun">=</span><span class="pln"> </span><span class="typ">Page</span><span class="pun">()</span></code></li><li class="L3"><code></code></li><li class="L4"><code><span class="pln"> </span><span class="com">#折线图定义、添加数据和配置项</span></code></li><li class="L5"><code><span class="pln"> line1 </span><span class="pun">=</span><span class="pln"> </span><span class="typ">Line</span><span class="pun">(</span><span class="str">"济南饭店评分统计"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"数据来源于大众点评TOP100"</span><span class="pun">,</span><span class="pln"> width</span><span class="pun">=</span><span class="lit">1200</span><span class="pun">)</span></code></li><li class="L6"><code><span class="pln"> line1</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"综合"</span><span class="pun">,</span><span class="pln"> list_y1</span><span class="pun">[</span><span class="lit">0</span><span class="pun">][:</span><span class="lit">20</span><span class="pun">],</span><span class="pln"> list_y1</span><span class="pun">[</span><span class="lit">1</span><span class="pun">][:</span><span class="lit">20</span><span class="pun">],</span><span class="pln"> is_label_show</span><span class="pun">=</span><span class="kwd">True</span><span class="pun">,</span><span class="pln"> is_more_utils</span><span class="pun">=</span><span class="kwd">True</span><span class="pun">)</span></code></li><li class="L7"><code></code></li><li class="L8"><code><span class="pln"> line2 </span><span class="pun">=</span><span class="pln"> </span><span class="typ">Line</span><span class="pun">(</span><span class="str">"青岛饭店评分统计"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"数据来源于大众点评TOP100"</span><span class="pun">,</span><span class="pln"> width</span><span class="pun">=</span><span class="lit">1200</span><span class="pun">)</span></code></li><li class="L9"><code><span class="pln"> line2</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"综合"</span><span class="pun">,</span><span class="pln"> list_y2</span><span class="pun">[</span><span class="lit">0</span><span class="pun">][:</span><span class="lit">20</span><span class="pun">],</span><span class="pln"> list_y2</span><span class="pun">[</span><span class="lit">1</span><span class="pun">][:</span><span class="lit">20</span><span class="pun">],</span><span class="pln"> is_label_show</span><span class="pun">=</span><span class="kwd">True</span><span class="pun">,</span><span class="pln"> is_more_utils</span><span class="pun">=</span><span class="kwd">True</span><span class="pun">)</span></code></li><li class="L0"><code></code></li><li class="L1"><code><span class="pln"> line3 </span><span class="pun">=</span><span class="pln"> </span><span class="typ">Line</span><span class="pun">(</span><span class="str">"威海饭店评分统计"</span><span class="pun">,</span><span class="pln"> </span><span class="str">"数据来源于大众点评TOP100"</span><span class="pun">,</span><span class="pln"> width</span><span class="pun">=</span><span class="lit">1200</span><span class="pun">)</span></code></li><li class="L2"><code><span class="pln"> line3</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="str">"综合"</span><span class="pun">,</span><span class="pln"> list_y3</span><span class="pun">[</span><span class="lit">0</span><span class="pun">][:</span><span class="lit">20</span><span class="pun">],</span><span class="pln"> list_y3</span><span class="pun">[</span><span class="lit">1</span><span class="pun">][:</span><span class="lit">20</span><span class="pun">],</span><span class="pln"> is_label_show</span><span class="pun">=</span><span class="kwd">True</span><span class="pun">,</span><span class="pln"> is_more_utils</span><span class="pun">=</span><span class="kwd">True</span><span class="pun">)</span></code></li><li class="L3"><code><span class="pln"> </span><span class="com">#页面添加图形</span></code></li><li class="L4"><code><span class="pln"> page</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="pln">line1</span><span class="pun">)</span></code></li><li class="L5"><code><span class="pln"> page</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="pln">line2</span><span class="pun">)</span></code></li><li class="L6"><code><span class="pln"> page</span><span class="pun">.</span><span class="kwd">add</span><span class="pun">(</span><span class="pln">line3</span><span class="pun">)</span></code></li><li class="L7"><code><span class="pln"> </span><span class="com">#生成HTML,数据可视化展示</span></code></li><li class="L8"><code><span class="pln"> page</span><span class="pun">.</span><span class="pln">render</span><span class="pun">(</span><span class="str">"山东3大城市TOP20商铺综合服务评分.html"</span><span class="pun">)</span></code></li><li class="L9"><code></code></li><li class="L0"><code><span class="kwd">if</span><span class="pln"> __name__ </span><span class="pun">==</span><span class="pln"> </span><span class="str">'__main__'</span><span class="pun">:</span></code></li><li class="L1"><code><span class="pln"> </span><span class="com">#评估3大城市商铺综合分数</span></code></li><li class="L2"><code><span class="pln"> shop1 </span><span class="pun">=</span><span class="pln"> evaluateshop</span><span class="pun">(</span><span class="str">"济南"</span><span class="pun">)</span></code></li><li class="L3"><code><span class="pln"> shop2 </span><span class="pun">=</span><span class="pln"> evaluateshop</span><span class="pun">(</span><span class="str">"青岛"</span><span class="pun">)</span></code></li><li class="L4"><code><span class="pln"> shop3 </span><span class="pun">=</span><span class="pln"> evaluateshop</span><span class="pun">(</span><span class="str">"威海"</span><span class="pun">)</span></code></li><li class="L5"><code></code></li><li class="L6"><code><span class="pln"> </span><span class="com">#建立数据列表画图,生成HTML</span></code></li><li class="L7"><code><span class="pln"> list_x </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L8"><code><span class="pln"> list_y1</span><span class="pun">,</span><span class="pln"> list_y2</span><span class="pun">,</span><span class="pln"> list_y3 </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[],</span><span class="pln"> </span><span class="pun">[]</span></code></li><li class="L9"><code><span class="pln"> </span><span class="kwd">for</span><span class="pln"> data</span><span class="pun">,</span><span class="pln"> data2</span><span class="pun">,</span><span class="pln"> data3 </span><span class="kwd">in</span><span class="pln"> zip</span><span class="pun">(</span><span class="pln">shop1</span><span class="pun">,</span><span class="pln"> shop2</span><span class="pun">,</span><span class="pln"> shop3</span><span class="pun">):</span></code></li><li class="L0"><code><span class="pln"> list_y1</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data</span><span class="pun">)</span></code></li><li class="L1"><code><span class="pln"> list_y2</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data2</span><span class="pun">)</span></code></li><li class="L2"><code><span class="pln"> list_y3</span><span class="pun">.</span><span class="pln">append</span><span class="pun">(</span><span class="pln">data3</span><span class="pun">)</span></code></li><li class="L3"><code><span class="pln"> plottop20</span><span class="pun">(</span><span class="pln">list_y1</span><span class="pun">,</span><span class="pln"> list_y2</span><span class="pun">,</span><span class="pln"> list_y3</span><span class="pun">)</span></code></li></ol></pre><h5 id="h5-3-1-4-3-3-"><a name="3.1.4.3.3 设置依赖包" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.3.3 设置依赖包</h5><p>选中项目,点击“文件”->“设置”,查看“Project Interpreter”,是否存在“pytest-runner”、“pyecharts-jupyter-installer”、“pyecharts”和“pyecharts-snapshot”依赖包,若无,则参照3.1.4.1.3进行安装。</p>
|
||
<h5 id="h5-3-1-4-3-4-"><a name="3.1.4.3.4 执行程序" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.3.4 执行程序</h5><p>选中“evaluatetop20.py”,点击右键选择“evaluatetop20”,执行程序,运行窗口显示打印信息为菜品分类、每种分类商铺数,“dataanalysis”目录下显示生成的HTML页面</p>
|
||
<p><img src="./水滴实验营_files/14-大城市top20商铺服务评分.png" alt=""></p>
|
||
<h5 id="h5-3-1-4-3-5-"><a name="3.1.4.3.5 查看可视化数据" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.1.4.3.5 查看可视化数据</h5><p>进入该程序文件所在文件夹D:\dazhongspider\dataanalysis,可以看到该HTML文件,双击打开,看到统计图表(如下述链接展示),点击右侧图表可以查看柱状图、数据视图等。</p>
|
||
<p><a href="https://jn1.is.shanhe.com/shuidilab/images/qilucasetrainning/analysis/%E5%B1%B1%E4%B8%9C3%E5%A4%A7%E5%9F%8E%E5%B8%82TOP20%E5%95%86%E9%93%BA%E7%BB%BC%E5%90%88%E6%9C%8D%E5%8A%A1%E8%AF%84%E5%88%86.html" title="山东3大城市TOP20商铺综合服务评分">https://jn1.is.shanhe.com/shuidilab/images/qilucasetrainning/analysis/山东3大城市TOP20商铺综合服务评分.html</a></p>
|
||
<p>点击最右侧各个图标,可以查看数据图表、折线图等。</p>
|
||
<h2 id="h2-3-2-r-csv-"><a name="3.2 R编程+CSV文件数据统计分析可视化" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.2 R编程+CSV文件数据统计分析可视化</h2><p>本部分利用R统计分析CSV文件中20大城市的星级商铺统计分析,统计指标同3.1.3.3,其他统计分析学生自行实践.</p>
|
||
<h3 id="h3-3-2-1-"><a name="3.2.1 新建代码" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.2.1 新建代码</h3><p>双击“RStudio”图标启动软件,在窗口内,点击“File”->“New File”->“R Script”。</p>
|
||
<p><img src="./水滴实验营_files/1-新建R脚本.png" alt=""></p>
|
||
<p>进入R脚本编辑界面,如图所示,输入如下代码到脚本文件中,请注意代码中的”C:/changedata/dazhongdata.csv”指的是清洗后的数据文件,请根据前面自己的设置进行路径及文件名修改。</p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">data</span><span class="pun"><-</span><span class="pln">read</span><span class="pun">.</span><span class="pln">csv</span><span class="pun">(</span><span class="pln">file</span><span class="pun">=</span><span class="str">"C:/changedata/dazhongdata.csv"</span><span class="pun">,</span><span class="pln">sep</span><span class="pun">=</span><span class="str">","</span><span class="pun">,</span><span class="pln">header</span><span class="pun">=</span><span class="pln">TRUE</span><span class="pun">,</span><span class="pln">encoding </span><span class="pun">=</span><span class="pln"> </span><span class="str">"utf-8"</span><span class="pun">)</span></code></li><li class="L1"><code><span class="pln">selectscore1</span><span class="pun"><-</span><span class="pln">subset</span><span class="pun">(</span><span class="pln">data</span><span class="pun">,</span><span class="pln">city</span><span class="pun">==</span><span class="str">"济南"</span><span class="pun">,</span><span class="kwd">select</span><span class="pun">=</span><span class="pln">c</span><span class="pun">(</span><span class="pln">shopName</span><span class="pun">,</span><span class="pln">tasteScore</span><span class="pun">,</span><span class="pln">environmentScore</span><span class="pun">,</span><span class="pln">serviceScore</span><span class="pun">))</span></code></li><li class="L2"><code><span class="pln">meanscore1</span><span class="pun"><-</span><span class="pln">rowMeans</span><span class="pun">(</span><span class="pln">selectscore1</span><span class="pun">[</span><span class="lit">2</span><span class="pun">:</span><span class="lit">4</span><span class="pun">])</span></code></li><li class="L3"><code><span class="pln">cdata1</span><span class="pun"><-</span><span class="pln">cbind</span><span class="pun">(</span><span class="pln">selectscore1</span><span class="pun">[</span><span class="lit">1</span><span class="pun">],</span><span class="pln">meanscore1</span><span class="pun">)</span></code></li><li class="L4"><code><span class="pln">top20_jinan</span><span class="pun"><-</span><span class="pln">cdata1</span><span class="pun">[</span><span class="pln">order</span><span class="pun">(</span><span class="pln">cdata1</span><span class="pun">[</span><span class="lit">2</span><span class="pun">],</span><span class="pln">decreasing </span><span class="pun">=</span><span class="pln"> TRUE</span><span class="pun">)[</span><span class="lit">1</span><span class="pun">:</span><span class="lit">20</span><span class="pun">],]</span></code></li><li class="L5"><code></code></li><li class="L6"><code><span class="kwd">print</span><span class="pun">(</span><span class="pln">top20_jinan</span><span class="pun">)</span></code></li><li class="L7"><code></code></li><li class="L8"><code><span class="pln">par</span><span class="pun">(</span><span class="pln">mfrow</span><span class="pun">=</span><span class="pln">c</span><span class="pun">(</span><span class="lit">3</span><span class="pun">,</span><span class="lit">1</span><span class="pun">))</span></code></li><li class="L9"><code><span class="pln">plot</span><span class="pun">(</span><span class="pln">factor</span><span class="pun">(</span><span class="pln">top20_jinan</span><span class="pun">[,</span><span class="lit">1</span><span class="pun">]),</span><span class="pln">top20_jinan</span><span class="pun">[,</span><span class="lit">2</span><span class="pun">],</span><span class="pln">xlable</span><span class="pun">=</span><span class="str">"济南TOP20商铺"</span><span class="pun">,</span><span class="pln">ylab</span><span class="pun">=</span><span class="str">"综合评分"</span><span class="pun">,</span><span class="pln">main</span><span class="pun">=</span><span class="str">"山东3大城市TOP20商铺综合评分"</span><span class="pun">)</span></code></li><li class="L0"><code></code></li><li class="L1"><code></code></li><li class="L2"><code><span class="pln">selectscore2</span><span class="pun"><-</span><span class="pln">subset</span><span class="pun">(</span><span class="pln">data</span><span class="pun">,</span><span class="pln">city</span><span class="pun">==</span><span class="str">"青岛"</span><span class="pun">,</span><span class="kwd">select</span><span class="pun">=</span><span class="pln">c</span><span class="pun">(</span><span class="pln">shopName</span><span class="pun">,</span><span class="pln">tasteScore</span><span class="pun">,</span><span class="pln">environmentScore</span><span class="pun">,</span><span class="pln">serviceScore</span><span class="pun">))</span></code></li><li class="L3"><code><span class="pln">meanscore2</span><span class="pun"><-</span><span class="pln">rowMeans</span><span class="pun">(</span><span class="pln">selectscore2</span><span class="pun">[</span><span class="lit">2</span><span class="pun">:</span><span class="lit">4</span><span class="pun">])</span></code></li><li class="L4"><code><span class="pln">cdata2</span><span class="pun"><-</span><span class="pln">cbind</span><span class="pun">(</span><span class="pln">selectscore2</span><span class="pun">[</span><span class="lit">1</span><span class="pun">],</span><span class="pln">meanscore2</span><span class="pun">)</span></code></li><li class="L5"><code><span class="pln">top20_qingdao</span><span class="pun"><-</span><span class="pln">cdata2</span><span class="pun">[</span><span class="pln">order</span><span class="pun">(</span><span class="pln">cdata2</span><span class="pun">[</span><span class="lit">2</span><span class="pun">],</span><span class="pln">decreasing </span><span class="pun">=</span><span class="pln"> TRUE</span><span class="pun">)[</span><span class="lit">1</span><span class="pun">:</span><span class="lit">20</span><span class="pun">],]</span></code></li><li class="L6"><code><span class="kwd">print</span><span class="pun">(</span><span class="pln">top20_qingdao</span><span class="pun">)</span></code></li><li class="L7"><code><span class="pln">plot</span><span class="pun">(</span><span class="pln">factor</span><span class="pun">(</span><span class="pln">top20_qingdao</span><span class="pun">[,</span><span class="lit">1</span><span class="pun">]),</span><span class="pln">top20_qingdao</span><span class="pun">[,</span><span class="lit">2</span><span class="pun">],</span><span class="pln">xlab</span><span class="pun">=</span><span class="str">"青岛TOP20商铺"</span><span class="pun">,</span><span class="pln">ylab</span><span class="pun">=</span><span class="str">"综合评分"</span><span class="pun">)</span></code></li><li class="L8"><code></code></li><li class="L9"><code><span class="pln">selectscore3</span><span class="pun"><-</span><span class="pln">subset</span><span class="pun">(</span><span class="pln">data</span><span class="pun">,</span><span class="pln">city</span><span class="pun">==</span><span class="str">"威海"</span><span class="pun">,</span><span class="kwd">select</span><span class="pun">=</span><span class="pln">c</span><span class="pun">(</span><span class="pln">shopName</span><span class="pun">,</span><span class="pln">tasteScore</span><span class="pun">,</span><span class="pln">environmentScore</span><span class="pun">,</span><span class="pln">serviceScore</span><span class="pun">))</span></code></li><li class="L0"><code><span class="pln">meanscore3</span><span class="pun"><-</span><span class="pln">rowMeans</span><span class="pun">(</span><span class="pln">selectscore3</span><span class="pun">[</span><span class="lit">2</span><span class="pun">:</span><span class="lit">4</span><span class="pun">])</span></code></li><li class="L1"><code><span class="pln">cdata3</span><span class="pun"><-</span><span class="pln">cbind</span><span class="pun">(</span><span class="pln">selectscore3</span><span class="pun">[</span><span class="lit">1</span><span class="pun">],</span><span class="pln">meanscore3</span><span class="pun">)</span></code></li><li class="L2"><code><span class="pln">top20_weihai</span><span class="pun"><-</span><span class="pln">cdata3</span><span class="pun">[</span><span class="pln">order</span><span class="pun">(</span><span class="pln">cdata3</span><span class="pun">[</span><span class="lit">2</span><span class="pun">],</span><span class="pln">decreasing </span><span class="pun">=</span><span class="pln"> TRUE</span><span class="pun">)[</span><span class="lit">1</span><span class="pun">:</span><span class="lit">20</span><span class="pun">],]</span></code></li><li class="L3"><code><span class="kwd">print</span><span class="pun">(</span><span class="pln">top20_weihai</span><span class="pun">)</span></code></li><li class="L4"><code><span class="pln">plot</span><span class="pun">(</span><span class="pln">factor</span><span class="pun">(</span><span class="pln">top20_weihai</span><span class="pun">[,</span><span class="lit">1</span><span class="pun">]),</span><span class="pln">top20_weihai</span><span class="pun">[,</span><span class="lit">2</span><span class="pun">],</span><span class="pln">xlab</span><span class="pun">=</span><span class="str">"威海TOP20商铺"</span><span class="pun">,</span><span class="pln">ylab</span><span class="pun">=</span><span class="str">"综合评分"</span><span class="pun">)</span></code></li></ol></pre><p>点击“保存”,弹出对话框,选择“UTF-8”编码,点击“确定”,弹出对话框,输入保存R脚本文件名称“top20”.</p>
|
||
<p><img src="./水滴实验营_files/2-编辑代码保存.png" alt=""></p>
|
||
<h3 id="h3-3-2-2-r-"><a name="3.2.2 执行R脚本" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.2.2 执行R脚本</h3><p>点击“source”按钮执行R脚本文件,在左下方控制台可以看到脚本中打印信息已输出,在右上方环境中可以看到脚本运行过程中各变量的值,在右下方显示执行结果,在plots标签页显示绘制的图形。</p>
|
||
<p><img src="./水滴实验营_files/3-运行结果.png" alt=""></p>
|
||
<p>在右下方工作区,点击“Zoom”可以放大到新窗口查看,点击“Export”可以导出图片,如图所示。</p>
|
||
<p><img src="./水滴实验营_files/4-运行结果图.png" alt=""></p>
|
||
<p><font color="red"> <strong>提示:</strong></font></p>
|
||
<ul>
|
||
<li>* 若执行脚本过程中出现“error in read.table(…….):more columns than column names”,表示某列未定义列名但有数据,请查看dazhongdata.csv数据表中最后一列后面的某列是不是没有列名但有一两行有数据,删除该列即可。</li><li>* 若执行脚本过程中出现“plot.new() : figure margins too large”错误,则将RStudio里的【Plots】窗口适当调大一点,或者把RStudio设置为全屏。如果仍然有该错误,则按下“Fn”不动后接着按下“F11”,将浏览器全屏。</li><li>* 若出现如下错误,则表示数据集没有相应城市的数据.</li></ul>
|
||
<p><img src="./水滴实验营_files/6-数据没有对应城市.png" alt=""></p>
|
||
<p>注释掉那部分代码,修改一页多图部分的代码,例如缺少一个城市的数据,则修改为</p>
|
||
<pre class="prettyprint linenums prettyprinted" style=""><i class="document-copy fa fa-files-o" style="display:none"></i><ol class="linenums"><li class="L0"><code><span class="pln">par</span><span class="pun">(</span><span class="pln">mfrow</span><span class="pun">=</span><span class="pln">c</span><span class="pun">(</span><span class="lit">2</span><span class="pun">,</span><span class="lit">1</span><span class="pun">))</span><span class="pln"> </span><span class="com"># 将一列3个图修改为1列2个图</span></code></li></ol></pre><h2 id="h2-3-3-"><a name="3.3 学生实训任务" class="reference-link"></a><span class="header-link octicon octicon-link"></span>3.3 学生实训任务</h2><p>自行设计新的数据统计分析内容,基于上述某个python程序进行修改,进行可视化展示,可以是柱状图、饼图等图形的任何一个或多个图形均可。</p>
|
||
<h1 id="h1-4-"><a name="4 小结" class="reference-link"></a><span class="header-link octicon octicon-link"></span>4 小结</h1><p>本章的综合实验案例涵盖了Python、MySQL、Navicat、Kettle、R等系统和软件的使用,技术上涉及数据采集、存储、查询、清洗和可视化分析等数据处理全流程,通过本章实验案例,学生可以更加熟悉软件使用,掌握数据处理的流程以及编程技术,可以帮助学生加深对各种技术的理解,体会项目开发的流程,提高各项技能。</p>
|
||
</div></div></div></div><!--for03356147647--><div id="evaluation" style="display: none;"><!--if--><div class="qc-desktop-com"><div class="comment"><div style="border: 1px solid rgb(204, 204, 204); padding-left: 10px; border-radius: 4px; margin-bottom: 10px; height: 45px; box-shadow: rgb(221, 221, 221) 0px 0px 2px 2px;"><p style="font-size: 12px;">给实验评分</p><span>:</span><span id="exp_course_score"></span></div><!--if--><div class="commentBtn"><p id="evaluate_tip"></p><input type="button" value="发表" name="send_comment" avalon-events="click:eclick_0_events46send95comment"></div></div><div class="avatarListWrap" style="height: auto; overflow-y: auto;"><div class="avatarList"><p class="avatarList-nocomment">暂无评论</p></div><!--ms-for:c in course_evaluate.comment_list | limitBy(100)--><!--ms-for-end:--><p style="margin-right: 20px;" class="qc-paging" id="expdesltop_view_data_page"></p></div></div><!--if--><!-- 随材管理 --><div style="display: none;"><div style="padding: 20px 10px 10px; text-align: center; color: rgb(102, 102, 102); font-size: 14px;">暂无实验随材</div><!--if--></div><!-- 视频管理 --><div style="display: none;"><!--if--></div><!-- 代码管理 --><div style="display: none; padding: 10px;"><input type="file" id="guac_upload_file" style="opacity: 0; width: 0px;"><!--if--><!--
|
||
<button ms-if="show_vm.code_data" style="margin:10px;width:100px;background-color:#0dc49b"
|
||
ms-on-click="events.code_upload()" class="layui-btn">
|
||
上传文件
|
||
</button>
|
||
--><!--if--><!--if--><div style="padding: 20px 10px 10px; text-align: center; color: rgb(102, 102, 102); font-size: 14px;">实验环境尚未启动,无法上传/下载文件</div><!--if--><!--
|
||
<div ms-if="show_vm.code_data&&show_vm.show" class="trying-to-load">
|
||
<div class="load-container load1">
|
||
<div class="loader">Loading...</div>
|
||
</div>
|
||
</div>
|
||
<div style="padding:20px 10px 10px;text-align: center;color:#666;font-size: 14px;" ms-if="show_vm.title"
|
||
ms-html="show_vm.title"></div>
|
||
<div ms-if="d.type=='code'&&!show_vm.show" style="padding: 5px">
|
||
<table style="margin:0" class="create-materials-table" ms-if="step_code_list.length !=0">
|
||
<tr>
|
||
<th>文件名称</th>
|
||
<th>大小(KB)</th>
|
||
</tr>
|
||
<tr ms-for="(index, val) in @step_code_list">
|
||
<td>
|
||
<a ms-if = "val.filesize >0" style="color: #00a5c1;cursor:pointer" ms-attr="{href:val.href,download:val.filename}"
|
||
ms-html="val.filename"></a>
|
||
<a ms-if = "val.filesize ==0 || val.filesize <0" style="color: #00a5c1;cursor:pointer"
|
||
ms-html="val.filename"></a>
|
||
</td>
|
||
<td ms-html="val.filesize"></td>
|
||
</tr>
|
||
</table>
|
||
</div>
|
||
--></div></div><!--for03356147647--><div id="report" style="display: none;"><!--if--><!--if--><div id="report_data_form" class="qc-desktop-com"><div id="course"><!--ms-for:field in fields_report--><div class="icode-form-group clearfix"><p style="font-size: 14px;"><label>完成情况</label><!--ms-for:val in field.data--><label><input type="radio" checked="" name="radio" value="0" avalon-events="change:echange_0_events46completionFun40val46value41"><span>全部完成</span></label><!--for157171563893--><label><input type="radio" name="radio" value="1" avalon-events="change:echange_0_events46completionFun40val46value41"><span>部分完成</span></label><!--for157171563893--><label><input type="radio" name="radio" value="2" avalon-events="change:echange_0_events46completionFun40val46value41"><span>未完成</span></label><!--for157171563893--><!--ms-for-end:--></p><!--if--></div><!--for696174317876--><div class="icode-form-group clearfix"><!--if--><div style="margin: 0px 28px 0px 20px;" class="ueditor row" change-class="row "><div class="layui-upload"><button type="button" class="layui-btn layui-btn-normal" style="padding: 2px; width: 64px; height: 24px; line-height: 24px; margin: 0px;" id="testList" change-class="">附件上传</button><div class="layui-upload-list"><table class="layui-table" id="report_attachment" style="overflow: auto; max-height: 200px;"><thead><tr><th>文件名</th><th>大小</th><th>状态</th><th>操作</th></tr></thead><tbody id="demoList"><!--ms-for:val in report_data.attach_list--><!--ms-for-end:--></tbody></table></div><div id="testListAction"></div></div><label class="checkLabel" style="font-size: 14px; margin-left: 0px; margin-bottom: 10px;">报告内容</label><!--if--><script type="text/plain" style="width: 0px; height: 500px;" id="content"></script><!--if--></div></div><!--for696174317876--><!--ms-for-end:--><div style="height: 60px; display: flex; justify-content: center; align-items: center;"><!--if--><button class="layui-btn layui-btn-primary" change-class="layui-btn layui-btn-primary " avalon-events="click:eclick_0_events46save95report4041">保存</button><button class="layui-btn" change-class="layui-btn " avalon-events="click:eclick_0_events46submit95report4041">提交</button></div><!-- style ="background-color:ccc;color:#fff" --></div></div><!-- 随材管理 --><div style="display: none;"><div style="padding: 20px 10px 10px; text-align: center; color: rgb(102, 102, 102); font-size: 14px;">暂无实验随材</div><!--if--></div><!-- 视频管理 --><div style="display: none;"><!--if--></div><!-- 代码管理 --><div style="display: none; padding: 10px;"><input type="file" id="guac_upload_file" style="opacity: 0; width: 0px;"><!--if--><!--
|
||
<button ms-if="show_vm.code_data" style="margin:10px;width:100px;background-color:#0dc49b"
|
||
ms-on-click="events.code_upload()" class="layui-btn">
|
||
上传文件
|
||
</button>
|
||
--><!--if--><!--if--><div style="padding: 20px 10px 10px; text-align: center; color: rgb(102, 102, 102); font-size: 14px;">实验环境尚未启动,无法上传/下载文件</div><!--if--><!--
|
||
<div ms-if="show_vm.code_data&&show_vm.show" class="trying-to-load">
|
||
<div class="load-container load1">
|
||
<div class="loader">Loading...</div>
|
||
</div>
|
||
</div>
|
||
<div style="padding:20px 10px 10px;text-align: center;color:#666;font-size: 14px;" ms-if="show_vm.title"
|
||
ms-html="show_vm.title"></div>
|
||
<div ms-if="d.type=='code'&&!show_vm.show" style="padding: 5px">
|
||
<table style="margin:0" class="create-materials-table" ms-if="step_code_list.length !=0">
|
||
<tr>
|
||
<th>文件名称</th>
|
||
<th>大小(KB)</th>
|
||
</tr>
|
||
<tr ms-for="(index, val) in @step_code_list">
|
||
<td>
|
||
<a ms-if = "val.filesize >0" style="color: #00a5c1;cursor:pointer" ms-attr="{href:val.href,download:val.filename}"
|
||
ms-html="val.filename"></a>
|
||
<a ms-if = "val.filesize ==0 || val.filesize <0" style="color: #00a5c1;cursor:pointer"
|
||
ms-html="val.filename"></a>
|
||
</td>
|
||
<td ms-html="val.filesize"></td>
|
||
</tr>
|
||
</table>
|
||
</div>
|
||
--></div></div><!--for03356147647--><div id="material" style="display: none;"><!--if--><!--if--><!--if--><!-- 随材管理 --><div style="display: none;"><div style="padding: 20px 10px 10px; text-align: center; color: rgb(102, 102, 102); font-size: 14px;">暂无实验随材</div><div style="padding: 5px;"><!--if--></div></div><!-- 视频管理 --><div style="display: none;"><!--if--></div><!-- 代码管理 --><div style="display: none; padding: 10px;"><input type="file" id="guac_upload_file" style="opacity: 0; width: 0px;"><!--if--><!--
|
||
<button ms-if="show_vm.code_data" style="margin:10px;width:100px;background-color:#0dc49b"
|
||
ms-on-click="events.code_upload()" class="layui-btn">
|
||
上传文件
|
||
</button>
|
||
--><!--if--><!--if--><div style="padding: 20px 10px 10px; text-align: center; color: rgb(102, 102, 102); font-size: 14px;">实验环境尚未启动,无法上传/下载文件</div><!--if--><!--
|
||
<div ms-if="show_vm.code_data&&show_vm.show" class="trying-to-load">
|
||
<div class="load-container load1">
|
||
<div class="loader">Loading...</div>
|
||
</div>
|
||
</div>
|
||
<div style="padding:20px 10px 10px;text-align: center;color:#666;font-size: 14px;" ms-if="show_vm.title"
|
||
ms-html="show_vm.title"></div>
|
||
<div ms-if="d.type=='code'&&!show_vm.show" style="padding: 5px">
|
||
<table style="margin:0" class="create-materials-table" ms-if="step_code_list.length !=0">
|
||
<tr>
|
||
<th>文件名称</th>
|
||
<th>大小(KB)</th>
|
||
</tr>
|
||
<tr ms-for="(index, val) in @step_code_list">
|
||
<td>
|
||
<a ms-if = "val.filesize >0" style="color: #00a5c1;cursor:pointer" ms-attr="{href:val.href,download:val.filename}"
|
||
ms-html="val.filename"></a>
|
||
<a ms-if = "val.filesize ==0 || val.filesize <0" style="color: #00a5c1;cursor:pointer"
|
||
ms-html="val.filename"></a>
|
||
</td>
|
||
<td ms-html="val.filesize"></td>
|
||
</tr>
|
||
</table>
|
||
</div>
|
||
--></div></div><!--for03356147647--><div id="video" style="display: none;"><!--if--><!--if--><!--if--><!-- 随材管理 --><div style="display: none;"><div style="padding: 20px 10px 10px; text-align: center; color: rgb(102, 102, 102); font-size: 14px;">暂无实验随材</div><!--if--></div><!-- 视频管理 --><div style="display: none;"><div class="video_manage" style="padding: 5px;"><!--if--><div class="notdata"><img src="./水滴实验营_files/notdata.png">
|
||
暂无视频
|
||
</div></div></div><!-- 代码管理 --><div style="display: none; padding: 10px;"><input type="file" id="guac_upload_file" style="opacity: 0; width: 0px;"><!--if--><!--
|
||
<button ms-if="show_vm.code_data" style="margin:10px;width:100px;background-color:#0dc49b"
|
||
ms-on-click="events.code_upload()" class="layui-btn">
|
||
上传文件
|
||
</button>
|
||
--><!--if--><!--if--><div style="padding: 20px 10px 10px; text-align: center; color: rgb(102, 102, 102); font-size: 14px;">实验环境尚未启动,无法上传/下载文件</div><!--if--><!--
|
||
<div ms-if="show_vm.code_data&&show_vm.show" class="trying-to-load">
|
||
<div class="load-container load1">
|
||
<div class="loader">Loading...</div>
|
||
</div>
|
||
</div>
|
||
<div style="padding:20px 10px 10px;text-align: center;color:#666;font-size: 14px;" ms-if="show_vm.title"
|
||
ms-html="show_vm.title"></div>
|
||
<div ms-if="d.type=='code'&&!show_vm.show" style="padding: 5px">
|
||
<table style="margin:0" class="create-materials-table" ms-if="step_code_list.length !=0">
|
||
<tr>
|
||
<th>文件名称</th>
|
||
<th>大小(KB)</th>
|
||
</tr>
|
||
<tr ms-for="(index, val) in @step_code_list">
|
||
<td>
|
||
<a ms-if = "val.filesize >0" style="color: #00a5c1;cursor:pointer" ms-attr="{href:val.href,download:val.filename}"
|
||
ms-html="val.filename"></a>
|
||
<a ms-if = "val.filesize ==0 || val.filesize <0" style="color: #00a5c1;cursor:pointer"
|
||
ms-html="val.filename"></a>
|
||
</td>
|
||
<td ms-html="val.filesize"></td>
|
||
</tr>
|
||
</table>
|
||
</div>
|
||
--></div></div><!--for03356147647--><div id="file" style="display: none;"><!--if--><!--if--><!--if--><!-- 随材管理 --><div style="display: none;"><div style="padding: 20px 10px 10px; text-align: center; color: rgb(102, 102, 102); font-size: 14px;">暂无实验随材</div><!--if--></div><!-- 视频管理 --><div style="display: none;"><!--if--></div><!-- 代码管理 --><div style="display: none; padding: 10px;"><input type="file" id="guac_upload_file" style="opacity: 0; width: 0px;"><!--if--><!--
|
||
<button ms-if="show_vm.code_data" style="margin:10px;width:100px;background-color:#0dc49b"
|
||
ms-on-click="events.code_upload()" class="layui-btn">
|
||
上传文件
|
||
</button>
|
||
--><!--if--><!--if--><div style="padding: 20px 10px 10px; text-align: center; color: rgb(102, 102, 102); font-size: 14px;">实验环境尚未启动,无法上传/下载文件</div><!--if--><!--
|
||
<div ms-if="show_vm.code_data&&show_vm.show" class="trying-to-load">
|
||
<div class="load-container load1">
|
||
<div class="loader">Loading...</div>
|
||
</div>
|
||
</div>
|
||
<div style="padding:20px 10px 10px;text-align: center;color:#666;font-size: 14px;" ms-if="show_vm.title"
|
||
ms-html="show_vm.title"></div>
|
||
<div ms-if="d.type=='code'&&!show_vm.show" style="padding: 5px">
|
||
<table style="margin:0" class="create-materials-table" ms-if="step_code_list.length !=0">
|
||
<tr>
|
||
<th>文件名称</th>
|
||
<th>大小(KB)</th>
|
||
</tr>
|
||
<tr ms-for="(index, val) in @step_code_list">
|
||
<td>
|
||
<a ms-if = "val.filesize >0" style="color: #00a5c1;cursor:pointer" ms-attr="{href:val.href,download:val.filename}"
|
||
ms-html="val.filename"></a>
|
||
<a ms-if = "val.filesize ==0 || val.filesize <0" style="color: #00a5c1;cursor:pointer"
|
||
ms-html="val.filename"></a>
|
||
</td>
|
||
<td ms-html="val.filesize"></td>
|
||
</tr>
|
||
</table>
|
||
</div>
|
||
--></div></div><!--for03356147647--><!--ms-for-end:--></div></div></div><div class="expDesktop_con"><div id="vm_progress" class="hide" change-class="hide"><canvas style="width:90px;height:90px;" id="c"></canvas><input class="hide" type="range" id="r" min="0" max="100" value="0"></div><div class="console_bg_img" change-class=""><img src="./水滴实验营_files/console_bg_img.png"></div><div class="expDesktop_con_wrap clearfix"><div class="fl videoCon" id="displayvm" style="position: relative; height: 956px;"><div id="display" class="ui-layout-center" style="width: 100%;"></div><div class="container"><!-- HTML5 Canvas --><div id="noVNC_screen" style="display:none"><canvas id="noVNC_canvas" class="mid-canvas" width="0" height="0">
|
||
Canvas not supported.
|
||
</canvas></div><div class="window-overlay" style="display:none"><div class="modal"><div class="modal-header"><h4>
|
||
复制粘贴命令
|
||
<a class="close"><span class="icon-close icon-Large"></span></a></h4></div><div class="modal-content"><form class="form form-horizontal"><fieldset><div class="modal-content alert alert-danger"><h6>将内容粘贴至文本框中,最大支持2000个字符,暂不支持中文等非标准键盘值特殊字符。</h6></div><div class="item"><div class="control-label">文本内容</div><div class="controls"><textarea class="copyText" rows="10" cols="50" maxlength="2000" name="copyText"></textarea></div></div><div class="form-actions"><input class="btn btn-primary" type="submit" value="提交"><input class="btn btn-cancel" type="button" value="取消"></div></fieldset></form></div></div></div></div><div id="displaynotebook" class="ui-layout-center"><iframe class="interact-notebook" id="interact-notebook" src="./水滴实验营_files/saved_resource.html"></iframe></div></div></div></div></div><div id="toolbar-options" class="hidden"><a title="复制" href="https://edu.shuidilab.cn/console/expdesktop/5665#" name="textcopy"><img style="width: 22px;height: 20px;margin-top: 0px;margin-left: -1px;" src="./水滴实验营_files/copytext.png" alt="复制"></a></div><button id="btntoolbar" style="position: absolute; left: -99999px; top: 0px;"></button></div><script src="./水滴实验营_files/require.min.js.下载"></script><script async="" src="./水滴实验营_files/main-console.min.98fbde9e.js.下载"></script><div class="tool-container tool-bottom toolbar-primary" style="opacity: 0; display: none; top: -120px;"><div class="tool-items"><a title="复制" href="https://edu.shuidilab.cn/console/expdesktop/5665#" name="textcopy" class="tool-item"><img style="width: 22px;height: 20px;margin-top: 0px;margin-left: -1px;" src="./水滴实验营_files/copytext.png" alt="复制"></a></div><div class="arrow"></div></div><div class="layui-layer-move"></div><div class="qc-modal fadeOut" style=""></div></body></html> |