利用Python爬取高德地图全国地铁站点信息

利用Python中的requests库进行地铁站点信息的获取,同时将数据保存在本机excel中

# 首先引入所需要的包 import requests from bs4 import BeautifulSoup import pandas as pd import json # 发送 GET 请求获取网页内容 url = 'http://map.amap.com/subway/index.html' response = requests.get(url) # 第一步：爬取两个 div 中的城市数据（包括 ID 和拼音），生成城市集合 if response.status_code == 200: # 解码 response_content = response.content.decode('utf-8') # 使用 Beautiful Soup 解析网页内容 soup = BeautifulSoup(response_content, 'html.parser') # 从这里开始，你可以使用 Beautiful Soup 对象（soup）来提取所需的信息 # 例如，查找标题 title = soup.title # 通过Beautiful Soup来找到城市信息元素，并提取这个元素的信息 for soup_a in soup.find('div', class_='city-list fl').find_all('a'): city_name_py = soup_a['cityname'] city_id = soup_a['id'] city_name_ch = soup_a.get_text() city_info_list.append({'name_py': city_name_py, 'id': city_id, 'name_ch': city_name_ch}) # 获取未显示出来的城市列表 for soup_a in soup.find('div', class_='more-city-list').find_all('a'): city_name_py = soup_a['cityname'] city_id = soup_a['id'] city_name_ch = soup_a.get_text() city_info_list.append({'name_py': city_name_py, 'id': city_id, 'name_ch': city_name_ch}) print(city_info_list) else: print("无法获取网页内容") for city_info in city_info_list: city_id = city_info.get("id") city_name = city_info.get("name_py") city_name_ch = city_info.get("name_ch") print("开始爬取城市" + city_name_ch + "的数据") city_lines_list = [] # 第二步：遍历城市集合，构造每一个城市的 url,并下载数据 # 构造每个城市的url url = "http://map.amap.com/service/subway?_1717380520536&srhdata=" + city_id + '_drw_' + city_name + '.json' res = requests.get(url) content = res.content.decode('utf-8') # 将内容字符串转换成json对象 content_json = json.loads(content) # 提取该城市的所有地铁线list line_info_list = content_json.get("l") # 第三步：开始处理每一个地铁线，提取内容到dataframe中 for line_info in line_info_list: # 地铁线名字 line_name = line_info["kn"] # 处理地铁线站点 df_per_zd = pd.DataFrame(line_info["st"]) df_per_zd = df_per_zd[['n', 'sl', 'poiid', 'sp', 't', 'su', 'sid']] df_per_zd['gd经度'] = df_per_zd['sl'].apply(lambda x: x.split(',')[0]) df_per_zd['gd纬度'] = df_per_zd['sl'].apply(lambda x: x.split(',')[1]) df_per_zd.drop('sl', axis=1, inplace=True) df_per_zd['路线名称'] = line_info['ln'] df_per_zd['城市名称'] = city_name_ch df_per_zd.rename(columns={"n": "站点名称", "poiid": "POI编号", "sp": "拼音名称", "t": "换乘标志 1:换乘，0:不可换乘", "su": "su", "sid": "sid编号"}, inplace=True) # 先将这条地铁线处理过的dataframe存起来，我们后面给他放到一张表里 city_lines_list.append(df_per_zd) # 这段代码就是将地铁线数据列表聚合到一张表里，形成每个城市的地铁站数据 city_subway_data = pd.concat(city_lines_list, ignore_index=True) # 第四步：将处理好的文件保存为xlsx city_subway_data.to_excel(city_name_ch + '.xlsx', sheet_name='Sheet1')