落网是个非常不错的独立音乐介绍网站，但是上面提供的歌曲仅供试听，没有提供下载链接。相信不少同学都想过有没工具把歌曲下载下来呢？答案是：有。 我最早是使用遨游浏览器，它里面提供一个插件（资源探测器），只要点开，就能看到当前浏览器页面中有没有可下载的链接。当时觉得还挺方便的，就是 要一首一首地去下，还是有些麻烦。当然火狐浏览器里也有个插件（NetVideoHunter），功能上也是类似。 对于学习过python的同学，都知道它用来爬取网络资源上非常方便、强大。前段时间本人写过使用使用python抓取落网期刊图片，但觉得还是不够实用， 要是能够把任意一个（或者所有）期刊资源都下载到本地，那该多好！！于是，自己花了段时间去观察落网页面的HTML源码，发现期刊的曲目链接其实是隐藏 在audio标签内的（比如第一期第一首曲目的链接：http://mp3-cdn.luoo.net/low/luoo/radio1/1.mp3）。
# -*- coding:gbk -*- ''' Created on 2017年2月26日 # # 爬取落网期刊信息（包括标题，介绍，音频文件等） # @author: ysx ''' from __future__ import unicode_literals from Queue import Queue import os import re import sys import threading import uuid import bs4 from bs4.element import NavigableString import requests reload(sys) sys.setdefaultencoding('gbk') # 下载路径 BASE_DIR = r'F:\luonet' # 下载资源超时时间 TIMEOUT = 10 # 当前最大期刊号 MAX_VOL_NO = 906 # 下载期刊使用最大线程数 MAX_THREAD_NUM = 10 def parse_vol_num(vol_url): '''给定期刊Url,判定是第几期 ''' try: return vol_url.split('/')[-1] except: raise RuntimeError('给定期刊Url[%s]不合法..') def check_path_avail(path): '''实际运行过程中，按曲目名去命名mp3文件，会出现写入报IOError（原因是歌曲名中包含?等非法字符）''' check_result = True if not os.path.exists(path): try: open(path, 'wb') except IOError: print '当前路径:[%s]无法写入..'%path check_result = False return check_result def download_source(url, path): print('下载资源:[%s]'%url) resp = requests.get(url, timeout=TIMEOUT) resp.raise_for_status() # 下载出现问题，立即抛出异常 with open(path, 'wb') as audio_handle: for chunk in resp.iter_content(10000): audio_handle.write(chunk) class LuoVolInfo(object): '''落网期刊类''' def __init__(self, vol_url, vol_title, vol_desc, vol_image_url, basedir=None): self.vol_num = parse_vol_num(vol_url) self.vol_title = vol_title self.vol_desc = vol_desc self.vol_image_url = vol_image_url self.vol_list = [] self._create_basedir(basedir) def append_audio(self, luoAudioInfo): '''追加曲目条目''' self.vol_list.append(luoAudioInfo) def _create_basedir(self, basedir): if basedir is None: basedir = BASE_DIR self._dirname = 'Vol.%s %s'%(self.vol_num, self.vol_title) basedir = os.path.join(basedir, self._dirname) os.makedirs(basedir) self.basedir = basedir def download_vol_image(self): '''下载专题封面''' self.image_path = os.path.join(self.basedir, self._dirname+'.jpg') if not check_path_avail(self.image_path): # 重命名文件 self.image_path = os.path.join(self.basedir, '未命名-期刊封面.jpg') download_source(self.vol_image_url, self.image_path) def download_vol_desc(self): '''下载专题描述''' self.desc_path = os.path.join(self.basedir, '专题说明.txt') with open(self.desc_path, 'w') as file_handle: file_handle.write(self.vol_desc.encode('gbk')) # 写入曲目列表 file_handle.write(os.linesep) file_handle.write('>曲目列表') file_handle.write(os.linesep) if self.vol_list: for vol in self.vol_list: file_handle.write(vol.audio_name) file_handle.write(os.linesep) def download_all(self): print '1.下载专题封面' self.download_vol_image() print '2.下载专题描述' self.download_vol_desc() # 下载曲目过程中，等待时间过长，尝试使用多线程加速下载 print '3.下载专题曲目' # 生成任务队列 audio_dl_queue = Queue() for vol in self.vol_list: audio_dl_queue.put_nowait(vol) # 定义线程目标函数 def target_vol_dl(): while True: audio = audio_dl_queue.get() if audio is None: break audio.download() audio_dl_queue.task_done() # 添加毒丸，并初始化线程池 thread_pool = [] for i in xrange(MAX_THREAD_NUM): audio_dl_queue.put_nowait(None) for i in xrange(MAX_THREAD_NUM): t = threading.Thread(target=target_vol_dl) t.setDaemon(True) t.start() thread_pool.append(t) for t in thread_pool: t.join() del thread_pool[:] class LuoAudioInfo(object): '''落网电台曲目类 ''' # 曲目下载链接模板 audio_src_template = 'http://mp3-cdn.luoo.net/low/luoo/radio%d/d.mp3' # 备用链接，曲目下载链接存在两种格式 audio_src_template2 = 'http://mp3-cdn.luoo.net/low/luoo/radio%d/%d.mp3' def __init__(self, audio_index, audio_name, audio_artist, audio_image_url, vol_url, basedir=None): self.audio_index = audio_index self.audio_name = audio_name self.audio_artist = audio_artist self.audio_image_url = audio_image_url self.vol_url = vol_url self._create_audio_basedir(basedir) self._parse_mp3_url() def _parse_mp3_url(self): '''给定期刊Url,判定是第几期 ''' try: vol_num = parse_vol_num(self.vol_url) except: raise RuntimeError('给定期刊Url[%s]不合法..') self.audio_mp3_url = self.audio_src_template%(int(vol_num.encode('gbk')), int(self.audio_index)) self.audio_mp3_url2 = self.audio_src_template2%(int(vol_num.encode('gbk')), int(self.audio_index)) print '解析后的mp3-Url:[%s]'%self.audio_mp3_url def _create_audio_basedir(self, basedir): '''创建曲目下载目录 ''' if basedir is None: basedir = BASE_DIR _dirname = self.audio_index basedir = os.path.join(basedir, _dirname) os.makedirs(basedir) self.basedir = basedir def download(self): '''下载当前曲目的所有资源''' self._download_image_or_audio() self._download_image_or_audio(is_mp3=False) def _download_image_or_audio(self, is_mp3=True): '''下载图片或mp3''' if is_mp3: write_path = os.path.join(self.basedir, self.audio_name) + '.mp3' request_url = self.audio_mp3_url else: write_path = os.path.join(self.basedir, self.audio_name) + '.jpg' request_url = self.audio_image_url # 检查待写入路径是否合法 if not check_path_avail(write_path): # 重命名文件 new_name = uuid.uuid1() write_path = os.path.join(self.basedir, '%s.%s'%(new_name, 'mp3' if is_mp3 else 'jpg')) try: download_source(request_url, write_path) except: if request_url==self.audio_mp3_url: print '下载超时:%s'%(request_url) print '尝试使用链接2：[%s]'%(self.audio_mp3_url2) download_source(self.audio_mp3_url2, write_path) def __repr__(self): return '<落网期刊曲目> 序号:[%s], 曲目名:[%s], 曲目艺术家:[%s], 曲目封面Url:[%s], 曲目下载链接:[%s]'%( self.audio_index, self.audio_name, self.audio_artist, self.audio_image_url, self.audio_mp3_url) def download_vol_content(vol_url): '''下载包括（1.期刊标题、图片；2.期刊介绍；3.期刊音频） @param vol_url: 任意一个期刊Url，如果不存在（页面响应404错误），返回False @param if_download_audio: 是否下载期刊下的音频（默认False） @return: Boolean True 成功 ''' print('当前下载期刊链接：[%s]'%vol_url) resp = requests.get(vol_url) resp.raise_for_status() # 下载出现问题，立即抛出异常 resp.encoding = 'utf-8' soup = bs4.BeautifulSoup(resp.text, 'lxml') # 先判定是否返回404错误页面 error_msg_ele = soup.select('div.error-msg > h1') if error_msg_ele and len(error_msg_ele): print '页面返回404，也许该期刊不存在..' return False print print '1.解析【期刊标题】' vol_title_ele = soup.select('h1.vol-name > span.vol-title') if vol_title_ele and len(vol_title_ele): vol_title = vol_title_ele[0].string print '本期标题：[%s]'%vol_title print print '2.解析【期刊说明】' vol_desc_ele = soup.select('div.vol-desc') if vol_desc_ele and len(vol_desc_ele): vol_desc = ''.join(list(vol_desc_ele[0].strings)) print '本期说明：[%s]'%vol_desc print print '3.解析【期刊图片】' vol_image_ele = soup.select('div.vol-cover-wrapper > img.vol-cover') if vol_image_ele and len(vol_image_ele): vol_image_url = vol_image_ele[0]['src'] print '图片链接：[%s]'%vol_image_url luo_vol_obj = LuoVolInfo(vol_url, vol_title, vol_desc, vol_image_url, ) print '4.解析【曲目信息】' vol_audio_ul_ele = soup.select('div.vol-tracklist > ul') if vol_audio_ul_ele and len(vol_audio_ul_ele): print vol_audio_ul_ele[0].children for audio_li_ele in vol_audio_ul_ele[0].children: if type(audio_li_ele) is NavigableString: # 跳过字符串节点 continue audio_li_soup = bs4.BeautifulSoup(audio_li_ele.prettify(), 'lxml') audio_name = audio_li_soup.select('div.track-wrapper > a.trackname')[0].string.strip() print '曲目名称：[%s]'%audio_name # 获取当前曲目序号 pattern = re.compile(r'(\d+).*') audio_index = pattern.match(audio_name).groups()[0] audio_artist = audio_li_soup.select('div.track-wrapper > span.artist')[0].string.strip() print '曲目作家：[%s]'%audio_artist audio_image_url = audio_li_soup.select('div.track-wrapper > a.btn-action-share.icon-share')[0]['data-img'].strip() print '曲目封面Url：[%s]'%audio_image_url audio_obj = LuoAudioInfo(audio_index, audio_name, audio_artist, audio_image_url, vol_url, basedir=luo_vol_obj.basedir) print repr(audio_obj) print # 追加至期刊曲目列表 luo_vol_obj.append_audio(audio_obj) luo_vol_obj.download_all() return True def main(): def print_useage(): print ''' >使用说明： 1.下载所有期刊： luonet_download_tool.py all 2.下载特定期刊（比如下载最新的906）： luonet_download_tool.py 906 ''' sys.exit() if len(sys.argv) == 1: print_useage() vol_url_template = 'http://www.luoo.net/music/%s' if len(sys.argv) == 2: dl_flag = sys.argv[1] if dl_flag == 'all': print '======================' print '>开始下载所有期刊:[%s-%s]'%(1, MAX_VOL_NO) print '======================' for i in xrange(1, MAX_VOL_NO): vol_url = vol_url_template%i download_vol_content(vol_url) elif dl_flag and dl_flag.isdigit(): print '======================' print '>下载指定期刊号:[%s]'%int(dl_flag) print '======================' obj_vol = int(dl_flag) download_vol_content(vol_url_template%obj_vol) else: print_useage() if __name__ == '__main__': main()