判断文件或者文件夹是否存在
if(os.path.exists(rootdir) == False)
创建文件夹
os.mkdir(rootdir)
调用系统命令
os.system(cmd)
字典循环
for key,value in dict.items()
打开文件并读取内容进行处理
fd = open('xxxx.txt', encoding='utf-8')
for line in fd:
print line
fd.close()
创建文件并写入内容
fd = open('xxxx.txt', 'a+', encoding='utf-8')
fd.write('aaaaa' + '\n')
fd.close()
使用xlrd读取EXCEL
导入
import xlrd
打开excel
data = xlrd.open_workbook('demo.xls') #注意这里的workbook首字母是小写
查看文件中包含sheet的名称
data.sheet_names()
得到第一个工作表，或者通过索引顺序 或 工作表名称
table = data.sheets()[0]
table = data.sheet_by_index(0)
table = data.sheet_by_name(u'Sheet1')
获取行数和列数
nrows = table.nrows
ncols = table.ncols
获取整行和整列的值（数组）
table.row_values(i)
table.col_values(i)
循环行,得到索引的列表
for rownum in range(table.nrows):
print table.row_values(rownum)
单元格
cell_A1 = table.cell(0,0).value
cell_C4 = table.cell(2,3).value
分别使用行列索引
cell_A1 = table.row(0)[0].value
cell_A2 = table.col(1)[0].value
简单的写入
row = 0
col = 0
ctype = 1 # 类型 0 empty,1 string, 2 number, 3 date, 4 boolean, 5 error
value = 'lixiaoluo'
xf = 0 # 扩展的格式化 (默认是0)
table.put_cell(row, col, ctype, value, xf)
table.cell(0,0) # 文本:u'lixiaoluo'
table.cell(0,0).value # 'lixiaoluo'
使用xlwt写入EXCEL
导入xlwt
import xlwt
新建一个excel文件
file = xlwt.Workbook() #注意这里的Workbook首字母是大写，无语吧
新建一个sheet
table = file.add_sheet('sheet name')
写入数据table.write(行,列,value)
table.write(0,0,'test')
如果对一个单元格重复操作，会引发
returns error:
# Exception: Attempt to overwrite cell:
# sheetname=u'sheet 1' rowx=0 colx=0
所以在打开时加cell_overwrite_ok=True解决
table = file.add_sheet('sheet name',cell_overwrite_ok=True)
保存文件
file.save('demo.xls')
另外，使用style
style = xlwt.XFStyle() #初始化样式
font = xlwt.Font() #为样式创建字体
font.name = 'Times New Roman'
font.bold = True
style.font = font #为样式设置字体
table.write(0, 0, 'some bold Times text', style) # 使用样式
命令行getopt
try:
options,args = getopt.getopt(sys.argv[1:],"hp:i:",["help","ip=","port="])
except getopt.GetoptError:
sys.exit()
for name,value in options:
if name in ("-h","--help"):
usage()
if name in ("-i","--ip"):
print(value)
if name in ("-p","--port"):
print(value)
简单爬虫
import requests
AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36'
HEADERS = {
'User-Agent': AGENT,
'Content-Type':'application/x-www-form-urlencoded; charset=UTF-8',
'X-Requested-With':'XMLHttpRequest',
'Accept':'*/*'
session = requests.session()
#模拟登录
postdata = {
'defaults':'xxx',
'fromLogin':'xxx',
'userName':'xxx',
'password':'xxxx'
}
url = 'xxxxxxxx'
login_info = session.post(url, headers = HEADERS, data = postdata,verify = False)
if(login_info.status_code == requests.codes.ok):
print('login success')
return True
else:
print('login err')
return False
}
#下载html页面
def downloadUrl(rootdir, url, orgid, page):
html = session.get(url, headers=global_config.HEADERS, verify=False)
if(html.text[1:7] == 'script'):
print(html.text)
return "err"
if(len(html.text) < 60):
return "err"
sample = open(rootdir + "/" + str(orgid) + '_' + str(page) + ".html", "w", encoding='utf-8')
sample.write(html.text)
sample.close()
return 'ok'
解析JOSN文件内容
def scrapy_by_file(json_file_name):
#读取JSON文件的内容
text = open(json_file_name, encoding='utf-8').read()
#特殊处理,去除从WINDOWS系统带过来的BOM特殊字符
if text.startswith(u'\ufeff'):
text = text.encode('utf8')[3:].decode('utf8')
#将文本内容的JSON数据转换成自定义的JSON对象
try:
json_data = json.loads(text)
except:
print(json_file_name)
return
for row in json_data['rows']:
def scrapy_by_row(row):
try:
orgid = row['organization']['id']
familyid = row['censusRegisterFamily']['id']
except:
print('errrr')
return
scrapy_by_row(row)
遍历文件夹
#遍历目录(rootdir) 遍历到的每个文件都执行dirFunc
def waklThroughDir(rootdir, dirFunc):
for parent, dirnames, filenames in os.walk(rootdir):
for filename in filenames:
print(filename)
#获取后缀为txt的文件
if(filename.split('.')[-1] == 'html'):
dirFunc(os.path.join(parent, filename))
采集温州房产网基本信息
# -*- coding: utf-8 -*-
import re
import requests
import time
#-----------------------------用于解析的正则表达式常量------------------------------------------------------------------
#解析页数
PAGE_NUM = '共找到 (.*?) 符合条件的记录'
#解析小区名称
NAME = 'texttext_title"><ahref(.*?)</a></div><divclass="texttext_moreinfo">'
#解析小区价格
PRICE = 'class="hot_price">(.*?)</span>'
#解析小区地址
ADDRESS = 'text_moreinfo">(.*?)</div><divclass="texttext_moreinfo"><span>'
#文件生成路径
ROOTDIR = 'F:\\test\\'
#-----------------------------模拟请求的头部信息，否则将被识别出是程序抓包而被拦截--------------------------------------
HEADERS = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate, sdch',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36',
'Host': 'www.0577home.net',
'Upgrade-Insecure-Requests': '1'
}
#-----------------------------抓取某一页的房产信息，pageNo为页号--------------------------------------------------------
def getHouseListByPageno(pageNo):
#建立一个连接用于后续发起请求
session = requests.session()
url = 'http://www.0577home.net/xiaoqu/list_0_0_0_0_0_0_0_' + str(pageNo) + '.html'
houseList = session.get(url, headers = HEADERS, verify = False)
#以写入模式打开文件
fh = open(ROOTDIR + "houseList_pageNo" + str(pageNo) + ".txt", 'w' ,encoding='utf-8')
#将movieList写入文件
fh.write(houseList.text)
#关闭文件
fh.close()
#-------------------------------获取需要抓取的页面总数------------------------------------------------------------------
def getPageNum():
#打开已经下载好的第一页房产内容
f = open(ROOTDIR + 'houseList_pageNo1.txt', encoding='utf-8')
#获取文件内容
rawContent = f.read()
#用正则表达式解析页面内容
pageNum = re.findall(PAGE_NUM, rawContent)
#返回页面号
return int(pageNum[0]) / 20 + 1
def parseHouseListToFile(srcFile, dstFile):
#打开待解析的文件
f = open(srcFile, encoding='utf-8')
#读取文件内容以备解析
rawContent = f.read()
p = re.compile('\s+')
content = re.sub(p, '', rawContent)
dnames = re.findall(NAME, content)
names = []
for dname in dnames:
idx = dname.rfind('>')
names.append(dname[idx + 1:])
prices = re.findall(PRICE, content)
daddress = re.findall(ADDRESS, content)
address = []
for daddr in daddress:
id = daddr.rfind('>')
address.append(daddr[id + 1:])
i = 0
for x in names:
#写入时用'$'做分割，结尾加上回车符
dstFile.write(names[i] + '$' + prices[i] + '$' + address[i] + '\n')
i = i + 1
#-------------------------------主函数，下载并解析房产信息--------------------------------------------------------------
if __name__ == '__main__':
#---------------------抓取页面-----------------------------
#抓取第一页房产信息
getHouseListByPageno(1)
#通过第一页房产信息获取总共要抓取的页面数量
pageNum = getPageNum()
#抓取剩余的页面
for i in range(2, int(pageNum) + 1):
getHouseListByPageno(str(i))
#---------------------解析页面-----------------------------
#获取当前年月日
localtime = time.strftime('%Y%m%d', time.localtime(time.time()))
#创建一个文件，文件名前面带上年月日
f = open(ROOTDIR + localtime + '_houseList.txt', 'a+', encoding='utf-8')
#解析所有的页面
#for k in range(1, int(pageNum) + 1):
for k in range(1, 115):
parseHouseListToFile(ROOTDIR + "houseList_pageNo" + str(k) + ".txt", f)
#关闭文件
f.close()
采集温州房产网详细信息
# -*- coding: utf-8 -*-
import re
import requests
import time
import os
#-----------------------------用于解析的正则表达式常量------------------------------------------------------------------
#解析页数
PAGE_NUM = '共找到 (.*?) 符合条件的记录'
#解析小区名称
NAME = 'texttext_title"><ahref(.*?)</a></div><divclass="texttext_moreinfo">'
#解析小区价格
PRICE = 'class="hot_price">(.*?)</span>'
#解析小区地址
ADDRESS = 'text_moreinfo">(.*?)</div><divclass="texttext_moreinfo"><span>'
#解析小区编号
ID = 'class="picdiv_left"><ahref="http://www.0577home.net/xiaoqu/(.*?).html'
#解析小区所属区域
LOCATION = '<div><a>所属区域：</a><span>(.*?)</span></div>'
#解析小区占地面积
AREA = '<div><a>占地面积：</a><span>(.*?)</span></div>'
#解析小区绿化率
GREENINGRATE = '<div><a>绿化率：</a><span>(.*?)</span></div>'
#解析小区楼总数
LAYER = '<div><a>楼总数：</a><span>(.*?)</span></div>'
#解析小区物业类型
TYPE = '<div><a>物业类型：</a><span>(.*?)</span></div>'
#解析小区所属小学
PRIMARYSCHOOL = '<div><a>所属小学：</a><span>(.*?)</span></div>'
#解析小区总建筑面积
BUILDINGAREA = '<div><a>总建筑面积：</a><span>(.*?)</span></div>'
#解析小区容积率
PLOTRATIO = '<div><a>容积率：</a><span>(.*?)</span></div>'
#解析小区开发商
DEVEPLOPER = '<div><a>开发商：</a><span>(.*?)</span></div>'
#文件生成路径
ROOTDIR = 'F:\\test\\'
#-----------------------------模拟请求的头部信息，否则将被识别出是程序抓包而被拦截--------------------------------------
HEADERS = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate, sdch',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36',
'Host': 'www.0577home.net',
'Upgrade-Insecure-Requests': '1'
}
#-----------------------------抓取某一页的房产信息，pageNo为页号--------------------------------------------------------
def getHouseListByPageno(pageNo):
#建立一个连接用于后续发起请求
session = requests.session()
url = 'http://www.0577home.net/xiaoqu/list_0_0_0_0_0_0_0_' + str(pageNo) + '.html'
houseList = session.get(url, headers = HEADERS, verify = False)
#以写入模式打开文件
fh = open(ROOTDIR + "houseList_pageNo" + str(pageNo) + ".txt", 'w' ,encoding='utf-8')
#将movieList写入文件
fh.write(houseList.text)
#关闭文件
fh.close()
def getHouseInfoByPageno(pageNo, k):
if(os.path.exists(ROOTDIR + "houseInfo_pageNo" + str(pageNo) + ".html")):
return
print('downloading !, count %s, page %s' % (str(k), str(pageNo)))
#建立一个连接用于后续发起请求
session = requests.session()
url = 'http://www.0577home.net/xiaoqu/detail_' + str(pageNo) + '.html'
houseList = session.get(url, headers = HEADERS, verify = False)
#以写入模式打开文件
fh = open(ROOTDIR + "houseInfo_pageNo" + str(pageNo) + ".html", 'w' ,encoding='utf-8')
#将movieList写入文件
fh.write(houseList.text)
#关闭文件
fh.close()
#-------------------------------获取需要抓取的页面总数------------------------------------------------------------------
def getPageNum():
#打开已经下载好的第一页房产内容
f = open(ROOTDIR + 'houseList_pageNo1.txt', encoding='utf-8')
#获取文件内容
rawContent = f.read()
#用正则表达式解析页面内容
pageNum = re.findall(PAGE_NUM, rawContent)
#返回页面号
return int(pageNum[0]) / 20 + 1
def parseHouseInfo(srcFile):
#打开待解析的文件
f = open(srcFile, encoding='utf-8')
#读取文件内容以备解析
content = f.read()
# p = re.compile('\s+')
# content = re.sub(p, '', rawContent)
location = re.findall(LOCATION, content)[0]
location = location.split(' ')
category1 = location[0]
category2 = location[1]
area = re.findall(AREA, content)[0]
greeningrate = re.findall(GREENINGRATE, content)[0]
layer = re.findall(LAYER, content)[0]
type = re.findall(TYPE, content)[0]
primaryschool = re.findall(PRIMARYSCHOOL, content)[0]
buildingarea = re.findall(BUILDINGAREA, content)[0]
plotratio = re.findall(PLOTRATIO, content)[0]
developer = re.findall(DEVEPLOPER, content)[0]
f.close()
return (category1, category2, area, greeningrate, layer, type, primaryschool, buildingarea, plotratio, developer)
def parseHouseListToFile(srcFile, dstFile):
#打开待解析的文件
f = open(srcFile, encoding='utf-8')
#读取文件内容以备解析
rawContent = f.read()
p = re.compile('\s+')
content = re.sub(p, '', rawContent)
dnames = re.findall(NAME, content)
names = []
for dname in dnames:
idx = dname.rfind('>')
names.append(dname[idx + 1:])
prices = re.findall(PRICE, content)
daddress = re.findall(ADDRESS, content)
ids = re.findall(ID, content)
address = []
for daddr in daddress:
id = daddr.rfind('>')
address.append(daddr[id + 1:])
i = 0
f.close()
for x in names:
#写入时用'$'做分割，结尾加上回车符
dstFile.write(names[i] + '$' + prices[i] + '$' + address[i] + '$' + ids[i] + '\n')
i = i + 1
#-------------------------------主函数，下载并解析房产信息--------------------------------------------------------------
if __name__ == '__main__':
#---------------------抓取页面-----------------------------
#抓取第一页房产信息
# getHouseListByPageno(1)
# #通过第一页房产信息获取总共要抓取的页面数量
# pageNum = getPageNum()
# #抓取剩余的页面
# for i in range(2, int(pageNum) + 1):
# getHouseListByPageno(str(i))
#---------------------解析页面-----------------------------
#获取当前年月日
localtime = time.strftime('%Y%m%d', time.localtime(time.time()))
#创建一个文件，文件名前面带上年月日
f = open(ROOTDIR + localtime + '_houseList.txt', 'a+', encoding='utf-8')
#解析所有的页面
#for k in range(1, int(pageNum) + 1):
for k in range(1, 115):
parseHouseListToFile(ROOTDIR + "houseList_pageNo" + str(k) + ".txt", f)
#关闭文件
f.close()
f = open(ROOTDIR + localtime + '_houseList.txt', encoding='utf-8')
fd = open(ROOTDIR + localtime + '_houseInfo.txt', 'w', encoding='utf-8')
k = 0
for line in f:
data = line.strip('\n')
data = data.split('$')
idx = data[3]
getHouseInfoByPageno(idx, k)
houseInfo = parseHouseInfo(ROOTDIR + "houseInfo_pageNo" + str(idx) + ".html")
print(str(k) + "$".join(data) + '$' + "$".join(houseInfo))
fd.write("$".join(data) + '$' + "$".join(houseInfo) + '\n')
k += 1
f.close()
fd.close()
读取csv文件
with open('job.csv', 'r') as f:
reader = csv.reader(f)
for row in reader:
print(row)
写入csv文件
#创建CSV文件并写入第一行
def createCsv(file):
if not os.path.exists(file):
csvfile = open(file, 'a+', encoding='utf-8', newline='')
writer = csv.writer(csvfile)
writer.writerow(paramname)
else:
csvfile = open(file, 'a+', newline='')
writer = csv.writer(csvfile)
return writer
python调用JAVA
import sys
import jpype
name = sys.argv[1]
jarpath = '/home/dsadm/why/python'
jpype.startJVM(jpype.getDefaultJVMPath(), "-Djava.ext.dirs=%s" % jarpath)
DECRYPT = jpype.JClass('why.fmrt.decrypt.DECRYPT')
upperName =DECRYPT.decrypt(name)
print(upperName)
jpype.shutdownJVM()
简单验证码破解
from urllib.request import urlretrieve
from urllib.request import urlopen
from bs4 import BeautifulSoup
import subprocess
import requests
from PIL import Image
from PIL import ImageOps
def cleanImage(imagePath):
image = Image.open(imagePath)
image = image.point(lambda x: 0 if x<143 else 255)
borderImage = ImageOps.expand(image,border=20,fill='white')
borderImage.save(imagePath)
html = urlopen("http://www.pythonscraping.com/humans-only")
bsObj = BeautifulSoup(html, "html.parser")
#Gather prepopulated form values
imageLocation = bsObj.find("img", {"title": "Image CAPTCHA"})["src"]
formBuildId = bsObj.find("input", {"name":"form_build_id"})["value"]
captchaSid = bsObj.find("input", {"name":"captcha_sid"})["value"]
captchaToken = bsObj.find("input", {"name":"captcha_token"})["value"]
captchaUrl = "http://pythonscraping.com"+imageLocation
urlretrieve(captchaUrl, "captcha.jpg")
cleanImage("captcha.jpg")
p = subprocess.Popen(["tesseract", "captcha.jpg", "captcha"], stdout=
subprocess.PIPE,stderr=subprocess.PIPE)
p.wait()
f = open("captcha.txt", "r")
#Clean any whitespace characters
captchaResponse = f.read().replace(" ", "").replace("\n", "")
print("Captcha solution attempt: "+captchaResponse)
if len(captchaResponse) == 5:
params = {"captcha_token":captchaToken, "captcha_sid":captchaSid,
"form_id":"comment_node_page_form", "form_build_id": formBuildId,
"captcha_response":captchaResponse, "name":"Ryan Mitchell",
"subject": "I come to seek the Grail",
"comment_body[und][0][value]":
"...and I am definitely not a bot"}
r = requests.post("http://www.pythonscraping.com/comment/reply/10",
data=params)
responseObj = BeautifulSoup(r.text)
if responseObj.find("div", {"class":"messages"}) is not None:
print(responseObj.find("div", {"class":"messages"}).get_text())
else:
print("There was a problem reading the CAPTCHA correctly!")
滑块验证码破解
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.action_chains import ActionChains
import PIL.Image as image
import time,re, random
import requests
try:
from StringIO import StringIO
except ImportError:
from io import StringIO
#爬虫模拟的浏览器头部信息
agent = 'Mozilla/5.0 (Windows NT 5.1; rv:33.0) Gecko/20100101 Firefox/33.0'
headers = {
'User-Agent': agent
}
# 根据位置对图片进行合并还原
# filename:图片
# location_list:图片位置
#内部两个图片处理函数的介绍
#crop函数带的参数为(起始点的横坐标，起始点的纵坐标，宽度，高度）
#paste函数的参数为(需要修改的图片，粘贴的起始点的横坐标，粘贴的起始点的纵坐标）
def get_merge_image(filename,location_list):
#打开图片文件
im = image.open(filename)
#创建新的图片,大小为260*116
new_im = image.new('RGB', (260,116))
im_list_upper=[]
im_list_down=[]
# 拷贝图片
for location in location_list:
#上面的图片
if location['y']==-58:
im_list_upper.append(im.crop((abs(location['x']),58,abs(location['x'])+10,166)))
#下面的图片
if location['y']==0:
im_list_down.append(im.crop((abs(location['x']),0,abs(location['x'])+10,58)))
new_im = image.new('RGB', (260,116))
x_offset = 0
#黏贴图片
for im in im_list_upper:
new_im.paste(im, (x_offset,0))
x_offset += im.size[0]
x_offset = 0
for im in im_list_down:
new_im.paste(im, (x_offset,58))
x_offset += im.size[0]
return new_im
#下载并还原图片
# driver:webdriver
# div:图片的div
def get_image(driver,div):
#找到图片所在的div
background_images=driver.find_elements_by_xpath(div)
location_list=[]
imageurl=''
#图片是被CSS按照位移的方式打乱的,我们需要找出这些位移,为后续还原做好准备
for background_image in background_images:
location={}
#在html里面解析出小图片的url地址，还有长高的数值
location['x']=int(re.findall("background-image: url\(\"(.*)\"\); background-position: (.*)px (.*)px;",background_image.get_attribute('style'))[0][1])
location['y']=int(re.findall("background-image: url\(\"(.*)\"\); background-position: (.*)px (.*)px;",background_image.get_attribute('style'))[0][2])
imageurl=re.findall("background-image: url\(\"(.*)\"\); background-position: (.*)px (.*)px;",background_image.get_attribute('style'))[0][0]
location_list.append(location)
#替换图片的后缀,获得图片的URL
imageurl=imageurl.replace("webp","jpg")
#获得图片的名字
imageName = imageurl.split('/')[-1]
#获得图片
session = requests.session()
r = session.get(imageurl, headers = headers, verify = False)
#下载图片
with open(imageName, 'wb') as f:
f.write(r.content)
f.close()
#重新合并还原图片
image=get_merge_image(imageName, location_list)
return image
#对比RGB值
def is_similar(image1,image2,x,y):
pass
#获取指定位置的RGB值
pixel1=image1.getpixel((x,y))
pixel2=image2.getpixel((x,y))
for i in range(0,3):
# 如果相差超过50则就认为找到了缺口的位置
if abs(pixel1[i]-pixel2[i])>=50:
return False
return True
#计算缺口的位置
def get_diff_location(image1,image2):
i=0
# 两张原始图的大小都是相同的260*116
# 那就通过两个for循环依次对比每个像素点的RGB值
# 如果相差超过50则就认为找到了缺口的位置
for i in range(0,260):
for j in range(0,116):
if is_similar(image1,image2,i,j)==False:
return i
#根据缺口的位置模拟x轴移动的轨迹
def get_track(length):
pass
list=[]
#间隔通过随机范围函数来获得,每次移动一步或者两步
x=random.randint(1,3)
#生成轨迹并保存到list内
while length-x>=5:
list.append(x)
length=length-x
x=random.randint(1,3)
#最后五步都是一步步移动
for i in range(length):
list.append(1)
return list
#滑动验证码破解程序
def main():
#打开火狐浏览器
driver = webdriver.Firefox()
#用火狐浏览器打开网页
driver.get("http://www.geetest.com/exp_embed")
#等待页面的上元素刷新出来
WebDriverWait(driver, 30).until(lambda the_driver: the_driver.find_element_by_xpath("//div[@class='gt_slider_knob gt_show']").is_displayed())
WebDriverWait(driver, 30).until(lambda the_driver: the_driver.find_element_by_xpath("//div[@class='gt_cut_bg gt_show']").is_displayed())
WebDriverWait(driver, 30).until(lambda the_driver: the_driver.find_element_by_xpath("//div[@class='gt_cut_fullbg gt_show']").is_displayed())
#下载图片
image1=get_image(driver, "//div[@class='gt_cut_bg gt_show']/div")
image2=get_image(driver, "//div[@class='gt_cut_fullbg gt_show']/div")
#计算缺口位置
loc=get_diff_location(image1, image2)
#生成x的移动轨迹点
track_list=get_track(loc)
#找到滑动的圆球
element=driver.find_element_by_xpath("//div[@class='gt_slider_knob gt_show']")
location=element.location
#获得滑动圆球的高度
y=location['y']
#鼠标点击元素并按住不放
print ("第一步,点击元素")
ActionChains(driver).click_and_hold(on_element=element).perform()
time.sleep(0.15)
print ("第二步，拖动元素")
track_string = ""
for track in track_list:
#不能移动太快,否则会被认为是程序执行
track_string = track_string + "{%d,%d}," % (track, y - 445)
#xoffset=track+22:这里的移动位置的值是相对于滑动圆球左上角的相对值，而轨迹变量里的是圆球的中心点，所以要加上圆球长度的一半。
#yoffset=y-445:这里也是一样的。不过要注意的是不同的浏览器渲染出来的结果是不一样的，要保证最终的计算后的值是22，也就是圆球高度的一半
ActionChains(driver).move_to_element_with_offset(to_element=element, xoffset=track+22, yoffset=y-445).perform()
#间隔时间也通过随机函数来获得,间隔不能太快,否则会被认为是程序执行
time.sleep(random.randint(10,50)/100)
print (track_string)
#xoffset=21，本质就是向后退一格。这里退了5格是因为圆球的位置和滑动条的左边缘有5格的距离
ActionChains(driver).move_to_element_with_offset(to_element=element, xoffset=21, yoffset=y-445).perform()
time.sleep(0.1)
ActionChains(driver).move_to_element_with_offset(to_element=element, xoffset=21, yoffset=y-445).perform()
time.sleep(0.1)
ActionChains(driver).move_to_element_with_offset(to_element=element, xoffset=21, yoffset=y-445).perform()
time.sleep(0.1)
ActionChains(driver).move_to_element_with_offset(to_element=element, xoffset=21, yoffset=y-445).perform()
time.sleep(0.1)
ActionChains(driver).move_to_element_with_offset(to_element=element, xoffset=21, yoffset=y-445).perform()
print ("第三步，释放鼠标")
#释放鼠标
ActionChains(driver).release(on_element=element).perform()
time.sleep(3)
#点击验证
# submit = driver.find_element_by_xpath("//div[@class='gt_ajax_tip success']")
# print(submit.location)
# time.sleep(5)
#关闭浏览器,为了演示方便,暂时注释掉.
#driver.quit()
#主函数入口
if __name__ == '__main__':
pass
main()
python构建web页面
import os
import tornado.httpserver
import tornado.ioloop
import tornado.options
import tornado.web
from view import *
from tornado.options import define, options
define("port", default=8000, help="run on the given port", type=int)
class Application(tornado.web.Application):
def __init__(self):
handlers = [
(r"/", Indexhandler),
]
settings = dict(
template_path=os.path.join(os.path.dirname(__file__), 'templates'),
autoescape=None,
debug=False,
)
tornado.web.Application.__init__(self, handlers, **settings)
if __name__ == "__main__":
tornado.options.parse_command_line()
http_server = tornado.httpserver.HTTPServer(Application(), xheaders=True)
http_server.listen(options.port)
tornado.ioloop.IOLoop.instance().start()
定时任务
#! /usr/bin/env python
# coding=utf-8
import time, os, sched
# 第一个参数确定任务的时间，返回从某个特定的时间到现在经历的秒数
# 第二个参数以某种人为的方式衡量时间
schedule = sched.scheduler(time.time, time.sleep)
def perform_command(cmd, inc):
# 安排inc秒后再次运行自己，即周期运行
schedule.enter(inc, 0, perform_command, (cmd, inc))
os.system(cmd)
def timming_exe(cmd, inc=60):
# enter用来安排某事件的发生时间，从现在起第n秒开始启动
schedule.enter(inc, 0, perform_command, (cmd, inc))
# 持续运行，直到计划时间队列变成空为止
schedule.run()
#每隔一天调用getMovieList.py程序
timming_exe("getMovieList.py", 60 * 60 * 24)
通过百度地图API，标准化地址
from urllib.request import urlopen
from urllib.parse import urlencode
from urllib.error import URLError
import json
class xBaiduMap:
def __init__(self, key='mgf2Gxr7EgnfPVQnpClZnsug'):
self.host = 'http://api.map.baidu.com'
self.path = '/geocoder?'
self.param = {'address': None, 'output': 'json', 'key': key, 'location': None, 'city': None}
def getLocation(self, address, city=None):
rlt = self.geocoding('address', address, city)
if rlt != None:
l = rlt['result']
if isinstance(l, list):
return None
return l['location']['lat'], l['location']['lng']
def getAddress(self, lat, lng):
rlt = self.geocoding('location', "{0},{1}".format(lat, lng))
if rlt != None:
l = rlt['result']
#return l['formatted_address']
# Here you can get more details about the location with 'addressComponent' key
ld=rlt['result']['addressComponent']
return (ld['city']+';'+ld['district']+';'+ld['street']+";"+ld['street_number'])
def geocoding(self, key, value, city=None):
if key == 'location':
if 'city' in self.param:
del self.param['city']
if 'address' in self.param:
del self.param['address']
elif key == 'address':
if 'location' in self.param:
del self.param['location']
if city == None and 'city' in self.param:
del self.param['city']
else:
self.param['city'] = city
self.param[key] = value
try:
r = urlopen(self.host + self.path + urlencode(self.param)).read()
except URLError:
print ("URLError")
return None
str_response = r.decode('utf-8')
rlt = json.loads(str_response)
if rlt['status'] == 'OK':
return rlt
else:
print ("Decoding Failed")
return None
多进程
import multiprocessing
for process_id in range(PROCESS_NUM):
p = multiprocessing.Process(target=worker, args=(process_id,))
jobs.append(p)
p.start()
文件切割小程序
def split_file(file_name, file_num):
#文件已经存在
if(os.path.exists("split_0.txt")):
return
#统计文件的总行数
count = -1
file = open(file_name, encoding='utf-8')
for count, line in enumerate(file):
pass
count += 1
file.close()
#每个文件的行数
count_per_file = count / file_num
#创建file_num个新文件
for i in range(file_num):
file = open("split_" + str(i) + ".txt", 'w', encoding='utf-8')
file.close()
#分割成file_num个新文件
file = open(file_name, encoding='utf-8')
count = -1
for count, line in enumerate(file):
file_index = (int)(count /count_per_file)
sub_file = open("split_" + str(file_index) + ".txt", "a+", encoding='utf-8')
if(sub_file != None):
sub_file.write(line)
python操作DB2
import ibm_db
con = ibm_db.connect("DATABASE=FMRT;HOSTNAME=XX.XX.XX.XX;PORT=60000;PORTOCOL=TCPIP;UID=db2inst1;PWD=db2inst1;", "", "")
sql = getSql(inputfile)
stmt = ibm_db.exec_immediate(con, sql)
result = ibm_db.fetch_both(stmt)
rowidx = 0
while (result):
#DO SOMETHING
result = ibm_db.fetch_both(stmt)
ibm_db.close(con)
jieba中文分词
import jieba
seg_list = jieba.cut("我来到北京清华大学", cut_all=True)
for line in seg_list:
print(line)
print("Full Mode: " + "/ ".join(seg_list)) # 全模式
seg_list = jieba.cut("我来到北京清华大学", cut_all=False)
print("Default Mode: " + "/ ".join(seg_list)) # 精确模式
seg_list = jieba.cut("他来到了网易杭研大厦") # 默认是精确模式
print(", ".join(seg_list))
seg_list = jieba.cut_for_search("小明硕士毕业于中国科学院计算所，后在日本京都大学深造") # 搜索引擎模式
print(", ".join(seg_list))
月末判断
import calendar
import sys
def isMonthEnd(datetime):
year = int(datetime[0:4])
month = int(datetime[4:6])
day = int(datetime[6:8])
wday, monthrange = calendar.monthrange(year, month)
if(day == monthrange):
return 1
else:
return 0
isMonthEnd(sys.argv[1])
移除中文分隔符
cmd = "sed ':a;N;$ s/\\r\\n//g;ba' " + oldfile + " > " + newfile
os.system(cmd)
多线程
# -*- coding: utf-8 -*-
"""
thread
~~~~~~~~~~~~~~~~
Thread framework
:copyright: (c) 2016 by why.
:license: MIT, see LICENSE for more details.
"""
import threading
class Threadconfig():
def __init__(self, thread_size):
self.thread_size = thread_size
def topen(self):
self.thread_tasks = []
def build(self, func, **kwargs):
self.thread_task = threading.Thread(target=func, kwargs=(kwargs))
self.thread_tasks.append(self.thread_task)
def run(self):
for thread_task in self.thread_tasks:
thread_task.setDaemon(True)
thread_task.start()
while 1:
alive = False
for thread_num in range(0, self.thread_size):
alive = alive or self.thread_tasks[thread_num].isAlive()
if not alive:
break
def __del__(self):
self.thread_tasks = []
