Python实现多线程抓取妹子图
作者:hebedich 发布时间:2022-10-02 15:39:25
标签:Python,多线程,抓取妹子图
心血来潮写了个多线程抓妹子图,虽然代码还是有一些瑕疵,但是还是记录下来,分享给大家。
Pic_downloader.py
# -*- coding: utf-8 -*-
"""
Created on Fri Aug 07 17:30:58 2015
@author: Dreace
"""
import urllib2
import sys
import time
import os
import random
from multiprocessing.dummy import Pool as ThreadPool
type_ = sys.getfilesystemencoding()
def rename():
return time.strftime("%Y%m%d%H%M%S")
def rename_2(name):
if len(name) == 2:
name = '0' + name + '.jpg'
elif len(name) == 1:
name = '00' + name + '.jpg'
else:
name = name + '.jpg'
return name
def download_pic(i):
global count
global time_out
if Filter(i):
try:
content = urllib2.urlopen(i,timeout = time_out)
url_content = content.read()
f = open(repr(random.randint(10000,999999999)) + "_" + rename_2(repr(count)),"wb")
f.write(url_content)
f.close()
count += 1
except Exception, e:
print i + "下载超时,跳过!".decode("utf-8").encode(type_)
def Filter(content):
for line in Filter_list:
line=line.strip('\n')
if content.find(line) == -1:
return True
def get_pic(url_address):
global pic_list
try:
str_ = urllib2.urlopen(url_address, timeout = time_out).read()
url_content = str_.split("\"")
for i in url_content:
if i.find(".jpg") != -1:
pic_list.append(i)
except Exception, e:
print "获取图片超时,跳过!".decode("utf-8").encode(type_)
MAX = 2
count = 0
time_out = 60
thread_num = 30
pic_list = []
page_list = []
Filter_list = ["imgsize.ph.126.net","img.ph.126.net","img2.ph.126.net"]
dir_name = "C:\Photos\\"+rename()
os.makedirs(dir_name)
os.chdir(dir_name)
start_time = time.time()
url_address = "http://sexy.faceks.com/?page="
for i in range(1,MAX + 1):
page_list.append(url_address + repr(i))
page_pool = ThreadPool(thread_num)
page_pool.map(get_pic,page_list)
print "获取到".decode("utf-8").encode(type_),len(pic_list),"张图片,开始下载!".decode("utf-8").encode(type_)
pool = ThreadPool(thread_num)
pool.map(download_pic,pic_list)
pool.close()
pool.join()
print count,"张图片保存在".decode("utf-8").encode(type_) + dir_name
print "共耗时".decode("utf-8").encode(type_),time.time() - start_time,"s"
我们来看下一个网友的作品
#coding: utf-8 #############################################################
# File Name: main.py
# Author: mylonly
# mail: mylonly@gmail.com
# Created Time: Wed 11 Jun 2014 08:22:12 PM CST
#########################################################################
#!/usr/bin/python
import re,urllib2,HTMLParser,threading,Queue,time
#各图集入口链接
htmlDoorList = []
#包含图片的Hmtl链接
htmlUrlList = []
#图片Url链接Queue
imageUrlList = Queue.Queue(0)
#捕获图片数量
imageGetCount = 0
#已下载图片数量
imageDownloadCount = 0
#每个图集的起始地址,用于判断终止
nextHtmlUrl = ''
#本地保存路径
localSavePath = '/data/1920x1080/'
#如果你想下你需要的分辨率的,请修改replace_str,有如下分辨率可供选择1920x1200,1980x1920,1680x1050,1600x900,1440x900,1366x768,1280x1024,1024x768,1280x800
replace_str = '1920x1080'
replaced_str = '960x600'
#内页分析处理类
class ImageHtmlParser(HTMLParser.HTMLParser):
def __init__(self):
self.nextUrl = ''
HTMLParser.HTMLParser.__init__(self)
def handle_starttag(self,tag,attrs):
global imageUrlList
if(tag == 'img' and len(attrs) > 2 ):
if(attrs[0] == ('id','bigImg')):
url = attrs[1][1]
url = url.replace(replaced_str,replace_str)
imageUrlList.put(url)
global imageGetCount
imageGetCount = imageGetCount + 1
print url
elif(tag == 'a' and len(attrs) == 4):
if(attrs[0] == ('id','pageNext') and attrs[1] == ('class','next')):
global nextHtmlUrl
nextHtmlUrl = attrs[2][1];
#首页分析类
class IndexHtmlParser(HTMLParser.HTMLParser):
def __init__(self):
self.urlList = []
self.index = 0
self.nextUrl = ''
self.tagList = ['li','a']
self.classList = ['photo-list-padding','pic']
HTMLParser.HTMLParser.__init__(self)
def handle_starttag(self,tag,attrs):
if(tag == self.tagList[self.index]):
for attr in attrs:
if (attr[1] == self.classList[self.index]):
if(self.index == 0):
#第一层找到了
self.index = 1
else:
#第二层找到了
self.index = 0
print attrs[1][1]
self.urlList.append(attrs[1][1])
break
elif(tag == 'a'):
for attr in attrs:
if (attr[0] == 'id' and attr[1] == 'pageNext'):
self.nextUrl = attrs[1][1]
print 'nextUrl:',self.nextUrl
break
#首页Hmtl解析器
indexParser = IndexHtmlParser()
#内页Html解析器
imageParser = ImageHtmlParser()
#根据首页得到所有入口链接
print '开始扫描首页...'
host = 'http://desk.zol.com.cn'
indexUrl = '/meinv/'
while (indexUrl != ''):
print '正在抓取网页:',host+indexUrl
request = urllib2.Request(host+indexUrl)
try:
m = urllib2.urlopen(request)
con = m.read()
indexParser.feed(con)
if (indexUrl == indexParser.nextUrl):
break
else:
indexUrl = indexParser.nextUrl
except urllib2.URLError,e:
print e.reason
print '首页扫描完成,所有图集链接已获得:'
htmlDoorList = indexParser.urlList
#根据入口链接得到所有图片的url
class getImageUrl(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
def run(self):
for door in htmlDoorList:
print '开始获取图片地址,入口地址为:',door
global nextHtmlUrl
nextHtmlUrl = ''
while(door != ''):
print '开始从网页%s获取图片...'% (host+door)
if(nextHtmlUrl != ''):
request = urllib2.Request(host+nextHtmlUrl)
else:
request = urllib2.Request(host+door)
try:
m = urllib2.urlopen(request)
con = m.read()
imageParser.feed(con)
print '下一个页面地址为:',nextHtmlUrl
if(door == nextHtmlUrl):
break
except urllib2.URLError,e:
print e.reason
print '所有图片地址均已获得:',imageUrlList
class getImage(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
def run(self):
global imageUrlList
print '开始下载图片...'
while(True):
print '目前捕获图片数量:',imageGetCount
print '已下载图片数量:',imageDownloadCount
image = imageUrlList.get()
print '下载文件路径:',image
try:
cont = urllib2.urlopen(image).read()
patter = '[0-9]*\.jpg';
match = re.search(patter,image);
if match:
print '正在下载文件:',match.group()
filename = localSavePath+match.group()
f = open(filename,'wb')
f.write(cont)
f.close()
global imageDownloadCount
imageDownloadCount = imageDownloadCount + 1
else:
print 'no match'
if(imageUrlList.empty()):
break
except urllib2.URLError,e:
print e.reason
print '文件全部下载完成...'
get = getImageUrl()
get.start()
print '获取图片链接线程启动:'
time.sleep(2)
download = getImage()
download.start()
print '下载图片链接线程启动:'
批量抓取指定网页上的所有图片
# -*- coding:utf-8 -*-
# coding=UTF-8
import os,urllib,urllib2,re
url = u"http://image.baidu.com/search/index?tn=baiduimage&ipn=r&ct=201326592&cl=2&lm=-1&st=-1&fm=index&fr=&sf=1&fmq=&pv=&ic=0&nc=1&z=&se=1&showtab=0&fb=0&width=&height=&face=0&istype=2&ie=utf-8&word=python&oq=python&rsp=-1"
outpath = "t:\\"
def getHtml(url):
webfile = urllib.urlopen(url)
outhtml = webfile.read()
print outhtml
return outhtml
def getImageList(html):
restr=ur'('
restr+=ur'http:\/\/[^\s,"]*\.jpg'
restr+=ur'|http:\/\/[^\s,"]*\.jpeg'
restr+=ur'|http:\/\/[^\s,"]*\.png'
restr+=ur'|http:\/\/[^\s,"]*\.gif'
restr+=ur'|http:\/\/[^\s,"]*\.bmp'
restr+=ur'|https:\/\/[^\s,"]*\.jpeg'
restr+=ur'|https:\/\/[^\s,"]*\.jpeg'
restr+=ur'|https:\/\/[^\s,"]*\.png'
restr+=ur'|https:\/\/[^\s,"]*\.gif'
restr+=ur'|https:\/\/[^\s,"]*\.bmp'
restr+=ur')'
htmlurl = re.compile(restr)
imgList = re.findall(htmlurl,html)
print imgList
return imgList
def download(imgList, page):
x = 1
for imgurl in imgList:
filepathname=str(outpath+'pic_%09d_%010d'%(page,x)+str(os.path.splitext(urllib2.unquote(imgurl).decode('utf8').split('/')[-1])[1])).lower()
print '[Debug] Download file :'+ imgurl+' >> '+filepathname
urllib.urlretrieve(imgurl,filepathname)
x+=1
def downImageNum(pagenum):
page = 1
pageNumber = pagenum
while(page <= pageNumber):
html = getHtml(url)#获得url指向的html内容
imageList = getImageList(html)#获得所有图片的地址,返回列表
download(imageList,page)#下载所有的图片
page = page+1
if __name__ == '__main__':
downImageNum(1)
![](https://www.aspxhome.com/images/zang.png)
![](https://www.aspxhome.com/images/jiucuo.png)
猜你喜欢
- 基本原理使用Adodb.Stream读二进制文件然后进行解析,然后返回一数组第一个元素为类型(BMP JPG PNG GIF SWF)第二个
- 前言最近工作中需要开发前端操作远程虚拟机的功能,简称WebShell. 基于当前的技术栈为react+django,调研了一会发现大部分的后
- 在Python中的break语句终止当前循环,继续执行下一个语句,就像C语言中的break一样。break最常见的用途是当一些外
- 我们都一定对比过编程的轻松与简单性。虽然我们都确认php和perl是最容易学习和编程的语言,但我仍旧想知道,如果用php、asp、jsp以及
- 这里我不想采用诸如ubuntu下的apt-get install方式进行python的安装,而是在linux下采用源码包的方式进行pytho
- 一、基本概念查找(Searching)就是根据给定的某个值,在查找表中确定一个其关键字等于给定值的数据元素(或记录)。查找表(Search
- PDOStatement::setAttributePDOStatement::setAttribute — 设置一个语句属性(PHP 5
- 前言最近助教改作业导出的成绩表格跟老师给的名单顺序不一致,脑壳一亮就用pandas写了个脚本自动吧原始导出的成绩誊写到老师给的名单中了哈哈哈
- 做WEB前台开发也有两年多,在开发中经常要涉及到DOM编程,我想大家也跟我一样,经常要处理各种DOM编程的操作,今天我就拿一个lightbo
- 注意: 在搭建网络的时候用carpool2D的时候,让高度和宽度方向不同池化时,用如下:nn.MaxPool2d(kernel_size=2
- NOMONEYDAY123114321-45324212-102250331008为了符合阅读习惯,最终报表希望是如下格式:NOMONTUE
- 对象Javascript 根本上是和对象相关的。数组是对象。函数是对象。对象是对象。那什么是对象呢?对象是名-值对的集合。名是字符串,值可以
- 1、路径https://www.lfd.uci.edu/~gohlke/pythonlibs/PS:网上说有时候报404,解决办法是换浏览器
- 鲲之鹏的技术人员将在本文介绍一种通过模拟操作微信App的方式采集指定公众号的所有历史数据的方法。通过我们抓包分析发现,微信公众号的历史数据是
- 目录1. 注册百度云账号2. 安 * aidu python api3.调用摄像头4.完整测试程序1. 注册百度云账号注册百度智能云,提交申请
- 背景 background css 说明 background-image:url(&q
- 又有人说设session.timeout=99999。这种同样不行,session有最大时间限制。我经过测试发现最大值为24小时,也就是说你
- 您是否常常在做网页的过程中发现一个问题呢?当图片上传的时候,如果图片太大 ,就会把网页撑破,唯一做的就要先把它用软件缩小,再上传上
- 前端代码要做到简洁易读、高效,还要考虑后端嵌套的方便性。前段时间做了一个导航,把整个制作过程重现,希望对大家有帮助。看到这样的导航,你会怎么
- 想必大家都知道MSSQL中SA权限是什么,可以说是至高无上。今天我就它的危害再谈点儿,我所讲的是配合NBSI上传功能得到WebShell。在