Ⅰ python使用自定义的cookie和header抓取下载网页图片
#-*-coding:UTF-8-*-
importrequests
__author__='lpe234'
defmain():
url=''
cookies={
'login':'yes',
'cookie2':'shdjd',
'cookie3':'gdhjej'
}
headers={
'content-type':'application/json',
'User-Agent':'Mozilla/5.0(Macintosh;IntelMacOSX10_12_6)AppleWebKit/537.36(KHTML,likeGecko)Chrome/60.0.3112.113Safari/537.36'
}
resp=requests.get(url=url,cookies=cookies,headers=headers)
printresp.content
if__name__=='__main__':
main()
Ⅱ python怎么爬取网页的背景图
测试环境：Windows10+Python3.5.2 打开命令提示符(管理员)，输入pip3 install requests安装requests模块 import requests,re#获取网页r = requests.get('http://www.bilibili.com')#正则表达式获取标题res = re.search(r'([^
Ⅲ python抓取网页上图片
正则表达式匹配的url有错误
for x in add:
print x # 这里可以看到报错的时候是 url 错误
dirpath = os.path.join('C:\\Users\\lilinan\\Desktop\\新建文件夹','%s.jpg' % t)
urllib.request.urlretrieve(x,dirpath)
t+=1
Ⅳ 想用python爬取网页上的图片，但无法用select()方法定位图片的源地址
是的可以撒入爬去获得。
Ⅳ Python3 打开一张网络图片
不要用io.StringIO，改用io.BytesIO
Ⅵ python 网络爬虫，怎么自动保存图片
defdownload_poster_image(movie):#定义一个下载图片函数
src=movie#取出它的url
r=requests.get(src)#去获取这个url
fname=url.split('/')[-1]#从url里面获取这个文件名
withopen(fname,'wb')asf:#应答的内容写进去并且使用二进制
f.write(s.content)
movie['poster-path']=fname
res=requests.get(url)
img_url=res.xpath('//img[@class="q-img-item"]/@src')
movie=img_url
download_poster_image(movie)#执行函数
Ⅶ 一个关于python正则表达式获取网页图片地址的问题
<img\sheight="100"[^>]+src="[^"]+"[^>]+>替换为空
Ⅷ 如何用 Python 爬取社交网络
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Author: Administrator
# @Date: 2015-10-31 15:45:27
# @Last Modified by: Administrator
# @Last Modified time: 2015-11-23 16:57:31
import requests
import sys
import json
import re
reload(sys)
sys.setdefaultencoding('utf-8')
#获取到匹配字符的字符串
def find(pattern,test):
finder = re.search(pattern, test)
start = finder.start()
end = finder.end()
return test[start:end-1]
cookies = {
'_ga':'GA1.2.10sdfsdfsdf', '_za':'8d570b05-b0b1-4c96-a441-faddff34',
'q_c1':'23ddd234234',
'_xsrf':'234id':'"ZTE3NWY2ZTsdfsdfsdfWM2YzYxZmE=|1446435757|"',
'z_c0':'"=|14464e234767|"',
'__utmt':'1', '__utma':'51854390.109883802f8.1417518721.1447917637.144c7922009.4',
'__utmb':'518542340.4.10.1447922009', '__utmc':'51123390', '__utmz':'5185435454sdf06.1.1.utmcsr=hu.com|utmcgcn=(referral)|utmcmd=referral|utmcct=/',
'__utmv':'51854340.1d200-1|2=registration_date=2028=1^3=entry_date=201330318=1'}
headers = {'user-agent':
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.111 Safari/537.36',
'referer':'http://www.hu.com/question/following',
'host':'www.hu.com','Origin':'http://www.hu.com',
'Content-Type':'application/x-www-form-urlencoded; charset=UTF-8',
'Connection':'keep-alive','X-Requested-With':'XMLHttpRequest','Content-Length':'81',
'Accept-Encoding':'gzip,deflate','Accept-Language':'zh-CN,zh;q=0.8','Connection':'keep-alive'
}
#多次访问之后，其实一加载时加载20个问题，具体参数传输就是offset，以20递增
dicc = {"offset":60}
n=20
b=0
# 与爬取图片相同的是，往下拉的时候也会发送http请求返回json数据，但是不同的是，像模拟登录首页不同的是除了
# 发送form表单的那些东西后，知乎是拒绝了我的请求了，刚开始以为是headers上的拦截，往headers添加浏览器
# 访问是的headers那些信息添加上，发现还是拒绝访问。
#想了一下，应该是cookie原因。这个加载的请求和模拟登录首页不同
#所以补上其他的cookies信息，再次请求，请求成功。
for x in xrange(20,460,20):
n = n+20
b = b+20
dicc['offset'] = x
formdata = {'method':'next','params':'{"offset":20}','_xsrf':''}
#传输需要json串，和python的字典是有区别的，需要转换
formdata['params'] = json.mps(dicc)
# print json.mps(dicc)
# print dicc
circle = requests.post("http://www.hu.com/node/ProfileFollowedQuestionsV2",
cookies=cookies,data=formdata,headers=headers)
#response内容 其实爬过一次之后就大同小异了。 都是
#问题返回的json串格式
# {"r":0,
# "msg": ["<div class=\"zm-profile-section-item zg-clear\">\n
# <span class=\"zm-profile-vote-count\">\n<div class=\"zm-profile-vote-num\">205K<\/div>\n
# <div class=\"zm-profile-vote-type\">\u6d4f\u89c8<\/div>\n
# <\/span>\n<div class=\"zm-profile-section-main\">\n
# <h2 class=\"zm-profile-question\">\n
# <a class=\"question_link\" target=\"_blank\" href=\"\/question\/21719532\">
# \u4ec0\u4e48\u4fc3\u4f7f\u4f60\u8d70\u4e0a\u72ec\u7acb\u5f00\u53d1\u8005\u4e4b\u8def\uff1f<\/a>\n
# <\/h2>\n<div class=\"meta zg-gray\">\n<a data-follow=\"q:link\" class=\"follow-link zg-unfollow meta-item\"
# href=\"javascript:;\" id=\"sfb-868760\">
# <i class=\"z-icon-follow\"><\/i>\u53d6\u6d88\u5173\u6ce8<\/a>\n<span class=\"zg-bull\">•<\/span>\n63 \u4e2a\u56de\u7b54\n<span class=\"zg-bull\">•<\/span>\n3589 \u4eba\u5173\u6ce8\n<\/div>\n<\/div>\n<\/div>",
# "<div class=\"zm-profile-section-item zg-clear\">\n
# <span class=\"zm-profile-vote-count\">\n
# <div class=\"zm-profile-vote-num\">157K<\/div>\n
# <div class=\"zm-profile-vote-type\">\u6d4f\u89c8<\/div>\n
# <\/span>\n<div class=\"zm-profile-section-main\">\n
# <h2 class=\"zm-profile-question\">\n
# <a class=\"question_link\" target=\"_blank\" href=\"\/question\/31764065\">
# \u672c\u79d1\u6e23\u6821\u7684\u5b66\u751f\u5982\u4f55\u8fdb\u5165\u7f8e\u5e1d\u725b\u6821\u8bfbPhD\uff1f<\/a>\n
# <\/h2>\n<div class=\"meta zg-gray\">\n
# <a data-follow=\"q:link\" class=\"follow-link zg-unfollow meta-item\" href=\"javascript:;\" id=\"sfb-4904877\">
# <i class=\"z-icon-follow\"><\/i>\u53d6\u6d88\u5173\u6ce8<\/a>\n<span class=\"zg-bull\">•
# <\/span>\n112 \u4e2a\u56de\u7b54\n<span class=\"zg-bull\">•<\/span>\n1582 \u4eba\u5173\u6ce8\n
# <\/div>\n<\/div>\n<\/div>"]}
# print circle.content
#同样json串需要自己 转换成字典后使用
jsondict = json.loads(circle.text)
msgstr = jsondict['msg']
# print len(msgstr)
#根据自己所需要的提取信息规则写出正则表达式
pattern = 'question\/.*?/a>'
try:
for y in xrange(0,20):
wholequestion = find(pattern, msgstr[y])
pattern2 = '>.*?<'
finalquestion = find(pattern2, wholequestion).replace('>','')
print str(b+y)+" "+finalquestion
#当问题已经访问完后再传参数 抛出异常 此时退出循环
except Exception, e:
print "全部%s个问题" %(b+y)
break