获取代理IP

# coding=utf8
import requests
from bs4 import BeautifulSoup
import re

user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5)'
headers = {'User-Agent': user_agent}


def getListProxies():
    session = requests.session()
    page = session.get("http://www.xicidaili.com/nn", headers=headers)
    soup = BeautifulSoup(page.text, 'lxml')

    proxyList = []
    taglist = soup.find_all('tr', attrs={'class': re.compile("(odd)|()")})
    for trtag in taglist:
        tdlist = trtag.find_all('td')
        proxy = {'http': tdlist[1].string + ':' + tdlist[2].string,
                 'https': tdlist[1].string + ':' + tdlist[2].string}
        proxyList.append(proxy)
        # 设定代理ip个数
        if len(proxyList) >= 10:
            break

    return proxyList

if __name__ == '__main__':
    res = getListProxies()
    print(res)

在 requests 中，我们同样可以很方便的设置代理服务器。如果需要使用代理，只需要用添加其他
属性（headers、cookies）一样方式，在请求中提供 proxies 属性即可。
proxies 属性接收一个字典，其中字典的键为代理服务器的协议(http 或是 https)，字典的值则为
代理服务器的地址和端口。

使用代理

# coding=utf8
import requests
from bs4 import BeautifulSoup
import re

header = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
    'Connection': 'keep-alive',
    'Accept-Language': 'zh-CN,zh;q=0.9',
    'Accept': 'text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01',
}
proxies = {
    'https': 'https://58.56.149.198:53281'
}


def ip(isProxies=False):
    '''
    不开代理
    :return:
    '''
    if isProxies:
        print('代理')
        wbdata = requests.get('https://www.sogou.com/web?query=ip', proxies=proxies, headers=header)
    else:
        print('未代理')
        wbdata = requests.get('https://www.sogou.com/web?query=ip', headers=header)
    soup = BeautifulSoup(wbdata.content, 'lxml')
    ip = soup.select_one('div.vrwrap > div.ipvr > div.localip > strong')
    print(ip.get_text())


if __name__ == '__main__':
    res = ip(True)