Python网络爬虫实战之十：利用API进行数据采集

{
   "resultcode":"200",
   "reason":"Return Successd!",
   "result":{
      "area":"江苏省苏州市",
      "location":"电信"
   }
}

<?xml version="1.0" encoding="utf-8" ?> 
<root>
    <resultcode>200</resultcode> 
    <reason>Return Successd!</reason> 
    <result>
        <area>江苏省苏州市</area> 
        <location>电信</location> 
    </result>
</root>

import json

jsonString = '{"arrayOfNums":[{"number":0},{"number":1},{"number":2}],"arrayOfFruits":[{"fruit":"apple"},{"fruit":"banana"},{"fruit":"pear"}]}'
jsonObj = json.loads(jsonString)
print(jsonObj.get("arrayOfNums"))
print(jsonObj.get("arrayOfNums")[1])
print(jsonObj.get("arrayOfNums")[1].get("number") + jsonObj.get("arrayOfNums")[2].get("number"))
print(jsonObj.get("arrayOfFruits")[2].get("fruit"))

from urllib import urlencode
import urllib
import json

# 配置您申请的APPKey
appkey = "XXXXXXXXXXXXXXXXXXXXXXXX"


# 根据城市查询天气
def queryWeather(appkey, m="GET", city="广州", dtype="json"):
    url = "http://v.juhe.cn/weather/index"
    params = {
        "cityname": city,  # 要查询的城市，如：温州、上海、北京
        "key": appkey,  # 应用APPKEY(应用详细页查询)
        "dtype": dtype,  # 返回数据的格式,xml或json，默认json
    }
    params = urlencode(params, )
    if m == "GET":
        f = urllib.urlopen("%s?%s" % (url, params))
    else:
        f = urllib.urlopen(url, params)

    content = f.read()
    res = json.loads(content)
    if res:
        error_code = res["error_code"]
        if error_code == 0:
            # 成功请求
            return res["result"]
        else:
            print "%s:%s" % (res["error_code"], res["reason"])
    else:
        print "request api error"


weather = queryWeather(appkey, "GET")
print weather
print urllib.unquote(weather.get("sk").get("wind_direction"))

from urllib import urlopen
import json


def getCountry(ipAddress, appkey):
    response = urlopen("http://apis.juhe.cn/ip/ip2addr?ip=" + ipAddress + "&key=" + appkey).read().decode('utf-8')
    responseJson = json.loads(response)
    return responseJson.get("area")


# 配置您申请的APPKey
appkey = "84bd1042092e7b0e3265483f46febc80"
print(getCountry("61.135.169.121", appkey))

pip install sinaweibopy

from weibo import APIClient
import webbrowser

## 1、个人微博的账号信息
APP_KEY = 'XXXXXX'
APP_SECRET = 'XXXXXXXXXXXXXXXXXXXXXXXX'
CALLBACK_URL = 'http://f.dataguru.cn'

## 2、请求授权
# 2.1
client = APIClient(app_key=APP_KEY, app_secret=APP_SECRET, redirect_uri=CALLBACK_URL)
url = client.get_authorize_url()
# print(url)
# https://api.weibo.com/oauth2/authorize?redirect_uri=http%3A//f.dataguru.cn&response_type=code&client_id=2337575664
# 2.2
# 打开申请授权的网页，点击同意授权后会跳转到之前设置的回调网页（即CALLBACK_URL）
# 在回调页的浏览器地址栏里获取code（动态变化），用于第二步调用oauth2/access_token接口，获取授权后的access token
webbrowser.open_new(url)
# http://f.dataguru.cn/?code=6240f86a9c757ef6ea985cd28647f05a
code = '6240f86a9c757ef6ea985cd28647f05a'

## 3、获得授权
# 获取token 和 token的生命周期
r = client.request_access_token(code)
# print(r)
access_token = r.access_token
# print(access_token)
expires_in = r.expires_in
# print(expires_in)

## 4、为以后的API请求设置token
client.set_access_token(access_token, expires_in)

## 5、获取当前登录用户及其所关注（授权）用户的最新微博 statuses/home_timeline
# https://api.weibo.com/2/statuses/home_timeline.json
statuses = client.statuses.home_timeline.get(count=10)['statuses']
# print(statuses[1])
length = len(statuses)
print(length)
# 输出了部分信息
for i in range(0, length):
    print(u'昵称：' + statuses[i]['user']['screen_name'])
    print(u'简介：' + statuses[i]['user']['description'])
    print(u'位置：' + statuses[i]['user']['location'])
    print(u'微博：' + statuses[i]['text'])

## 6、获取最新的提到登录用户的微博列表，即@我的微博 statuses/mentions
# https://api.weibo.com/2/statuses/mentions.json
statuses = client.statuses.mentions.get()['statuses']
# print(statuses[1])
length = len(statuses)
print(length)
# 输出了部分信息
for i in range(0, length):
    print(u'昵称：' + statuses[i]['user']['screen_name'])
    print(u'简介：' + statuses[i]['user']['description'])
    print(u'位置：' + statuses[i]['user']['location'])
    print(u'微博：' + statuses[i]['text'])
    print(u'时间：' + statuses[i]['created_at'])

pip install weibopy

from weibopy import WeiboOauth2
import webbrowser

## 1、个人微博的账号信息
APP_KEY = 'XXXXXX'
APP_SECRET = 'XXXXXXXXXXXX'
CALLBACK_URL = 'http://f.dataguru.cn'

## 2、请求授权
# 2.1
client = WeiboOauth2(APP_KEY, APP_SECRET, CALLBACK_URL)
authorize_url = client.authorize_url
print(authorize_url)
# https://api.weibo.com/oauth2/authorize?redirect_uri=http%3A//f.dataguru.cn&response_type=code&client_id=2337575664
# 2.2
# 打开申请授权的网页，点击同意授权后会跳转到之前设置的回调网页（即CALLBACK_URL）
# 在回调页的浏览器地址栏里获取code（动态变化），用于第二步调用oauth2/access_token接口，获取授权后的access token
webbrowser.open_new(authorize_url)
# http://f.dataguru.cn/?code=4b156593e9dfdd16279bbcc9eb7817bf
code = '4b156593e9dfdd16279bbcc9eb7817bf'

## 3、获得授权
# 获取token 和 token的生命周期
r = client.auth_access(code)
# print(r)
access_token = r.get("access_token")
# print(access_token)
expires_in = r.expires_in
# print(expires_in)

## 4、为以后的API请求设置token
from weibopy import WeiboClient

client = WeiboClient(access_token)

## 5、获取当前登录用户及其所关注（授权）用户的最新微博 statuses/home_timeline
# https://api.weibo.com/2/statuses/home_timeline.json
result = client.get(suffix="statuses/home_timeline.json")
statuses = result.get("statuses")
# print(statuses[0])
length = len(statuses)
print(length)
# 输出了部分信息
for i in range(0, length):
    print(u'昵称：' + statuses[i]['user']['screen_name'])
    print(u'简介：' + statuses[i]['user']['description'])
    print(u'位置：' + statuses[i]['user']['location'])
    print(u'微博：' + statuses[i]['text'])

## 6、获取最新的提到登录用户的微博列表，即@我的微博 statuses/mentions
# https://api.weibo.com/2/statuses/mentions.json
result = client.get(suffix="statuses/mentions.json")
statuses = result.get("statuses")
# print(statuses[0])
length = len(statuses)
print(length)
# 输出了部分信息
for i in range(0, length):
    print(u'昵称：' + statuses[i]['user']['screen_name'])
    print(u'简介：' + statuses[i]['user']['description'])
    print(u'位置：' + statuses[i]['user']['location'])
    print(u'微博：' + statuses[i]['text'])
    print(u'时间：' + statuses[i]['created_at'])