Web scraping error when I try to create a request
Question:
I am trying to scraping some data from a movie web (Each page has 20 records). Main steps that I do is:
- create a request
- get the response from the web
- download
It seems I have some error in step 1, but I really don’t know how to fix it. Url of each page is like
https://movie.douban.com/j/chart/top_list?type=5&interval_id=100:90&action=&start=0&limit=20"(first page)
import urllib.parse
import urllib.request
import json
def create_request(page):
base_url = 'https://movie.douban.com/j/chart/top_list?type=5&interval_id=100:90&action=&'
data = {
'start': (page-1)*20,
'limit': 20
}
data = urllib.parse.urlencode(data)
url = base_url + data
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML,
like Gecko) Chrome/106.0.0.0 Safari/537.36'
}
request = urllib.request.Request(url=url,data=data,headers=headers)
return request
def get_content(request):
response = urllib.request.urlopen(request)
content = response.read().decode('utf-8')
return content
def down_load(page, content):
with open('douban_' + str(page) + '.json', 'w', encoding = 'utf-8') as fp:
fp.write(content)
if __name__ == '__main__':
start_page = int(input('Start Page pls:'))
end_page = int(input('End page pls:'))
for page in range(start_page, end_page+1):
request = create_request(page)
content = get_content(request)
down_load(page, content)
The prompt is:
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-4-1a3d2d29485f> in <module>
62 request = create_request(page)
63 # 获取响应的数据
---> 64 content = get_content(request)
65 # 下载
66 down_load(page, content)
<ipython-input-4-1a3d2d29485f> in get_content(request)
44
45 def get_content(request):
---> 46 response = urllib.request.urlopen(request)
47 content = response.read().decode('utf-8')
48 return content
~Anaconda3liburllibrequest.py in urlopen(url, data, timeout, cafile, capath, cadefault, context)
220 else:
221 opener = _opener
--> 222 return opener.open(url, data, timeout)
223
224 def install_opener(opener):
~Anaconda3liburllibrequest.py in open(self, fullurl, data, timeout)
520 for processor in self.process_request.get(protocol, []):
521 meth = getattr(processor, meth_name)
--> 522 req = meth(req)
523
524 sys.audit('urllib.Request', req.full_url, req.data, req.headers, req.get_method())
~Anaconda3liburllibrequest.py in do_request_(self, request)
1279 msg = "POST data should be bytes, an iterable of bytes, "
1280 "or a file object. It cannot be of type str."
-> 1281 raise TypeError(msg)
1282 if not request.has_header('Content-type'):
1283 request.add_unredirected_header(
TypeError: POST data should be bytes, an iterable of bytes, or a file object. It cannot be of type str.
Answers:
The error message is telling you what the issue is. Your data needs to be a bytes object and not a string. I also think that you shouldn’t add the data to the end of the url like you are in the create_request function.
Try this instead.
import urllib.parse
import urllib.request
import json
def create_request(page):
base_url = 'https://movie.douban.com/j/chart/top_list?type=5&interval_id=100:90&action=&'
data = {
'start': (page-1)*20,
'limit': 20
}
data = json.dumps(data).encode() # <-- changed this
url = base_url # <-- changed this
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ppleWebKit/537.36 (KHTML,like Gecko) Chrome/106.0.0.0 Safari/537.36'}
request = urllib.request.Request(url=url,data=data,headers=headers)
return request
def get_content(request):
response = urllib.request.urlopen(request)
content = response.read().decode('utf-8')
return content
def down_load(page, content):
with open('douban_' + str(page) + '.json', 'w', encoding = 'utf-8') as fp:
fp.write(content)
if __name__ == '__main__':
start_page = int(input('Start Page pls:'))
end_page = int(input('End page pls:'))
for page in range(start_page, end_page+1):
request = create_request(page)
content = get_content(request)
down_load(page, content)
I tested it by entering '15' as the start page and this was part of the output.
[
{
"rating": ["9.4", "50"],
"rank": 1,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p511118051.jpg",
"is_playable": false,
"id": "1295644",
"types": ["剧情", "动作", "犯罪"],
"regions": ["法国", "美国"],
"title": "这个杀手不太冷",
"url": "https://movie.douban.com/subject/1295644/",
"release_date": "1994-09-14",
"actor_count": 21,
"vote_count": 2180528,
"score": "9.4",
"actors": [
"让·雷诺",
"娜塔莉·波特曼",
"加里·奥德曼",
"丹尼·爱罗",
"彼得·阿佩尔",
"迈克尔·巴达鲁科",
"艾伦·格里尼",
"伊丽莎白·瑞根",
"卡尔·马图斯维奇",
"弗兰克·赛格",
"麦温",
"乔治·马丁",
"罗伯特·拉萨多",
"亚当·布斯奇",
"马里奥·托迪斯科",
"萨米·纳塞利",
"让·雨果·安格拉德",
"埃莱娜·卡多纳",
"沈晓谦",
"大塚明夫",
"Keith A. Glascoe"
],
"is_watched": false
},
{
"rating": ["9.3", "50"],
"rank": 2,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p2642829472.jpg",
"is_playable": true,
"id": "1291552",
"types": ["剧情", "动作", "奇幻", "冒险"],
"regions": ["美国", "新西兰"],
"title": "指环王3：王者无敌",
"url": "https://movie.douban.com/subject/1291552/",
"release_date": "2004-03-12",
"actor_count": 67,
"vote_count": 755927,
"score": "9.3",
"actors": [
"伊利亚·伍德",
"西恩·奥斯汀",
"维果·莫腾森",
"奥兰多·布鲁姆",
"伊恩·麦克莱恩",
"肖恩·宾",
"多米尼克·莫纳汉",
"丽芙·泰勒",
"约翰·贝西",
"凯特·布兰切特",
"比利·博伊德",
"萨德文·布罗菲",
"阿利斯泰尔·布朗宁",
"马尔顿·索克斯",
"伯纳德·希尔",
"伊安·霍姆",
"布鲁斯·霍普金斯",
"Ian Hughes",
"劳伦斯·马克奥雷",
"诺埃尔·阿普利比",
"布雷特·麦肯齐",
"Alexandra Astin",
"Sarah McLeod",
"Maisy McLeod-Riera",
"约翰·诺贝尔",
"Paul Norell",
"米兰达·奥图",
"布鲁斯·菲利普斯",
"沙恩·朗吉",
"约翰·瑞斯-戴维斯",
"Todd Rippon",
"安迪·瑟金斯",
"Harry Sinclair",
"乔尔·托贝克",
"卡尔·厄本",
"史蒂芬·乌瑞",
"雨果·维文",
"大卫·文翰",
"阿兰·霍华德",
"萨拉·贝克",
"Robert Pollock",
"佩特·史密斯",
"杰德·布罗菲",
"菲利普·格里夫",
"布拉德·道里夫",
"克里斯托弗·李",
"布鲁斯·斯宾斯",
"吉诺·阿赛维多",
"Jarl Benzon",
"Jørn Benzon",
"Robert Catto",
"Michael Elsworth",
"彼得·杰克逊",
"Sandro Kopp",
"安德鲁·莱斯尼",
"约瑟夫·米卡-亨特",
"亨利·莫腾森",
"克雷格·帕克",
"克里斯蒂安·瑞沃斯",
"迈克尔·斯曼内科 ",
"霍华德·肖",
"约翰·斯蒂芬森",
"理查德·泰勒",
"张磊",
"姜广涛",
"周志强",
"金永钢"
],
"is_watched": false
},
{
"rating": ["9.2", "50"],
"rank": 3,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p462657443.jpg",
"is_playable": true,
"id": "1851857",
"types": ["剧情", "动作", "科幻", "犯罪", "惊悚"],
"regions": ["美国", "英国"],
"title": "蝙蝠侠：黑暗骑士",
"url": "https://movie.douban.com/subject/1851857/",
"release_date": "2008-07-14",
"actor_count": 40,
"vote_count": 997680,
"score": "9.2",
"actors": [
"克里斯蒂安·贝尔",
"希斯·莱杰",
"艾伦·艾克哈特",
"迈克尔·凯恩",
"玛吉·吉伦哈尔",
"加里·奥德曼",
"摩根·弗里曼",
"莫尼克·加布里埃拉·库尔内",
"罗恩·迪恩",
"基里安·墨菲",
"黄经汉",
"内斯特·卡博内尔",
"埃里克·罗伯茨",
"里特奇·科斯特",
"安东尼·迈克尔·豪尔",
"基斯·斯扎拉巴基克",
"柯林·麦克法兰",
"约书亚·哈尔托",
"美琳达·麦格劳",
"内森·甘宝",
"迈克尔·维约",
"迈克尔·斯托扬诺夫",
"威廉·斯米利",
"丹尼·高德林",
"迈克尔·加·怀特",
"马修·奥尼尔",
"威廉·菲克纳",
"欧鲁米基·欧拉乌米",
"格雷格·比姆",
"爱利克·赫尔曼",
"毕崔斯·罗森",
"文森佐·尼克力",
"陈冠希",
"亚当·卡莱佩里斯",
"罗南·萨莫尔斯",
"玛丽扎·卡布蕾拉",
"Tina Simmons",
"大卫·达斯马齐连",
"利萨·麦克阿里斯特",
"理查德·迪兰"
],
"is_watched": false
},
{
"rating": ["9.2", "45"],
"rank": 4,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p2640236255.jpg",
"is_playable": true,
"id": "1291572",
"types": ["剧情", "动作", "奇幻", "冒险"],
"regions": ["美国", "新西兰"],
"title": "指环王2：双塔奇兵",
"url": "https://movie.douban.com/subject/1291572/",
"release_date": "2003-04-25",
"actor_count": 58,
"vote_count": 706830,
"score": "9.2",
"actors": [
"伊利亚·伍德",
"西恩·奥斯汀",
"多米尼克·莫纳汉",
"奥兰多·布鲁姆",
"维果·莫腾森",
"伊恩·麦克莱恩",
"凯特·布兰切特",
"比利·博伊德",
"杰德·布罗菲",
"丽芙·泰勒",
"安迪·瑟金斯",
"米兰达·奥图",
"雨果·维文",
"布拉德·道里夫",
"卡卢姆·吉廷斯",
"伯纳德·希尔",
"布鲁斯·霍普金斯",
"克里斯托弗·李",
"纳撒尼尔·利斯",
"罗宾·马尔科姆",
"约翰·贝西",
"萨拉·贝克",
"克雷格·帕克",
"布鲁斯·菲利普斯",
"Robert Pollock",
"约翰·瑞斯-戴维斯",
"奥利维亚·田內",
"卡尔·厄本",
"史蒂芬·乌瑞",
"大卫·文翰",
"菲利普·格里夫",
"肖恩·宾",
"约翰·诺贝尔",
"Jarl Benzon",
"Jørn Benzon",
"Ben Britton",
"阿利斯泰尔·布朗宁",
"Sean Button",
"Robert Catto",
"罗德尼·库克",
"卡尔洛斯·德林克沃特",
"西奥沙福瓦",
"本·弗兰舍姆",
"Jonathan Harding",
"Lucas Hayward",
"丹·汉纳",
"保罗·霍姆斯",
"杰森霍德",
"彼得·杰克逊",
"Sam Kelly",
"Sandro Kopp",
"姜广涛",
"张磊",
"陆建艺",
"金永钢",
"劳伦斯·马克奥雷",
"布鲁克·奥佩斯",
"周志强"
],
"is_watched": false
},
{
"rating": ["9.1", "45"],
"rank": 5,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p2373910134.jpg",
"is_playable": true,
"id": "1291571",
"types": ["剧情", "动作", "奇幻", "冒险"],
"regions": ["新西兰", "美国"],
"title": "指环王1：护戒使者",
"url": "https://movie.douban.com/subject/1291571/",
"release_date": "2002-04-04",
"actor_count": 54,
"vote_count": 797471,
"score": "9.1",
"actors": [
"伊利亚·伍德",
"西恩·奥斯汀",
"伊恩·麦克莱恩",
"维果·莫腾森",
"奥兰多·布鲁姆",
"多米尼克·莫纳汉",
"比利·博伊德",
"克里斯托弗·李",
"马尔顿·索克斯",
"梅根·爱德华兹",
"伊安·霍姆",
"凯特·布兰切特",
"阿兰·霍华德",
"马克·弗格森",
"肖恩·宾",
"萨拉·贝克",
"劳伦斯·马克奥雷",
"安迪·瑟金斯",
"彼得·麦肯齐",
"伊恩·穆内",
"克雷格·帕克",
"卡梅隆·罗德",
"约翰·瑞斯-戴维斯",
"丽芙·泰勒",
"大卫·韦瑟莱",
"雨果·维文",
"菲利普·格里夫",
"威廉·约翰逊",
"伊丽莎白·穆迪",
"布莱恩·瑟金特",
"杰德·布罗菲",
"诺曼·凯茨",
"兰德尔·威廉·库克",
"萨比恩·克洛森",
"西奥沙福瓦",
"本·弗兰舍姆",
"彼得·杰克逊",
"艾伦·李",
"诺埃尔·阿普利比",
"Michael Elsworth",
"Sarah McLeod",
"马丁·桑德森",
"Harry Sinclair",
"吉诺·阿赛维多",
"Jarl Benzon",
"Jørn Benzon",
"Ben Britton",
"Jonathan Harding",
"Taea Hartwell",
"Sam Kelly",
"姜广涛",
"张磊",
"周志强",
"布雷特·麦肯齐"
],
"is_watched": false
},
{
"rating": ["9.1", "45"],
"rank": 6,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p451926968.jpg",
"is_playable": false,
"id": "1291843",
"types": ["动作", "科幻"],
"regions": ["美国"],
"title": "黑客帝国",
"url": "https://movie.douban.com/subject/1291843/",
"release_date": "2000-01-14",
"actor_count": 25,
"vote_count": 791898,
"score": "9.1",
"actors": [
"基努·里维斯",
"劳伦斯·菲什伯恩",
"凯瑞-安·莫斯",
"雨果·维文",
"格洛丽亚·福斯特",
"乔·潘托里亚诺",
"马库斯·钟",
"朱利安·阿拉汗加",
"马特·多兰",
"贝琳达·麦克洛里",
"安东尼雷派克",
"罗伯特·泰勒",
"阿达·尼科德莫",
"罗温·维特",
"塔玛拉·布朗",
"纳塔莉·特珍",
"比尔·扬",
"克里斯·斯科特",
"纳许·埃哲顿",
"户田惠子",
"沈晓谦",
"童自荣",
"保罗·戈达德",
"金永钢",
"张磊"
],
"is_watched": false
},
{
"rating": ["9.3", "50"],
"rank": 7,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p2215886505.jpg",
"is_playable": true,
"id": "1295399",
"types": ["动作", "冒险", "剧情"],
"regions": ["日本"],
"title": "七武士",
"url": "https://movie.douban.com/subject/1295399/",
"release_date": "1954-04-26",
"actor_count": 48,
"vote_count": 184460,
"score": "9.3",
"actors": [
"三船敏郎",
"志村乔",
"津岛惠子",
"岛崎雪子",
"藤原釜足",
"加东大介",
"木村功",
"千秋实",
"宫口精二",
"小杉义男",
"左卜全",
"稻叶义男",
"土屋嘉男",
"高堂国典",
"东野英治郎",
"上田吉二郎",
"多多良纯",
"渡边笃",
"小川虎之助",
"山形勋",
"上山草人",
"清水元",
"高木新平",
"大友伸",
"高原骏雄",
"稻垣三郎",
"堺左千夫",
"千石规子",
"本间文子",
"大久保正信",
"伊藤实",
"大村千吉",
"广濑正一",
"宇野晃司",
"谷晃",
"中岛春雄",
"清水美恵",
"熊谷卓三",
"夏木顺平",
"岩本弘司",
"堤康久",
"马野都留子",
"森今日子",
"加藤武",
"仲代达矢",
"宇津井健",
"山本廉",
"东静子"
],
"is_watched": false
},
{
"rating": ["9.0", "45"],
"rank": 8,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p1512562287.jpg",
"is_playable": true,
"id": "3742360",
"types": ["剧情", "喜剧", "动作", "西部"],
"regions": ["中国大陆", "中国香港"],
"title": "让子弹飞",
"url": "https://movie.douban.com/subject/3742360/",
"release_date": "2010-12-16",
"actor_count": 32,
"vote_count": 1592874,
"score": "9.0",
"actors": [
"姜文",
"葛优",
"周润发",
"刘嘉玲",
"陈坤",
"张默",
"姜武",
"周韵",
"廖凡",
"姚橹",
"邵兵",
"苗圃",
"冯小刚",
"胡军",
"马珂",
"白冰",
"杜奕衡",
"李静",
"胡明",
"危笑",
"杨奇雨",
"赵铭",
"宣晓鸣",
"绀野千春",
"郭俊立",
"马元",
"姜洪齐",
"岳红",
"吴昔果",
"沙瑀",
"陈磊",
"文祥"
],
"is_watched": false
},
{
"rating": ["9.0", "45"],
"rank": 9,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p1910926158.jpg",
"is_playable": true,
"id": "1292000",
"types": ["剧情", "动作", "悬疑", "惊悚"],
"regions": ["美国", "德国", "意大利"],
"title": "搏击俱乐部",
"url": "https://movie.douban.com/subject/1292000/",
"release_date": "1999-09-10",
"actor_count": 30,
"vote_count": 800974,
"score": "9.0",
"actors": [
"爱德华·诺顿",
"布拉德·皮特",
"海伦娜·伯翰·卡特",
"扎克·格雷尼尔",
"米特·洛夫",
"杰瑞德·莱托",
"艾恩·贝利",
"里奇蒙德·阿奎特",
"乔治·马奎尔",
"Ezra Buzzington",
"Carl Ciarfalio",
"斯图尔特·布拉姆博格",
"马特·温斯顿",
"劳伦·桑切斯",
"Tyrone R. Livingston",
"保罗·卡拉夫特斯",
"Scotch Ellis Loring",
"Andi Carnick",
"迈克尔·肖姆斯·维尔斯",
"Markus Redmond",
"大卫·安德鲁斯",
"鲍勃·斯蒂芬森",
"欧根妮·邦杜兰特",
"凯文·斯科特·麦克",
"Bennie Moore",
"埃文·米兰德",
"乔伊·比索奈特",
"Christina Cabot",
"大卫·李·史密斯",
"霍特·麦克卡兰尼"
],
"is_watched": false
},
{
"rating": ["8.9", "45"],
"rank": 10,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p1633113220.jpg",
"is_playable": false,
"id": "1300299",
"types": ["剧情", "动作", "犯罪", "悬疑", "惊悚"],
"regions": ["韩国"],
"title": "杀人回忆",
"url": "https://movie.douban.com/subject/1300299/",
"release_date": "2003-05-02",
"actor_count": 49,
"vote_count": 661779,
"score": "8.9",
"actors": [
"宋康昊",
"金相庆",
"金雷夏",
"宋在浩",
"边希峰",
"高瑞熙",
"柳泰浩",
"朴努植",
"朴海日",
"全美善",
"徐永嬅",
"崔钟律",
"刘承睦",
"申贤宗",
"李在应",
"郑仁仙",
"吴龙",
"朴真宇",
"朴泰京",
"沈成宝",
"朴镇宇",
"廉惠兰",
"李东勇",
"赵德济",
"申文成",
"孙康国",
"李大贤",
"李玉珠",
"刘仁秀",
"千明宰",
"郭秀贞",
"曹文义",
"朴贤英",
"申贤胜",
"權炳吉",
"金周灵",
"金荷景",
"金熙珍",
"崔铉基",
"李江山",
"孙镇浩",
"白奉基",
"刘琴",
"禹高娜",
"申云燮",
"金景来",
"李勋京",
"李多一",
"金景来"
],
"is_watched": false
},
{
"rating": ["8.9", "45"],
"rank": 11,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p2004174709.jpg",
"is_playable": true,
"id": "1294639",
"types": ["动作", "传记", "剧情", "历史", "战争"],
"regions": ["美国"],
"title": "勇敢的心",
"url": "https://movie.douban.com/subject/1294639/",
"release_date": "1995-05-18",
"actor_count": 24,
"vote_count": 537785,
"score": "8.9",
"actors": [
"梅尔·吉布森",
"苏菲·玛索",
"布莱恩·考克斯",
"詹姆斯·科兹莫",
"辛·劳洛",
"凯瑟琳·麦克马克",
"安古斯·麦克菲登",
"布莱丹·格里森",
"詹姆斯·霍纳",
"彼得·穆兰",
"汤米·弗拉纳根",
"艾伦·阿姆斯特朗",
"大卫·奥哈拉",
"帕特里克·麦高汉",
"伊安·邦纳",
"梅丽·卡尔维",
"史蒂芬·比灵顿",
"迈克尔·伯恩",
"亚力克斯·诺顿",
"杰拉德·迈克索利",
"詹姆斯·鲁宾逊",
"辛·麦金利",
"Donal Gibson",
"鲁珀特·范西塔特"
],
"is_watched": false
},
{
"rating": ["8.8", "45"],
"rank": 12,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p2219011938.jpg",
"is_playable": true,
"id": "1291543",
"types": ["动作", "喜剧", "犯罪", "奇幻"],
"regions": ["中国大陆", "中国香港"],
"title": "功夫",
"url": "https://movie.douban.com/subject/1291543/",
"release_date": "2004-12-23",
"actor_count": 33,
"vote_count": 1032748,
"score": "8.8",
"actors": [
"周星驰",
"元秋",
"元华",
"黄圣依",
"梁小龙",
"陈国坤",
"田启文",
"林子聪",
"林雪",
"冯克安",
"释彦能",
"冯小刚",
"袁祥仁",
"张一白",
"赵志凌",
"董志华",
"何文辉",
"陈凯师",
"贾康熙",
"林子善",
"任珈锐",
"王仕颖",
"冯勉恒",
"范湉湉",
"杨能",
"黄锐民",
"姜瑰瑾",
"丁小龙",
"张明明",
"钱升玮",
"林志泰",
"阮昊天",
"程守一"
],
"is_watched": false
},
{
"rating": ["8.8", "45"],
"rank": 13,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p2634997853.jpg",
"is_playable": true,
"id": "1652587",
"types": ["动作", "科幻", "冒险"],
"regions": ["美国"],
"title": "阿凡达",
"url": "https://movie.douban.com/subject/1652587/",
"release_date": "2010-01-04",
"actor_count": 30,
"vote_count": 1304529,
"score": "8.8",
"actors": [
"萨姆·沃辛顿",
"佐伊·索尔达娜",
"西格妮·韦弗",
"史蒂芬·朗",
"米歇尔·罗德里格兹",
"吉奥瓦尼·瑞比西",
"乔·大卫·摩尔 ",
"希·庞德",
"韦斯·斯塔迪",
"拉兹·阿隆索",
"迪利普·劳",
"马特·杰拉德",
"肖恩·安东尼·莫兰",
"季冠霖",
"詹姆斯·霍纳",
"彼得·门萨",
"泰瑞·诺塔里",
"陆建艺",
"凯文·多曼",
"黛布拉·威尔逊",
"斯科特·劳伦斯",
"詹姆斯·皮特",
"雅克布·托毛里",
"凯尔森·亨德森",
"伍迪·舒尔茨",
"彼得·狄龙",
"肖恩·帕特里克·墨菲",
"大卫·范·霍恩",
"凯利·基尔戈",
"杰森·怀特"
],
"is_watched": false
},
{
"rating": ["9.1", "45"],
"rank": 14,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p2446408045.jpg",
"is_playable": true,
"id": "1291936",
"types": ["动作", "科幻", "动画"],
"regions": ["日本"],
"title": "攻壳机动队",
"url": "https://movie.douban.com/subject/1291936/",
"release_date": "1995-11-18",
"actor_count": 16,
"vote_count": 130822,
"score": "9.1",
"actors": [
"田中敦子",
"大塚明夫",
"山寺宏一",
"大木民夫",
"家弓家正",
"玄田哲章",
"山内雅人",
"小川真司",
"宫本充",
"小高三良",
"后藤敦",
"山路和弘",
"佐藤政道",
"Lia Sargent",
"布莱恩·科兰斯顿",
"松山鹰志"
],
"is_watched": false
},
{
"rating": ["8.8", "45"],
"rank": 15,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p1596085504.jpg",
"is_playable": true,
"id": "1298070",
"types": ["动作", "冒险", "奇幻"],
"regions": ["美国"],
"title": "加勒比海盗",
"url": "https://movie.douban.com/subject/1298070/",
"release_date": "2003-11-21",
"actor_count": 34,
"vote_count": 811264,
"score": "8.8",
"actors": [
"约翰尼·德普",
"杰弗里·拉什",
"奥兰多·布鲁姆",
"凯拉·奈特莉",
"杰克·达文波特",
"乔纳森·普雷斯",
"李·阿伦伯格",
"大卫·拜利",
"马丁·科勒巴",
"乔迪·卡巴勒罗",
"劳伦·马赫",
"安格斯·巴尼特",
"露欣达·戴泽克",
"陆建艺",
"Giles New",
"小伊萨克·C.辛格尔顿",
"麦肯锡·克鲁克",
"Christian Martin",
"凯文·麦克纳利",
"瓦妮莎·布兰奇",
"内杰·亚当森",
"戴兰·史密斯",
"盖伊·西纳",
"达米恩·奥哈尔",
"David Patykewich",
"保罗·基斯",
"罗比·基",
"卢克·德·沃夫松",
"佐伊·索尔达娜",
"乔·格里萨菲",
"格雷格·埃利斯",
"特雷瓦·埃迪内",
"特雷沃·格德达德",
"杰拉德·J·雷耶斯"
],
"is_watched": false
},
{
"rating": ["8.8", "45"],
"rank": 16,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p2597918718.jpg",
"is_playable": true,
"id": "1295409",
"types": ["剧情", "喜剧", "动作", "犯罪"],
"regions": ["中国香港"],
"title": "纵横四海",
"url": "https://movie.douban.com/subject/1295409/",
"release_date": "1991-02-02",
"actor_count": 11,
"vote_count": 395931,
"score": "8.8",
...
Would replacing urllib.parse.urlencode(data) with urllib.parse.urlencode(data).encode(‘utf-8’) in the create_request function have helped here as well?
I am trying to scraping some data from a movie web (Each page has 20 records). Main steps that I do is:
- create a request
- get the response from the web
- download
It seems I have some error in step 1, but I really don’t know how to fix it. Url of each page is like
https://movie.douban.com/j/chart/top_list?type=5&interval_id=100:90&action=&start=0&limit=20"(first page)
import urllib.parse
import urllib.request
import json
def create_request(page):
base_url = 'https://movie.douban.com/j/chart/top_list?type=5&interval_id=100:90&action=&'
data = {
'start': (page-1)*20,
'limit': 20
}
data = urllib.parse.urlencode(data)
url = base_url + data
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML,
like Gecko) Chrome/106.0.0.0 Safari/537.36'
}
request = urllib.request.Request(url=url,data=data,headers=headers)
return request
def get_content(request):
response = urllib.request.urlopen(request)
content = response.read().decode('utf-8')
return content
def down_load(page, content):
with open('douban_' + str(page) + '.json', 'w', encoding = 'utf-8') as fp:
fp.write(content)
if __name__ == '__main__':
start_page = int(input('Start Page pls:'))
end_page = int(input('End page pls:'))
for page in range(start_page, end_page+1):
request = create_request(page)
content = get_content(request)
down_load(page, content)
The prompt is:
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-4-1a3d2d29485f> in <module>
62 request = create_request(page)
63 # 获取响应的数据
---> 64 content = get_content(request)
65 # 下载
66 down_load(page, content)
<ipython-input-4-1a3d2d29485f> in get_content(request)
44
45 def get_content(request):
---> 46 response = urllib.request.urlopen(request)
47 content = response.read().decode('utf-8')
48 return content
~Anaconda3liburllibrequest.py in urlopen(url, data, timeout, cafile, capath, cadefault, context)
220 else:
221 opener = _opener
--> 222 return opener.open(url, data, timeout)
223
224 def install_opener(opener):
~Anaconda3liburllibrequest.py in open(self, fullurl, data, timeout)
520 for processor in self.process_request.get(protocol, []):
521 meth = getattr(processor, meth_name)
--> 522 req = meth(req)
523
524 sys.audit('urllib.Request', req.full_url, req.data, req.headers, req.get_method())
~Anaconda3liburllibrequest.py in do_request_(self, request)
1279 msg = "POST data should be bytes, an iterable of bytes, "
1280 "or a file object. It cannot be of type str."
-> 1281 raise TypeError(msg)
1282 if not request.has_header('Content-type'):
1283 request.add_unredirected_header(
TypeError: POST data should be bytes, an iterable of bytes, or a file object. It cannot be of type str.
The error message is telling you what the issue is. Your data needs to be a bytes object and not a string. I also think that you shouldn’t add the data to the end of the url like you are in the create_request function.
Try this instead.
import urllib.parse
import urllib.request
import json
def create_request(page):
base_url = 'https://movie.douban.com/j/chart/top_list?type=5&interval_id=100:90&action=&'
data = {
'start': (page-1)*20,
'limit': 20
}
data = json.dumps(data).encode() # <-- changed this
url = base_url # <-- changed this
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ppleWebKit/537.36 (KHTML,like Gecko) Chrome/106.0.0.0 Safari/537.36'}
request = urllib.request.Request(url=url,data=data,headers=headers)
return request
def get_content(request):
response = urllib.request.urlopen(request)
content = response.read().decode('utf-8')
return content
def down_load(page, content):
with open('douban_' + str(page) + '.json', 'w', encoding = 'utf-8') as fp:
fp.write(content)
if __name__ == '__main__':
start_page = int(input('Start Page pls:'))
end_page = int(input('End page pls:'))
for page in range(start_page, end_page+1):
request = create_request(page)
content = get_content(request)
down_load(page, content)
I tested it by entering '15' as the start page and this was part of the output.
[
{
"rating": ["9.4", "50"],
"rank": 1,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p511118051.jpg",
"is_playable": false,
"id": "1295644",
"types": ["剧情", "动作", "犯罪"],
"regions": ["法国", "美国"],
"title": "这个杀手不太冷",
"url": "https://movie.douban.com/subject/1295644/",
"release_date": "1994-09-14",
"actor_count": 21,
"vote_count": 2180528,
"score": "9.4",
"actors": [
"让·雷诺",
"娜塔莉·波特曼",
"加里·奥德曼",
"丹尼·爱罗",
"彼得·阿佩尔",
"迈克尔·巴达鲁科",
"艾伦·格里尼",
"伊丽莎白·瑞根",
"卡尔·马图斯维奇",
"弗兰克·赛格",
"麦温",
"乔治·马丁",
"罗伯特·拉萨多",
"亚当·布斯奇",
"马里奥·托迪斯科",
"萨米·纳塞利",
"让·雨果·安格拉德",
"埃莱娜·卡多纳",
"沈晓谦",
"大塚明夫",
"Keith A. Glascoe"
],
"is_watched": false
},
{
"rating": ["9.3", "50"],
"rank": 2,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p2642829472.jpg",
"is_playable": true,
"id": "1291552",
"types": ["剧情", "动作", "奇幻", "冒险"],
"regions": ["美国", "新西兰"],
"title": "指环王3：王者无敌",
"url": "https://movie.douban.com/subject/1291552/",
"release_date": "2004-03-12",
"actor_count": 67,
"vote_count": 755927,
"score": "9.3",
"actors": [
"伊利亚·伍德",
"西恩·奥斯汀",
"维果·莫腾森",
"奥兰多·布鲁姆",
"伊恩·麦克莱恩",
"肖恩·宾",
"多米尼克·莫纳汉",
"丽芙·泰勒",
"约翰·贝西",
"凯特·布兰切特",
"比利·博伊德",
"萨德文·布罗菲",
"阿利斯泰尔·布朗宁",
"马尔顿·索克斯",
"伯纳德·希尔",
"伊安·霍姆",
"布鲁斯·霍普金斯",
"Ian Hughes",
"劳伦斯·马克奥雷",
"诺埃尔·阿普利比",
"布雷特·麦肯齐",
"Alexandra Astin",
"Sarah McLeod",
"Maisy McLeod-Riera",
"约翰·诺贝尔",
"Paul Norell",
"米兰达·奥图",
"布鲁斯·菲利普斯",
"沙恩·朗吉",
"约翰·瑞斯-戴维斯",
"Todd Rippon",
"安迪·瑟金斯",
"Harry Sinclair",
"乔尔·托贝克",
"卡尔·厄本",
"史蒂芬·乌瑞",
"雨果·维文",
"大卫·文翰",
"阿兰·霍华德",
"萨拉·贝克",
"Robert Pollock",
"佩特·史密斯",
"杰德·布罗菲",
"菲利普·格里夫",
"布拉德·道里夫",
"克里斯托弗·李",
"布鲁斯·斯宾斯",
"吉诺·阿赛维多",
"Jarl Benzon",
"Jørn Benzon",
"Robert Catto",
"Michael Elsworth",
"彼得·杰克逊",
"Sandro Kopp",
"安德鲁·莱斯尼",
"约瑟夫·米卡-亨特",
"亨利·莫腾森",
"克雷格·帕克",
"克里斯蒂安·瑞沃斯",
"迈克尔·斯曼内科 ",
"霍华德·肖",
"约翰·斯蒂芬森",
"理查德·泰勒",
"张磊",
"姜广涛",
"周志强",
"金永钢"
],
"is_watched": false
},
{
"rating": ["9.2", "50"],
"rank": 3,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p462657443.jpg",
"is_playable": true,
"id": "1851857",
"types": ["剧情", "动作", "科幻", "犯罪", "惊悚"],
"regions": ["美国", "英国"],
"title": "蝙蝠侠：黑暗骑士",
"url": "https://movie.douban.com/subject/1851857/",
"release_date": "2008-07-14",
"actor_count": 40,
"vote_count": 997680,
"score": "9.2",
"actors": [
"克里斯蒂安·贝尔",
"希斯·莱杰",
"艾伦·艾克哈特",
"迈克尔·凯恩",
"玛吉·吉伦哈尔",
"加里·奥德曼",
"摩根·弗里曼",
"莫尼克·加布里埃拉·库尔内",
"罗恩·迪恩",
"基里安·墨菲",
"黄经汉",
"内斯特·卡博内尔",
"埃里克·罗伯茨",
"里特奇·科斯特",
"安东尼·迈克尔·豪尔",
"基斯·斯扎拉巴基克",
"柯林·麦克法兰",
"约书亚·哈尔托",
"美琳达·麦格劳",
"内森·甘宝",
"迈克尔·维约",
"迈克尔·斯托扬诺夫",
"威廉·斯米利",
"丹尼·高德林",
"迈克尔·加·怀特",
"马修·奥尼尔",
"威廉·菲克纳",
"欧鲁米基·欧拉乌米",
"格雷格·比姆",
"爱利克·赫尔曼",
"毕崔斯·罗森",
"文森佐·尼克力",
"陈冠希",
"亚当·卡莱佩里斯",
"罗南·萨莫尔斯",
"玛丽扎·卡布蕾拉",
"Tina Simmons",
"大卫·达斯马齐连",
"利萨·麦克阿里斯特",
"理查德·迪兰"
],
"is_watched": false
},
{
"rating": ["9.2", "45"],
"rank": 4,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p2640236255.jpg",
"is_playable": true,
"id": "1291572",
"types": ["剧情", "动作", "奇幻", "冒险"],
"regions": ["美国", "新西兰"],
"title": "指环王2：双塔奇兵",
"url": "https://movie.douban.com/subject/1291572/",
"release_date": "2003-04-25",
"actor_count": 58,
"vote_count": 706830,
"score": "9.2",
"actors": [
"伊利亚·伍德",
"西恩·奥斯汀",
"多米尼克·莫纳汉",
"奥兰多·布鲁姆",
"维果·莫腾森",
"伊恩·麦克莱恩",
"凯特·布兰切特",
"比利·博伊德",
"杰德·布罗菲",
"丽芙·泰勒",
"安迪·瑟金斯",
"米兰达·奥图",
"雨果·维文",
"布拉德·道里夫",
"卡卢姆·吉廷斯",
"伯纳德·希尔",
"布鲁斯·霍普金斯",
"克里斯托弗·李",
"纳撒尼尔·利斯",
"罗宾·马尔科姆",
"约翰·贝西",
"萨拉·贝克",
"克雷格·帕克",
"布鲁斯·菲利普斯",
"Robert Pollock",
"约翰·瑞斯-戴维斯",
"奥利维亚·田內",
"卡尔·厄本",
"史蒂芬·乌瑞",
"大卫·文翰",
"菲利普·格里夫",
"肖恩·宾",
"约翰·诺贝尔",
"Jarl Benzon",
"Jørn Benzon",
"Ben Britton",
"阿利斯泰尔·布朗宁",
"Sean Button",
"Robert Catto",
"罗德尼·库克",
"卡尔洛斯·德林克沃特",
"西奥沙福瓦",
"本·弗兰舍姆",
"Jonathan Harding",
"Lucas Hayward",
"丹·汉纳",
"保罗·霍姆斯",
"杰森霍德",
"彼得·杰克逊",
"Sam Kelly",
"Sandro Kopp",
"姜广涛",
"张磊",
"陆建艺",
"金永钢",
"劳伦斯·马克奥雷",
"布鲁克·奥佩斯",
"周志强"
],
"is_watched": false
},
{
"rating": ["9.1", "45"],
"rank": 5,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p2373910134.jpg",
"is_playable": true,
"id": "1291571",
"types": ["剧情", "动作", "奇幻", "冒险"],
"regions": ["新西兰", "美国"],
"title": "指环王1：护戒使者",
"url": "https://movie.douban.com/subject/1291571/",
"release_date": "2002-04-04",
"actor_count": 54,
"vote_count": 797471,
"score": "9.1",
"actors": [
"伊利亚·伍德",
"西恩·奥斯汀",
"伊恩·麦克莱恩",
"维果·莫腾森",
"奥兰多·布鲁姆",
"多米尼克·莫纳汉",
"比利·博伊德",
"克里斯托弗·李",
"马尔顿·索克斯",
"梅根·爱德华兹",
"伊安·霍姆",
"凯特·布兰切特",
"阿兰·霍华德",
"马克·弗格森",
"肖恩·宾",
"萨拉·贝克",
"劳伦斯·马克奥雷",
"安迪·瑟金斯",
"彼得·麦肯齐",
"伊恩·穆内",
"克雷格·帕克",
"卡梅隆·罗德",
"约翰·瑞斯-戴维斯",
"丽芙·泰勒",
"大卫·韦瑟莱",
"雨果·维文",
"菲利普·格里夫",
"威廉·约翰逊",
"伊丽莎白·穆迪",
"布莱恩·瑟金特",
"杰德·布罗菲",
"诺曼·凯茨",
"兰德尔·威廉·库克",
"萨比恩·克洛森",
"西奥沙福瓦",
"本·弗兰舍姆",
"彼得·杰克逊",
"艾伦·李",
"诺埃尔·阿普利比",
"Michael Elsworth",
"Sarah McLeod",
"马丁·桑德森",
"Harry Sinclair",
"吉诺·阿赛维多",
"Jarl Benzon",
"Jørn Benzon",
"Ben Britton",
"Jonathan Harding",
"Taea Hartwell",
"Sam Kelly",
"姜广涛",
"张磊",
"周志强",
"布雷特·麦肯齐"
],
"is_watched": false
},
{
"rating": ["9.1", "45"],
"rank": 6,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p451926968.jpg",
"is_playable": false,
"id": "1291843",
"types": ["动作", "科幻"],
"regions": ["美国"],
"title": "黑客帝国",
"url": "https://movie.douban.com/subject/1291843/",
"release_date": "2000-01-14",
"actor_count": 25,
"vote_count": 791898,
"score": "9.1",
"actors": [
"基努·里维斯",
"劳伦斯·菲什伯恩",
"凯瑞-安·莫斯",
"雨果·维文",
"格洛丽亚·福斯特",
"乔·潘托里亚诺",
"马库斯·钟",
"朱利安·阿拉汗加",
"马特·多兰",
"贝琳达·麦克洛里",
"安东尼雷派克",
"罗伯特·泰勒",
"阿达·尼科德莫",
"罗温·维特",
"塔玛拉·布朗",
"纳塔莉·特珍",
"比尔·扬",
"克里斯·斯科特",
"纳许·埃哲顿",
"户田惠子",
"沈晓谦",
"童自荣",
"保罗·戈达德",
"金永钢",
"张磊"
],
"is_watched": false
},
{
"rating": ["9.3", "50"],
"rank": 7,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p2215886505.jpg",
"is_playable": true,
"id": "1295399",
"types": ["动作", "冒险", "剧情"],
"regions": ["日本"],
"title": "七武士",
"url": "https://movie.douban.com/subject/1295399/",
"release_date": "1954-04-26",
"actor_count": 48,
"vote_count": 184460,
"score": "9.3",
"actors": [
"三船敏郎",
"志村乔",
"津岛惠子",
"岛崎雪子",
"藤原釜足",
"加东大介",
"木村功",
"千秋实",
"宫口精二",
"小杉义男",
"左卜全",
"稻叶义男",
"土屋嘉男",
"高堂国典",
"东野英治郎",
"上田吉二郎",
"多多良纯",
"渡边笃",
"小川虎之助",
"山形勋",
"上山草人",
"清水元",
"高木新平",
"大友伸",
"高原骏雄",
"稻垣三郎",
"堺左千夫",
"千石规子",
"本间文子",
"大久保正信",
"伊藤实",
"大村千吉",
"广濑正一",
"宇野晃司",
"谷晃",
"中岛春雄",
"清水美恵",
"熊谷卓三",
"夏木顺平",
"岩本弘司",
"堤康久",
"马野都留子",
"森今日子",
"加藤武",
"仲代达矢",
"宇津井健",
"山本廉",
"东静子"
],
"is_watched": false
},
{
"rating": ["9.0", "45"],
"rank": 8,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p1512562287.jpg",
"is_playable": true,
"id": "3742360",
"types": ["剧情", "喜剧", "动作", "西部"],
"regions": ["中国大陆", "中国香港"],
"title": "让子弹飞",
"url": "https://movie.douban.com/subject/3742360/",
"release_date": "2010-12-16",
"actor_count": 32,
"vote_count": 1592874,
"score": "9.0",
"actors": [
"姜文",
"葛优",
"周润发",
"刘嘉玲",
"陈坤",
"张默",
"姜武",
"周韵",
"廖凡",
"姚橹",
"邵兵",
"苗圃",
"冯小刚",
"胡军",
"马珂",
"白冰",
"杜奕衡",
"李静",
"胡明",
"危笑",
"杨奇雨",
"赵铭",
"宣晓鸣",
"绀野千春",
"郭俊立",
"马元",
"姜洪齐",
"岳红",
"吴昔果",
"沙瑀",
"陈磊",
"文祥"
],
"is_watched": false
},
{
"rating": ["9.0", "45"],
"rank": 9,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p1910926158.jpg",
"is_playable": true,
"id": "1292000",
"types": ["剧情", "动作", "悬疑", "惊悚"],
"regions": ["美国", "德国", "意大利"],
"title": "搏击俱乐部",
"url": "https://movie.douban.com/subject/1292000/",
"release_date": "1999-09-10",
"actor_count": 30,
"vote_count": 800974,
"score": "9.0",
"actors": [
"爱德华·诺顿",
"布拉德·皮特",
"海伦娜·伯翰·卡特",
"扎克·格雷尼尔",
"米特·洛夫",
"杰瑞德·莱托",
"艾恩·贝利",
"里奇蒙德·阿奎特",
"乔治·马奎尔",
"Ezra Buzzington",
"Carl Ciarfalio",
"斯图尔特·布拉姆博格",
"马特·温斯顿",
"劳伦·桑切斯",
"Tyrone R. Livingston",
"保罗·卡拉夫特斯",
"Scotch Ellis Loring",
"Andi Carnick",
"迈克尔·肖姆斯·维尔斯",
"Markus Redmond",
"大卫·安德鲁斯",
"鲍勃·斯蒂芬森",
"欧根妮·邦杜兰特",
"凯文·斯科特·麦克",
"Bennie Moore",
"埃文·米兰德",
"乔伊·比索奈特",
"Christina Cabot",
"大卫·李·史密斯",
"霍特·麦克卡兰尼"
],
"is_watched": false
},
{
"rating": ["8.9", "45"],
"rank": 10,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p1633113220.jpg",
"is_playable": false,
"id": "1300299",
"types": ["剧情", "动作", "犯罪", "悬疑", "惊悚"],
"regions": ["韩国"],
"title": "杀人回忆",
"url": "https://movie.douban.com/subject/1300299/",
"release_date": "2003-05-02",
"actor_count": 49,
"vote_count": 661779,
"score": "8.9",
"actors": [
"宋康昊",
"金相庆",
"金雷夏",
"宋在浩",
"边希峰",
"高瑞熙",
"柳泰浩",
"朴努植",
"朴海日",
"全美善",
"徐永嬅",
"崔钟律",
"刘承睦",
"申贤宗",
"李在应",
"郑仁仙",
"吴龙",
"朴真宇",
"朴泰京",
"沈成宝",
"朴镇宇",
"廉惠兰",
"李东勇",
"赵德济",
"申文成",
"孙康国",
"李大贤",
"李玉珠",
"刘仁秀",
"千明宰",
"郭秀贞",
"曹文义",
"朴贤英",
"申贤胜",
"權炳吉",
"金周灵",
"金荷景",
"金熙珍",
"崔铉基",
"李江山",
"孙镇浩",
"白奉基",
"刘琴",
"禹高娜",
"申云燮",
"金景来",
"李勋京",
"李多一",
"金景来"
],
"is_watched": false
},
{
"rating": ["8.9", "45"],
"rank": 11,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p2004174709.jpg",
"is_playable": true,
"id": "1294639",
"types": ["动作", "传记", "剧情", "历史", "战争"],
"regions": ["美国"],
"title": "勇敢的心",
"url": "https://movie.douban.com/subject/1294639/",
"release_date": "1995-05-18",
"actor_count": 24,
"vote_count": 537785,
"score": "8.9",
"actors": [
"梅尔·吉布森",
"苏菲·玛索",
"布莱恩·考克斯",
"詹姆斯·科兹莫",
"辛·劳洛",
"凯瑟琳·麦克马克",
"安古斯·麦克菲登",
"布莱丹·格里森",
"詹姆斯·霍纳",
"彼得·穆兰",
"汤米·弗拉纳根",
"艾伦·阿姆斯特朗",
"大卫·奥哈拉",
"帕特里克·麦高汉",
"伊安·邦纳",
"梅丽·卡尔维",
"史蒂芬·比灵顿",
"迈克尔·伯恩",
"亚力克斯·诺顿",
"杰拉德·迈克索利",
"詹姆斯·鲁宾逊",
"辛·麦金利",
"Donal Gibson",
"鲁珀特·范西塔特"
],
"is_watched": false
},
{
"rating": ["8.8", "45"],
"rank": 12,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p2219011938.jpg",
"is_playable": true,
"id": "1291543",
"types": ["动作", "喜剧", "犯罪", "奇幻"],
"regions": ["中国大陆", "中国香港"],
"title": "功夫",
"url": "https://movie.douban.com/subject/1291543/",
"release_date": "2004-12-23",
"actor_count": 33,
"vote_count": 1032748,
"score": "8.8",
"actors": [
"周星驰",
"元秋",
"元华",
"黄圣依",
"梁小龙",
"陈国坤",
"田启文",
"林子聪",
"林雪",
"冯克安",
"释彦能",
"冯小刚",
"袁祥仁",
"张一白",
"赵志凌",
"董志华",
"何文辉",
"陈凯师",
"贾康熙",
"林子善",
"任珈锐",
"王仕颖",
"冯勉恒",
"范湉湉",
"杨能",
"黄锐民",
"姜瑰瑾",
"丁小龙",
"张明明",
"钱升玮",
"林志泰",
"阮昊天",
"程守一"
],
"is_watched": false
},
{
"rating": ["8.8", "45"],
"rank": 13,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p2634997853.jpg",
"is_playable": true,
"id": "1652587",
"types": ["动作", "科幻", "冒险"],
"regions": ["美国"],
"title": "阿凡达",
"url": "https://movie.douban.com/subject/1652587/",
"release_date": "2010-01-04",
"actor_count": 30,
"vote_count": 1304529,
"score": "8.8",
"actors": [
"萨姆·沃辛顿",
"佐伊·索尔达娜",
"西格妮·韦弗",
"史蒂芬·朗",
"米歇尔·罗德里格兹",
"吉奥瓦尼·瑞比西",
"乔·大卫·摩尔 ",
"希·庞德",
"韦斯·斯塔迪",
"拉兹·阿隆索",
"迪利普·劳",
"马特·杰拉德",
"肖恩·安东尼·莫兰",
"季冠霖",
"詹姆斯·霍纳",
"彼得·门萨",
"泰瑞·诺塔里",
"陆建艺",
"凯文·多曼",
"黛布拉·威尔逊",
"斯科特·劳伦斯",
"詹姆斯·皮特",
"雅克布·托毛里",
"凯尔森·亨德森",
"伍迪·舒尔茨",
"彼得·狄龙",
"肖恩·帕特里克·墨菲",
"大卫·范·霍恩",
"凯利·基尔戈",
"杰森·怀特"
],
"is_watched": false
},
{
"rating": ["9.1", "45"],
"rank": 14,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p2446408045.jpg",
"is_playable": true,
"id": "1291936",
"types": ["动作", "科幻", "动画"],
"regions": ["日本"],
"title": "攻壳机动队",
"url": "https://movie.douban.com/subject/1291936/",
"release_date": "1995-11-18",
"actor_count": 16,
"vote_count": 130822,
"score": "9.1",
"actors": [
"田中敦子",
"大塚明夫",
"山寺宏一",
"大木民夫",
"家弓家正",
"玄田哲章",
"山内雅人",
"小川真司",
"宫本充",
"小高三良",
"后藤敦",
"山路和弘",
"佐藤政道",
"Lia Sargent",
"布莱恩·科兰斯顿",
"松山鹰志"
],
"is_watched": false
},
{
"rating": ["8.8", "45"],
"rank": 15,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p1596085504.jpg",
"is_playable": true,
"id": "1298070",
"types": ["动作", "冒险", "奇幻"],
"regions": ["美国"],
"title": "加勒比海盗",
"url": "https://movie.douban.com/subject/1298070/",
"release_date": "2003-11-21",
"actor_count": 34,
"vote_count": 811264,
"score": "8.8",
"actors": [
"约翰尼·德普",
"杰弗里·拉什",
"奥兰多·布鲁姆",
"凯拉·奈特莉",
"杰克·达文波特",
"乔纳森·普雷斯",
"李·阿伦伯格",
"大卫·拜利",
"马丁·科勒巴",
"乔迪·卡巴勒罗",
"劳伦·马赫",
"安格斯·巴尼特",
"露欣达·戴泽克",
"陆建艺",
"Giles New",
"小伊萨克·C.辛格尔顿",
"麦肯锡·克鲁克",
"Christian Martin",
"凯文·麦克纳利",
"瓦妮莎·布兰奇",
"内杰·亚当森",
"戴兰·史密斯",
"盖伊·西纳",
"达米恩·奥哈尔",
"David Patykewich",
"保罗·基斯",
"罗比·基",
"卢克·德·沃夫松",
"佐伊·索尔达娜",
"乔·格里萨菲",
"格雷格·埃利斯",
"特雷瓦·埃迪内",
"特雷沃·格德达德",
"杰拉德·J·雷耶斯"
],
"is_watched": false
},
{
"rating": ["8.8", "45"],
"rank": 16,
"cover_url": "https://img9.doubanio.com/view/photo/s_ratio_poster/public/p2597918718.jpg",
"is_playable": true,
"id": "1295409",
"types": ["剧情", "喜剧", "动作", "犯罪"],
"regions": ["中国香港"],
"title": "纵横四海",
"url": "https://movie.douban.com/subject/1295409/",
"release_date": "1991-02-02",
"actor_count": 11,
"vote_count": 395931,
"score": "8.8",
...
Would replacing urllib.parse.urlencode(data) with urllib.parse.urlencode(data).encode(‘utf-8’) in the create_request function have helped here as well?