How to get a json response with scrapy?
Question:
I am trying to replicate this same block of code with scrapy’s default request library as it faster and much more efficient. I am trying to retrieve data from the steamstore api:
import requests
import json
url = "http://store.steampowered.com/api/appdetails/"
appid = 10
name = "Counter-Strike"
parameters = {"appids": appid}
response = requests.get(url=url, params=parameters)
json_data = response.json()
json_app_data = json_data[str(appid)]
if json_app_data['success']:
data = json_app_data['data']
else:
data = {'name': name, 'steam_appid': appid}
print(data)
Answers:
Actually, scrapy didn’t know what’s params? meaning it can’t take params as parameter.So alternatively, you can do that is to inject the entire api url in scrapy Request method.
Example:
import scrapy
import json
from scrapy.crawler import CrawlerProcess
class TestSpider(scrapy.Spider):
name = 'test'
def start_requests(self):
headers= {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36'}
api_url='https://store.steampowered.com/api/appdetails/?appids=10'
yield scrapy.Request(
url= api_url,
method='GET',
headers=headers,
callback=self.parse
)
def parse(self, response):
json_app_data = json.loads(response.body)
print(json_app_data['10']['data'])
if __name__ == "__main__":
process = CrawlerProcess(TestSpider)
process.crawl()
process.start()
Output:
{'type': 'game', 'name': 'Counter-Strike', 'steam_appid': 10, 'required_age': 0, 'is_free': False, 'detailed_description': "Play the world's number
1 online action game. Engage in an incredibly realistic brand of terrorist warfare in this wildly popular team-based game. Ally with teammates to complete strategic missions. Take out enemy sites. Rescue hostages. Your role affects your team's success. Your team's success affects your role.", 'about_the_game': "Play the world's number 1 online action game. Engage in an incredibly realistic brand of terrorist warfare in this wildly popular team-based game. Ally with teammates to complete strategic missions. Take out enemy sites. Rescue hostages. Your role affects your team's success. Your team's success affects your role.", 'short_description': "Play the world's number 1 online action game. Engage in an incredibly realistic brand of terrorist warfare in this wildly popular team-based game. Ally with teammates to complete strategic missions. Take out enemy sites. Rescue hostages. Your role affects your team's success. Your team's success affects your role.", 'supported_languages': 'English<strong>*</strong>, French<strong>*</strong>, German<strong>*</strong>, Italian<strong>*</strong>, Spanish - Spain<strong>*</strong>, Simplified Chinese<strong>*</strong>, Traditional Chinese<strong>*</strong>, Korean<strong>*</strong><br><strong>*</strong>languages with full audio support', 'header_image': 'https://cdn.akamai.steamstatic.com/steam/apps/10/header.jpg?t=1602535893', 'website': None, 'pc_requirements': {'minimum': 'rnttt<p><strong>Minimum:</strong> 500 mhz processor, 96mb ram, 16mb video card, Windows XP, Mouse, Keyboard, Internet Connection<br /></p>rnttt<p><strong>Recommended:</strong> 800 mhz processor, 128mb ram, 32mb+ video card, Windows XP, Mouse, Keyboard, Internet Connection<br /></p>rnttt'}, 'mac_requirements': {'minimum': 'Minimum: OS X Snow Leopard 10.6.3, 1GB RAM, 4GB Hard Drive Space,NVIDIA GeForce 8 or higher, ATI X1600 or higher, or Intel HD 3000 or higher Mouse,
Keyboard, Internet Connection'}, 'linux_requirements': {'minimum': 'Minimum: Linux Ubuntu 12.04, Dual-core from Intel or AMD at 2.8 GHz, 1GB Memory, nVidia GeForce 8600/9600GT, ATI/AMD Radeaon HD2600/3600 (Graphic Drivers: nVidia 310, AMD 12.11), OpenGL 2.1, 4GB Hard Drive Space, OpenAL Compatible Sound Card'}, 'developers': ['Valve'], 'publishers': ['Valve'], 'price_overview': {'currency': 'USD', 'initial': 549, 'final': 549, 'discount_percent': 0, 'initial_formatted': '', 'final_formatted': '$5.49 USD'}, 'packages': [574941, 7], 'package_groups': [{'name': 'default', 'title': 'Buy Counter-Strike', 'description': '', 'selection_text': 'Select a purchase option', 'save_text': '', 'display_type': 0, 'is_recurring_subscription': 'false', 'subs': [{'packageid': 7, 'percent_savings_text': ' ', 'percent_savings': 0, 'option_text': 'Counter-Strike: Condition Zero - $5.49 USD', 'option_description': '', 'can_get_free_license': '0', 'is_free_license': False, 'price_in_cents_with_discount': 549}, {'packageid': 574941, 'percent_savings_text': ' ', 'percent_savings': 0, 'option_text': 'Counter-Strike - Commercial License - $5.49 USD', 'option_description': '', 'can_get_free_license': '0', 'is_free_license': False, 'price_in_cents_with_discount': 549}]}], 'platforms': {'windows': True, 'mac': True, 'linux': True}, 'metacritic': {'score': 88, 'url': 'https://www.metacritic.com/game/pc/counter-strike?ftag=MCD-06-10aaa1f'}, 'categories': [{'id': 1, 'description': 'Multi-player'}, {'id': 49, 'description': 'PvP'}, {'id': 36, 'description': 'Online PvP'}, {'id': 37, 'description': 'Shared/Split Screen PvP'}, {'id': 8, 'description': 'Valve Anti-Cheat enabled'}], 'genres': [{'id': '1', 'description': 'Action'}], 'screenshots': [{'id': 0, 'path_thumbnail': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000000132.600x338.jpg?t=1602535893', 'path_full': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000000132.1920x1080.jpg?t=1602535893'}, {'id': 1, 'path_thumbnail': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000000133.600x338.jpg?t=1602535893', 'path_full': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000000133.1920x1080.jpg?t=1602535893'}, {'id': 2, 'path_thumbnail': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000000134.600x338.jpg?t=1602535893', 'path_full': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000000134.1920x1080.jpg?t=1602535893'}, {'id': 3, 'path_thumbnail': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000000135.600x338.jpg?t=1602535893',
'path_full': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000000135.1920x1080.jpg?t=1602535893'}, {'id': 4, 'path_thumbnail': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000000136.600x338.jpg?t=1602535893', 'path_full': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000000136.1920x1080.jpg?t=1602535893'}, {'id': 5, 'path_thumbnail': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002540.600x338.jpg?t=1602535893', 'path_full': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002540.1920x1080.jpg?t=1602535893'}, {'id': 6, 'path_thumbnail': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002539.600x338.jpg?t=1602535893', 'path_full': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002539.1920x1080.jpg?t=1602535893'}, {'id': 7, 'path_thumbnail': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002538.600x338.jpg?t=1602535893', 'path_full': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002538.1920x1080.jpg?t=1602535893'}, {'id': 8, 'path_thumbnail': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002537.600x338.jpg?t=1602535893', 'path_full': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002537.1920x1080.jpg?t=1602535893'}, {'id': 9, 'path_thumbnail': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002536.600x338.jpg?t=1602535893', 'path_full': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002536.1920x1080.jpg?t=1602535893'}, {'id': 10, 'path_thumbnail': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002541.600x338.jpg?t=1602535893', 'path_full':
'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002541.1920x1080.jpg?t=1602535893'}, {'id': 11, 'path_thumbnail': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002542.600x338.jpg?t=1602535893', 'path_full': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002542.1920x1080.jpg?t=1602535893'}, {'id': 12, 'path_thumbnail': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002543.600x338.jpg?t=1602535893', 'path_full': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002543.1920x1080.jpg?t=1602535893'}], 'recommendations': {'total': 122748}, 'release_date': {'coming_soon': False, 'date': '1 Nov, 2000'}, 'support_info': {'url': 'http://steamcommunity.com/app/10', 'email': ''}, 'background': 'https://cdn.akamai.steamstatic.com/steam/apps/10/page_bg_generated_v6b.jpg?t=1602535893',
'background_raw': 'https://cdn.akamai.steamstatic.com/steam/apps/10/page_bg_generated.jpg?t=1602535893', 'content_descriptors': {'ids': [2, 5], 'notes': 'Includes intense violence and blood.'}}
2022-07-26 18:32:01 [scrapy.core.engine] INFO: Closing spider (finished)
2022-07-26 18:32:01 [scrapy.statscollectors] INFO: Dumping Scrapy stats:
{'downloader/request_bytes': 324,
'downloader/request_count': 1,
'downloader/request_method_count/GET': 1,
'downloader/response_bytes': 2192,
'downloader/response_count': 1,
'downloader/response_status_count/200': 1
I am trying to replicate this same block of code with scrapy’s default request library as it faster and much more efficient. I am trying to retrieve data from the steamstore api:
import requests
import json
url = "http://store.steampowered.com/api/appdetails/"
appid = 10
name = "Counter-Strike"
parameters = {"appids": appid}
response = requests.get(url=url, params=parameters)
json_data = response.json()
json_app_data = json_data[str(appid)]
if json_app_data['success']:
data = json_app_data['data']
else:
data = {'name': name, 'steam_appid': appid}
print(data)
Actually, scrapy didn’t know what’s params? meaning it can’t take params as parameter.So alternatively, you can do that is to inject the entire api url in scrapy Request method.
Example:
import scrapy
import json
from scrapy.crawler import CrawlerProcess
class TestSpider(scrapy.Spider):
name = 'test'
def start_requests(self):
headers= {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36'}
api_url='https://store.steampowered.com/api/appdetails/?appids=10'
yield scrapy.Request(
url= api_url,
method='GET',
headers=headers,
callback=self.parse
)
def parse(self, response):
json_app_data = json.loads(response.body)
print(json_app_data['10']['data'])
if __name__ == "__main__":
process = CrawlerProcess(TestSpider)
process.crawl()
process.start()
Output:
{'type': 'game', 'name': 'Counter-Strike', 'steam_appid': 10, 'required_age': 0, 'is_free': False, 'detailed_description': "Play the world's number
1 online action game. Engage in an incredibly realistic brand of terrorist warfare in this wildly popular team-based game. Ally with teammates to complete strategic missions. Take out enemy sites. Rescue hostages. Your role affects your team's success. Your team's success affects your role.", 'about_the_game': "Play the world's number 1 online action game. Engage in an incredibly realistic brand of terrorist warfare in this wildly popular team-based game. Ally with teammates to complete strategic missions. Take out enemy sites. Rescue hostages. Your role affects your team's success. Your team's success affects your role.", 'short_description': "Play the world's number 1 online action game. Engage in an incredibly realistic brand of terrorist warfare in this wildly popular team-based game. Ally with teammates to complete strategic missions. Take out enemy sites. Rescue hostages. Your role affects your team's success. Your team's success affects your role.", 'supported_languages': 'English<strong>*</strong>, French<strong>*</strong>, German<strong>*</strong>, Italian<strong>*</strong>, Spanish - Spain<strong>*</strong>, Simplified Chinese<strong>*</strong>, Traditional Chinese<strong>*</strong>, Korean<strong>*</strong><br><strong>*</strong>languages with full audio support', 'header_image': 'https://cdn.akamai.steamstatic.com/steam/apps/10/header.jpg?t=1602535893', 'website': None, 'pc_requirements': {'minimum': 'rnttt<p><strong>Minimum:</strong> 500 mhz processor, 96mb ram, 16mb video card, Windows XP, Mouse, Keyboard, Internet Connection<br /></p>rnttt<p><strong>Recommended:</strong> 800 mhz processor, 128mb ram, 32mb+ video card, Windows XP, Mouse, Keyboard, Internet Connection<br /></p>rnttt'}, 'mac_requirements': {'minimum': 'Minimum: OS X Snow Leopard 10.6.3, 1GB RAM, 4GB Hard Drive Space,NVIDIA GeForce 8 or higher, ATI X1600 or higher, or Intel HD 3000 or higher Mouse,
Keyboard, Internet Connection'}, 'linux_requirements': {'minimum': 'Minimum: Linux Ubuntu 12.04, Dual-core from Intel or AMD at 2.8 GHz, 1GB Memory, nVidia GeForce 8600/9600GT, ATI/AMD Radeaon HD2600/3600 (Graphic Drivers: nVidia 310, AMD 12.11), OpenGL 2.1, 4GB Hard Drive Space, OpenAL Compatible Sound Card'}, 'developers': ['Valve'], 'publishers': ['Valve'], 'price_overview': {'currency': 'USD', 'initial': 549, 'final': 549, 'discount_percent': 0, 'initial_formatted': '', 'final_formatted': '$5.49 USD'}, 'packages': [574941, 7], 'package_groups': [{'name': 'default', 'title': 'Buy Counter-Strike', 'description': '', 'selection_text': 'Select a purchase option', 'save_text': '', 'display_type': 0, 'is_recurring_subscription': 'false', 'subs': [{'packageid': 7, 'percent_savings_text': ' ', 'percent_savings': 0, 'option_text': 'Counter-Strike: Condition Zero - $5.49 USD', 'option_description': '', 'can_get_free_license': '0', 'is_free_license': False, 'price_in_cents_with_discount': 549}, {'packageid': 574941, 'percent_savings_text': ' ', 'percent_savings': 0, 'option_text': 'Counter-Strike - Commercial License - $5.49 USD', 'option_description': '', 'can_get_free_license': '0', 'is_free_license': False, 'price_in_cents_with_discount': 549}]}], 'platforms': {'windows': True, 'mac': True, 'linux': True}, 'metacritic': {'score': 88, 'url': 'https://www.metacritic.com/game/pc/counter-strike?ftag=MCD-06-10aaa1f'}, 'categories': [{'id': 1, 'description': 'Multi-player'}, {'id': 49, 'description': 'PvP'}, {'id': 36, 'description': 'Online PvP'}, {'id': 37, 'description': 'Shared/Split Screen PvP'}, {'id': 8, 'description': 'Valve Anti-Cheat enabled'}], 'genres': [{'id': '1', 'description': 'Action'}], 'screenshots': [{'id': 0, 'path_thumbnail': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000000132.600x338.jpg?t=1602535893', 'path_full': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000000132.1920x1080.jpg?t=1602535893'}, {'id': 1, 'path_thumbnail': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000000133.600x338.jpg?t=1602535893', 'path_full': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000000133.1920x1080.jpg?t=1602535893'}, {'id': 2, 'path_thumbnail': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000000134.600x338.jpg?t=1602535893', 'path_full': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000000134.1920x1080.jpg?t=1602535893'}, {'id': 3, 'path_thumbnail': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000000135.600x338.jpg?t=1602535893',
'path_full': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000000135.1920x1080.jpg?t=1602535893'}, {'id': 4, 'path_thumbnail': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000000136.600x338.jpg?t=1602535893', 'path_full': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000000136.1920x1080.jpg?t=1602535893'}, {'id': 5, 'path_thumbnail': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002540.600x338.jpg?t=1602535893', 'path_full': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002540.1920x1080.jpg?t=1602535893'}, {'id': 6, 'path_thumbnail': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002539.600x338.jpg?t=1602535893', 'path_full': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002539.1920x1080.jpg?t=1602535893'}, {'id': 7, 'path_thumbnail': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002538.600x338.jpg?t=1602535893', 'path_full': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002538.1920x1080.jpg?t=1602535893'}, {'id': 8, 'path_thumbnail': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002537.600x338.jpg?t=1602535893', 'path_full': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002537.1920x1080.jpg?t=1602535893'}, {'id': 9, 'path_thumbnail': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002536.600x338.jpg?t=1602535893', 'path_full': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002536.1920x1080.jpg?t=1602535893'}, {'id': 10, 'path_thumbnail': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002541.600x338.jpg?t=1602535893', 'path_full':
'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002541.1920x1080.jpg?t=1602535893'}, {'id': 11, 'path_thumbnail': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002542.600x338.jpg?t=1602535893', 'path_full': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002542.1920x1080.jpg?t=1602535893'}, {'id': 12, 'path_thumbnail': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002543.600x338.jpg?t=1602535893', 'path_full': 'https://cdn.akamai.steamstatic.com/steam/apps/10/0000002543.1920x1080.jpg?t=1602535893'}], 'recommendations': {'total': 122748}, 'release_date': {'coming_soon': False, 'date': '1 Nov, 2000'}, 'support_info': {'url': 'http://steamcommunity.com/app/10', 'email': ''}, 'background': 'https://cdn.akamai.steamstatic.com/steam/apps/10/page_bg_generated_v6b.jpg?t=1602535893',
'background_raw': 'https://cdn.akamai.steamstatic.com/steam/apps/10/page_bg_generated.jpg?t=1602535893', 'content_descriptors': {'ids': [2, 5], 'notes': 'Includes intense violence and blood.'}}
2022-07-26 18:32:01 [scrapy.core.engine] INFO: Closing spider (finished)
2022-07-26 18:32:01 [scrapy.statscollectors] INFO: Dumping Scrapy stats:
{'downloader/request_bytes': 324,
'downloader/request_count': 1,
'downloader/request_method_count/GET': 1,
'downloader/response_bytes': 2192,
'downloader/response_count': 1,
'downloader/response_status_count/200': 1