Scrapy download Images and rename the image as md5 hash
Question:
I have a Scrapy spider which is working as far as scraping is concerned but I am having issues during downloading the images. I want to download the images and rename them as md5 hash for example: c69/96d/f0d/c6996df0d9d852f1f39fcb7074ace625.jpg also I’d like to add the md5 name of the image to my JSON output as well. I tried converting the image urls to md5 hash in the spider and then renaming the image file in the HouzzImagePipeline by accessing the item['path'] but it is not downloading the images. Here is the spider:
import scrapy
import json
import hashlib
def make_path(urls):
img_path = []
for url in urls:
image_url_hash = hashlib.md5(url.encode()).hexdigest()
img_path.append(
image_url_hash[:3]
+ "/"
+ image_url_hash[3:6]
+ "/"
+ image_url_hash[6:9]
+ "/"
+ image_url_hash
)
return img_path
class HouzzSimilar(scrapy.Spider):
name = "houzz_crawler"
custom_settings = {
"LOG_FILE": "houzz_spider.log",
"IMAGES_STORE": "houzz_images",
"FEEDS": {
"houzz.json": {
"format": "json",
}
},
"ITEM_PIPELINES": {
"houzz_crawler.pipelines.HouzzImagePipeline": 1,
},
}
headers = {
"authority": "www.houzz.com",
"accept": "*/*",
"accept-language": "en,ru;q=0.9",
"content-type": "application/x-www-form-urlencoded; charset=UTF-8",
"origin": "https://www.houzz.com",
"referer": "https://www.houzz.com/photos/columbus-ave-residence-contemporary-bathroom-new-york-phvw-vp~160668148",
"rrid": "70402547-c900-47f7-a913-8e1cbc9aa0c3",
"sec-ch-ua": '"Chromium";v="110", "Not A(Brand";v="24", "YaBrowser";v="23"',
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": '"Linux"',
"sec-fetch-dest": "empty",
"sec-fetch-mode": "cors",
"sec-fetch-site": "same-origin",
"user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 YaBrowser/23.3.1.906 (beta) Yowser/2.5 Safari/537.36",
"x-csrf-token": "i8B5ykgX-eprPj5yAHSxOng08Pa4qAr2Z0TQ",
"x-hz-request": "true",
"x-ol-exp-id": "clhhdi4wu00003y71rnvty395",
"x-ol-exp-name": "Photo - View",
"x-ol-ext-device-id": "23a3cfb8-7a04-4462-af71-d98689271533",
"x-ol-ext-session-id": "782c0a90-8925-409f-90c1-f47798e0426e",
"x-ol-product": "Houzz",
"x-ol-product-variant": "Houzz US",
"x-ol-session-id": "782c0a90-8925-409f-90c1-f47798e0426e",
"x-requested-with": "XMLHttpRequest",
}
cookies = {
"v": "1683311076_f9d9a715-f45b-42dc-bc6d-7da75774a57f_9bda9dd500ca1e5119bbecaba51e53f0",
"vct": "en-US-vxnkSVVkSBzkSVVkCR%2FkSVVk8B%2FkSVVk4R3kSVVk4h3kSVVk",
"_gcl_au": "1.1.17413922.1683311086",
"crossdevicetracking": "915374c0-439c-46a1-bbf2-3a2aaa487e69",
"_pin_unauth": "dWlkPU16Y3dNbVF6T0dNdE1tWTBOaTAwWTJSa0xUazVZakV0TXprek5XWm1ZV014WWprMw",
"_sp_id.c905": "5af74097-a6bb-46e7-8d14-35ff6d738f39.1683317411.2.1683359810.1683317411.13ad94c9-5560-4fbf-963f-b63e32f2124d",
"g_state": '{"i_p":1684144918349,"i_l":3}',
"browseResultSetGridWidth": "554",
"_gid": "GA1.2.1176067560.1683652076",
"ln_or": "eyIzODE1NzE2IjoiZCJ9",
"_csrf": "G_nV-Kaa7rlqgTwnueAXkJtj",
"jdv": "t7WOzUb2vHLZtWVVHSk%2BXJEWN7ua9zR%2FUkXpY9RYDUW00hxMyur5c%2Bzn6M%2BqQADtWOInJpmlQA37Gxp0L267jdj74Iwe",
"documentWidth": "1318",
"_uetsid": "0bf41840ee8c11edac06995ca98afa3c",
"_uetvid": "1e07d960eb7211ed880b7db3cdc86191",
"_derived_epik": "dj0yJnU9NFBDc3RuOExta3NiM2xfaV9WS0RYbVVLRS1lRVpycDEmbj1tVE1RRUtOUjYwYU1Kalp0el9mNTBBJm09OCZ0PUFBQUFBR1JiUmprJnJtPTgmcnQ9QUFBQUFHUmJSamsmc3A9NQ",
"IR_gbd": "houzz.com",
"IR_5454": "1683703358356%7C0%7C1683703358356%7C%7C",
"_ga": "GA1.2.1658927820.1683311086",
"_dc_gtm_UA-3519678-1": "1",
"_ga_PB0RC2CT7B": "GS1.1.1683703353.11.1.1683704001.59.0.0",
"hzd": "70402547-c900-47f7-a913-8e1cbc9aa0c3%3A%3A%3A%3A%3ASeeMoreIdeas",
}
base_url = "https://www.houzz.com/photos/home-design-ideas-phbr0-bp~"
similar_ideas_api_url = "https://www.houzz.com/j/getSimilarSpaces"
def start_requests(self):
yield scrapy.Request(
url=self.base_url, headers=self.headers, callback=self.parse_ideas
)
def parse_ideas(self, response):
ideas = response.css("a.hz-photo-card__ratio-box::attr(href)").extract()
for idea in ideas:
yield scrapy.Request(
url=idea, headers=self.headers, callback=self.parse_project_url
)
def parse_project_url(self, response):
data = response.css('script[id="hz-ctx"] ::text').get()
json_data = json.loads(data)
space_id = json_data["data"]["pageContentData"]["spaceId"]
space = json_data["data"]["stores"]["data"]["SpaceStore"]["data"][space_id]
project_id = space["projectId"]
space_url = space["url"]
raw_project_url = (
space_url.split("~")[0].replace("phvw", "pj").replace("vp", "vj")
)
project_url = raw_project_url + "~" + str(project_id)
yield scrapy.Request(
url=project_url, headers=self.headers, callback=self.parse_project_idea
)
def parse_project_idea(self, response):
idea_board = response.css(
"div.hz-prj-container.hz-prj-container__photos.clearfix ::attr(href)"
).extract()
for idea_link in idea_board:
yield scrapy.Request(
url=idea_link,
headers=self.headers,
callback=self.parse_idea_details,
)
def parse_idea_details(self, response):
item = {}
item["ideadId"] = response.url.split("~")[-1]
item["ideaUrl"] = response.url
item["Title"] = response.css(
"h1.hz-view-photo__space-info__title.text-bold::text"
).get()
item["imageURL"] = response.css(
"div.view-photo-image-pane > img::attr(src)"
).extract()
item["image_urls"] = item["imageURL"].copy()
item["similarIdeas"] = []
item["path"] = make_path(
response.css("div.view-photo-image-pane > img::attr(src)").extract()
)
spaceId = response.url.split("~")[-1]
body = f"spaceId={spaceId}&fromItem=0&itemsPerPage=10&contentDescriptor=%7B%22t%22%3A1%2C%22et%22%3A3%2C%22id%22%3A160668148%7D"
yield scrapy.Request(
url=self.similar_ideas_api_url,
method="POST",
cookies=self.cookies,
headers=self.headers,
body=body,
cb_kwargs={"item": item},
callback=self.get_similar_ideas_urls,
)
def get_similar_ideas_urls(self, response, item=None):
data = response.json()["spaceData"]["spaces"]
space_keys = list(data.keys())
space_urls = set([data[key]["url"] for key in space_keys])
yield scrapy.Request(
url=space_urls.pop(),
headers=self.headers,
cb_kwargs={"item": item, "space_urls": space_urls},
callback=self.parse_similar_ideas,
)
def parse_similar_ideas(self, response, item=None, space_urls=None):
# add the image urls to the top master list as well as locally.
image_urls = response.css(
"div.view-photo-image-pane > img::attr(src)"
).extract()
item["image_urls"] += image_urls
item["similarIdeas"].append(
{
"ideaId": response.url.split("~")[-1],
"ideaUrl": response.url,
"Title": response.css(
"h1.hz-view-photo__space-info__title.text-bold::text"
).get(),
"image_urls": image_urls,
"path": make_path(image_urls),
}
)
if len(space_urls) > 0:
yield scrapy.Request(
url=space_urls.pop(),
headers=self.headers,
cb_kwargs={"item": item, "space_urls": space_urls},
dont_filter=True,
callback=self.parse_similar_ideas,
)
else:
yield item
ImagePipeline:
class HouzzImagePipeline(ImagesPipeline): # Inherit the ImagePipeline class
def get_media_requests(self, item, info):
for image_url in item["image_urls"]:
yield scrapy.Request(image_url)
def file_path(self, request, response=None, info=None, *, item=None):
image_filename = item["path"].split("/")[-1] + ".jpg"
return image_filename
def item_completed(self, results, item, info):
# once the item is complete you can delete the master
# image_urls list and rename the temporary one
item["image_urls"] = item["imageURL"]
del item["imageURL"]
return item
How can I download and rename the image as md5 hash and add that name of the image to my JSON output as well?
Answers:
You have it perfect in your spider file, now all you need to do is use the same calculation you used in your file for creating each of the paths, but in your pipeline in the file_path method. Another tip I might offer is to add a ".jpg" to the end of the path so that others know that they are images and devices know how to display them correctly.
for example:
In your spider file simply add the ".jpg"
def make_path(urls):
img_path = []
for url in urls:
image_url_hash = hashlib.md5(url.encode()).hexdigest()
img_path.append(
image_url_hash[:3]
+ "/"
+ image_url_hash[3:6]
+ "/"
+ image_url_hash[6:9]
+ "/"
+ image_url_hash + ".jpg" # <--- here
)
return img_path
Then create a similar function that performs the same calculation to use in the image pipeline on the request.url, which is the same url that that you used to calculate the hash in the spider module.
def make_path(url):
image_url_hash = hashlib.md5(url.encode()).hexdigest()
return (
image_url_hash[:3]
+ "/"
+ image_url_hash[3:6]
+ "/"
+ image_url_hash[6:9]
+ "/"
+ image_url_hash + ".jpg"
)
class HouzzCrawlerPipeline:
def process_item(self, item, spider):
return item
class HouzzImagePipeline(ImagesPipeline): # Inherit the ImagePipeline class
def get_media_requests(self, item, info):
for image_url in item["image_urls"]:
yield scrapy.Request(image_url)
def file_path(self, request, response=None, info=None, *, item=None):
return make_path(request.url)
def item_completed(self, results, item, info):
# once the item is complete you can delete the master
# image_urls list and rename the temporary one
item["image_urls"] = item["imageURL"]
del item["imageURL"]
return item
houzz.json
[
{
"ideadId": "190897519",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-sunroom-minneapolis-phvw-vp~190897519",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/31d19f3a033dd7e3_9-7355/home-design.jpg"
],
"similarIdeas": [
{
"ideaId": "190897544",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-sunroom-minneapolis-phvw-vp~190897544",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/33d164b0033dd818_9-9818/home-design.jpg"
],
"path": ["0ff/eb3/b31/0ffeb3b31aad95578cf08487ae4c3ebb.jpg"]
},
{
"ideaId": "190897616",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-living-room-minneapolis-phvw-vp~190897616",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/6391ca76033dd8af_9-9530/home-design.jpg"
],
"path": ["e95/bc8/57f/e95bc857f0db27aeecebcd525dc96522.jpg"]
},
{
"ideaId": "190897549",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-landscape-minneapolis-phvw-vp~190897549",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/3c11c067033dd820_9-7414/home-design.jpg"
],
"path": ["37a/0c8/023/37a0c8023ec9242846d3a0c672b855a2.jpg"]
},
{
"ideaId": "190897563",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-landscape-minneapolis-phvw-vp~190897563",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/23619369033dd852_9-9924/home-design.jpg"
],
"path": ["af8/916/5e8/af89165e8452bd36dcdf6bbea374d1e5.jpg"]
},
{
"ideaId": "190897731",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-living-room-minneapolis-phvw-vp~190897731",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/e4a1cefa033dd8b6_9-7565/home-design.jpg"
],
"path": ["06d/859/673/06d859673df39bb5bd08202fd5a93ac2.jpg"]
},
{
"ideaId": "132721833",
"ideaUrl": "https://www.houzz.com/photos/sandy-house-sunroom-beach-style-sunroom-boston-phvw-vp~132721833",
"Title": "Sandy House- Sunroom",
"image_urls": [
"https://st.hzcdn.com/simgs/pictures/sunrooms/sandy-house-sunroom-lda-architecture-and-interiors-img~d5113a510bec7e64_9-3301-1-a6e1f10.jpg"
],
"path": ["282/23e/a88/28223ea883c816fe7e8ba8bfbf580554.jpg"]
},
{
"ideaId": "127987936",
"ideaUrl": "https://www.houzz.com/photos/blackfoot-pass-modern-sunroom-minneapolis-phvw-vp~127987936",
"Title": "Blackfoot Pass",
"image_urls": [
"https://st.hzcdn.com/simgs/pictures/sunrooms/blackfoot-pass-sustainable-nine-design-build-img~9461cf950b9d2921_9-6357-1-aff52a6.jpg"
],
"path": ["074/cc7/9bb/074cc79bb9b63fc999a022703adfaf59.jpg"]
},
{
"ideaId": "190897519",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-sunroom-minneapolis-phvw-vp~190897519",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/31d19f3a033dd7e3_9-7355/home-design.jpg"
],
"path": ["330/5e1/942/3305e1942fab6ae30fc4a294c46b72ca.jpg"]
},
{
"ideaId": "190897530",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-dining-room-minneapolis-phvw-vp~190897530",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/50716cfd033dd800_9-7384/home-design.jpg"
],
"path": ["fbc/c99/f98/fbcc99f98a6fe92b256e33c05443f3b8.jpg"]
},
{
"ideaId": "41776862",
"ideaUrl": "https://www.houzz.com/photos/luxurious-family-home-traditional-sunroom-omaha-phvw-vp~41776862",
"Title": "Luxurious Family Home",
"image_urls": [
"https://st.hzcdn.com/simgs/pictures/sunrooms/luxurious-family-home-frasier-martis-architects-p-c-img~e9a131ea061ecca8_9-6315-1-ff6fd38.jpg"
],
"path": ["ed4/83a/243/ed483a24375dae95e4b12a11f58b5012.jpg"]
},
{
"ideaId": "190897523",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-porch-minneapolis-phvw-vp~190897523",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/df51067a033dd7f0_9-7367/home-design.jpg"
],
"path": ["fb2/54d/9ec/fb254d9ecf3db875b7748454a4b0ecdb.jpg"]
},
{
"ideaId": "190897579",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-traditional-pool-minneapolis-phvw-vp~190897579",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/f541034b033dd8a0_9-9206/home-design.jpg"
],
"path": ["bbc/b12/5d4/bbcb125d4b815b6d48d710029091f95e.jpg"]
},
{
"ideaId": "111461960",
"ideaUrl": "https://www.houzz.com/photos/contemporary-retreat-contemporary-sunroom-portland-maine-phvw-vp~111461960",
"Title": "Contemporary Retreat",
"image_urls": [
"https://st.hzcdn.com/simgs/pictures/sunrooms/contemporary-retreat-maguire-construction-inc-img~53314a9b0a709781_9-5668-1-7c5fbc3.jpg"
],
"path": ["95a/96c/134/95a96c134a4ef06cd551d7e75cf42521.jpg"]
},
{
"ideaId": "190897577",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-porch-minneapolis-phvw-vp~190897577",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/5251035a033dd898_9-7536/home-design.jpg"
],
"path": ["7e6/a97/826/7e6a97826c40ff3037799b0812555008.jpg"]
},
{
"ideaId": "190897555",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-minneapolis-phvw-vp~190897555",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/e511795f033dd831_9-9502/home-design.jpg"
],
"path": ["060/555/925/0605559256313afbe75ff8537cfd044a.jpg"]
},
{
"ideaId": "39997663",
"ideaUrl": "https://www.houzz.com/photos/sun-room-addition-beach-style-sunroom-new-york-phvw-vp~39997663",
"Title": "Sun Room Addition",
"image_urls": [
"https://st.hzcdn.com/simgs/pictures/sunrooms/sun-room-addition-roam-architecture-img~4c21437805fb75ab_9-0889-1-e553b47.jpg"
],
"path": ["e15/5b4/fa7/e155b4fa74375b36d988325cb078e518.jpg"]
},
{
"ideaId": "190897560",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-pool-minneapolis-phvw-vp~190897560",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/a5b14567033dd840_9-9263/home-design.jpg"
],
"path": ["f7d/739/e6b/f7d739e6b88b3c3cd4db17bce7dea47c.jpg"]
},
{
"ideaId": "190897564",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-landscape-minneapolis-phvw-vp~190897564",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/a35160cb033dd859_9-7473/home-design.jpg"
],
"path": ["90a/ccf/1a6/90accf1a6e618bd656ca914a0ccb6f6d.jpg"]
},
{
"ideaId": "190897570",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-landscape-minneapolis-phvw-vp~190897570",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/10018340033dd881_9-7511/home-design.jpg"
],
"path": ["7f5/5ff/47d/7f55ff47ddd2bb7121d788e0f139b53b.jpg"]
},
{
"ideaId": "49362491",
"ideaUrl": "https://www.houzz.com/photos/15-nottingham-rd-transitional-sunroom-charleston-phvw-vp~49362491",
"Title": "15 Nottingham Rd",
"image_urls": [
"https://st.hzcdn.com/simgs/pictures/sunrooms/15-nottingham-rd-element-construction-partners-img~ee11bace06cf27e5_9-2610-1-3d53f33.jpg"
],
"path": ["0b0/bc7/5f7/0b0bc75f79b779b0165f7cb6b073706b.jpg"]
},
{
"ideaId": "190897534",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-bedroom-minneapolis-phvw-vp~190897534",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/c3d15f6d033dd809_9-9517/home-design.jpg"
],
"path": ["5eb/537/0a9/5eb5370a969522787ead4a91563ceb82.jpg"]
},
{
"ideaId": "190897580",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-home-office-minneapolis-phvw-vp~190897580",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/ab9181f4033dd8a8_9-9415/home-design.jpg"
],
"path": ["f59/1ee/158/f591ee15806ed059f80774975b250424.jpg"]
},
{
"ideaId": "190897538",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-dining-room-minneapolis-phvw-vp~190897538",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/f841c451033dd810_9-7399/home-design.jpg"
],
"path": ["27d/2a0/166/27d2a01666df03a2ab7df261833f0247.jpg"]
},
{
"ideaId": "143265498",
"ideaUrl": "https://www.houzz.com/photos/modern-farmhouse-upstate-farmhouse-sunroom-new-york-phvw-vp~143265498",
"Title": "Modern Farmhouse-Upstate",
"image_urls": [
"https://st.hzcdn.com/simgs/pictures/sunrooms/modern-farmhouse-upstate-crisp-architects-img~928174ca0ce5acd3_9-0181-1-2d9c0aa.jpg"
],
"path": ["3fe/961/929/3fe961929ac41fc2d7d8095d735510df.jpg"]
},
{
"ideaId": "190897565",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-landscape-minneapolis-phvw-vp~190897565",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/8b51b447033dd862_9-9915/home-design.jpg"
],
"path": ["7fd/442/d1a/7fd442d1a0acfe3877751724b46edd32.jpg"]
},
{
"ideaId": "190897527",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-home-office-minneapolis-phvw-vp~190897527",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/7ea1bdc0033dd7f8_9-7375/home-design.jpg"
],
"path": ["3d9/b3c/c49/3d9b3cc49fa9b756d5851bcb33ab858f.jpg"]
},
{
"ideaId": "42287112",
"ideaUrl": "https://www.houzz.com/photos/mt-rain-house-contemporary-sunroom-baltimore-phvw-vp~42287112",
"Title": "Mt Rain House",
"image_urls": [
"https://st.hzcdn.com/simgs/pictures/sunrooms/mt-rain-house-place-architecturedesign-img~32814511062a33e6_9-4616-1-b02d57a.jpg"
],
"path": ["635/976/0d1/6359760d15b7989fe2943b856cbf80a2.jpg"]
},
{
"ideaId": "190897571",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-landscape-minneapolis-phvw-vp~190897571",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/bc11298c033dd888_9-0082/home-design.jpg"
],
"path": ["19f/39f/e82/19f39fe824b9d1552003624f4ad03e50.jpg"]
},
{
"ideaId": "190897562",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-traditional-pool-minneapolis-phvw-vp~190897562",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/40e1c0b2033dd84a_9-9285/home-design.jpg"
],
"path": ["69a/de2/aa0/69ade2aa0dbdee10eb4454ee883ad57c.jpg"]
},
{
"ideaId": "190897817",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-living-room-minneapolis-phvw-vp~190897817",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/6ee1fda5033dd8c0_9-7580/home-design.jpg"
],
"path": ["740/83a/e5a/74083ae5a4173ae4869443e750605123.jpg"]
},
{
"ideaId": "190897554",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-porch-minneapolis-phvw-vp~190897554",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/20b1c3c0033dd829_9-7424/home-design.jpg"
],
"path": ["6bc/421/116/6bc421116602bad60ed01461b24736c2.jpg"]
},
{
"ideaId": "190897566",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-traditional-pool-minneapolis-phvw-vp~190897566",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/4dc141d4033dd869_9-9342/home-design.jpg"
],
"path": ["547/c8d/548/547c8d548ca08225cdc3f3bf847bb4a4.jpg"]
},
{
"ideaId": "101615580",
"ideaUrl": "https://www.houzz.com/photos/greenwich-contemporary-transformation-traditional-sunroom-phvw-vp~101615580",
"Title": "Greenwich Contemporary Transformation",
"image_urls": [
"https://st.hzcdn.com/simgs/pictures/sunrooms/greenwich-contemporary-transformation-grayson-de-vere-interior-design-and-staging-img~fc4111e609b2d1e1_9-8048-1-926a5b7.jpg"
],
"path": ["57c/10c/e71/57c10ce71e54ff650a7d032d58eed4ec.jpg"]
},
{
"ideaId": "47662917",
"ideaUrl": "https://www.houzz.com/photos/traditional-sunroom-traditional-sunroom-phvw-vp~47662917",
"Title": "Traditional Sunroom",
"image_urls": [
"https://st.hzcdn.com/simgs/pictures/sunrooms/traditional-sunroom-martin-horn-inc-img~59c1427b06af9e65_9-9991-1-a7ac9a4.jpg"
],
"path": ["127/989/b0c/127989b0cd3366148f3f7bc0c863714a.jpg"]
},
{
"ideaId": "190897573",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-landscape-minneapolis-phvw-vp~190897573",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/953128bd033dd890_9-9851/home-design.jpg"
],
"path": ["311/1cd/00b/3111cd00b768a1167510c1d766d3024d.jpg"]
},
{
"ideaId": "190897557",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-exterior-minneapolis-phvw-vp~190897557",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/24117c2d033dd838_9-9517/home-design.jpg"
],
"path": ["262/7d8/003/2627d80032dca419d5d0f60042ea5003.jpg"]
},
{
"ideaId": "190897569",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-patio-minneapolis-phvw-vp~190897569",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/c071fe7c033dd879_9-7504/home-design.jpg"
],
"path": ["ea2/eeb/0ba/ea2eeb0ba1ff2f580fc9e24c3678f8bc.jpg"]
}
],
"path": ["330/5e1/942/3305e1942fab6ae30fc4a294c46b72ca.jpg"]
}
]
I have a Scrapy spider which is working as far as scraping is concerned but I am having issues during downloading the images. I want to download the images and rename them as md5 hash for example: c69/96d/f0d/c6996df0d9d852f1f39fcb7074ace625.jpg also I’d like to add the md5 name of the image to my JSON output as well. I tried converting the image urls to md5 hash in the spider and then renaming the image file in the HouzzImagePipeline by accessing the item['path'] but it is not downloading the images. Here is the spider:
import scrapy
import json
import hashlib
def make_path(urls):
img_path = []
for url in urls:
image_url_hash = hashlib.md5(url.encode()).hexdigest()
img_path.append(
image_url_hash[:3]
+ "/"
+ image_url_hash[3:6]
+ "/"
+ image_url_hash[6:9]
+ "/"
+ image_url_hash
)
return img_path
class HouzzSimilar(scrapy.Spider):
name = "houzz_crawler"
custom_settings = {
"LOG_FILE": "houzz_spider.log",
"IMAGES_STORE": "houzz_images",
"FEEDS": {
"houzz.json": {
"format": "json",
}
},
"ITEM_PIPELINES": {
"houzz_crawler.pipelines.HouzzImagePipeline": 1,
},
}
headers = {
"authority": "www.houzz.com",
"accept": "*/*",
"accept-language": "en,ru;q=0.9",
"content-type": "application/x-www-form-urlencoded; charset=UTF-8",
"origin": "https://www.houzz.com",
"referer": "https://www.houzz.com/photos/columbus-ave-residence-contemporary-bathroom-new-york-phvw-vp~160668148",
"rrid": "70402547-c900-47f7-a913-8e1cbc9aa0c3",
"sec-ch-ua": '"Chromium";v="110", "Not A(Brand";v="24", "YaBrowser";v="23"',
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": '"Linux"',
"sec-fetch-dest": "empty",
"sec-fetch-mode": "cors",
"sec-fetch-site": "same-origin",
"user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 YaBrowser/23.3.1.906 (beta) Yowser/2.5 Safari/537.36",
"x-csrf-token": "i8B5ykgX-eprPj5yAHSxOng08Pa4qAr2Z0TQ",
"x-hz-request": "true",
"x-ol-exp-id": "clhhdi4wu00003y71rnvty395",
"x-ol-exp-name": "Photo - View",
"x-ol-ext-device-id": "23a3cfb8-7a04-4462-af71-d98689271533",
"x-ol-ext-session-id": "782c0a90-8925-409f-90c1-f47798e0426e",
"x-ol-product": "Houzz",
"x-ol-product-variant": "Houzz US",
"x-ol-session-id": "782c0a90-8925-409f-90c1-f47798e0426e",
"x-requested-with": "XMLHttpRequest",
}
cookies = {
"v": "1683311076_f9d9a715-f45b-42dc-bc6d-7da75774a57f_9bda9dd500ca1e5119bbecaba51e53f0",
"vct": "en-US-vxnkSVVkSBzkSVVkCR%2FkSVVk8B%2FkSVVk4R3kSVVk4h3kSVVk",
"_gcl_au": "1.1.17413922.1683311086",
"crossdevicetracking": "915374c0-439c-46a1-bbf2-3a2aaa487e69",
"_pin_unauth": "dWlkPU16Y3dNbVF6T0dNdE1tWTBOaTAwWTJSa0xUazVZakV0TXprek5XWm1ZV014WWprMw",
"_sp_id.c905": "5af74097-a6bb-46e7-8d14-35ff6d738f39.1683317411.2.1683359810.1683317411.13ad94c9-5560-4fbf-963f-b63e32f2124d",
"g_state": '{"i_p":1684144918349,"i_l":3}',
"browseResultSetGridWidth": "554",
"_gid": "GA1.2.1176067560.1683652076",
"ln_or": "eyIzODE1NzE2IjoiZCJ9",
"_csrf": "G_nV-Kaa7rlqgTwnueAXkJtj",
"jdv": "t7WOzUb2vHLZtWVVHSk%2BXJEWN7ua9zR%2FUkXpY9RYDUW00hxMyur5c%2Bzn6M%2BqQADtWOInJpmlQA37Gxp0L267jdj74Iwe",
"documentWidth": "1318",
"_uetsid": "0bf41840ee8c11edac06995ca98afa3c",
"_uetvid": "1e07d960eb7211ed880b7db3cdc86191",
"_derived_epik": "dj0yJnU9NFBDc3RuOExta3NiM2xfaV9WS0RYbVVLRS1lRVpycDEmbj1tVE1RRUtOUjYwYU1Kalp0el9mNTBBJm09OCZ0PUFBQUFBR1JiUmprJnJtPTgmcnQ9QUFBQUFHUmJSamsmc3A9NQ",
"IR_gbd": "houzz.com",
"IR_5454": "1683703358356%7C0%7C1683703358356%7C%7C",
"_ga": "GA1.2.1658927820.1683311086",
"_dc_gtm_UA-3519678-1": "1",
"_ga_PB0RC2CT7B": "GS1.1.1683703353.11.1.1683704001.59.0.0",
"hzd": "70402547-c900-47f7-a913-8e1cbc9aa0c3%3A%3A%3A%3A%3ASeeMoreIdeas",
}
base_url = "https://www.houzz.com/photos/home-design-ideas-phbr0-bp~"
similar_ideas_api_url = "https://www.houzz.com/j/getSimilarSpaces"
def start_requests(self):
yield scrapy.Request(
url=self.base_url, headers=self.headers, callback=self.parse_ideas
)
def parse_ideas(self, response):
ideas = response.css("a.hz-photo-card__ratio-box::attr(href)").extract()
for idea in ideas:
yield scrapy.Request(
url=idea, headers=self.headers, callback=self.parse_project_url
)
def parse_project_url(self, response):
data = response.css('script[id="hz-ctx"] ::text').get()
json_data = json.loads(data)
space_id = json_data["data"]["pageContentData"]["spaceId"]
space = json_data["data"]["stores"]["data"]["SpaceStore"]["data"][space_id]
project_id = space["projectId"]
space_url = space["url"]
raw_project_url = (
space_url.split("~")[0].replace("phvw", "pj").replace("vp", "vj")
)
project_url = raw_project_url + "~" + str(project_id)
yield scrapy.Request(
url=project_url, headers=self.headers, callback=self.parse_project_idea
)
def parse_project_idea(self, response):
idea_board = response.css(
"div.hz-prj-container.hz-prj-container__photos.clearfix ::attr(href)"
).extract()
for idea_link in idea_board:
yield scrapy.Request(
url=idea_link,
headers=self.headers,
callback=self.parse_idea_details,
)
def parse_idea_details(self, response):
item = {}
item["ideadId"] = response.url.split("~")[-1]
item["ideaUrl"] = response.url
item["Title"] = response.css(
"h1.hz-view-photo__space-info__title.text-bold::text"
).get()
item["imageURL"] = response.css(
"div.view-photo-image-pane > img::attr(src)"
).extract()
item["image_urls"] = item["imageURL"].copy()
item["similarIdeas"] = []
item["path"] = make_path(
response.css("div.view-photo-image-pane > img::attr(src)").extract()
)
spaceId = response.url.split("~")[-1]
body = f"spaceId={spaceId}&fromItem=0&itemsPerPage=10&contentDescriptor=%7B%22t%22%3A1%2C%22et%22%3A3%2C%22id%22%3A160668148%7D"
yield scrapy.Request(
url=self.similar_ideas_api_url,
method="POST",
cookies=self.cookies,
headers=self.headers,
body=body,
cb_kwargs={"item": item},
callback=self.get_similar_ideas_urls,
)
def get_similar_ideas_urls(self, response, item=None):
data = response.json()["spaceData"]["spaces"]
space_keys = list(data.keys())
space_urls = set([data[key]["url"] for key in space_keys])
yield scrapy.Request(
url=space_urls.pop(),
headers=self.headers,
cb_kwargs={"item": item, "space_urls": space_urls},
callback=self.parse_similar_ideas,
)
def parse_similar_ideas(self, response, item=None, space_urls=None):
# add the image urls to the top master list as well as locally.
image_urls = response.css(
"div.view-photo-image-pane > img::attr(src)"
).extract()
item["image_urls"] += image_urls
item["similarIdeas"].append(
{
"ideaId": response.url.split("~")[-1],
"ideaUrl": response.url,
"Title": response.css(
"h1.hz-view-photo__space-info__title.text-bold::text"
).get(),
"image_urls": image_urls,
"path": make_path(image_urls),
}
)
if len(space_urls) > 0:
yield scrapy.Request(
url=space_urls.pop(),
headers=self.headers,
cb_kwargs={"item": item, "space_urls": space_urls},
dont_filter=True,
callback=self.parse_similar_ideas,
)
else:
yield item
ImagePipeline:
class HouzzImagePipeline(ImagesPipeline): # Inherit the ImagePipeline class
def get_media_requests(self, item, info):
for image_url in item["image_urls"]:
yield scrapy.Request(image_url)
def file_path(self, request, response=None, info=None, *, item=None):
image_filename = item["path"].split("/")[-1] + ".jpg"
return image_filename
def item_completed(self, results, item, info):
# once the item is complete you can delete the master
# image_urls list and rename the temporary one
item["image_urls"] = item["imageURL"]
del item["imageURL"]
return item
How can I download and rename the image as md5 hash and add that name of the image to my JSON output as well?
You have it perfect in your spider file, now all you need to do is use the same calculation you used in your file for creating each of the paths, but in your pipeline in the file_path method. Another tip I might offer is to add a ".jpg" to the end of the path so that others know that they are images and devices know how to display them correctly.
for example:
In your spider file simply add the ".jpg"
def make_path(urls):
img_path = []
for url in urls:
image_url_hash = hashlib.md5(url.encode()).hexdigest()
img_path.append(
image_url_hash[:3]
+ "/"
+ image_url_hash[3:6]
+ "/"
+ image_url_hash[6:9]
+ "/"
+ image_url_hash + ".jpg" # <--- here
)
return img_path
Then create a similar function that performs the same calculation to use in the image pipeline on the request.url, which is the same url that that you used to calculate the hash in the spider module.
def make_path(url):
image_url_hash = hashlib.md5(url.encode()).hexdigest()
return (
image_url_hash[:3]
+ "/"
+ image_url_hash[3:6]
+ "/"
+ image_url_hash[6:9]
+ "/"
+ image_url_hash + ".jpg"
)
class HouzzCrawlerPipeline:
def process_item(self, item, spider):
return item
class HouzzImagePipeline(ImagesPipeline): # Inherit the ImagePipeline class
def get_media_requests(self, item, info):
for image_url in item["image_urls"]:
yield scrapy.Request(image_url)
def file_path(self, request, response=None, info=None, *, item=None):
return make_path(request.url)
def item_completed(self, results, item, info):
# once the item is complete you can delete the master
# image_urls list and rename the temporary one
item["image_urls"] = item["imageURL"]
del item["imageURL"]
return item
houzz.json
[
{
"ideadId": "190897519",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-sunroom-minneapolis-phvw-vp~190897519",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/31d19f3a033dd7e3_9-7355/home-design.jpg"
],
"similarIdeas": [
{
"ideaId": "190897544",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-sunroom-minneapolis-phvw-vp~190897544",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/33d164b0033dd818_9-9818/home-design.jpg"
],
"path": ["0ff/eb3/b31/0ffeb3b31aad95578cf08487ae4c3ebb.jpg"]
},
{
"ideaId": "190897616",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-living-room-minneapolis-phvw-vp~190897616",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/6391ca76033dd8af_9-9530/home-design.jpg"
],
"path": ["e95/bc8/57f/e95bc857f0db27aeecebcd525dc96522.jpg"]
},
{
"ideaId": "190897549",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-landscape-minneapolis-phvw-vp~190897549",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/3c11c067033dd820_9-7414/home-design.jpg"
],
"path": ["37a/0c8/023/37a0c8023ec9242846d3a0c672b855a2.jpg"]
},
{
"ideaId": "190897563",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-landscape-minneapolis-phvw-vp~190897563",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/23619369033dd852_9-9924/home-design.jpg"
],
"path": ["af8/916/5e8/af89165e8452bd36dcdf6bbea374d1e5.jpg"]
},
{
"ideaId": "190897731",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-living-room-minneapolis-phvw-vp~190897731",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/e4a1cefa033dd8b6_9-7565/home-design.jpg"
],
"path": ["06d/859/673/06d859673df39bb5bd08202fd5a93ac2.jpg"]
},
{
"ideaId": "132721833",
"ideaUrl": "https://www.houzz.com/photos/sandy-house-sunroom-beach-style-sunroom-boston-phvw-vp~132721833",
"Title": "Sandy House- Sunroom",
"image_urls": [
"https://st.hzcdn.com/simgs/pictures/sunrooms/sandy-house-sunroom-lda-architecture-and-interiors-img~d5113a510bec7e64_9-3301-1-a6e1f10.jpg"
],
"path": ["282/23e/a88/28223ea883c816fe7e8ba8bfbf580554.jpg"]
},
{
"ideaId": "127987936",
"ideaUrl": "https://www.houzz.com/photos/blackfoot-pass-modern-sunroom-minneapolis-phvw-vp~127987936",
"Title": "Blackfoot Pass",
"image_urls": [
"https://st.hzcdn.com/simgs/pictures/sunrooms/blackfoot-pass-sustainable-nine-design-build-img~9461cf950b9d2921_9-6357-1-aff52a6.jpg"
],
"path": ["074/cc7/9bb/074cc79bb9b63fc999a022703adfaf59.jpg"]
},
{
"ideaId": "190897519",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-sunroom-minneapolis-phvw-vp~190897519",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/31d19f3a033dd7e3_9-7355/home-design.jpg"
],
"path": ["330/5e1/942/3305e1942fab6ae30fc4a294c46b72ca.jpg"]
},
{
"ideaId": "190897530",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-dining-room-minneapolis-phvw-vp~190897530",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/50716cfd033dd800_9-7384/home-design.jpg"
],
"path": ["fbc/c99/f98/fbcc99f98a6fe92b256e33c05443f3b8.jpg"]
},
{
"ideaId": "41776862",
"ideaUrl": "https://www.houzz.com/photos/luxurious-family-home-traditional-sunroom-omaha-phvw-vp~41776862",
"Title": "Luxurious Family Home",
"image_urls": [
"https://st.hzcdn.com/simgs/pictures/sunrooms/luxurious-family-home-frasier-martis-architects-p-c-img~e9a131ea061ecca8_9-6315-1-ff6fd38.jpg"
],
"path": ["ed4/83a/243/ed483a24375dae95e4b12a11f58b5012.jpg"]
},
{
"ideaId": "190897523",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-porch-minneapolis-phvw-vp~190897523",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/df51067a033dd7f0_9-7367/home-design.jpg"
],
"path": ["fb2/54d/9ec/fb254d9ecf3db875b7748454a4b0ecdb.jpg"]
},
{
"ideaId": "190897579",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-traditional-pool-minneapolis-phvw-vp~190897579",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/f541034b033dd8a0_9-9206/home-design.jpg"
],
"path": ["bbc/b12/5d4/bbcb125d4b815b6d48d710029091f95e.jpg"]
},
{
"ideaId": "111461960",
"ideaUrl": "https://www.houzz.com/photos/contemporary-retreat-contemporary-sunroom-portland-maine-phvw-vp~111461960",
"Title": "Contemporary Retreat",
"image_urls": [
"https://st.hzcdn.com/simgs/pictures/sunrooms/contemporary-retreat-maguire-construction-inc-img~53314a9b0a709781_9-5668-1-7c5fbc3.jpg"
],
"path": ["95a/96c/134/95a96c134a4ef06cd551d7e75cf42521.jpg"]
},
{
"ideaId": "190897577",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-porch-minneapolis-phvw-vp~190897577",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/5251035a033dd898_9-7536/home-design.jpg"
],
"path": ["7e6/a97/826/7e6a97826c40ff3037799b0812555008.jpg"]
},
{
"ideaId": "190897555",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-minneapolis-phvw-vp~190897555",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/e511795f033dd831_9-9502/home-design.jpg"
],
"path": ["060/555/925/0605559256313afbe75ff8537cfd044a.jpg"]
},
{
"ideaId": "39997663",
"ideaUrl": "https://www.houzz.com/photos/sun-room-addition-beach-style-sunroom-new-york-phvw-vp~39997663",
"Title": "Sun Room Addition",
"image_urls": [
"https://st.hzcdn.com/simgs/pictures/sunrooms/sun-room-addition-roam-architecture-img~4c21437805fb75ab_9-0889-1-e553b47.jpg"
],
"path": ["e15/5b4/fa7/e155b4fa74375b36d988325cb078e518.jpg"]
},
{
"ideaId": "190897560",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-pool-minneapolis-phvw-vp~190897560",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/a5b14567033dd840_9-9263/home-design.jpg"
],
"path": ["f7d/739/e6b/f7d739e6b88b3c3cd4db17bce7dea47c.jpg"]
},
{
"ideaId": "190897564",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-landscape-minneapolis-phvw-vp~190897564",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/a35160cb033dd859_9-7473/home-design.jpg"
],
"path": ["90a/ccf/1a6/90accf1a6e618bd656ca914a0ccb6f6d.jpg"]
},
{
"ideaId": "190897570",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-landscape-minneapolis-phvw-vp~190897570",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/10018340033dd881_9-7511/home-design.jpg"
],
"path": ["7f5/5ff/47d/7f55ff47ddd2bb7121d788e0f139b53b.jpg"]
},
{
"ideaId": "49362491",
"ideaUrl": "https://www.houzz.com/photos/15-nottingham-rd-transitional-sunroom-charleston-phvw-vp~49362491",
"Title": "15 Nottingham Rd",
"image_urls": [
"https://st.hzcdn.com/simgs/pictures/sunrooms/15-nottingham-rd-element-construction-partners-img~ee11bace06cf27e5_9-2610-1-3d53f33.jpg"
],
"path": ["0b0/bc7/5f7/0b0bc75f79b779b0165f7cb6b073706b.jpg"]
},
{
"ideaId": "190897534",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-bedroom-minneapolis-phvw-vp~190897534",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/c3d15f6d033dd809_9-9517/home-design.jpg"
],
"path": ["5eb/537/0a9/5eb5370a969522787ead4a91563ceb82.jpg"]
},
{
"ideaId": "190897580",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-home-office-minneapolis-phvw-vp~190897580",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/ab9181f4033dd8a8_9-9415/home-design.jpg"
],
"path": ["f59/1ee/158/f591ee15806ed059f80774975b250424.jpg"]
},
{
"ideaId": "190897538",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-dining-room-minneapolis-phvw-vp~190897538",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/f841c451033dd810_9-7399/home-design.jpg"
],
"path": ["27d/2a0/166/27d2a01666df03a2ab7df261833f0247.jpg"]
},
{
"ideaId": "143265498",
"ideaUrl": "https://www.houzz.com/photos/modern-farmhouse-upstate-farmhouse-sunroom-new-york-phvw-vp~143265498",
"Title": "Modern Farmhouse-Upstate",
"image_urls": [
"https://st.hzcdn.com/simgs/pictures/sunrooms/modern-farmhouse-upstate-crisp-architects-img~928174ca0ce5acd3_9-0181-1-2d9c0aa.jpg"
],
"path": ["3fe/961/929/3fe961929ac41fc2d7d8095d735510df.jpg"]
},
{
"ideaId": "190897565",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-landscape-minneapolis-phvw-vp~190897565",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/8b51b447033dd862_9-9915/home-design.jpg"
],
"path": ["7fd/442/d1a/7fd442d1a0acfe3877751724b46edd32.jpg"]
},
{
"ideaId": "190897527",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-home-office-minneapolis-phvw-vp~190897527",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/7ea1bdc0033dd7f8_9-7375/home-design.jpg"
],
"path": ["3d9/b3c/c49/3d9b3cc49fa9b756d5851bcb33ab858f.jpg"]
},
{
"ideaId": "42287112",
"ideaUrl": "https://www.houzz.com/photos/mt-rain-house-contemporary-sunroom-baltimore-phvw-vp~42287112",
"Title": "Mt Rain House",
"image_urls": [
"https://st.hzcdn.com/simgs/pictures/sunrooms/mt-rain-house-place-architecturedesign-img~32814511062a33e6_9-4616-1-b02d57a.jpg"
],
"path": ["635/976/0d1/6359760d15b7989fe2943b856cbf80a2.jpg"]
},
{
"ideaId": "190897571",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-landscape-minneapolis-phvw-vp~190897571",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/bc11298c033dd888_9-0082/home-design.jpg"
],
"path": ["19f/39f/e82/19f39fe824b9d1552003624f4ad03e50.jpg"]
},
{
"ideaId": "190897562",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-traditional-pool-minneapolis-phvw-vp~190897562",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/40e1c0b2033dd84a_9-9285/home-design.jpg"
],
"path": ["69a/de2/aa0/69ade2aa0dbdee10eb4454ee883ad57c.jpg"]
},
{
"ideaId": "190897817",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-living-room-minneapolis-phvw-vp~190897817",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/6ee1fda5033dd8c0_9-7580/home-design.jpg"
],
"path": ["740/83a/e5a/74083ae5a4173ae4869443e750605123.jpg"]
},
{
"ideaId": "190897554",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-porch-minneapolis-phvw-vp~190897554",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/20b1c3c0033dd829_9-7424/home-design.jpg"
],
"path": ["6bc/421/116/6bc421116602bad60ed01461b24736c2.jpg"]
},
{
"ideaId": "190897566",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-traditional-pool-minneapolis-phvw-vp~190897566",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/4dc141d4033dd869_9-9342/home-design.jpg"
],
"path": ["547/c8d/548/547c8d548ca08225cdc3f3bf847bb4a4.jpg"]
},
{
"ideaId": "101615580",
"ideaUrl": "https://www.houzz.com/photos/greenwich-contemporary-transformation-traditional-sunroom-phvw-vp~101615580",
"Title": "Greenwich Contemporary Transformation",
"image_urls": [
"https://st.hzcdn.com/simgs/pictures/sunrooms/greenwich-contemporary-transformation-grayson-de-vere-interior-design-and-staging-img~fc4111e609b2d1e1_9-8048-1-926a5b7.jpg"
],
"path": ["57c/10c/e71/57c10ce71e54ff650a7d032d58eed4ec.jpg"]
},
{
"ideaId": "47662917",
"ideaUrl": "https://www.houzz.com/photos/traditional-sunroom-traditional-sunroom-phvw-vp~47662917",
"Title": "Traditional Sunroom",
"image_urls": [
"https://st.hzcdn.com/simgs/pictures/sunrooms/traditional-sunroom-martin-horn-inc-img~59c1427b06af9e65_9-9991-1-a7ac9a4.jpg"
],
"path": ["127/989/b0c/127989b0cd3366148f3f7bc0c863714a.jpg"]
},
{
"ideaId": "190897573",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-landscape-minneapolis-phvw-vp~190897573",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/953128bd033dd890_9-9851/home-design.jpg"
],
"path": ["311/1cd/00b/3111cd00b768a1167510c1d766d3024d.jpg"]
},
{
"ideaId": "190897557",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-exterior-minneapolis-phvw-vp~190897557",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/24117c2d033dd838_9-9517/home-design.jpg"
],
"path": ["262/7d8/003/2627d80032dca419d5d0f60042ea5003.jpg"]
},
{
"ideaId": "190897569",
"ideaUrl": "https://www.houzz.com/photos/vintage-stone-french-country-patio-minneapolis-phvw-vp~190897569",
"Title": "VINTAGE STONE",
"image_urls": [
"https://st.hzcdn.com/simgs/c071fe7c033dd879_9-7504/home-design.jpg"
],
"path": ["ea2/eeb/0ba/ea2eeb0ba1ff2f580fc9e24c3678f8bc.jpg"]
}
],
"path": ["330/5e1/942/3305e1942fab6ae30fc4a294c46b72ca.jpg"]
}
]