Home Depot Scraper

import requests
from bs4 import BeautifulSoup
import json

product_data = []

payload = {
    'api_key': 'YOUR_API_KEY', #add your key
    'url': 'https://www.homedepot.com/b/Appliances-Refrigerators/N-5yc1vZc3pi?catStyle=ShowProducts&NCNI-5&searchRedirect=refrigerators&semanticToken=i10r10r00f22000000000_202311261341369949425674627_us-east4-5qn1%20i10r10r00f22000000000%20%3E%20rid%3A%7B945c050322f005b6254c2457daf503cb%7D%3Arid%20st%3A%7Brefrigerators%7D%3Ast%20ml%3A%7B24%7D%3Aml%20ct%3A%7Brefrigerator%7D%3Act%20nr%3A%7Brefrigerator%7D%3Anr%20nf%3A%7Bn%2Fa%7D%3Anf%20qu%3A%7Brefrigerator%7D%3Aqu%20ie%3A%7B0%7D%3Aie%20qr%3A%7Brefrigerator%7D%3Aqr&Nao=24',
    'country_code': 'us',
    'render': 'true'
}

response = requests.get('https://api.scraperapi.com', params=payload)
soup = BeautifulSoup(response.content, 'html.parser')

products = soup.find_all(class_='browse-search__pod')
for product in products:
    product_data.append({
        'price': product.find(class_='price-format__main-price').text,
        'model': product.find(class_='sui-mr-1').text.replace('Model#', '').strip(),
        'brand': product.find(class_='product-header__title__brand--bold--ey0fs').text.strip(),
        'description': product.find(class_='product-header__title-product--ey0fs').text,
        'url': product.find('div', attrs={'data-testid': 'product-header'}).find('a')['href'],
    })

with open('homedepot-data.json', 'w') as f:
    json.dump(product_data, f)