How to Scrape Redfin: Using Code and No Code Approaches

import asyncio
import json

from playwright.async_api import async_playwright

location = "Washington, DC"
max_pagination = 2

async def extract_data(page, selector) -> list:
    """
    Parsing details from the listing page

    Args:
        page: webpage of the browser
        selector: selector for the div containing property details

    Returns:
        list: details of homes for sale
    """

    # Initializing selectors and xpaths
    next_page_selector = "[data-rf-test-id='react-data-paginate-next']"
    price_selector = "[class='homecardV2Price']"
    specification_selector = "[class='HomeStatsV2 font-size-small ']"
    address_selector = "[class='homeAddressV2']"

    # List to save the details of properties
    homes_for_sale = []

    # Paginating through each page
    for _ in range(max_pagination):
        # Waiting for the page to finish loading
        await page.wait_for_load_state("load")

        # Extracting the elements
        all_visible_elements = page.locator(selector)
        all_visible_elements_count = await all_visible_elements.count()

        for index in range(all_visible_elements_count):
            # Hovering the element to load the price
            inner_element = all_visible_elements.nth(index=index)
            await inner_element.hover()
            inner_element = all_visible_elements.nth(index=index)

            # Extracting necessary data
            price = await inner_element.locator(price_selector).inner_text() if await inner_element.locator(price_selector).count() else None
            specifications = await inner_element.locator(specification_selector).inner_text() if await inner_element.locator(specification_selector).count() else None
            address = await inner_element.locator(address_selector).inner_text() if await inner_element.locator(address_selector).count() else None

            # Removing extra spaces and unicode characters
            price = clean_data(price)
            specifications = clean_data(specifications)
            address = clean_data(address)

            data_to_save = {
                "price": price,
                "specifications": specifications,
                "address": address,
            }

            homes_for_sale.append(data_to_save)

        next_page = page.locator(next_page_selector)
        await next_page.hover()
        if not await next_page.count():
            break

        # Clicking the next page button
        await next_page.click()

    save_data(homes_for_sale, "Data.json")

async def run(playwright) -> None:
    # Initializing the browser and creating a new page.
    browser = await playwright.firefox.launch(headless=False)
    context = await browser.new_context()
    page = await context.new_page()

    await page.set_viewport_size({"width": 1920, "height": 1080})
    page.set_default_timeout(120000)

    # Navigating to the homepage
    await page.goto("https://www.Redfin.com/", wait_until="domcontentloaded")
    await page.wait_for_load_state("load")
    await page.wait_for_load_state(timeout=60000)

    # Initializing the xpath and selector
    xpath_search_box = "[placeholder='City, Address, School, Agent, ZIP']"
    listing_div_selector = "[class='bottomV2 ']"

    # Clicking the input field to enter the location and navigating to the listing page
    await page.locator(xpath_search_box).click()
    await page.locator(xpath_search_box).fill(location)
    await page.locator(xpath_search_box).press("Enter")

    # Waiting until the list of properties is loaded
    await page.wait_for_selector(listing_div_selector)

    await extract_data(page, listing_div_selector)

    await context.close()
    await browser.close()

def clean_data(data: str) -> str:
    """
    Cleaning data by removing extra white spaces and Unicode characters

    Args:
        data (str): data to be cleaned

    Returns:
        str: cleaned string
    """
    if not data:
        return ""
    cleaned_data = " ".join(data.split()).strip()
     cleaned_data = cleaned_data.encode("ascii", "ignore").decode("ascii")
    return cleaned_data

def save_data(product_page_data: list, filename: str):
    """Converting a list of dictionaries to JSON format

    Args:
        product_page_data (list): details of each product
        filename (str): name of the JSON file
    """
    with open(filename, "w") as outfile:
        json.dump(product_page_data, outfile, indent=4)

async def main() -> None:
    async with async_playwright() as playwright:
        await run(playwright)

if __name__ == "__main__":
    asyncio.run(main())

const { chromium, firefox } = require('playwright');
const fs = require('fs');

const location = "Washington,DC";
const maxPagination = 2;

/**
* Save data as list of dictionaries
as json file
* @param {object} data
*/
function saveData(data) {
    let dataStr = JSON.stringify(data, null, 2)
    fs.writeFile("DataJS.json", dataStr, 'utf8', function (err) {
        if (err) {
            console.log("An error occurred while writing JSON Object to File.");
            return console.log(err);
        }
        console.log("JSON file has been saved.");
    });
}

function cleanData(data) {
    if (!data) {
        return;
    }
    // removing extra spaces and unicode characters
    let cleanedData = data.split(/\s+/).join(" ").trim();
    cleanedData = cleanedData.replace(/[^\x00-\x7F]/g, "");
    return cleanedData;
}

/**
* The data extraction function used to extract
necessary data from the element.
* @param {HtmlElement} innerElement
* @returns
*/
async function extractData(innerElement) {

    async function extractData(data) {
        let count = await data.count();
        if (count) {
            return await data.innerText()
        }
        return null
    };
    // intializing xpath and selectors

    priceSelector = "[class='homecardV2Price']"
    specificationSelector = "[class='HomeStatsV2 font-size-small ']"
    addressSelector = "[class='homeAddressV2']"

    // Extracting necessary data
    let price = innerElement.locator(priceSelector);
    price = await extractData(price);

    let specifications = innerElement.locator(specificationSelector);
    specifications = await extractData(specifications);

    let address = innerElement.locator(addressSelector);
    address = await extractData(address)

    // cleaning data
    price = cleanData(price)
    specifications = cleanData(specifications)
    address = cleanData(address)

    extractedData = {
        "price": price,
        "specifications":specifications,
        'address': address
    }
    console.log(extractData)

    return extractedData
}
/**
* The main function initiate a browser object and handle the navigation.
*/
async function run() {
    // intializing browser and creating new page
    const browser = await chromium.launch({ headless: false});
    const context = await browser.newContext();
    const page = await context.newPage();

    // initializing xpaths and selectors
    const xpathSearchBox = "[placeholder='City, Address, School, Agent, ZIP']";
    const listingDivSelector = "[class='bottomV2 ']";
    const xpathNextPage = "[data-rf-test-id='react-data-paginate-next']";

    // Navigating to the home page
    await page.goto('https://www.Redfin.com/', {
    waitUntil: 'domcontentloaded',
    timeout: 60000,
    });

    // Clicking the input field to enter the location
    await page.waitForSelector(xpathSearchBox, { timeout: 60000 });
    await page.click(xpathSearchBox);
    await page.fill(xpathSearchBox, location);
    await page.keyboard.press('Enter');

    // Wait until the list of properties is loaded
    await page.waitForSelector(listingDivSelector);

    // to store the extracted data
    let data = [];
    // navigating through pagination
    for (let pageNum = 0; pageNum < maxPagination; pageNum++) {

        await page.waitForLoadState("load", { timeout: 120000 });
        await page.waitForTimeout(10);

        let allVisibleElements = page.locator(listingDivSelector);
        allVisibleElementsCount = await allVisibleElements.count()
        // going through each listing element
        for (let index = 0; index < allVisibleElementsCount; index++) {

            await page.waitForTimeout(2000);
            await page.waitForLoadState("load");
            let innerElement = await allVisibleElements.nth(index);
            await innerElement.hover();

            innerElement = await allVisibleElements.nth(index);
            let dataToSave = await extractData(innerElement);
            data.push(dataToSave);
        };
        //to load next page
        let nextPage = page.locator(xpathNextPage);
        await nextPage.hover();
        if (await nextPage.count()) {
            await nextPage.click();
        }
        else { break };
    };
    saveData(data);
    await context.close();
    await browser.close();
};

run();