How to Scrape Realtor.com: Code and No Code Approaches

import asyncio
import json
import logging
from playwright.async_api import async_playwright

LOCATION = "Texas City,TX"
MAX_PAGINATION = 2

async def extract_text(innerElement, selector):
    locator = innerElement.locator(selector)
    return await locator.inner_text() if await locator.count() else None

def clean_data(data: str) -> str:
    if not data:
        return ""
    cleanedData = " ".join(data.split()).strip()
    return cleanedData.encode("ascii", "ignore").decode("ascii")

async def extract_home_data(page, selector):
    homes = []

    for _ in selectors:
        text = await extract_text(page, selector)
        homes.append(clean_data(text))

    return homes

async def extract_data(page, selector, location, max_pagination) -> list:
    homes_for_sale = []

    for pageNumber in range(max_pagination):
        await page.wait_for_load_state("load")

        all_visible_elements = page.locator(selector)
        all_visible_elements_count = await all_visible_elements.count()

        for index in range(all_visible_elements_count):
            inner_element = all_visible_elements.nth(index=index)
            await inner_element.hover()

            selectors = [
                "[data-testid='card-price']",
                "[data-testid='property-meta-beds']",
                "[data-testid='property-meta-baths']",
                "[data-testid='property-meta-sqft']",
                "[data-testid='property-meta-lot-size']",
                "[data-testid='card-address']"
            ]

            price, beds, baths, sqft, lot_size, address = await extract_home_data(inner_element, selectors)

            homes_for_sale.append({
                "price": price,
                "no of Beds": beds,
                "no of Baths": baths,
                "Sqft": sqft,
                "Acre Lot": lot_size,
                "address": address
            })

        nextPage = page.locator("[aria-label='Go to next page']")
        if not await nextPage.count():
            break
        await nextPage.click()

    save_data(homes_for_sale, "HomesData.json")

async def run(playwright, location):
    browser = await playwright.firefox.launch(headless=False)
    context = await browser.new_context()
    page = await context.new_page()

    await page.set_viewport_size({"width": 1920, "height": 1080})
    page.set_default_timeout(120000)

    await page.goto("https://www.realtor.com/", wait_until="domcontentloaded")
    await page.wait_for_load_state("load")

    search_box = "[id='search-bar']"
    await page.locator(search_box).click()
    await page.locator(search_box).fill(location)
    await page.locator(search_box).press("Enter")

    listing_div_selector = "[data-testid='card-content']"
    await page.wait_for_selector(listing_div_selector)

    await extract_data(page, listing_div_selector, location, MAX_PAGINATION)

    await context.close()
    await browser.close()

def save_data(product_page_data: list, filename: str):
    with open(filename, "w") as outfile:
        json.dump(product_page_data, outfile, indent=4)

async def main(location):
    async with async_playwright() as playwright:
        await run(playwright, location)

if __name__ == "__main__":
    asyncio.run(main(LOCATION))`

const { chromium, firefox } = require('playwright');
const fs = require('fs');

const location = "Texas City,TX";
const maxPagination = 2;

/**
* Save data as list of dictionaries
as json file
* @param {object} data
*/
function saveData(data) {
    let dataStr = JSON.stringify(data, null, 2)
    fs.writeFile("DataJS.json", dataStr, 'utf8', function (err) {
        if (err) {
            console.log("An error occurred while writing JSON Object to File.");
            return console.log(err);
        }
        console.log("JSON file has been saved.");
    });
}

function cleanData(data) {
    if (!data) {
        return;
}
    // removing extra spaces and unicode characters
    let cleanedData = data.split(/\s+/).join(" ").trim();
    cleanedData = cleanedData.replace(/[^\x00-\x7F]/g, "");
    return cleanedData;
}

/**
* The data extraction function used to extract
necessary data from the element.
* @param {HtmlElement} innerElement
* @returns
*/
async function extractData(innerElement) {

    async function extractData(data) {
        let count = await data.count();
        if (count) {
            return await data.innerText()
        }
        return null
    };
    // intializing xpath and selectors

    const priceSelector = "[data-testid='card-price']"
    const bedSelector = "[data-testid='property-meta-beds']"
    const bathSelector = "[data-testid='property-meta-baths']"
    const sqftSelector = "[data-testid='property-meta-sqft']"
    const acreSelector = "[data-testid='property-meta-lot-size']"
    const addressSelector = "[data-testid='card-address']"

    // Extracting necessary data
    let price = innerElement.locator(priceSelector);
    price = await extractData(price);

    let noOfBeds = innerElement.locator(bedSelector);
    noOfBeds = await extractData(noOfBeds);

    let noOfBaths = innerElement.locator(bathSelector);
    noOfBaths = await extractData(noOfBaths);

    let areaInSqft = innerElement.locator(sqftSelector);
    areaInSqft = await extractData(areaInSqft);
    if (areaInSqft) {
        areaInSqft = areaInSqft.split('\n')[0];
    } else {
        areaInSqft = null;
    }

    let areaInAcreLot = innerElement.locator(acreSelector);
    areaInAcreLot = await extractData(areaInAcreLot);
    if (areaInAcreLot) {
        areaInAcreLot = areaInAcreLot.split('\n')[0];
    } else {
        areaInAcreLot = null;
    }

    let address = innerElement.locator(addressSelector);
    address = await extractData(address)

    // cleaning data
    price = cleanData(price)
    noOfBeds = cleanData(noOfBeds)
    noOfBaths = cleanData(noOfBaths)
    areaInAcreLot = cleanData(areaInAcreLot)
    areaInSqft = cleanData(areaInSqft)
    address = cleanData(address)

    extractedData = {
        "price": price,
        "no of Beds": noOfBeds,
        "no of Baths": noOfBaths,
        "Sqft": areaInSqft,
        'Acre Lot': areaInAcreLot,
        'address': address
    }
    console.log(extractData)

    return extractedData
}
/**
* The main function initiate a browser object and handle the navigation.
*/
async function run() {
    // intializing browser and creating new page
    const browser = await chromium.launch({ headless: false});
    const context = await browser.newContext();
    const page = await context.newPage();

    // initializing xpaths and selectors
    const xpathSearchBox = "[id = 'search-bar']";
    const listingDivSelector = "[data-testid='card-content']";
    const totalResultsSelector = "[data-testid='total-results']";
    const xpathNextPage = "[aria-label='Go to next page']";

    // Navigating to the home page
    await page.goto('https://www.realtor.com/', {
    waitUntil: 'domcontentloaded',
    timeout: 60000,
  });

    // Clicking the input field to enter the location
    await page.waitForSelector(xpathSearchBox, { timeout: 60000 });
    await page.click(xpathSearchBox);
    await page.fill(xpathSearchBox, location);
    await page.keyboard.press('Enter');

    // Wait until the list of properties is loaded
    await page.waitForSelector(listingDivSelector);

    const totalResultCount = await page.locator(totalResultsSelector).innerText();
    console.log(`Total results found - ${totalResultCount} for location - ${location}`);

    // to store the extracted data
    let data = [];
    // navigating through pagination
    for (let pageNum = 0; pageNum < maxPagination; pageNum++) {

        await page.waitForLoadState("load", { timeout: 120000 });
        await page.waitForTimeout(10);

        let allVisibleElements = page.locator(listingDivSelector);
        allVisibleElementsCount = await allVisibleElements.count()
        // going through each listing element
        for (let index = 0; index < allVisibleElementsCount; index++) {

            await page.waitForTimeout(2000);
            await page.waitForLoadState("load");
            let innerElement = await allVisibleElements.nth(index);
            await innerElement.hover();

            innerElement = await allVisibleElements.nth(index);
            let dataToSave = await extractData(innerElement);
            data.push(dataToSave);
        };
        //to load next page
        let nextPage = page.locator(xpathNextPage);
        await nextPage.hover();
        if (await nextPage.count()) {
            await nextPage.click();
        }
        else { break };
    };
    saveData(data);
    await context.close();
    await browser.close();
};

run();