How to Scrape Amazon Reviews: Using Code and No Code Approaches

import asyncio
import json

from playwright.async_api import async_playwright

url = " https://www.amazon.com/Crocs-Unisex-Classic-Black-Women/dp/B0014C2NBC/ref=sr_1_3?crid=2WI0QM6BYFKLZ&keywords=crocs&qid=1696915791&s=amazon-devices&sprefix=croc%2Camazon-devices%2C449&sr=1-3"
max_pagination = 2


async def extract_data(page) -> list:
    """
    Parsing details from the listing page

    Args:
        page: webpage of the browser

    Returns:
        list: details of homes for sale
    """

    # Initializing selectors and xpaths
    seemore_selector = "//div[@id='reviews-medley-footer']//a"
    div_selector = "[class='a-section celwidget']"
    next_page_selector = "[class='a-last']"
    name_xpath = "//a[@class='a-profile']//span[@class='a-profile-name']"
    rate_xpath = "//a//i[contains(@class,'review-rating')]/span"
    review_title_xpath = "//a[contains(@class, 'review-title')]/span[2]"
    review_date_xpath = "//span[contains(@class,'review-date')]"
    review_text_xpath = "[data-hook='review-body']"
    # navigating to reviewpage
    review_page_locator = page.locator(seemore_selector)
    await review_page_locator.hover()
    await review_page_locator.click()

    # List to save the details of properties
    amazon_reviews_ratings = []

    # Paginating through each page
    for _ in range(max_pagination):
        # Waiting for the page to finish loading
        await page.wait_for_load_state("load")

        # Extracting the elements
        review_cards = page.locator(div_selector)
        cards_count = await review_cards.count()
        for index in range(cards_count):
            # Hovering the element to load the price
            inner_element = review_cards.nth(index=index)
            await inner_element.hover()
            inner_element = review_cards.nth(index=index)
            
            # Extracting necessary data
            name = await inner_element.locator(name_xpath).inner_text() if await inner_element.locator(name_xpath).count() else None
            rate = await inner_element.locator(rate_xpath).inner_text() if await inner_element.locator(rate_xpath).count() else None
            review_title = await inner_element.locator(review_title_xpath).inner_text() if await inner_element.locator(review_title_xpath).count() else None
            review_date = await inner_element.locator(review_date_xpath).inner_text() if await inner_element.locator(review_date_xpath).count() else None
            review_text = await inner_element.locator(review_text_xpath).inner_text() if await inner_element.locator(review_text_xpath).count() else None

            # Removing extra spaces and unicode characters
            name = clean_data(name)
            rate = clean_data(rate)
            review_title = clean_data(review_title)
            review_date = clean_data(review_date)
            review_text = clean_data(review_text)

            data_to_save = {
                "reviewer_name": name,
                "rate": rate,
                "review_title": review_title,
                "review_date": review_date,
                "review_text": review_text
            }

            amazon_reviews_ratings.append(data_to_save)
        next_page_locator = page.locator(next_page_selector)

        # Check if the "Next Page" button exists
        if await next_page_locator.count() > 0:
            await next_page_locator.hover()
            await next_page_locator.click()
        else:
            break

    save_data(amazon_reviews_ratings, "Data.json")


async def run(playwright) -> None:
    # Initializing the browser and creating a new page.
    browser = await playwright.firefox.launch(headless=False)
    context = await browser.new_context()
    page = await context.new_page()

    await page.set_viewport_size({"width": 1920, "height": 1080})
    page.set_default_timeout(120000)

    # Navigating to the homepage
    await page.goto(url, wait_until="domcontentloaded")

    await extract_data(page)

    await context.close()
    await browser.close()


def clean_data(data: str) -> str:
    """
    Cleaning data by removing extra white spaces and Unicode characters

    Args:
        data (str): data to be cleaned

    Returns:
        str: cleaned string
    """
    if not data:
        return ""
    cleaned_data = " ".join(data.split()).strip()
    cleaned_data = cleaned_data.encode("ascii", "ignore").decode("ascii")
    return cleaned_data


def save_data(product_page_data: list, filename: str):
    """Converting a list of dictionaries to JSON format

    Args:
        product_page_data (list): details of each product
        filename (str): name of the JSON file
    """
    with open(filename, "w") as outfile:
        json.dump(product_page_data, outfile, indent=4)


async def main() -> None:
    async with async_playwright() as playwright:
        await run(playwright)


if __name__ == "__main__":
    asyncio.run(main())

const { chromium, firefox } = require('playwright');
const fs = require('fs');

const url = "https://www.amazon.com/Crocs-Unisex-Classic-Black-Women/dp/B0014C2NBC/ref=sr_1_3?crid=2WI0QM6BYFKLZ&keywords=crocs&qid=1696915791&s=amazon-devices&sprefix=croc%2Camazon-devices%2C449&sr=1-3";
const maxPagination = 2;

/**
* Save data as list of dictionaries
as json file
* @param {object} data
*/
function saveData(data) {
    let dataStr = JSON.stringify(data, null, 2)
    fs.writeFile("data.json", dataStr, 'utf8', function (err) {
        if (err) {
            console.log("An error occurred while writing JSON Object to File.");
            return console.log(err);
        }
        console.log("JSON file has been saved.");
    });
}

function cleanData(data) {
    if (!data) {
        return;
    }
    // removing extra spaces and unicode characters
    let cleanedData = data.split(/\s+/).join(" ").trim();
    cleanedData = cleanedData.replace(/[^\x00-\x7F]/g, "");
    return cleanedData;
}

/**
* The data extraction function used to extract
necessary data from the element.
* @param {HtmlElement} innerElement
* @returns
*/
async function extractData(innerElement) {

    async function extractData(data) {
        let count = await data.count();
        if (count) {
            return await data.innerText()
        }
        return null
    };
    // intializing xpath and selectors
    const nameXpath = "//a[@class='a-profile']//span[@class='a-profile-name']";
    const rateXpath = "//a//i[contains(@class,'review-rating')]/span";
    const reviewTitleXpath = "//a[contains(@class, 'review-title')]/span[2]";
    const reviewDateXpath = "//span[contains(@class,'review-date')]";
    const reviewTextXpath = "[data-hook='review-body']";

    let name = innerElement.locator(nameXpath);
    name = await extractData(name);

    let rate = innerElement.locator(rateXpath);
    rate = await extractData(rate);

    let reviewTitle = innerElement.locator(reviewTitleXpath);
    reviewTitle = await extractData(reviewTitle);

    let reviewDate = innerElement.locator(reviewDateXpath);
    reviewDate = await extractData(reviewDate);

    let reviewText = innerElement.locator(reviewTextXpath);
    reviewText = await extractData(reviewText);

    // cleaning data
    name = cleanData(name)
    rate = cleanData(rate)
    reviewTitle = cleanData(reviewTitle)
    reviewDate = cleanData(reviewDate)
    reviewText = cleanData(reviewText)

    extractedData = {
        "reviewer's name": name,
        'rate': rate,
        'reviewTitle': reviewTitle,
        'reviewDate': reviewDate,
        'reviewText': reviewText
    }

    return extractedData
}
/**
* The main function initiate a browser object and handle the navigation.
*/
async function run() {
    // intializing browser and creating new page
    const browser = await firefox.launch({ headless: false });
    const context = await browser.newContext();
    const page = await context.newPage();

    await page.setViewportSize({ width: 1920, height: 1080 });
    page.setDefaultTimeout(120000);
    // intializing xpath and selectors
    const seeMoreSelector = "//div[@id='reviews-medley-footer']//a"
    const divSelector = "[class='a-section celwidget']"
    const nextPageSelector = "[class='a-last']"


    // Navigating to the home page
    await page.goto(url, { waitUntil: 'domcontentloaded' });
    // Navigating to Review page
    const ReviewPageLocator = await page.$(seeMoreSelector);
    await ReviewPageLocator.hover();
    await ReviewPageLocator.click();

    // Wait until the list of properties is loaded
    await page.waitForSelector(divSelector);

    // to store the extracted data
    let data = [];
    // navigating through pagination
    for (let pageNum = 0; pageNum < maxPagination; pageNum++) {

        await page.waitForLoadState("load");
        await page.waitForTimeout(10);

        let reviewCards = page.locator(divSelector);
        reviewCardsCount = await reviewCards.count()
        // going through each listing element
        for (let index = 0; index < reviewCardsCount; index++) {

            await page.waitForTimeout(2000);
            await page.waitForLoadState("load");
            let innerElement = await reviewCards.nth(index);
            await innerElement.hover();

            innerElement = await reviewCards.nth(index);
            let dataToSave = await extractData(innerElement);
            data.push(dataToSave);
        };
        //to load next page
        const nextPageLocator = await page.$(nextPageSelector);

        // Check if the "Next Page" button exists
        if (nextPageLocator !== null) {
            await nextPageLocator.hover();
            await nextPageLocator.click();
        } else {
            // Exit the loop or perform other actions when the button is not found
            break;
        }
    };
    saveData(data);
    await context.close();
    await browser.close();
};

run();