How Web Scraping is used to Extract User Reviews from Google Chrome Online Store Extension?

# Using Selenium to extract Chrome web store reviews
from selenium import webdriver
import time
from bs4 import BeautifulSoup
test_url = 'https://chrome.google.com/webstore/detail/data-scraper-easy-web-scr/nndknepjnldbdbepjfgmncbggmopgden'
option = webdriver.ChromeOptions()
option.add_argument("--incognito")
chromedriver = r'chromedriver.exe'
browser = webdriver.Chrome(chromedriver, options=option)
browser.get(test_url)
html_source = browser.page_source

# extracting chrome extension name
soup=BeautifulSoup(html_source, "html.parser")
print(soup.find_all('h1',{'class','e-f-w'})[0].get_text())
total_users = soup.find_all('span',{'class','e-f-ih'})[0].get_text()
print(total_users.strip())
total_reviews = soup.find_all('div',{'class','nAtiRe'})[0].get_text()
print(total_reviews.strip())
# rating value
meta=soup.find_all('meta')
for val in meta:
    try:
        if val['itemprop']=='ratingValue':
            print(val['content'])
    except:
        pass
#Output
    'Data Scraper - Easy Web Scraping'
    '200,000+ users'
    '562'
  4.080071174377224

# clicking reviews button
element = browser.find_element_by_xpath('//*[@id=":25"]/div/div')
element.click()
time.sleep(5)
# scrolling till end of the pag
from selenium.webdriver.common.keys import Keys
html = browser.find_element_by_tag_name('html')
html.send_keys(Keys.END)
html_source = browser.page_source
browser.close()
soup=BeautifulSoup(html_source, 'html.parser')

review_author_list_src = soup.find_all('span', {'class','comment-thread-displayname'})
review_author_name_list = []

for val in review_author_list_src:
    try:
        review_author_name_list.append(val.get_text())
    except:
        pass
review_author_name_list[:10]
#Output
['Bryan Bloom',
 'Sudhakar Kadavasal',
 'Lauren Rich',
 '�yvind Andr� Sandberg',
 'Paul Adamson',
 'Phoebe Staab',
 'Frank Mathmann',
 'Bobby Thomas',
 'Kevin Humphrey',
 'David Wills']

# extracting review dates
date_src = soup.find_all('span',{'class', 'ba-Eb-Nf'})
date_src
date_list = []
for val in date_src:
    date_list.append(val.get_text())
date_list[:10]
# Output
['Modified Feb 7, 2019',
 'Modified Jan 8, 2019',
 'Modified Dec 31, 2018',
 'Modified Jan 4, 2019',
 'Modified Dec 14, 2018',
 'Modified Feb 5, 2019',
 'Modified Dec 13, 2018',
 'Modified Jan 16, 2019',
 'Modified Nov 29, 2018',
 'Modified Nov 16, 2018']

# extracting review star rating
star_rating_src = soup.find_all('div', {'class','rsw-stars'})
star_rating_list = []
for val in star_rating_src:
    try:
        star_rating_list.append(val['aria-label'])
    except:
        pass
star_rating_list[:10]
# Output
['5 stars',
 '5 stars',
 '5 stars',
 '5 stars',
 '5 stars',
 '5 stars',
 '5 stars',
 '2 stars',
 '5 stars',
 '5 stars']

# extracting review content
review_content_src = soup.find_all('div',{'class', 'ba-Eb-ba'})
review_content_list = []
for val in review_content_src:
    review_content_list.append(val.get_text())
review_content_list[:3]
# Output
['This is one of the first times ever writing a review, but I HAD to.  This is the most awesome, easy-to-use, and amazing extension ever.  Literally saves hundreds of hours.  Thank you!',
 'Loved it. It automatically detected the data structure suited for the website and that helped me in learning how to use the tool without having to read the tutorial! Beautifully written tool. Kudos.',
 'Great tool for mining data. We used Data Miner to extract data from the Medicare.gov website for an upcoming mailing to nursing homes and assisted living facilities. It can comb through a number of pages in a matter of seconds, extracting thousands of rows into one concise spreadsheet. I would highly recommend this product to any business looking to obtain data for any purpose - mailing, email campaign, etc. Thank you Data Miner!']