一休の口コミをスクレイピングしてみる【Python】

$ pip install selenium

$ pip install webdriver_manager

$ python3 -m notebook

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.common.exceptions import NoSuchElementException
from webdriver_manager.chrome import ChromeDriverManager
import pandas
import time

options = Options()
driver = webdriver.Chrome(ChromeDriverManager().install(), options=options)

url = 'https://www.ikyu.com/00000662/#review-modal'
driver.get(url)

# 「口コミをもっと見る」のボタンがなくなるまでクリックを続ける
while True:
    try:
        element = driver.find_element_by_css_selector('.WordOfMouth-ikyu__showMore--21ctT')
        time.sleep(1)
        if element.is_displayed():
            element.click()
        else:
            break
    except NoSuchElementException:
        # これ以上読み込めなくても終了
        break

body = driver.find_elements_by_xpath("//dd[@class='WordOfMouth-ikyu__reviewContentsFusuma--TZ3NI']/p[1]")
date = driver.find_elements_by_class_name('WordOfMouth-ikyu__postedDate--1YawZ')

l=[]
for d, b,in zip(date, body):
    data={}
    data["口コミ"]=b.text
    data["投稿日"]=d.text
    l.append(data)

# ドライバーを終了させる
driver.quit()

df=pandas.DataFrame(l)
df.to_csv("一休.csv", encoding='utf_8_sig')
df

====== WebDriver manager ======

  # 省略

selenium.common.exceptions.WebDriverException: Message: unknown error: Chrome failed to start: crashed.
  (unknown error: DevToolsActivePort file doesn't exist)
  (The process started from chrome location /usr/bin/google-chrome is no longer running, so ChromeDriver is assuming that Chrome has crashed.)

options = Options()
options.add_argument("--headless")
options.add_argument("--no-sandbox")
driver = webdriver.Chrome(ChromeDriverManager().install(), options=options)

$ google-chrome
Running as root without --no-sandbox is not supported. See https://crbug.com/638180.