Kyobo Bookstore review crawl is not available.

I'd like to get a list of Naver bestsellers (yes24) and go to the Kyobo Bookstore page of the book and get a Clover review (one-line review) from the page It's not working.

Only the annotated parts do not work properly. The code is executed, but it is printed as blank. I want to know how to get the review correctly.

//from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
from bs4 import BeautifulSoup
import time


url = "https://book.naver.com/bestsell/bestseller_list.naver"

driver = webdriver.Chrome(ChromeDriverManager().install())
driver.implicitly_wait(30)

driver.get(url)
bsObject = BeautifulSoup(driver.page_source, 'html.parser')

book_page_urls = []
for index in range(0, 25):
    dl_data = bsObject.find('dt', {'id':"book_title_"+str(index)})
    link = dl_data.select('a')[0].get('href')
    book_page_urls.append(link)

for index, book_page_url in enumerate(book_page_urls):

    driver.get(book_page_url)
    bsObject = BeautifulSoup(driver.page_source, 'html.parser')

    # Title, isbn extraction
    title = bsObject.find('meta', {'property':'og:title'}).get('content')
    isbn = bsObject.select_one('#container > div.spot > div.book_info > div.book_info_inner > div:nth-child(3)').text

    # index, title, isbn output
    print(index+1, title, isbn)

    kyobo_page_urls = []
    for index in range(0, 1) :
        kyobo_review_data = bsObject.find_all('div', {'class':"npay_wrap"})[1]
        kyobo_review_link = kyobo_review_data.select('a')[0].get('href')
        kyobo_page_urls.append(kyobo_review_link)

        print(kyobo_review_link)


    # The part of the problem that doesn't go around
    for index, kyobo_page_url in enumerate(kyobo_page_urls):

        driver.get(kyobo_page_url)
        bsObject = BeautifulSoup(driver.page_source, 'html.parser')

        a = bsObject.select('div.comment_wrap')

        for asd in a : 
            kyobo_review = asd.select_one('dd.comment > div.txt').text

            print(kyobo_review)

python crawling

2022-09-20 08:58

1 Answers

#from selenium.webdriver.common.by import By
#a = bsObject.select('div.comment_wrap')
#a = driver.find_element(By.CSS_SELECTOR, 'div.nav_page > ul.tab_detail_content > li:nth-of-type(3)')
#a.click()
driver.get(kyobo_page_url + '#review')
time.sleep(20)
a = bsObject.select('dd.comment > div.txt')
for asd in a : 
    #kyobo_review = asd.select_one('dd.comment > div.txt').text
    kyobo_review = asd.text
    print(f'kyobo_review = {kyobo_review}')

2022-09-20 08:58

If you have any answers or tips

Popular Tags

python x 4647

android x 1593

java x 1494

javascript x 1427

c x 927

c++ x 878

ruby-on-rails x 696

php x 692

python3 x 685

html x 656