파이썬 크롤링 코드 질문이 있어서 올립니다.

게시물 수정 , 삭제는 로그인 필요

뉴스 제목과 링크까지는 크롤링이 완료가 되는데 본문만 빈칸인 채로 크롤링이 진행됩니다.

도움을 주시면 감사하겠습니다.

------------------------------------------------

#-*-coding:utf-8 -*-

from bs4 import BeautifulSoup

from datetime import datetime

import requests

import pandas as pd

import re

#한글깨짐 방지

import sys

import io

#sys.stdout = io.TextIOWrapper(sys.stdout.detach(),encoding = 'utf-8')

#sys.stderr = io.TextIOWrapper(sys.stderr.detach(),encoding = 'utf-8')

title_text = []

link_text = []

source_text = []

contents_text = []

result = {}

RESULT_PATH = '/Users/user/Desktop'

now = datetime.now()

def contents_cleansing(contents):

    first_cleansing_contents = re.sub('.*?','',str(contents)).strip()

    second_cleansing_contents = re.sub('.*?','',first_cleansing_contents).strip()

    

    third_cleansing_contents = re.sub('<.+?>','',second_cleansing_contents).strip()

    contents_text.append(third_cleansing_contents)

    #print(contents_text)

    

def crawler(maxpage,query,sort,s_date,e_date):

    s_from = s_date.replace(".","")

    e_to = e_date.replace(".","")

    page = 1

    maxpage_t = (int(maxpage)-1)*10+1

    while page <= maxpage_t:

        url = "https://search.naver.com/search.naver?where=news&query=" + query + "&sort="+sort+"&ds=" + s_date + "&de=" + e_date + "&nso=so%3Ar%2Cp%3Afrom" + s_from + "to" + e_to + "%2Ca%3A&start=" + str(page)

        response = requests.get(url)

        html = response.text

    

        soup = BeautifulSoup(html,'html.parser')

    

        atags = soup.find_all('a', 'news_tit')

        for atag in atags:

            title = atag.get('title')

            title_text.append(title)

            link_text.append(atag['href'])

        

        source_list = soup.find_all('a','info press')

        for source_list in source_list:

            source_text.append(source_list.text)

    

        contents_lists = soup.find_all('a','api_txt_lines dsc_txt_wrap')

        for contents_list in contents_lists:

            contents_cleansing(contents_list)

    

        result = {"title":title_text, "source":source_text, "contents":contents_text,"link":link_text}

        df = pd.DataFrame(result)

        page += 10

    

    outputFileName = '%s-%s-%s  %s시 %s분 %s초 merging.xlsx'%(now.year, now.month, now.day, now.hour, now.minute, now.second)

    df.to_excel(RESULT_PATH+outputFileName, sheet_name='sheet1')

def main():

    info_main = input("="*50+"\n"+"입력 형식에 맞게 입력해주세요."+"\n"+"시작하려면 Enter를 눌러주세요."+"\n"+"="*50)

    maxpage = input('최대 크롤링할 페이지 수를 입력하시오:')

    query = input("검색어 입력:")

    sort = input("뉴스 검색 방식 입력(관련도순=0 최신순=1 오래된순=2):")

    s_date = input("시작날짜 입력(2021.05.28):")

    e_date = input("끝날짜 입력(2021.06.01):")

    crawler(maxpage,query,sort,s_date,e_date)

main()