主页 / 问题 / 992047
Accepted
Сергей Богданов
Asked:2020-06-12 20:56:09 +0000 UTC2020-06-12 20:56:09 +0000 UTC

解析整个站点

  • 772

下面是代码。逐个链接解析站点页面，收集其上的所有链接并写入文件。如何使解析器随后跟踪每个链接并解析该页面上的所有链接并添加到文件中。最终目标是从整个站点收集链接。如果可能的话，举个例子。这是代码

    def get_books() -> typing.List[str]:
    rs = requests.get('https://......')
    root = BeautifulSoup(rs.text, 'html.parser')

    t = []
    for x in root.findAll('a'):
        try:
            t1 = x["href"].strip()
            if not 'http' in t1:
                t1 = 'https://....' + t1
            t.append(t1)
        except KeyError:
            pass
    return t


if __name__ == '__main__':
    books = get_books()
    f = io.open('parsed_data.htm', 'w', encoding='utf8')
    for line in books:
        f.write(line + "\n")
    f.close()

    tf = io.open('text_new.txt', 'w', encoding='utf8')
    with open('parsed_data.htm', 'r') as f:
        file = f.readlines()

    seen_item = []
    for line in file:
        if line not in seen_item:
            seen_item.append(line)
            tf.write(line)
    tf.close()
python-3.x

1 个回答

  1. Best Answer
    import requests
from bs4 import BeautifulSoup

def get_books(url):
    root = None
    try:
        rs = requests.get(url)
        root = BeautifulSoup(rs.text, 'html.parser')
    except:
        pass

    if root:
        temp = []
        for x in root.findAll('a'):
            try:
                t1 = x["href"].strip()
                if not 'http' in t1:
                    t1 = 'https://....' + t1
                temp.append(t1)

            except KeyError:
                pass

        return temp


if __name__ == '__main__':

    books = get_books('https://docs.python.org/2/library/webbrowser.html')

    print('before: ', len(books))

    for book in books: # Парсит все ссылки полученые за первый проход
        more_books = get_books(book)

    print('more_books: ', len(more_books))

    all_books = more_books + books
    print('all_books: ', len(all_books))


    with open('E:\\Python\\exampeles\\parsed_data.htm', 'w', encoding='utf8') as file:
        for line in all_books:
            file.write(line + "\n")

    with open('E:\\Python\\exampeles\\text_new.txt', 'w', encoding='utf8') as tf:
        seen_item = []
        for line in all_books:
            if line not in seen_item:
                seen_item.append(line)
                tf.write(line + "\n")
        print(len(seen_item))
    • -1

相关问题