Crawler series (6) use URllib and RE to crawl Baidu tieba

First of all, we use Chrome browser to open Baidu Tieba and enter key words in the input field to search. The example here is “Computer Bar”.

Next, we directly use the shortcut key Ctrl+U to open the source code of the web page, and carefully analyze the data we need to capture in each page

It is easy to find that the content of each post is contained in a <li> tag, and we can use regular expressions to match, including:

• Topic Name:r'href="/p/\d+" title="(.+?) "'
• Subject author:R 'title=" (.+?) "'
• Link address:r'href="/p/(\d+)"'
• Reply:R "title =" reply "> (\ d +) < '
• Date created:R 'title=" create time ">(.+?) < '

The core code is as follows:

import re
# Parse web source code, extract data
def parse_page(html) :
    # theme name
    titles = re.findall(r'href="/p/\d+" title="(.+?) "',html)
    # Theme author
    authods = re.findall(R 'title=" (.+?) "',html)
    # link address
    nums = re.findall(r'href="/p/(\d+)"',html)
    links = ['http://tieba.baidu.com/p/'+str(num) for num in nums]
    # Number of replies
    focus = re.findall(R "title =" reply "> (\ d +) ',html)
    # create time
    ctimes = re.findall(R 'title=" create time ">(.+?) < ',html)
    # Get results
    data = zip(titles,authods,links,focus,ctimes)
    # return result
    return data
Copy the code

The complete code is below, again quite simple, less than 100 lines

import urllib.request
import urllib.parse
import re
import json
import csv
import time
import random

Get the source code of the web page
def get_page(url) :
    headers = {
        'USER-AGENT':'the Mozilla / 5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
    }
    req = urllib.request.Request(url=url,headers=headers)
    response = urllib.request.urlopen(req)
    html = response.read().decode('utf-8')
    return html

# Parse web source code, extract data
def parse_page(html) :
    titles = re.findall(r'href="/p/\d+" title="(.+?) "',html)
    authods = re.findall(R 'title=" (.+?) "',html)
    nums = re.findall(r'href="/p/(\d+)"',html)
    links = ['http://tieba.baidu.com/p/'+str(num) for num in nums]
    focus = re.findall(R "title =" reply "> (\ d +) ',html)
    ctimes = re.findall(R 'title=" create time ">(.+?) < ',html)
    data = zip(titles,authods,links,focus,ctimes)
    return data

# Open file
def openfile(fm,fileName) :
    if fm == 'txt':
        return open(fileName+'.txt'.'w',encoding='utf-8')
    elif fm == 'json':
        return open(fileName+'.json'.'w',encoding='utf-8')
    elif fm == 'csv':
        return open(fileName+'.csv'.'w',encoding='utf-8',newline=' ')
    else:
        return None

Save the data to a file
def save2file(fm,fd,data) :
    if fm == 'txt':
        for item in data:
            fd.write('----------------------------------------\n')
            fd.write('the title: + str(item[0]) + '\n')
            fd.write('authod：' + str(item[1]) + '\n')
            fd.write('the link: + str(item[2]) + '\n')
            fd.write('the focus: + str(item[3]) + '\n')
            fd.write('ctime：' + str(item[4]) + '\n')
    if fm == 'json':
        temp = ('title'.'authod'.'link'.'focus'.'ctime')
        for item in data:
            json.dump(dict(zip(temp,item)),fd,ensure_ascii=False)
    if fm == 'csv':
        writer = csv.writer(fd)
        for item in data:
            writer.writerow(item)

# Start crawling the page
def crawl() :
    kw = input('Please enter the theme post bar name:')
    base_url = 'http://tieba.baidu.com/f?kw=' + urllib.parse.quote(kw) + '&ie=utf-8&pn={page}'
    fm = input('Please input file save format (TXT, JSON, CSV) :')
    whilefm! ='txt' andfm! ='json' andfm! ='csv':
        fm = input('Input error, please re-enter file save format (TXT, JSON, CSV) :')
    fd = openfile(fm,kw)
    page = 0
    total_page = int(re.findall((\d+),get_page(base_url.format(page=str(0))))0])
    print('Start crawling')
    while page < total_page:
        print('Climbing to number one'.int(page/50+1), '页.......')
        html = get_page(base_url.format(page=str(page)))
        data = parse_page(html)
        save2file(fm,fd,data)
        page += 50
        time.sleep(random.random())
    fd.close()
    print('End of crawl')

if __name__ == '__main__':
    crawl()
Copy the code