I implemented a prototype novel website in Python

preface

Related software

Crawl data

CREATE TABLE `novel` (
   `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'Increment primary key',
   `title` varchar(100) NOT NULL COMMENT 'title',
   `content` text NOT NULL COMMENT 'content',
   PRIMARY KEY (`id`)
 ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8Copy the code

# database driver
pip install pymysql
Database connection pool
pip install DBUtilsCopy the code

# -*- coding: UTF-8 -*-
# import requests library
import requests
# Import file operation library

import codecs
from bs4 import BeautifulSoup
import sys
import mysql_DBUtils
from mysql_DBUtils import MyPymysqlPool
import importlib
importlib.reload(sys)


Assign a request header to the request to mimic chrome
headers = {
    'User-Agent': 'the Mozilla / 5.0 (Windows NT 10.0; Win64; X64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36'}
server = 'http://www.biquge.cm'
# star change address
book = 'http://www.biquge.cm/2/2042/'
# define the DB
mysql = MyPymysqlPool("dbMysql")


Get chapter content
def get_contents(chapter):
    req = requests.get(url=chapter)
    html = req.content
    html_doc = str(html, 'gbk')
    bf = BeautifulSoup(html_doc, 'html.parser')
    texts = bf.find_all('div', id="content")
    # get the content of div tag id attribute content \xa0 is unbroken whitespace  
    content = texts[0].text.replace('\xa0' * 4, '\n')
    return content


Write to database
def write_db(chapter, content):
    sql = "INSERT INTO novel (title, content) VALUES(%(title)s, %(content)s);"
    param = {"title": chapter, "content": content}
    mysql.insert(sql, param)


# main method
def main():
    res = requests.get(book, headers=headers)
    html = res.content
    html_doc = str(html, 'gbk')
    # Use HTML. Parser
    soup = BeautifulSoup(html_doc, 'html.parser')
    Get all chapters
    a = soup.find('div', id='list').find_all('a')
    print('Total chapters: %d' % len(a))
    for each in a:
        try:
            chapter = server + each.get('href')
            content = get_contents(chapter)
            chapter = each.string
            write_db(chapter, content)
        except Exception as e:
            print(e)
    mysql.dispose()


if __name__ == '__main__':
    main()Copy the code

The Web implementation

• Object relational maps for creating models
• Perfect management interface designed for end users
• First-class URL design
• Designer-friendly template language
• Caching system

Create a project

pip install Django
# Create a project
python  django-admin.py startproject itstyle
# change directory
cd itstyle
# to create the App
python manage.py startapp novelCopy the code

The default port is 8000
python manage.py runserverCopy the code

python manage.py runserver 8001Copy the code

The project structure

│ ├─ │ Settings. Py │ ├─ Novel │ Settings# Basic configuration│ │ urls. Py# URL mapping│ │ wsgi.py │ │ ├─templates# Related page│ will. HTML# section│ novel_list. HTML# First page of the novel├ ─ utils │ │ dbMysqlConfig CNFDatabase configuration parameters│ │ encoder. Py# code class│ │ mysql_DBUtils. PyDatabase connection pool└ ─ view │ index. Py# Background businessCopy the code

Note the main points

RESTful style

from django.conf.urls import url
from django.urls import path
from view import index

urlpatterns = [
    # Starchange
    path(' ', index.main),  # new
    # chapter page regular match
    path('chapter/<int:novel_id>/', index.chapter),    # new
]Copy the code

from django.http import HttpResponse
from django.shortcuts import render
from utils.mysql_DBUtils import mysql


# Starchange chapter list
def main(request):
    sql = "SELECT id,title FROM novel LIMIT 10;"
    result = mysql.getAll(sql)
    # result = json.dumps(result, cls=MyEncoder, ensure_ascii=False, indent=4)
    # result = json.loads(result)
    context = {'novel_list': result}
    return render(request, 'novel_list.html',  context)


# def chapter(request):
# id = request.GET['id']
# sql = "SELECT content FROM novel where id = %(id)s;"
# param = {"id": id}
# result = mysql.getOne(sql, param)
# context = {'novel': result}
# return render(request, 'novel.html', context)

' ''a single chapter Here novel_id corresponding urls. Py of < int: novel_id > you can visit: http://localhost:8000/chapter/1/ '' '
def chapter(request, novel_id):
    sql = "SELECT title,content FROM novel where id = %(id)s;"
    param = {"id": novel_id}
    result = mysql.getOne(sql, param)
    context = {'novel': result}
    return render(request, 'novel.html', context)Copy the code

The list shows

{% for novel in novel_list %}
    <a href="/chapter/{{novel.id}} "><li>{{ novel.title }}</li></a>
{% endfor %}Copy the code

summary

Add Teacher 2643804531 as a friend, she will send you the past information, timely send you the project oh

611481448, this is our group number, if you have any questions can consult in the group