An old school friend’s story triggered the NodeJS crawler

(2018.08.31 Supplement)

Crawler scheme analysis:

Open the main page you want to crawl to, right-click a document under Doc File and choose “Check”. In the browser developer tool (Windows F12 can be directly opened), you can see the URL of the document and the DOM structure of the entire page under Element.

Using the same method, you can open one of the articles, right click where the article is crawling to view the Dom structure of the article and retrieve the content.

A brief idea:

Locate the document in Table– Doc File under Table– take out the url of the article under Doc File– get the content of the article according to the URL — select the DOM where the content we want to crawl to — format and save it in TXT.

The node packages used are as follows:

var http = require('http');
var https = require('https')
var fs = require('fs');
var cheerio = require('cheerio');
var urlHeader = 'https://wwwn.cdc.gov'
var urlFather = 'https://wwwn.cdc.gov/nchs/nhanes/search/datapage.aspx?Component=Laboratory&CycleBeginYear=2013'/ / the initial urllet count = 0;
function findUrlList(x,callback){
    https.get(x, function (res) {   
        var html = ' '; Var titles = []; res.setEncoding('utf-8'); // Listen to the data event, one piece ata time res.on('data'.function(chunk) { html += chunk; }); // Listen for the end event. If the HTML of the entire page content is retrieved, the callback res.on() is executed.'end'.function() { var $ = cheerio.load(html); Var urlArr = []; $('#GridView1 tbody tr').each(function (index, item) {
            let url = urlHeader + $(this).children().next().children('a').attr("href")
            startRequest(url)
            urlArr.push(url)
         })
        console.log(urlArr.length)
        callback()
        });

    }).on('error'.function (err) {
        console.log(err);
    });
}

functionStartRequest (x) {// Use the HTTP module to make a get request to the server.function (res) {   
        var html = ' '; Var titles = []; res.setEncoding('utf-8'); // Listen to the data event, one piece ata time res.on('data'.function(chunk) { html += chunk; }); // Listen for the end event. If the HTML of the entire page content is retrieved, the callback res.on() is executed.'end'.function() { var $ = cheerio.load(html); Var news_item = {// Get the title of the article: $('div #PageHeader h2').text().trim(),
            url: 'Article Address:'+x,
            firstParagraph: 'First paragraph: \n'+ $('#Component_Description').next().text().trim(),
            codeBookAndFrequencies: 'Table information: \n'+ $('#Codebook').children().text().trim() }; savedContent($,news_item); // Store the content and title of each article}); }).on('error'.function(err) { console.error(err); }); } // This function is used to store local news content resourcesfunction savedContent($, news_item) {
    count++;
    let x = '['+count+'] ' + '\n';
    x += news_item.url;
    x = x + '\n'; 
    x += news_item.firstParagraph;
    x = x + '\n'; 
    x += news_item.codeBookAndFrequencies;
    x = x + '\n'; 
    x += '------------------------------------------------------------ \n'; 
    x = x + '\n'; 
    x = x + '\n'; // Add the news text paragraph by paragraph to the /data folder and name the file fs.appendFile('./data/' + news_item.title + '.txt', x, 'utf-8'.function (err) {
        if(err) { console.log(err); }}); } // startRequest(url); FindUrlList (urlFather,() => {console.log()'work done')})Copy the code

Run the command line node app.js to see the result!

Crawler TXT result check:

API reference: