background
With the increasing complexity of the front end, there are many packaging tools, such as Grunt and gulp. To later Webpack and Parcel. However, many scaffolding tools, such as VUe-CLI, have helped us integrate the use of build tools. Sometimes we may not know the internal implementation. Understanding how these tools work can help us better understand and use them, as well as apply them to our projects.
Some knowledge points
Before we start building the wheel, we need to do a little bit of stocking up on the facts.
Modular knowledge
The first is module related knowledge, mainly es6 modules and commonJS modular specification. See CommonJS, AMD/CMD, ES6 Modules and WebPack for more details. Now all we need to know is:
es6 modulesIs a way to determine module dependencies at compile time.CommonJSAccording to the module specification of “.js “, Node will wrap the contents of JS files in the header and add them in the header during the compilation process(function (export, require, modules, __filename, __dirname){\nI added it to the tail\n};. This allows us to use these parameters within a single JS file.
AST Basics
What is an abstract syntax tree?
In computer science, an abstract syntax tree (AST for short), or syntax tree, is a tree-like representation of the abstract syntactic structure of source code, specifically the source code of a programming language. Each node in the tree represents a structure in the source code. The syntax is “abstract” because it does not represent every detail that occurs in real grammar.
You can use Esprima to convert code into an AST. The first abstract syntax tree that a piece of code converts to is an object that has a top-level type attribute Program, and the second property body is an array. Each item in the body array is an object that contains all of the statement descriptions:
typeKind: specifies the keyword of a variable declaration. Var declaration: specifies the array of declared contents, each of which is also an objecttype: Describes the type of the statement ID: describes the object of the variable nametypeName: specifies the name of the variable init: initializes the variable value objecttype: Type value: indicates the value"is tree"Row without quotation marks:"\"is tree"\"QuotedCopy the codeGet into the business
Webpack is easy to pack
With that in mind, let’s take a look at a simple WebPack packaging process. First we define three files:
// index.js
import a from './test'
console.log(a)
// test.js
import b from './message'
const a = 'hello' + b
export default a
// message.js
const b = 'world'
export default b
Copy the codeThis is a simple way to define an index.js reference to test.js; Test.js internally references message.js. Take a look at the packaged code:
(function (modules) {
var installedModules = {};
function __webpack_require__(moduleId) {
if (installedModules[moduleId]) {
return installedModules[moduleId].exports;
}
var module = installedModules[moduleId] = {
i: moduleId,
l: false.exports: {}}; modules[moduleId].call(module.exports, module.module.exports, __webpack_require__);
// Flag the module as loaded
module.l = true;
// Return the exports of the module
return module.exports;
}
// expose the modules object (__webpack_modules__)
__webpack_require__.m = modules;
// expose the module cache
__webpack_require__.c = installedModules;
// define getter function for harmony exports
__webpack_require__.d = function (exports, name, getter) {
if(! __webpack_require__.o(exports, name)) {Object.defineProperty(exports, name, {enumerable: true.get: getter}); }};// define __esModule on exports
__webpack_require__.r = function (exports) {
if (typeof Symbol! = ='undefined' && Symbol.toStringTag) {
Object.defineProperty(exports, Symbol.toStringTag, {value: 'Module'});
}
Object.defineProperty(exports, '__esModule', {value: true});
};
// create a fake namespace object
// mode & 1: value is a module id, require it
// mode & 2: merge all properties of value into the ns
// mode & 4: return value when already ns object
// mode & 8|1: behave like require
__webpack_require__.t = function (value, mode) {
/ * * * * * * /
if (mode & 1) value = __webpack_require__(value);
if (mode & 8) return value;
if ((mode & 4) && typeof value === 'object' && value && value.__esModule) return value;
var ns = Object.create(null);
__webpack_require__.r(ns);
Object.defineProperty(ns, 'default', {enumerable: true.value: value});
if (mode & 2 && typeofvalue ! ='string') for (var key in value) __webpack_require__.d(ns, key, function (key) {
return value[key];
}.bind(null, key));
return ns;
};
// getDefaultExport function for compatibility with non-harmony modules
__webpack_require__.n = function (module) {
var getter = module && module.__esModule ?
function getDefault() {
return module['default'];
} :
function getModuleExports() {
return module;
};
__webpack_require__.d(getter, 'a', getter);
return getter;
};
// Object.prototype.hasOwnProperty.call
__webpack_require__.o = function (object, property) {
return Object.prototype.hasOwnProperty.call(object, property);
};
// __webpack_public_path__
__webpack_require__.p = "";
// Load entry module and return exports
return __webpack_require__(__webpack_require__.s = "./src/index.js"); ({})"./src/index.js": (function (module, __webpack_exports__, __webpack_require__) {
"use strict";
eval("__webpack_require__.r(__webpack_exports__); \n/* harmony import */ var _test__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! ./test */ \"./src/test.js\"); \n\n\nconsole.log(_test__WEBPACK_IMPORTED_MODULE_0__[\"default\"])\n\n\n//# sourceURL=webpack:///./src/index.js?");
}),
"./src/message.js": (function (module, __webpack_exports__, __webpack_require__) {
// ...
}),
"./src/test.js": (function (module, __webpack_exports__, __webpack_require__) {
// ...})});Copy the codeLooking messy? That’s okay. Let’s do this again. Here’s what we see at first glance:
(function(modules) {
// ...({})// ...
})
Copy the codeIt’s a self-executing function that passes in a modules object. What’s the format of modules object? The code above already gives us the answer:
{
"./src/index.js": (function (module, __webpack_exports__, __webpack_require__) {
// ...
}),
"./src/message.js": (function (module, __webpack_exports__, __webpack_require__) {
// ...
}),
"./src/test.js": (function (module, __webpack_exports__, __webpack_require__) {
// ...})}Copy the codeIt’s a path, a function, a key, a value pair. Inside the function is the code after the file we defined is transferred to ES5:
"use strict";
eval("__webpack_require__.r(__webpack_exports__); \n/* harmony import */ var _test__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! ./test */ \"./src/test.js\"); \n\n\nconsole.log(_test__WEBPACK_IMPORTED_MODULE_0__[\"default\"])\n\n\n//# sourceURL=webpack:///./src/index.js?");
Copy the codeAt this point, the structure is basically analyzed, and then we look at its execution. The code executed since the beginning of the execution function is:
__webpack_require__(__webpack_require__.s = "./src/index.js");
Copy the codeThe __webpack_require_ function is called and a moduleId argument is passed with “./ SRC /index.js”. Let’s look at the main implementation inside the function:
// Define the module format
var module = installedModules[moduleId] = {
i: moduleId, // moduleId
l: false.// Whether it has been cached
exports: {} // Export objects to provide mounting
};
modules[moduleId].call(module.exports, module.module.exports, __webpack_require__);
Copy the codeHere we call our modules function and pass in the __webpack_require__ function as an internal call. The module.exports argument acts as an export inside the function. Since test.js is referenced in index.js, the __webpack_require__ is used to load test.js:
var _test__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__("./src/test.js");
Copy the codeMessage.js is used inside test.js so message.js is loaded inside test.js. When message.js is executed, the result is returned because there are no dependencies:
var b = 'world'
__webpack_exports__["default"] = (b)
Copy the codeOnce the execution is complete, the next level returns to the root file index.js. Finally complete the entire file dependency processing. The whole time, we are working our way inwards into the number in the form of a dependency tree, and when we return the result, we start working our way back to the root.
Develop a simple Tinypack
With all this research in mind, let’s first consider what a basic package compilation tool can do.
- Convert ES6 syntax to ES5
- Handle module load dependencies
- Generate a JS file that can be loaded and executed in the browser
The first problem, converting syntax, we can actually do that with Babel. The core steps are:
- through
babylonGenerate AST - through
babel-coreRegenerate the AST source code
/** * get the file and parse it into ast syntax * @param filename // import file * @returns {*} */
function getAst (filename) {
const content = fs.readFileSync(filename, 'utf-8')
return babylon.parse(content, {
sourceType: 'module'}); }/** * compile * @param ast * @returns {*} */
function getTranslateCode(ast) {
const {code} = transformFromAst(ast, null, {
presets: ['env']});return code
}
Copy the codeThen we need to deal with module dependencies, and that needs to get a dependency view. Fortunately, babel-traverse provides a function to traverse the AST view and do the processing. You can obtain the dependency properties with ImportDeclaration:
function getDependence (ast) {
let dependencies = []
traverse(ast, {
ImportDeclaration: ({node}) = >{ dependencies.push(node.source.value); }})return dependencies
}
/** * Generate full file dependency mapping * @param fileName * @param entry * @returns {{fileName: *, dependence, code: *}} */
function parse(fileName, entry) {
let filePath = fileName.indexOf('.js') = = =- 1 ? fileName + '.js' : fileName
let dirName = entry ? ' ' : path.dirname(config.entry)
let absolutePath = path.join(dirName, filePath)
const ast = getAst(absolutePath)
return {
fileName,
dependence: getDependence(ast),
code: getTranslateCode(ast),
};
}
Copy the codeSo far, we only have the root file dependencies and compiled code. For example, our index.js relies on test.js but we didn’t know that test.js also relies on message.js, and their source code is not compiled. Therefore, we also need to do depth traversal to obtain the completed depth dependence:
/ * * * * to obtain depth queue dependence @ param main * @ returns []} {* * /
function getQueue(main) {
let queue = [main]
for (let asset of queue) {
asset.dependence.forEach(function (dep) {
let child = parse(dep)
queue.push(child)
})
}
return queue
}
Copy the codeSo at this point we’ve compiled and parsed all of our files. The last step is that we need to wrap the source code in accordance with the idea of Webpack. The first step is to generate a Modules object:
function bundle(queue) {
let modules = ' '
queue.forEach(function (mod) {
modules += ` '${mod.fileName}': function (require, module, exports) { ${mod.code}}, `
})
// ...
}
Copy the codeOnce you get modules, the next thing to do is wrap the entire file around it, register require, module.exports:
(function(modules) {
function require(fileName) {
// ...
}
require('${config.entry}');
})({${modules}})
Copy the codeInside the function, it just loops through the JS code for each dependent file, completing the code:
function bundle(queue) {
let modules = ' '
queue.forEach(function (mod) {
modules += ` '${mod.fileName}': function (require, module, exports) { ${mod.code}}, `
})
const result = `
(function(modules) {
function require(fileName) {
const fn = modules[fileName];
const module = { exports : {} };
fn(require, module, module.exports);
return module.exports;
}
require('${config.entry}'); ({})${modules}})
`;
return result;
}
Copy the codeThat’s about it, let’s pack it up and try it:
(function (modules) {
function require(fileName) {
const fn = modules[fileName];
const module = {exports: {}};
fn(require.module.module.exports);
return module.exports;
}
require('./src/index.js'); ({})'./src/index.js': function (require, module, exports) {
"use strict";
var _test = require("./test");
var _test2 = _interopRequireDefault(_test);
function _interopRequireDefault(obj) {
return obj && obj.__esModule ? obj : {default: obj};
}
console.log(_test2.default);
}, './test': function (require, module, exports) {
"use strict";
Object.defineProperty(exports, "__esModule", {
value: true
});
var _message = require("./message");
var _message2 = _interopRequireDefault(_message);
function _interopRequireDefault(obj) {
return obj && obj.__esModule ? obj : {default: obj};
}
var a = 'hello' + _message2.default;
exports.default = a;
}, './message': function (require, module, exports) {
"use strict";
Object.defineProperty(exports, "__esModule", {
value: true
});
var b = 'world'; exports.default = b; }})Copy the codeTest it again:
Well, basically a simple Tinypack is done.
Refer to the article
Abstract syntax tree
JS abstract syntax tree at a glance
The source code
All the source code for Tinypack has been uploaded to Github