Skip to content

Commit 096efc3

Browse files
sokrasokra
authored
Merge pull request #15801 from webpack/refactor-json-modules
refactor json modules
2 parents b8748cf + aa76e82 commit 096efc3

12 files changed

Lines changed: 167 additions & 26 deletions

File tree

‎lib/dependencies/JsonExportsDependency.js‎

Lines changed: 17 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -13,18 +13,21 @@ const NullDependency = require("./NullDependency");
1313
/** @typedef {import("../Dependency").ExportsSpec} ExportsSpec */
1414
/** @typedef {import("../Dependency").UpdateHashContext} UpdateHashContext */
1515
/** @typedef {import("../ModuleGraph")} ModuleGraph */
16+
/** @typedef {import("../json/JsonData")} JsonData */
1617
/** @typedef {import("../util/Hash")} Hash */
1718

1819
const getExportsFromData = data => {
1920
if (data && typeof data === "object") {
2021
if (Array.isArray(data)) {
21-
return data.map((item, idx) => {
22-
return {
23-
name: `${idx}`,
24-
canMangle: true,
25-
exports: getExportsFromData(item)
26-
};
27-
});
22+
return data.length < 100
23+
? data.map((item, idx) => {
24+
return {
25+
name: `${idx}`,
26+
canMangle: true,
27+
exports: getExportsFromData(item)
28+
};
29+
})
30+
: undefined;
2831
} else {
2932
const exports = [];
3033
for (const key of Object.keys(data)) {
@@ -42,12 +45,11 @@ const getExportsFromData = data => {
4245

4346
class JsonExportsDependency extends NullDependency {
4447
/**
45-
* @param {(string | ExportSpec)[]} exports json exports
48+
* @param {JsonData=} data json data
4649
*/
47-
constructor(exports) {
50+
constructor(data) {
4851
super();
49-
this.exports = exports;
50-
this._hashUpdate = undefined;
52+
this.data = data;
5153
}
5254

5355
get type() {
@@ -61,7 +63,7 @@ class JsonExportsDependency extends NullDependency {
6163
*/
6264
getExports(moduleGraph) {
6365
return {
64-
exports: this.exports,
66+
exports: getExportsFromData(this.data && this.data.get()),
6567
dependencies: undefined
6668
};
6769
}
@@ -73,23 +75,18 @@ class JsonExportsDependency extends NullDependency {
7375
* @returns {void}
7476
*/
7577
updateHash(hash, context) {
76-
if (this._hashUpdate === undefined) {
77-
this._hashUpdate = this.exports
78-
? JSON.stringify(this.exports)
79-
: "undefined";
80-
}
81-
hash.update(this._hashUpdate);
78+
this.data.updateHash(hash);
8279
}
8380

8481
serialize(context) {
8582
const { write } = context;
86-
write(this.exports);
83+
write(this.data);
8784
super.serialize(context);
8885
}
8986

9087
deserialize(context) {
9188
const { read } = context;
92-
this.exports = read();
89+
this.data = read();
9390
super.deserialize(context);
9491
}
9592
}
@@ -100,4 +97,3 @@ makeSerializable(
10097
);
10198

10299
module.exports = JsonExportsDependency;
103-
module.exports.getExportsFromData = getExportsFromData;

‎lib/json/JsonData.js‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,14 @@ class JsonData {
2424
}
2525
return this._data;
2626
}
27+
28+
updateHash(hash) {
29+
if (this._buffer === undefined && this._data !== undefined) {
30+
this._buffer = Buffer.from(JSON.stringify(this._data));
31+
}
32+
33+
if (this._buffer) return hash.update(this._buffer);
34+
}
2735
}
2836

2937
register(JsonData, "webpack/lib/json/JsonData", null, {

‎lib/json/JsonParser.js‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -41,15 +41,13 @@ class JsonParser extends Parser {
4141
typeof source === "object"
4242
? source
4343
: parseFn(source[0] === "\ufeff" ? source.slice(1) : source);
44-
45-
state.module.buildInfo.jsonData = new JsonData(data);
44+
const jsonData = new JsonData(data);
45+
state.module.buildInfo.jsonData = jsonData;
4646
state.module.buildInfo.strict = true;
4747
state.module.buildMeta.exportsType = "default";
4848
state.module.buildMeta.defaultObject =
4949
typeof data === "object" ? "redirect-warn" : false;
50-
state.module.addDependency(
51-
new JsonExportsDependency(JsonExportsDependency.getExportsFromData(data))
52-
);
50+
state.module.addDependency(new JsonExportsDependency(jsonData));
5351
return state;
5452
}
5553
}

‎test/MemoryLimitTestCases.test.js‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
"use strict";
2+
3+
require("./helpers/warmup-webpack");
4+
const path = require("path");
5+
const fs = require("graceful-fs");
6+
const rimraf = require("rimraf");
7+
const captureStdio = require("./helpers/captureStdio");
8+
const webpack = require("..");
9+
10+
const toMiB = bytes => `${Math.round(bytes / 1024 / 1024)}MiB`;
11+
const base = path.join(__dirname, "memoryLimitCases");
12+
const outputBase = path.join(__dirname, "js", "memoryLimit");
13+
const tests = fs
14+
.readdirSync(base)
15+
.filter(
16+
testName =>
17+
fs.existsSync(path.join(base, testName, "index.js")) ||
18+
fs.existsSync(path.join(base, testName, "webpack.config.js"))
19+
)
20+
.filter(testName => {
21+
const testDirectory = path.join(base, testName);
22+
const filterPath = path.join(testDirectory, "test.filter.js");
23+
if (fs.existsSync(filterPath) && !require(filterPath)()) {
24+
describe.skip(testName, () => it("filtered"));
25+
return false;
26+
}
27+
return true;
28+
});
29+
30+
describe("MemoryLimitTestCases", () => {
31+
jest.setTimeout(40000);
32+
let stderr;
33+
beforeEach(() => {
34+
stderr = captureStdio(process.stderr, true);
35+
if (global.gc) {
36+
global.gc();
37+
global.gc();
38+
}
39+
});
40+
afterEach(() => {
41+
stderr.restore();
42+
});
43+
tests.forEach(testName => {
44+
let testConfig = {
45+
heapSizeLimitBytes: 250 * 1024 * 1024
46+
};
47+
try {
48+
// try to load a test file
49+
testConfig = Object.assign(
50+
testConfig,
51+
require(path.join(base, testName, "test.config.js"))
52+
);
53+
} catch (e) {
54+
// ignored
55+
}
56+
it(`should build ${JSON.stringify(testName)} with heap limit of ${toMiB(
57+
testConfig.heapSizeLimitBytes
58+
)}`, done => {
59+
const outputDirectory = path.join(outputBase, testName);
60+
rimraf.sync(outputDirectory);
61+
fs.mkdirSync(outputDirectory, { recursive: true });
62+
let options = {
63+
mode: "development",
64+
entry: "./index",
65+
output: {
66+
filename: "bundle.js"
67+
}
68+
};
69+
if (fs.existsSync(path.join(base, testName, "webpack.config.js"))) {
70+
options = require(path.join(base, testName, "webpack.config.js"));
71+
}
72+
73+
(Array.isArray(options) ? options : [options]).forEach(options => {
74+
if (!options.context) options.context = path.join(base, testName);
75+
if (!options.output) options.output = options.output || {};
76+
if (!options.output.path) options.output.path = outputDirectory;
77+
if (!options.plugins) options.plugins = [];
78+
if (!options.optimization) options.optimization = {};
79+
if (options.optimization.minimize === undefined)
80+
options.optimization.minimize = false;
81+
});
82+
const heapSizeStart = process.memoryUsage().heapUsed;
83+
const c = webpack(options);
84+
const compilers = c.compilers ? c.compilers : [c];
85+
compilers.forEach(c => {
86+
const ifs = c.inputFileSystem;
87+
c.inputFileSystem = Object.create(ifs);
88+
c.inputFileSystem.readFile = function () {
89+
const args = Array.prototype.slice.call(arguments);
90+
const callback = args.pop();
91+
ifs.readFile.apply(
92+
ifs,
93+
args.concat([
94+
(err, result) => {
95+
if (err) return callback(err);
96+
if (!/\.(js|json|txt)$/.test(args[0]))
97+
return callback(null, result);
98+
callback(null, result.toString("utf-8").replace(/\r/g, ""));
99+
}
100+
])
101+
);
102+
};
103+
});
104+
c.run((err, stats) => {
105+
if (err) return done(err);
106+
if (/error$/.test(testName)) {
107+
expect(stats.hasErrors()).toBe(true);
108+
} else if (stats.hasErrors()) {
109+
return done(
110+
new Error(
111+
stats.toString({
112+
all: false,
113+
errors: true,
114+
errorStack: true,
115+
errorDetails: true
116+
})
117+
)
118+
);
119+
}
120+
const heapUsed = process.memoryUsage().heapUsed - heapSizeStart;
121+
if (heapUsed > testConfig.heapSizeLimitBytes) {
122+
return done(
123+
new Error(`Out of memory limit with ${toMiB(heapUsed)} heap used`)
124+
);
125+
}
126+
if (testConfig.validate) testConfig.validate(stats, stderr.toString());
127+
done();
128+
});
129+
});
130+
});
131+
});

‎test/memoryLimitCases/json/index.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
const ctx = require.context("./src", false, /\.json$/);

‎test/memoryLimitCases/json/src/1.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"type":"Topology","box":[-73.9958013,45.3984821,-73.4742952,45.7047897],"transform":{"scale":[0.0005225512024048059,0.00030692144288576825],"translate":[-73.9958013,45.3984821]},"objects":{"boundary":{"type":"Polygon","arcs":[[0]],"id":"relation/8508277","properties":{"admin_level":"6","alt_name:1":"Montréal","boundary":"administrative","name":"Agglomération de Montréal","name:en":"Urban agglomeration of Montreal","name:fr":"Agglomération de Montréal","type":"boundary","wikidata":"Q2826806","wikipedia":"fr:Agglomération de Montréal","id":"relation/8508277"}}},"arcs":[[[992,804],[-2,23],[-15,31],[-3,32],[4,45],[12,24],[2,14],[5,9],[3,8],[-4,7],[-23,-3],[-4,4],[-8,-1],[-5,-2],[-22,-7],[-18,-7],[-10,-1],[-8,-25],[-5,-18],[-6,-11],[-11,-9],[-18,-14],[-29,-31],[-25,-20],[-6,-5],[-53,-44],[-17,-21],[-14,-17],[-17,-22],[-3,-9],[-6,-16],[-5,-24],[-2,-6],[-6,-22],[-13,-25],[-11,-21],[-5,-11],[-2,-3],[-12,-28],[-1,-3],[-1,-25],[-11,-22],[-2,-3],[-1,-4],[-3,-8],[0,-2],[-4,-6],[-6,-6],[-23,-7],[-7,-3],[-6,-3],[-14,-11],[-6,-11],[-11,-7],[-7,-3],[-3,-1],[-16,-17],[-11,-8],[-8,-5],[-3,-5],[-9,-22],[-11,-3],[-11,-8],[-5,-10],[-5,-5],[-4,-1],[-10,-3],[-27,3],[-20,4],[-11,9],[-8,0],[-10,7],[-15,-5],[-5,0],[-21,8],[-20,0],[-2,-2],[-3,-1],[-3,-4],[-7,-12],[-3,-3],[-1,-1],[-2,-1],[-2,1],[-6,12],[-8,4],[-3,5],[-1,5],[-7,1],[-14,1],[-7,0],[-8,3],[-11,6],[-7,5],[-7,6],[-11,-4],[-11,-9],[-6,-7],[-7,-12],[-8,-11],[-7,-9],[-21,-21],[-19,-13],[-14,-19],[-10,-19],[-5,-16],[-7,-13],[-11,-26],[-14,-17],[-15,-20],[-10,-6],[-12,-4],[-4,0],[5,-17],[0,-3],[1,-2],[1,-6],[3,-10],[2,-12],[2,-9],[2,-9],[2,-5],[2,-19],[0,-25],[10,-13],[17,-16],[14,-14],[5,-6],[6,-7],[2,-2],[1,0],[1,-1],[1,0],[11,-5],[6,-3],[2,-1],[6,0],[16,1],[21,2],[12,5],[13,3],[3,2],[6,3],[2,2],[8,7],[12,5],[5,2],[3,0],[4,0],[6,-2],[18,-9],[13,-5],[25,-6],[36,-6],[29,-3],[9,-2],[22,-5],[7,11],[4,7],[5,7],[5,4],[1,1],[3,4],[7,5],[7,5],[8,4],[9,6],[8,5],[12,8],[49,5],[14,1],[5,1],[13,2],[45,1],[12,1],[12,0],[4,0],[8,-1],[11,-2],[8,-3],[9,-3],[12,-5],[3,-2],[6,-3],[18,-10],[10,-6],[9,-3],[5,-1],[1,0],[7,-1],[2,0],[12,1],[13,1],[16,1],[6,1],[7,1],[36,4],[24,4],[15,4],[20,7],[13,8],[7,5],[4,3],[14,14],[9,15],[7,21],[2,7],[3,26],[1,14],[-1,23],[0,5],[0,5],[0,21],[-2,7],[-2,7],[-5,16],[-2,23],[-1,6],[-4,10],[7,5],[2,1],[1,3],[1,1],[2,2],[3,4],[-1,5],[2,2],[-2,10],[-3,16],[-8,45],[-2,7],[0,3],[-2,9],[0,3],[-4,29],[-2,10],[19,25],[14,32],[10,25],[14,35],[1,4],[0,17],[3,18],[-4,33],[-2,25],[3,20],[4,12],[17,40],[9,21],[4,11],[10,33]]]}

‎test/memoryLimitCases/json/src/2.json‎

Lines changed: 1 addition & 0 deletions
Large diffs are not rendered by default.

‎test/memoryLimitCases/json/src/3.json‎

Lines changed: 1 addition & 0 deletions
Large diffs are not rendered by default.

‎test/memoryLimitCases/json/src/4.json‎

Lines changed: 1 addition & 0 deletions
Large diffs are not rendered by default.

‎test/memoryLimitCases/json/src/5.json‎

Lines changed: 1 addition & 0 deletions
Large diffs are not rendered by default.

0 commit comments

Comments
 (0)