Skip to content

Commit 7a30bce

Browse files
kuhekuhe
authored
fix(xml-builder): use xml 1.1 parsing behavior for entities (#7964)
1 parent 7babd8b commit 7a30bce

7 files changed

Lines changed: 112 additions & 13 deletions

File tree

‎clients/client-s3/test/e2e/s3-object-features.e2e.spec.ts‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,8 @@ describe("@aws-sdk/client-s3", () => {
100100
"\n \n",
101101
"a\r\n b\n c\r",
102102
"a\r\u0085 b\u0085",
103-
"a\r\u2028 b\u0085 c\u2028",
103+
"a\r\u2028 b\u0085 c\u2028 d\x15",
104+
"special_c0_\x15\x15\x15\x15",
104105
];
105106

106107
beforeAll(async () => {
@@ -126,16 +127,45 @@ describe("@aws-sdk/client-s3", () => {
126127
}
127128
});
128129

130+
it("can list objects with special character keys", async () => {
131+
const listObjects = await client.listObjects({
132+
Bucket,
133+
Prefix: "special_c0_",
134+
});
135+
136+
expect(listObjects.Contents?.length).toBeGreaterThanOrEqual(1);
137+
expect(listObjects.Contents?.[0].Key).toEqual("special_c0_\x15\x15\x15\x15");
138+
});
139+
129140
it("can delete keys containing special characters", async () => {
141+
const [firstHalf, secondHalf] = [keys.slice(0, 4), keys.slice(4)];
142+
130143
await client.deleteObjects({
131144
Bucket,
132145
Delete: {
133-
Objects: keys.map((Key) => ({
146+
Objects: firstHalf.map((Key) => ({
134147
Key,
135148
})),
136149
},
137150
});
138151

152+
// S3 cannot delete the keys using DeleteObjects because the keys would be part
153+
// of the XML payload and are rejected, specifically the \x15 C0 control char
154+
// not valid in XML 1.0.
155+
156+
// This is despite the keys being listable (in a supposedly XML-1.0 ListObjects response payload).
157+
// and individually deletable.
158+
const p = [];
159+
for (const key of secondHalf) {
160+
p.push(
161+
client.deleteObject({
162+
Bucket,
163+
Key: key,
164+
})
165+
);
166+
}
167+
await Promise.all(p);
168+
139169
await Promise.all(
140170
keys.map(async (Key) => {
141171
return client

‎packages-internal/xml-builder/package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,9 @@
33
"version": "3.972.19",
44
"description": "XML utilities for the AWS SDK",
55
"dependencies": {
6+
"@nodable/entities": "2.1.0",
67
"@smithy/types": "^4.14.1",
7-
"fast-xml-parser": "5.7.1",
8+
"fast-xml-parser": "5.7.2",
89
"tslib": "^2.6.2"
910
},
1011
"scripts": {

‎packages-internal/xml-builder/src/xml-parser.spec.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,28 @@ describe("xml parsing", () => {
217217
});
218218
});
219219

220+
it("should preserve control characters like \\x15 in text content despite xml 1.0 decl", () => {
221+
const xml = `<?xml version="1.0" encoding="UTF-8"?>
222+
<Response><Key>prefix&#x15;suffix</Key></Response>`;
223+
const object = parseXML(xml);
224+
expect(object).toEqual({
225+
Response: {
226+
Key: "prefix\x15suffix",
227+
},
228+
});
229+
});
230+
231+
it("should preserve control characters like \\x15 in text content despite xml 1.0 decl (browser)", () => {
232+
const xml = `<?xml version="1.0" encoding="UTF-8"?>
233+
<Response><Key>prefix&#x15;suffix</Key></Response>`;
234+
const object = parseXMLBrowser(xml);
235+
expect(object).toEqual({
236+
Response: {
237+
Key: "prefix\x15suffix",
238+
},
239+
});
240+
});
241+
220242
it("throws on parsing error", () => {
221243
const xmlSamples = [`<unclosed`, `<unmatched></matched>`];
222244

‎packages-internal/xml-builder/src/xml-parser.ts‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,29 @@
1+
// todo: package types are incorrect wrt exports.
2+
import type EntityDecoder from "@nodable/entities";
3+
import { COMMON_HTML, CURRENCY, XML } from "@nodable/entities";
14
import { XMLParser } from "fast-xml-parser";
25

6+
// todo: package types are incorrect wrt exports.
7+
const { EntityDecoder: EntityDecoderImpl } = require("@nodable/entities");
8+
9+
/**
10+
* Custom entity decoder that preserves C0 control characters (e.g. U+0015)
11+
* which XML 1.0 strict mode would strip. AWS services may return these
12+
* characters in responses despite XML 1.0 declaration.
13+
*
14+
* @internal
15+
*/
16+
const entityDecoder: EntityDecoder = new EntityDecoderImpl({
17+
namedEntities: { ...XML, ...COMMON_HTML, ...CURRENCY },
18+
numericAllowed: true,
19+
limit: {
20+
maxTotalExpansions: Infinity,
21+
},
22+
ncr: {
23+
xmlVersion: 1.1,
24+
},
25+
});
26+
327
/**
428
* Because this is only used against trusted server responses,
529
* we should not be setting any client side limits.
@@ -13,6 +37,27 @@ const parser = new XMLParser({
1337
maxTotalExpansions: Infinity,
1438
},
1539
htmlEntities: true,
40+
entityDecoder: {
41+
setExternalEntities: (entities: Record<string, string>): void => {
42+
entityDecoder.setExternalEntities(entities);
43+
},
44+
addInputEntities: (entities: Record<string, string>): void => {
45+
entityDecoder.addInputEntities(entities);
46+
},
47+
reset: (): void => {
48+
entityDecoder.reset();
49+
},
50+
decode: (text: string): string => {
51+
return entityDecoder.decode(text);
52+
},
53+
/**
54+
* The previous XML parser allowed C0 control chars despite
55+
* any presence of the xml 1.0 declaration `<?xml version="1.0" encoding="UTF-8"?>`.
56+
* We will avoid having the xml version set by the incoming payload
57+
* as a workaround.
58+
*/
59+
setXmlVersion: (version: string) => void {},
60+
},
1661
ignoreAttributes: false,
1762
ignoreDeclaration: true,
1863
parseTagValue: false,

‎private/aws-protocoltests-restxml-schema/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@
6262
"@smithy/util-stream": "^4.5.25",
6363
"@smithy/util-utf8": "^4.2.2",
6464
"entities": "2.2.0",
65-
"fast-xml-parser": "5.7.1",
65+
"fast-xml-parser": "5.7.2",
6666
"tslib": "^2.6.2"
6767
},
6868
"devDependencies": {

‎private/aws-protocoltests-restxml/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@
6464
"@smithy/util-utf8": "^4.2.2",
6565
"@smithy/uuid": "^1.1.2",
6666
"entities": "2.2.0",
67-
"fast-xml-parser": "5.7.1",
67+
"fast-xml-parser": "5.7.2",
6868
"tslib": "^2.6.2"
6969
},
7070
"devDependencies": {

‎yarn.lock‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1176,7 +1176,7 @@ __metadata:
11761176
concurrently: "npm:7.0.0"
11771177
downlevel-dts: "npm:0.10.1"
11781178
entities: "npm:2.2.0"
1179-
fast-xml-parser: "npm:5.7.1"
1179+
fast-xml-parser: "npm:5.7.2"
11801180
premove: "npm:4.0.0"
11811181
tslib: "npm:^2.6.2"
11821182
typescript: "npm:~5.8.3"
@@ -1236,7 +1236,7 @@ __metadata:
12361236
concurrently: "npm:7.0.0"
12371237
downlevel-dts: "npm:0.10.1"
12381238
entities: "npm:2.2.0"
1239-
fast-xml-parser: "npm:5.7.1"
1239+
fast-xml-parser: "npm:5.7.2"
12401240
premove: "npm:4.0.0"
12411241
tslib: "npm:^2.6.2"
12421242
typescript: "npm:~5.8.3"
@@ -25483,11 +25483,12 @@ __metadata:
2548325483
version: 0.0.0-use.local
2548425484
resolution: "@aws-sdk/xml-builder@workspace:packages-internal/xml-builder"
2548525485
dependencies:
25486+
"@nodable/entities": "npm:2.1.0"
2548625487
"@smithy/types": "npm:^4.14.1"
2548725488
"@tsconfig/recommended": "npm:1.0.1"
2548825489
concurrently: "npm:7.0.0"
2548925490
downlevel-dts: "npm:0.10.1"
25490-
fast-xml-parser: "npm:5.7.1"
25491+
fast-xml-parser: "npm:5.7.2"
2549125492
premove: "npm:4.0.0"
2549225493
tslib: "npm:^2.6.2"
2549325494
typescript: "npm:~5.8.3"
@@ -28557,7 +28558,7 @@ __metadata:
2855728558
languageName: node
2855828559
linkType: hard
2855928560

28560-
"@nodable/entities@npm:^2.1.0":
28561+
"@nodable/entities@npm:2.1.0, @nodable/entities@npm:^2.1.0":
2856128562
version: 2.1.0
2856228563
resolution: "@nodable/entities@npm:2.1.0"
2856328564
checksum: 10c0/5a4cba2b61a5b6c726328b18b1de6d033cae4a658a118644bf31e0bcbda126ea7b69385043dc556cf1ed859b9ca220e82b81b5e5c48ef1b519fb8ec104575dee
@@ -34581,17 +34582,17 @@ __metadata:
3458134582
languageName: node
3458234583
linkType: hard
3458334584

34584-
"fast-xml-parser@npm:5.7.1":
34585-
version: 5.7.1
34586-
resolution: "fast-xml-parser@npm:5.7.1"
34585+
"fast-xml-parser@npm:5.7.2":
34586+
version: 5.7.2
34587+
resolution: "fast-xml-parser@npm:5.7.2"
3458734588
dependencies:
3458834589
"@nodable/entities": "npm:^2.1.0"
3458934590
fast-xml-builder: "npm:^1.1.5"
3459034591
path-expression-matcher: "npm:^1.5.0"
3459134592
strnum: "npm:^2.2.3"
3459234593
bin:
3459334594
fxparser: src/cli/cli.js
34594-
checksum: 10c0/b8b54e33060da5fc5ce26fdc73c4728f18415f9be9a774f1406b03265a5b411b742c39dba0127c3f0f31fad5b3ee11f51be79aa16df160f69fd5e4b902bfbb85
34595+
checksum: 10c0/d48439ce0700add82f5e7c6ccc5a1f06483beb7cd8e88caa83c6406843e52f14988e60d05cbb3a86ffe07e073807674c807e0764d94a280e1c96d7e2011dae8e
3459534596
languageName: node
3459634597
linkType: hard
3459734598

0 commit comments

Comments
 (0)