From 1402b3c3da7e5c84a99761124a23a5cfb6961b55 Mon Sep 17 00:00:00 2001 From: Yushion Yang <2685310462@qq.com> Date: Fri, 23 Oct 2026 18:15:26 +0800 Subject: [PATCH 1/2] fix(csv-parse): discover delimiters in Unicode field data --- .../csv-parse/lib/utils/delimiter_discover.js | 1 + .../csv-parse/test/option.delimiter_auto.ts | 46 +++++++++++++++++++ .../test/utils.delimiter_discover.js | 18 ++++++++ 3 files changed, 65 insertions(+) diff --git a/packages/csv-parse/lib/utils/delimiter_discover.js b/packages/csv-parse/lib/utils/delimiter_discover.js index d55897885..e8a61ba23 100644 --- a/packages/csv-parse/lib/utils/delimiter_discover.js +++ b/packages/csv-parse/lib/utils/delimiter_discover.js @@ -33,6 +33,7 @@ const delimiter_discover = function (records, options) { for (let i = 0, l = record.length; i < l; i++) { // Count the character frequency const code = record.charCodeAt(i); + if (code >= info.length) continue; if (info[code].lines[line] === undefined) info[code].lines[line] = 0; info[code].lines[line]++; } diff --git a/packages/csv-parse/test/option.delimiter_auto.ts b/packages/csv-parse/test/option.delimiter_auto.ts index ebc0eff79..76866c009 100644 --- a/packages/csv-parse/test/option.delimiter_auto.ts +++ b/packages/csv-parse/test/option.delimiter_auto.ts @@ -38,6 +38,52 @@ describe("Option `delimiter_auto`", function () { ]); }); + it("sync preserves Unicode field content", function () { + parse_sync("é,東京,😀\nß,北京,🚀", { + delimiter_auto: true, + }).should.eql([ + ["é", "東京", "😀"], + ["ß", "北京", "🚀"], + ]); + }); + + it("stream discovers with split UTF-8 field characters", function (next) { + const expected = [ + ["姓名", "城市"], + ["张三", "北京"], + ["李四", "上海"], + ]; + const input = Buffer.from( + expected.map((record) => record.join(",")).join("\n"), + ); + const parser = parse({ delimiter_auto: { size: 9 } }, (err, records) => { + if (err) return next(err); + records.should.eql(expected); + next(); + }); + for (let i = 0; i < input.length; i++) { + parser.write(input.subarray(i, i + 1)); + } + parser.end(); + }); + + it("custom score receives only ASCII candidates with Unicode fields", function () { + const char_codes: number[] = []; + parse_sync("é:甲;β\nø:乙;γ", { + delimiter_auto: { + score: (info) => { + char_codes.push(info.char_code); + return info.char_code === ":".charCodeAt(0) ? 1 : 0; + }, + }, + }).should.eql([ + ["é", "甲;β"], + ["ø", "乙;γ"], + ]); + char_codes.length.should.eql(127); + char_codes.every((code) => code < 127).should.eql(true); + }); + it("sync custom score", function () { // The default score elects `;` because it is more preferred than `:` parse_sync("a:b;c\nd:e;f", { diff --git a/packages/csv-parse/test/utils.delimiter_discover.js b/packages/csv-parse/test/utils.delimiter_discover.js index 4157fb8d0..32b9daa0a 100644 --- a/packages/csv-parse/test/utils.delimiter_discover.js +++ b/packages/csv-parse/test/utils.delimiter_discover.js @@ -28,4 +28,22 @@ describe("Option `delimiter_auto`", function () { delimiter_discover("1,a.b,c.d\n2,e\n3,f,g").should.eql(","); delimiter_discover([["1,a.b,c.d"], ["2,e"], ["3,f,g"]]).should.eql(","); }); + + for (const [name, input] of [ + ["string", "é,東京,😀\nß,北京,🚀"], + ["buffer", Buffer.from("é,東京,😀\nß,北京,🚀")], + ["records", [["é,東京,😀"], ["ß,北京,🚀"]]], + ]) { + it(`ignores non-ASCII field characters in ${name} input`, function () { + delimiter_discover(input).should.eql(","); + }); + } + + it("ignores character code 127 outside the candidate range", function () { + delimiter_discover("\x7f;\x7f\n\x7f;\x7f").should.eql(";"); + }); + + it("still counts character code 126 as a candidate", function () { + delimiter_discover("~~,x\n~~,y").should.eql("~"); + }); }); From 1de61df45448f45a161d92bfd9112100461da44e Mon Sep 17 00:00:00 2001 From: Yushion Yang <2685310462@qq.com> Date: Fri, 23 Oct 2026 21:58:11 +0800 Subject: [PATCH 2/2] fix(csv-parse): respect input encoding during delimiter discovery --- packages/csv-parse/lib/api/index.js | 42 +++++-- .../csv-parse/test/option.delimiter_auto.ts | 103 ++++++++++++++++++ 2 files changed, 137 insertions(+), 8 deletions(-) diff --git a/packages/csv-parse/lib/api/index.js b/packages/csv-parse/lib/api/index.js index ca97838cd..3e2fb72a3 100644 --- a/packages/csv-parse/lib/api/index.js +++ b/packages/csv-parse/lib/api/index.js @@ -107,10 +107,32 @@ const transform = function (original_options = {}) { // Ensure that nextBuf is not concatenated a second time during buffer reconciliation nextBuf = undefined; // this.delimiterBufPrevious = delimiterBuf; + if (!end && bom && !bomSkipped && delimiterBuf.length < 3) { + this.state.delimiterBufPrevious = delimiterBuf; + return; + } if (end || delimiterBuf.length > delimiter_auto.size) { + let sample = delimiterBuf; + let sampleEncoding = encoding || "utf8"; + if (bom && !bomSkipped) { + for (const [bomEncoding, signature] of Object.entries(boms)) { + if ( + sample.length >= signature.length && + signature.compare(sample, 0, signature.length) === 0 + ) { + sample = sample.subarray(signature.length); + sampleEncoding = bomEncoding; + break; + } + } + } this.options.delimiter = [ Buffer.from( - delimiter_discover(delimiterBuf, this.options.delimiter_auto), + delimiter_discover( + sample.toString(sampleEncoding), + this.options.delimiter_auto, + ), + sampleEncoding, ), ]; this.state.previousBuf = delimiterBuf; @@ -141,16 +163,16 @@ const transform = function (original_options = {}) { if (bomSkipped === false) { if (bom === false) { this.state.bomSkipped = true; - } else if (buf.length < 3) { + } else if (buf.length < 3 && end === false) { // No enough data - if (end === false) { - // Wait for more data - this.state.previousBuf = buf; - return; - } + this.state.previousBuf = buf; + return; } else { for (const encoding in boms) { - if (boms[encoding].compare(buf, 0, boms[encoding].length) === 0) { + if ( + buf.length >= boms[encoding].length && + boms[encoding].compare(buf, 0, boms[encoding].length) === 0 + ) { // Skip BOM const bomLength = boms[encoding].length; this.state.bufBytesStart += bomLength; @@ -160,6 +182,10 @@ const transform = function (original_options = {}) { ...this.original_options, encoding: encoding, }); + // BOM normalization must retain the delimiter discovered above. + if (this.state.delimiterDiscovered) { + options.delimiter = this.options.delimiter; + } // Properties are merged with the existing options instance for (const key in options) { this.options[key] = options[key]; diff --git a/packages/csv-parse/test/option.delimiter_auto.ts b/packages/csv-parse/test/option.delimiter_auto.ts index 76866c009..1ddcd8d87 100644 --- a/packages/csv-parse/test/option.delimiter_auto.ts +++ b/packages/csv-parse/test/option.delimiter_auto.ts @@ -47,6 +47,109 @@ describe("Option `delimiter_auto`", function () { ]); }); + for (const encoding of ["utf16le", "ucs2"] as const) { + it(`sync uses explicit ${encoding} encoding`, function () { + parse_sync(Buffer.from("é;東京;😀\nß;北京;🚀", encoding), { + encoding, + delimiter_auto: true, + }).should.eql([ + ["é", "東京", "😀"], + ["ß", "北京", "🚀"], + ]); + }); + } + + it("sync preserves ASCII fields in UTF-16LE input", function () { + parse_sync(Buffer.from("a;b\nc;d", "utf16le"), { + encoding: "utf16le", + delimiter_auto: true, + }).should.eql([ + ["a", "b"], + ["c", "d"], + ]); + }); + + for (const encoding of ["utf8", "utf16le"] as const) { + it(`sync preserves the discovered delimiter after ${encoding} BOM detection`, function () { + parse_sync(Buffer.from("\ufeffé;東京\nß;北京", encoding), { + bom: true, + delimiter_auto: true, + }).should.eql([ + ["é", "東京"], + ["ß", "北京"], + ]); + }); + + it(`stream waits for a split ${encoding} BOM with a small discovery size`, function (next) { + const input = Buffer.from("\ufeffé;東京\nß;北京", encoding); + const parser = parse( + { bom: true, delimiter_auto: { size: 0 } }, + (err, records) => { + if (err) return next(err); + records.should.eql([ + ["é", "東京"], + ["ß", "北京"], + ]); + next(); + }, + ); + parser.write(input.subarray(0, 1)); + parser.write(input.subarray(1, 2)); + parser.end(input.subarray(2)); + }); + } + + it("stream discovers UTF-16LE with split code units", function (next) { + const expected = [ + ["姓名", "城市"], + ["张三", "北京"], + ["李四", "上海"], + ]; + const input = Buffer.from( + expected.map((record) => record.join(";")).join("\n"), + "utf16le", + ); + const parser = parse( + { encoding: "utf16le", delimiter_auto: { size: 23 } }, + (err, records) => { + if (err) return next(err); + records.should.eql(expected); + next(); + }, + ); + for (let i = 0; i < input.length; i++) { + parser.write(input.subarray(i, i + 1)); + } + parser.end(); + }); + + it("sync accepts a UTF-16LE BOM without records", function () { + parse_sync(Buffer.from("\ufeff", "utf16le"), { + bom: true, + delimiter_auto: true, + }).should.eql([]); + }); + + it("sync preserves a BOM when bom is false", function () { + parse_sync(Buffer.from("\ufeffa;b\nc;d", "utf16le"), { + encoding: "utf16le", + delimiter_auto: true, + }).should.eql([ + ["\ufeffa", "b"], + ["c", "d"], + ]); + }); + + it("sync preserves binary field output", function () { + parse_sync(Buffer.from("a;b\nc;d"), { + encoding: null, + delimiter_auto: true, + }).should.eql([ + [Buffer.from("a"), Buffer.from("b")], + [Buffer.from("c"), Buffer.from("d")], + ]); + }); + it("stream discovers with split UTF-8 field characters", function (next) { const expected = [ ["姓名", "城市"],