fix(csv-parse): handle Unicode delimiter discovery

packages/csv-parse/lib/api/index.js

@@ -107,10 +107,32 @@ const transform = function (original_options = {}) {
         // Ensure that nextBuf is not concatenated a second time during buffer reconciliation
         nextBuf = undefined;
         // this.delimiterBufPrevious = delimiterBuf;
+        if (!end && bom && !bomSkipped && delimiterBuf.length < 3) {
+          this.state.delimiterBufPrevious = delimiterBuf;
+          return;
+        }
         if (end || delimiterBuf.length > delimiter_auto.size) {
+          let sample = delimiterBuf;
+          let sampleEncoding = encoding || "utf8";
+          if (bom && !bomSkipped) {
+            for (const [bomEncoding, signature] of Object.entries(boms)) {
+              if (
+                sample.length >= signature.length &&
+                signature.compare(sample, 0, signature.length) === 0
+              ) {
+                sample = sample.subarray(signature.length);
+                sampleEncoding = bomEncoding;
+                break;
+              }
+            }
+          }
           this.options.delimiter = [
             Buffer.from(
-              delimiter_discover(delimiterBuf, this.options.delimiter_auto),
+              delimiter_discover(
+                sample.toString(sampleEncoding),
+                this.options.delimiter_auto,
+              ),
+              sampleEncoding,
             ),
           ];
           this.state.previousBuf = delimiterBuf;
@@ -141,16 +163,16 @@ const transform = function (original_options = {}) {
       if (bomSkipped === false) {
         if (bom === false) {
           this.state.bomSkipped = true;
-        } else if (buf.length < 3) {
+        } else if (buf.length < 3 && end === false) {
           // No enough data
-          if (end === false) {
-            // Wait for more data
-            this.state.previousBuf = buf;
-            return;
-          }
+          this.state.previousBuf = buf;
+          return;
         } else {
           for (const encoding in boms) {
-            if (boms[encoding].compare(buf, 0, boms[encoding].length) === 0) {
+            if (
+              buf.length >= boms[encoding].length &&
+              boms[encoding].compare(buf, 0, boms[encoding].length) === 0
+            ) {
               // Skip BOM
               const bomLength = boms[encoding].length;
               this.state.bufBytesStart += bomLength;
@@ -160,6 +182,10 @@ const transform = function (original_options = {}) {
                 ...this.original_options,
                 encoding: encoding,
               });
+              // BOM normalization must retain the delimiter discovered above.
+              if (this.state.delimiterDiscovered) {
+                options.delimiter = this.options.delimiter;
+              }
               // Properties are merged with the existing options instance
               for (const key in options) {
                 this.options[key] = options[key];

packages/csv-parse/lib/utils/delimiter_discover.js

@@ -33,6 +33,7 @@ const delimiter_discover = function (records, options) {
     for (let i = 0, l = record.length; i < l; i++) {
       // Count the character frequency
       const code = record.charCodeAt(i);
+      if (code >= info.length) continue;
       if (info[code].lines[line] === undefined) info[code].lines[line] = 0;
       info[code].lines[line]++;
     }

packages/csv-parse/test/option.delimiter_auto.ts

@@ -38,6 +38,155 @@ describe("Option `delimiter_auto`", function () {
     ]);
   });
 
+  it("sync preserves Unicode field content", function () {
+    parse_sync("é,東京,😀\nß,北京,🚀", {
+      delimiter_auto: true,
+    }).should.eql([
+      ["é", "東京", "😀"],
+      ["ß", "北京", "🚀"],
+    ]);
+  });
+
+  for (const encoding of ["utf16le", "ucs2"] as const) {
+    it(`sync uses explicit ${encoding} encoding`, function () {
+      parse_sync(Buffer.from("é;東京;😀\nß;北京;🚀", encoding), {
+        encoding,
+        delimiter_auto: true,
+      }).should.eql([
+        ["é", "東京", "😀"],
+        ["ß", "北京", "🚀"],
+      ]);
+    });
+  }
+
+  it("sync preserves ASCII fields in UTF-16LE input", function () {
+    parse_sync(Buffer.from("a;b\nc;d", "utf16le"), {
+      encoding: "utf16le",
+      delimiter_auto: true,
+    }).should.eql([
+      ["a", "b"],
+      ["c", "d"],
+    ]);
+  });
+
+  for (const encoding of ["utf8", "utf16le"] as const) {
+    it(`sync preserves the discovered delimiter after ${encoding} BOM detection`, function () {
+      parse_sync(Buffer.from("\ufeffé;東京\nß;北京", encoding), {
+        bom: true,
+        delimiter_auto: true,
+      }).should.eql([
+        ["é", "東京"],
+        ["ß", "北京"],
+      ]);
+    });
+
+    it(`stream waits for a split ${encoding} BOM with a small discovery size`, function (next) {
+      const input = Buffer.from("\ufeffé;東京\nß;北京", encoding);
+      const parser = parse(
+        { bom: true, delimiter_auto: { size: 0 } },
+        (err, records) => {
+          if (err) return next(err);
+          records.should.eql([
+            ["é", "東京"],
+            ["ß", "北京"],
+          ]);
+          next();
+        },
+      );
+      parser.write(input.subarray(0, 1));
+      parser.write(input.subarray(1, 2));
+      parser.end(input.subarray(2));
+    });
+  }
+
+  it("stream discovers UTF-16LE with split code units", function (next) {
+    const expected = [
+      ["姓名", "城市"],
+      ["张三", "北京"],
+      ["李四", "上海"],
+    ];
+    const input = Buffer.from(
+      expected.map((record) => record.join(";")).join("\n"),
+      "utf16le",
+    );
+    const parser = parse(
+      { encoding: "utf16le", delimiter_auto: { size: 23 } },
+      (err, records) => {
+        if (err) return next(err);
+        records.should.eql(expected);
+        next();
+      },
+    );
+    for (let i = 0; i < input.length; i++) {
+      parser.write(input.subarray(i, i + 1));
+    }
+    parser.end();
+  });
+
+  it("sync accepts a UTF-16LE BOM without records", function () {
+    parse_sync(Buffer.from("\ufeff", "utf16le"), {
+      bom: true,
+      delimiter_auto: true,
+    }).should.eql([]);
+  });
+
+  it("sync preserves a BOM when bom is false", function () {
+    parse_sync(Buffer.from("\ufeffa;b\nc;d", "utf16le"), {
+      encoding: "utf16le",
+      delimiter_auto: true,
+    }).should.eql([
+      ["\ufeffa", "b"],
+      ["c", "d"],
+    ]);
+  });
+
+  it("sync preserves binary field output", function () {
+    parse_sync(Buffer.from("a;b\nc;d"), {
+      encoding: null,
+      delimiter_auto: true,
+    }).should.eql([
+      [Buffer.from("a"), Buffer.from("b")],
+      [Buffer.from("c"), Buffer.from("d")],
+    ]);
+  });
+
+  it("stream discovers with split UTF-8 field characters", function (next) {
+    const expected = [
+      ["姓名", "城市"],
+      ["张三", "北京"],
+      ["李四", "上海"],
+    ];
+    const input = Buffer.from(
+      expected.map((record) => record.join(",")).join("\n"),
+    );
+    const parser = parse({ delimiter_auto: { size: 9 } }, (err, records) => {
+      if (err) return next(err);
+      records.should.eql(expected);
+      next();
+    });
+    for (let i = 0; i < input.length; i++) {
+      parser.write(input.subarray(i, i + 1));
+    }
+    parser.end();
+  });
+
+  it("custom score receives only ASCII candidates with Unicode fields", function () {
+    const char_codes: number[] = [];
+    parse_sync("é:甲;β\nø:乙;γ", {
+      delimiter_auto: {
+        score: (info) => {
+          char_codes.push(info.char_code);
+          return info.char_code === ":".charCodeAt(0) ? 1 : 0;
+        },
+      },
+    }).should.eql([
+      ["é", "甲;β"],
+      ["ø", "乙;γ"],
+    ]);
+    char_codes.length.should.eql(127);
+    char_codes.every((code) => code < 127).should.eql(true);
+  });
+
   it("sync custom score", function () {
     // The default score elects `;` because it is more preferred than `:`
     parse_sync("a:b;c\nd:e;f", {

packages/csv-parse/test/utils.delimiter_discover.js

@@ -28,4 +28,22 @@ describe("Option `delimiter_auto`", function () {
     delimiter_discover("1,a.b,c.d\n2,e\n3,f,g").should.eql(",");
     delimiter_discover([["1,a.b,c.d"], ["2,e"], ["3,f,g"]]).should.eql(",");
   });
+
+  for (const [name, input] of [
+    ["string", "é,東京,😀\nß,北京,🚀"],
+    ["buffer", Buffer.from("é,東京,😀\nß,北京,🚀")],
+    ["records", [["é,東京,😀"], ["ß,北京,🚀"]]],
+  ]) {
+    it(`ignores non-ASCII field characters in ${name} input`, function () {
+      delimiter_discover(input).should.eql(",");
+    });
+  }
+
+  it("ignores character code 127 outside the candidate range", function () {
+    delimiter_discover("\x7f;\x7f\n\x7f;\x7f").should.eql(";");
+  });
+
+  it("still counts character code 126 as a candidate", function () {
+    delimiter_discover("~~,x\n~~,y").should.eql("~");
+  });
 });