feat(spec)!: a URL with a special scheme is read for credentials the way a URL parser reads it, however many slashes follow the scheme

.changeset/url-credential-special-scheme-read-however-many-slashes.md

@@ -0,0 +1,32 @@
+---
+'@objectstack/spec': minor
+---
+
+feat(spec)!: a URL with a special scheme is read for credentials the way a URL parser reads it, however many slashes follow the scheme
+
+Clause-②: no (narrowing)
+Metadata-protocol: narrowed
+
+<!-- adr-0087: registered url-credential-special-scheme-read-however-many-slashes -->
+
+**BREAKING** accept-set narrowing at the two metadata doors that judge a URL credential: an authored connector instance (`DeclarativeConnectorEntrySchema` — `defineStack({ connectors })`, `os validate` / `os compile`, the artifact load, `PUT /api/v1/meta/connector/:name`) and a datasource driver config's URL keys (`config.url` on postgres, mysql, mongo and turso, turso `config.syncUrl`) at every door that parses the datasource.
+
+- **A special scheme's authority is read as a URL parser reads it.** For `http`, `https`, `ws`, `wss` and `ftp`, in any letter case, the authority starts after the whole run of `/` and `\` that follows the scheme, however long, none included. A connector string holding a userinfo there, and a datasource URL holding a userinfo password there, are refused (`422 INVALID_METADATA` at the save door, located at the string, naming the parts, never the value); a bare username in a datasource URL stays accepted.
+- **One grammar at both doors.** The connector entry and the datasource doors read the one userinfo grammar, and both read redactors rewrite exactly what the write refuses, at the position the write judged, serving every other character as stored.
+- **Unchanged.** Every other scheme keeps `scheme://` or a scheme-relative `//` (`postgres:`, `libsql:`, `mongodb:` and the rest carry no authority without `//`); `file:` carries no userinfo; a URL spelled `scheme://` reads exactly as before. A stored row is never re-judged: a save of its served form keeps the stored value, and an edit of the served value stores exactly the edit.
+
+## FROM → TO
+
+```ts
+// before — parsed green; now refused at providerConfig.baseUrl
+{ name: 'billing_api', label: 'Billing API', type: 'api', provider: 'rest',
+  providerConfig: { baseUrl: 'https:///billing:…@api.billing.example.com/v1' },
+  auth: { type: 'none' } },
+
+// after — the URL holds no credential, and the password is a reference
+{ name: 'billing_api', label: 'Billing API', type: 'api', provider: 'rest',
+  providerConfig: { baseUrl: 'https://api.billing.example.com/v1' },
+  auth: { type: 'basic', username: 'billing', credentialRef: 'BILLING_API_PASSWORD' } },
+```
+
+The same holds for `https:billing:…@host`, `https:/billing:…@host` and `https:\billing:…@host`, and for a datasource URL key: spell it `scheme://`, keep it credential-free (a bare username stays accepted) and bind the password through the Setup → Datasources connection form's secret field, or reference it with `external.credentialsRef`. There is deliberately no automatic rewrite: the value in the string may be the only copy of the secret. The upgrade guide entry is `url-credential-special-scheme-read-however-many-slashes`.

packages/spec/src/data/datasource-credential-redaction.test.ts

@@ -814,3 +814,62 @@ describe('driver identity: every accepted spelling of a builtin driver is redact
     }
   });
 });
+
+// A URL with a special scheme — `http`, `https`, `ws`, `wss`, `ftp`, in any
+// letter case — is read for a password the way a URL parser reads it, however
+// many `/` or `\` follow the scheme (none included); the datasource read
+// removes the password exactly where the write door judges it, at every
+// URL-bearing driver key, turso `syncUrl` included.
+describe('the datasource read rewrites a special-scheme URL\'s password where the write door judges it, however many slashes follow the scheme', () => {
+  const BS = String.fromCharCode(0x5c);
+  const KEYS: Array<[name: string, driver: string, key: string, make: (url: string) => Record<string, unknown>]> = [
+    ['postgres url', 'postgres', 'url', (url) => ({ url })],
+    ['mysql url', 'mysql', 'url', (url) => ({ url })],
+    ['mongodb url', 'mongodb', 'url', (url) => ({ url })],
+    ['turso url', 'turso', 'url', (url) => ({ url })],
+    ['turso syncUrl', 'turso', 'syncUrl', (syncUrl) => ({ url: 'file:./data/replica.db', syncUrl })],
+  ];
+  const FORMS: Array<[string, (s: string) => string]> = [
+    ['no slash', (s) => `${s}:`],
+    ['one slash', (s) => `${s}:/`],
+    ['a backslash', (s) => `${s}:${BS}`],
+    ['three slashes', (s) => `${s}:///`],
+    ['an uppercase scheme and no slash', (s) => `${s.toUpperCase()}:`],
+  ];
+  const leadsOf = (schemes: readonly string[]) =>
+    schemes.flatMap((scheme) => FORMS.map(([form, lead]) => [`\`${scheme}:\` with ${form}`, lead(scheme)] as [string, string]));
+  const SPECIAL = leadsOf(['http', 'https', 'ws', 'wss', 'ftp']);
+  const NO_AUTHORITY = leadsOf(['file', 'postgresql', 'libsql']).filter(([, lead]) => !lead.endsWith('///'));
+
+  /** Whether the write door refuses `config` for a password in the userinfo at `key`. */
+  const writeRefuses = (driver: string, key: string, config: Record<string, unknown>): boolean => {
+    const result = getDriverConfigSchema(driver)!.safeParse(config);
+    return !result.success && result.error.issues.some(
+      (i) => i.path.join('.') === key && i.message.includes('embeds a password in its userinfo'),
+    );
+  };
+
+  describe.each(KEYS)('%s', (_name, driver, key, make) => {
+    it.each(SPECIAL)('after %s: refused at the write door, served with the password removed, and the served form passes', (_label, lead) => {
+      const stored = make(`${lead}svc:[email protected]/prod`);
+      expect(writeRefuses(driver, key, stored), 'the write door refuses the stored URL').toBe(true);
+      const { config, redactedKeys } = redactDatasourceConfig(driver, stored);
+      expect(config[key]).toBe(`${lead}[email protected]/prod`);
+      expect(redactedKeys).toContain(key);
+      expect(writeRefuses(driver, key, config), 'the served URL passes the write door').toBe(false);
+    });
+
+    it.each([...SPECIAL, ...NO_AUTHORITY])('refused ⇔ rewritten after %s', (_label, lead) => {
+      for (const url of [`${lead}svc:[email protected]/prod`, `${lead}[email protected]/prod`, `${lead}db.example.com/prod`]) {
+        const config = make(url);
+        const rewritten = redactDatasourceConfig(driver, config).redactedKeys.includes(key);
+        expect(writeRefuses(driver, key, config), 'refused ⇔ rewritten').toBe(rewritten);
+      }
+    });
+
+    it.each(NO_AUTHORITY)('CONTROL: after %s there is no authority, so the URL is served as stored', (_label, lead) => {
+      const stored = make(`${lead}svc:[email protected]/prod`);
+      expect(redactDatasourceConfig(driver, stored).config[key]).toBe(stored[key]);
+    });
+  });
+});

packages/spec/src/data/datasource-credential-redaction.ts

@@ -400,7 +400,9 @@ export function redactableConfigKeys(driver: unknown): string[] {
  * write refuses: the string is judged as a URL parser reads it — trimmed of
  * its leading and trailing C0 control characters and spaces, with every ASCII
  * tab, LF and CR deleted — and rewritten at the matching offsets of the
- * authored string; a scheme-relative `//` authority counts; the userinfo ends
+ * authored string; a scheme-relative `//` authority counts, and a special
+ * scheme's (`http`, `https`, `ws`, `wss`, `ftp`) authority starts after
+ * however many `/` or `\` follow it, none included; the userinfo ends
  * at the LAST `@` in the authority —
  * a lazier boundary stopping at the first `@` would split
  * `postgres://u:p@ss@host/db` after `p`, leave `ss@host` in place, and publish

packages/spec/src/data/driver/common.zod.ts

@@ -201,10 +201,12 @@ export const URL_EMBEDDED_CREDENTIAL_REFUSED = (key: string): string =>
  * with every ASCII tab, LF and CR deleted — and the userinfo answered is that
  * cleaned text (why WHATWG parsing itself is refused is on that parse).
  *
- * A string with no `//` (sqlite/turso `file:` paths, `:memory:`) has no
- * authority and answers `undefined`. The FIRST `:` inside the returned
- * userinfo splits username from password; that split belongs to the two
- * accessors, not here.
+ * A special scheme (`http`, `https`, `ws`, `wss`, `ftp`, any letter case)
+ * starts the authority after however many `/` or `\` follow it, none
+ * included, as a URL parser does; any other string with no `//`
+ * (sqlite/turso `file:` paths, `:memory:`) has no authority and answers
+ * `undefined`. The FIRST `:` inside the returned userinfo splits username
+ * from password; that split belongs to the two accessors, not here.
  */
 function urlUserinfo(value: string): string | undefined {
   return urlUserinfoSpan(value)?.userinfo;

packages/spec/src/data/driver/driver-credential-refusal.test.ts

@@ -1302,3 +1302,82 @@ describe('datasource — bound credentialsRef + composed mongo config naming no
     expect(result.error!.issues.some((i) => i.message.includes('not accepted in the driver-options passthrough'))).toBe(true);
   });
 });
+
+// A URL with a special scheme — `http`, `https`, `ws`, `wss`, `ftp`, in any
+// letter case — is read for a password the way a URL parser reads it: its
+// authority starts after the whole run of `/` and `\` that follows the scheme,
+// however long, none included. Every other scheme keeps `scheme://` (or a
+// scheme-relative `//`), and `file:` carries no userinfo. Judged at every
+// URL-bearing driver key, turso `syncUrl` included.
+const BS = String.fromCharCode(0x5c);
+/** [form, the scheme and the run after it] — `s` is the scheme. */
+const SPECIAL_SCHEME_FORMS: Array<[string, (s: string) => string]> = [
+  ['no slash', (s) => `${s}:`],
+  ['one slash', (s) => `${s}:/`],
+  ['a backslash', (s) => `${s}:${BS}`],
+  ['three slashes', (s) => `${s}:///`],
+  ['a run of slashes and backslashes', (s) => `${s}:/${BS}/`],
+  ['an uppercase scheme and no slash', (s) => `${s.toUpperCase()}:`],
+  ['an uppercase scheme and three slashes', (s) => `${s.toUpperCase()}:///`],
+];
+const schemeLeadsOf = (schemes: readonly string[]) =>
+  schemes.flatMap((scheme) => SPECIAL_SCHEME_FORMS.map(([form, lead]) => [`\`${scheme}:\` with ${form}`, lead(scheme)] as [string, string]));
+const SPECIAL_SCHEME_LEADS = schemeLeadsOf(['http', 'https', 'ws', 'wss', 'ftp']);
+const NO_AUTHORITY_LEADS = schemeLeadsOf(['file', 'postgresql', 'mysql', 'mongodb', 'libsql']).filter(([, lead]) => !lead.endsWith('///'));
+
+describe.each(URL_FAMILY)('$name — a URL with a special scheme is read for a password the way a URL parser reads it, however many slashes follow the scheme', (f) => {
+  /** The userinfo refusal at `f.key` — other refusals at the same key (a driver's own URL grammar) are not this rule's. */
+  const userinfoRefusalAt = (config: Record<string, unknown>) => {
+    const result = f.schema.safeParse(config);
+    if (result.success) return undefined;
+    return result.error.issues.find(
+      (i) => i.path.join('.') === f.key && i.message.includes('embeds a password in its userinfo'),
+    );
+  };
+
+  it.each(SPECIAL_SCHEME_LEADS)('refuses `user:password@host` after %s', (_label, lead) => {
+    const issue = userinfoRefusalAt(f.make(`${lead}svc:[email protected]/prod`));
+    expect(issue, `refusal must be pathed at \`${f.key}\``).toBeDefined();
+    expect(issue!.code).toBe('custom');
+    expect(issue!.message).toContain('external.credentialsRef');
+  });
+
+  it.each(SPECIAL_SCHEME_LEADS)('CONTROL: a bare username after %s carries no password and is not refused for one', (_label, lead) => {
+    expect(userinfoRefusalAt(f.make(`${lead}[email protected]/prod`))).toBeUndefined();
+    expect(userinfoRefusalAt(f.make(`${lead}db.example.com/prod/a:b@c`))).toBeUndefined();
+  });
+
+  it.each(NO_AUTHORITY_LEADS)('CONTROL: no authority without `//` after a scheme that is not special, or `file:` — %s', (_label, lead) => {
+    expect(userinfoRefusalAt(f.make(`${lead}svc:[email protected]/prod`))).toBeUndefined();
+  });
+
+  it('CONTROL: the `//` forms keep their reading — a password after `scheme://` or `//` is refused, an empty `scheme:///` authority holds none', () => {
+    expect(userinfoRefusalAt(f.make(f.sample('svc:hunter2@')))).toBeDefined();
+    expect(userinfoRefusalAt(f.make('//svc:[email protected]/prod'))).toBeDefined();
+    for (const scheme of ['file', 'postgresql', 'libsql']) {
+      expect(userinfoRefusalAt(f.make(`${scheme}:///svc:[email protected]/prod`)), scheme).toBeUndefined();
+    }
+  });
+});
+
+describe('the shared userinfo parse reads a special scheme\'s authority however many slashes follow the scheme', () => {
+  it.each(SPECIAL_SCHEME_LEADS)('after %s: the password and the username are read', (_label, lead) => {
+    expect(urlUserinfoPassword(`${lead}u:p@h/db`)).toBe('p');
+    expect(urlUserinfoUsername(`${lead}u:p@h/db`)).toBe('u');
+    expect(urlUserinfoUsername(`${lead}u@h/db`)).toBe('u');
+    // CONTROL: no userinfo, and an `@` in the path, carry none.
+    expect(urlUserinfoUsername(`${lead}h/db`)).toBeUndefined();
+    expect(urlUserinfoUsername(`${lead}h/a:b@c`)).toBeUndefined();
+  });
+
+  it('after a special scheme the authority is read on the cleaned string, tab and newline characters deleted', () => {
+    const TAB = String.fromCharCode(0x09);
+    const LF = String.fromCharCode(0x0a);
+    expect(urlUserinfoPassword(`ht${TAB}tps:u:p@h/db`)).toBe('p');
+    expect(urlUserinfoPassword(` https:/${LF}/${BS}u:p@h/db`)).toBe('p');
+  });
+
+  it.each(NO_AUTHORITY_LEADS)('CONTROL: after %s there is no authority, so no userinfo', (_label, lead) => {
+    expect(urlUserinfoUsername(`${lead}u:p@h/db`)).toBeUndefined();
+  });
+});

packages/spec/src/data/driver/url-boundary.ts

@@ -81,6 +81,34 @@ export function urlAsParsed(value: string): UrlAsParsed {
 /** `scheme://` or a scheme-relative `//`: where an authority starts. */
 const AUTHORITY_START_RE = /^(?:[a-z][a-z0-9+.-]*:)?\/\//i;
 
+/** A leading `scheme:`, the scheme captured. */
+const SCHEME_RE = /^([a-z][a-z0-9+.-]*):/i;
+
+/**
+ * The special schemes (WHATWG URL Standard) after which a URL parser starts
+ * the authority, and so reads a userinfo, past however many `/` or `\`
+ * follow the scheme — none included. `file`, the remaining special scheme,
+ * carries no userinfo.
+ */
+const AUTHORITY_SPECIAL_SCHEMES: ReadonlySet<string> = new Set(['ftp', 'http', 'https', 'ws', 'wss']);
+
+/**
+ * The offset in `text` (a cleaned string, {@link urlAsParsed}) where its
+ * authority starts, or `undefined` when it has none: after a special scheme
+ * ({@link AUTHORITY_SPECIAL_SCHEMES}, in any letter case) and the whole run of
+ * `/` and `\` that follows it, however long, as a URL parser reads it;
+ * otherwise after `scheme://` or a scheme-relative `//`.
+ */
+function authorityStart(text: string): number | undefined {
+  const scheme = SCHEME_RE.exec(text);
+  if (scheme && AUTHORITY_SPECIAL_SCHEMES.has(scheme[1]!.toLowerCase())) {
+    let start = scheme[0].length;
+    while (start < text.length && (text[start] === '/' || text[start] === '\\')) start += 1;
+    return start;
+  }
+  return AUTHORITY_START_RE.exec(text)?.[0].length;
+}
+
 /** Where the userinfo of an authored string sits, and what a URL parser reads there. */
 export interface UrlUserinfoSpan {
   /** Offset in the authored string where the userinfo starts. */
@@ -105,27 +133,36 @@ export interface UrlUserinfoSpan {
  * Deliberately NOT `new URL()`: real DSNs take forms WHATWG parsing rejects or
  * mangles (postgres/mongo multi-host `user:pass@h1:5432,h2:5432/db` throws
  * `ERR_INVALID_URL`, bare `:memory:`, `file:` paths), and a parse that throws
- * on the exact inputs it must judge would fail open. The boundaries are
- * RFC 3986's, read on the cleaned string ({@link urlAsParsed}):
+ * on the exact inputs it must judge would fail open. The boundaries are read
+ * on the cleaned string ({@link urlAsParsed}):
  *
- *  - the authority is what follows `//` (scheme-relative included), up to the
- *    first `/`, `?` or `#` — a `:` or `@` in a path or query is never userinfo;
+ *  - for a special scheme — `http`, `https`, `ws`, `wss`, `ftp`, in any letter
+ *    case — the authority starts where a URL parser starts it: after the whole
+ *    run of `/` and `\` that follows the scheme, however long, none included
+ *    (`https:host`, `https:/host`, `https:\host` and `https:///host` each have
+ *    the authority `host`);
+ *  - for any other scheme, and with no scheme, the authority is RFC 3986's:
+ *    what follows `scheme://` or a scheme-relative `//`;
+ *  - either way the authority runs up to the first `/`, `?` or `#` — a `:` or
+ *    `@` in a path or query is never userinfo. A `\` does not end it: a URL
+ *    parser ends a special scheme's authority there, an RFC 3986 parser (the
+ *    libsql client's) does not, and the longer authority holds every userinfo
+ *    either reads;
  *  - userinfo ends at the LAST `@` in the authority (a malformed literal `@`
  *    inside a password must not decide how much of it goes unjudged).
  *
- * A string with no `//` (sqlite/turso `file:` paths, `:memory:`) has no
- * authority and answers `undefined`.
+ * A string with no authority — a non-special scheme with no `//` (sqlite/turso
+ * `file:` paths, `libsql:host`), `:memory:` — answers `undefined`.
  */
 export function urlUserinfoSpan(value: string): UrlUserinfoSpan | undefined {
   const { text, rawOffsets } = urlAsParsed(value);
-  const scheme = AUTHORITY_START_RE.exec(text);
-  if (!scheme) return undefined;
-  const rest = text.slice(scheme[0].length);
+  const userinfoStart = authorityStart(text);
+  if (userinfoStart === undefined) return undefined;
+  const rest = text.slice(userinfoStart);
   const authorityEnd = rest.search(/[/?#]/);
   const authority = authorityEnd === -1 ? rest : rest.slice(0, authorityEnd);
   const at = authority.lastIndexOf('@');
   if (at === -1) return undefined;
-  const userinfoStart = scheme[0].length;
   const userinfo = authority.slice(0, at);
   const colon = userinfo.indexOf(':');
   return {

packages/spec/src/integration/connector-credential-redaction.ts

@@ -75,9 +75,11 @@
  *     and any URL a kind declares later. A string is judged as a URL parser
  *     reads it — trimmed of its leading and trailing C0 control characters
  *     and spaces, with every ASCII tab, LF and CR deleted: the userinfo of an
- *     authority that starts at `scheme://` or a scheme-relative `//`,
- *     rewritten at the matching position of the authored string; and a
- *     credential query parameter in any string. What a credential part is,
+ *     authority that starts where a URL parser starts it — after a special
+ *     scheme (`http`, `https`, `ws`, `wss`, `ftp`) and however many `/` or `\`
+ *     follow it, none included; after `scheme://` or a scheme-relative `//`
+ *     otherwise — rewritten at the matching position of the authored string;
+ *     and a credential query parameter in any string. What a credential part is,
  *     and the walk that finds every string holding one, live in ONE place,
  *     `./connector-url-credentials.ts`, which the write door's refinement
  *     runs too: an authored URL holding one is refused, so only a row stored