diff --git a/scripts/gen-models.mjs b/scripts/gen-models.mjs index c517b26..3aafd40 100644 --- a/scripts/gen-models.mjs +++ b/scripts/gen-models.mjs @@ -8,6 +8,7 @@ const dir = process.env.INTERSCRIPT_ML_DIR ?? "../ml-models" const data = parse(readFileSync(`${dir}/models.yaml`, "utf8")) const fields = [ "id", + "kind", "task", "scripts", "precision", diff --git a/src/generated/models.json b/src/generated/models.json index ce4642c..8e1536f 100644 --- a/src/generated/models.json +++ b/src/generated/models.json @@ -3,11 +3,12 @@ "models": [ { "id": "khm-latn-1.0", + "kind": null, "task": "translit", "scripts": ["Khmr", "Latn"], "precision": "fp32", "filename": "khm-latn-1.0-fp32.zip", - "url": "https://github.com/interscript/interscript-ml/releases/download/khm-latn-1.0/khm-latn-1.0-fp32.zip", + "url": "https://github.com/interscript/interscript-models/releases/download/khm-latn-1.0/khm-latn-1.0-fp32.zip", "parts": null, "sha256": "55993d473a2ed9489058779cad7e115db05e4020085b71616468cfae4f2f65cb", "size": 1418009977, @@ -31,11 +32,12 @@ }, { "id": "urd-g2p-1.0", + "kind": null, "task": "g2p", "scripts": ["Arab", "IPA"], "precision": "fp32", "filename": "urd-g2p-1.0-fp32.zip", - "url": "https://github.com/interscript/interscript-ml/releases/download/urd-g2p-1.0/urd-g2p-1.0-fp32.zip", + "url": "https://github.com/interscript/interscript-models/releases/download/urd-g2p-1.0/urd-g2p-1.0-fp32.zip", "parts": null, "sha256": "9f0a248c7116aa7f6b9ac125fb9684a3556f3b8ce8b64567c05a5e7e7a018ebf", "size": 1418039605, @@ -59,11 +61,12 @@ }, { "id": "urd-diac-1.0", + "kind": null, "task": "diacritization", "scripts": ["Arab", "Arab"], "precision": "fp32", "filename": "urd-diac-1.0-fp32.zip", - "url": "https://github.com/interscript/interscript-ml/releases/download/urd-diac-1.0/urd-diac-1.0-fp32.zip", + "url": "https://github.com/interscript/interscript-models/releases/download/urd-diac-1.0/urd-diac-1.0-fp32.zip", "parts": null, "sha256": "c299ea44a4da4851a52283f8e42d2fb6f8c7a9c443ba7eb0270f97372e10b8ea", "size": 1417851421, @@ -82,6 +85,7 @@ }, { "id": "heb-diac-1.0", + "kind": null, "task": "diacritization", "scripts": ["Hebr", "Hebr"], "precision": "fp32", @@ -89,12 +93,12 @@ "url": null, "parts": [ { - "url": "https://github.com/interscript/interscript-ml/releases/download/heb-diac-1.0/heb-diac-1.0-fp32.zip.part-00", + "url": "https://github.com/interscript/interscript-models/releases/download/heb-diac-1.0/heb-diac-1.0-fp32.zip.part-00", "sha256": "5497f70cc3c96829e65f1eb0cc401c2d1ffeb4fa909acd4def824c2e019cbc37", "size": 1500000000 }, { - "url": "https://github.com/interscript/interscript-ml/releases/download/heb-diac-1.0/heb-diac-1.0-fp32.zip.part-01", + "url": "https://github.com/interscript/interscript-models/releases/download/heb-diac-1.0/heb-diac-1.0-fp32.zip.part-01", "sha256": "a7d2011ab4f9326a86e234cc49cb50daf82949e51efbf03a21d5796c46465e9d", "size": 1281139753 } @@ -121,6 +125,7 @@ }, { "id": "tha-g2p-base-1.0", + "kind": null, "task": "g2p", "scripts": ["Thai", "IPA"], "precision": "fp32", @@ -128,12 +133,12 @@ "url": null, "parts": [ { - "url": "https://github.com/interscript/interscript-ml/releases/download/tha-g2p-base-1.0/tha-g2p-base-1.0-fp32.zip.part-00", + "url": "https://github.com/interscript/interscript-models/releases/download/tha-g2p-base-1.0/tha-g2p-base-1.0-fp32.zip.part-00", "sha256": "c4525414e5dec91a54d055ca9f1acd51c198f81fbfc55977aa82c1fb038d58d7", "size": 1500000000 }, { - "url": "https://github.com/interscript/interscript-ml/releases/download/tha-g2p-base-1.0/tha-g2p-base-1.0-fp32.zip.part-01", + "url": "https://github.com/interscript/interscript-models/releases/download/tha-g2p-base-1.0/tha-g2p-base-1.0-fp32.zip.part-01", "sha256": "f5d11fbd75f92e8b5471add867418e2dd4f49a5e258b9ded2f3b5ffa630abe24", "size": 1281174343 } @@ -144,12 +149,12 @@ { "name": "per_teacher", "value": 1.25, - "source": "interscript/interscript-ml docs/RESULTS.md#tha-g2p-base-1.0" + "source": "interscript/interscript-models docs/RESULTS.md#tha-g2p-base-1.0" }, { "name": "per_student", "value": 3.53, - "source": "interscript/interscript-ml docs/RESULTS.md#tha-g2p-base-1.0" + "source": "interscript/interscript-models docs/RESULTS.md#tha-g2p-base-1.0" } ], "parity": { @@ -160,11 +165,12 @@ }, { "id": "tha-g2p-small-1.0", + "kind": null, "task": "g2p", "scripts": ["Thai", "IPA"], "precision": "int8", "filename": "tha-g2p-small-1.0-int8.zip", - "url": "https://github.com/interscript/interscript-ml/releases/download/tha-g2p-small-1.0/tha-g2p-small-1.0-int8.zip", + "url": "https://github.com/interscript/interscript-models/releases/download/tha-g2p-small-1.0/tha-g2p-small-1.0-int8.zip", "parts": null, "sha256": "f2cfa93a42d98aabf08fdc82dfc6bd1574e21993f36558aa581fbe29e0588a94", "size": 257468517, @@ -172,17 +178,17 @@ { "name": "per_teacher", "value": 4.43, - "source": "interscript/interscript-ml docs/RESULTS.md#tha-g2p-small-1.0" + "source": "interscript/interscript-models docs/RESULTS.md#tha-g2p-small-1.0" }, { "name": "per_student_beam4", "value": 12.06, - "source": "interscript/interscript-ml docs/RESULTS.md#tha-g2p-small-1.0" + "source": "interscript/interscript-models docs/RESULTS.md#tha-g2p-small-1.0" }, { "name": "per_student", "value": 2.85, - "source": "interscript/interscript-ml docs/RESULTS.md#tha-g2p-small-1.0" + "source": "interscript/interscript-models docs/RESULTS.md#tha-g2p-small-1.0" } ], "parity": { @@ -193,11 +199,12 @@ }, { "id": "tha-g2p-small-1.0-int4", + "kind": null, "task": "g2p", "scripts": ["Thai", "IPA"], "precision": "int4", "filename": "tha-g2p-small-1.0-int4.zip", - "url": "https://github.com/interscript/interscript-ml/releases/download/tha-g2p-small-1.0/tha-g2p-small-1.0-int4.zip", + "url": "https://github.com/interscript/interscript-models/releases/download/tha-g2p-small-1.0/tha-g2p-small-1.0-int4.zip", "parts": null, "sha256": "e5d12a3d65c6ac39fa08f52c975c99ff7a51c3e7b7aca7fbb461d2de2345e4cb", "size": 201958073, @@ -205,17 +212,17 @@ { "name": "per_teacher", "value": 4.43, - "source": "interscript/interscript-ml docs/RESULTS.md#tha-g2p-small-1.0" + "source": "interscript/interscript-models docs/RESULTS.md#tha-g2p-small-1.0" }, { "name": "per_student_beam4", "value": 12.06, - "source": "interscript/interscript-ml docs/RESULTS.md#tha-g2p-small-1.0" + "source": "interscript/interscript-models docs/RESULTS.md#tha-g2p-small-1.0" }, { "name": "per_student", "value": 2.85, - "source": "interscript/interscript-ml docs/RESULTS.md#tha-g2p-small-1.0" + "source": "interscript/interscript-models docs/RESULTS.md#tha-g2p-small-1.0" } ], "parity": { @@ -226,11 +233,12 @@ }, { "id": "heb-diac-small-1.0", + "kind": null, "task": "diacritization", "scripts": ["Hebr", "Hebr"], "precision": "fp32", "filename": "heb-diac-small-1.0-fp32.zip", - "url": "https://github.com/interscript/interscript-ml/releases/download/heb-diac-small-1.0/heb-diac-small-1.0-fp32.zip", + "url": "https://github.com/interscript/interscript-models/releases/download/heb-diac-small-1.0/heb-diac-small-1.0-fp32.zip", "parts": null, "sha256": "905c53873d5a62bbe6252c6ec0ecef5f37e96e78cb01e8cd99b6b24c4d4d74ad", "size": 1418663344, @@ -238,12 +246,12 @@ { "name": "der_teacher_greedy", "value": 24.79, - "source": "interscript/interscript-ml docs/RESULTS.md#heb-diac-small-1.0" + "source": "interscript/interscript-models docs/RESULTS.md#heb-diac-small-1.0" }, { "name": "der_student_greedy", "value": 30.37, - "source": "interscript/interscript-ml docs/RESULTS.md#heb-diac-small-1.0" + "source": "interscript/interscript-models docs/RESULTS.md#heb-diac-small-1.0" } ], "parity": { @@ -254,6 +262,7 @@ }, { "id": "fas-g2p-1.0", + "kind": null, "task": "g2p", "scripts": ["Arab", "Latn"], "precision": "fp32", @@ -261,12 +270,12 @@ "url": null, "parts": [ { - "url": "https://github.com/interscript/interscript-ml/releases/download/fas-g2p-1.0/fas-g2p-1.0-fp32.zip.part-00", + "url": "https://github.com/interscript/interscript-models/releases/download/fas-g2p-1.0/fas-g2p-1.0-fp32.zip.part-00", "sha256": "6c475e53f3c7ef3edcef5a24d2d81f51d13ddf4c417de98440943c9a32b03e55", "size": 1500000000 }, { - "url": "https://github.com/interscript/interscript-ml/releases/download/fas-g2p-1.0/fas-g2p-1.0-fp32.zip.part-01", + "url": "https://github.com/interscript/interscript-models/releases/download/fas-g2p-1.0/fas-g2p-1.0-fp32.zip.part-01", "sha256": "c5656c649412f41e201c567e002514d31390b636d4650d9dcb6dfed1973fb3d5", "size": 1280254270 } @@ -277,12 +286,12 @@ { "name": "cer", "value": 1.6, - "source": "interscript/interscript-ml docs/RESULTS.md#fas-g2p-1.0" + "source": "interscript/interscript-models docs/RESULTS.md#fas-g2p-1.0" }, { "name": "sb_homograph_ezafe_norm", "value": 77.34, - "source": "interscript/interscript-ml docs/RESULTS.md#fas-g2p-1.0" + "source": "interscript/interscript-models docs/RESULTS.md#fas-g2p-1.0" } ], "parity": { @@ -293,11 +302,12 @@ }, { "id": "ara-diac-small-1.0", + "kind": null, "task": "diacritization", "scripts": ["Arab", "Arab"], "precision": "fp32", "filename": "ara-diac-small-1.0-fp32.zip", - "url": "https://github.com/interscript/interscript-ml/releases/download/ara-diac-small-1.0/ara-diac-small-1.0-fp32.zip", + "url": "https://github.com/interscript/interscript-models/releases/download/ara-diac-small-1.0/ara-diac-small-1.0-fp32.zip", "parts": null, "sha256": "8b149beb04294a2e482af876a34ae0fca3e87412606116fae1a376845fd4111a", "size": 1418126835, @@ -305,22 +315,22 @@ { "name": "der_teacher_greedy", "value": 1.3205, - "source": "interscript/interscript-ml docs/RESULTS.md#ara-diac-small-1.0" + "source": "interscript/interscript-models docs/RESULTS.md#ara-diac-small-1.0" }, { "name": "der_student_greedy", "value": 3.658, - "source": "interscript/interscript-ml docs/RESULTS.md#ara-diac-small-1.0" + "source": "interscript/interscript-models docs/RESULTS.md#ara-diac-small-1.0" }, { "name": "der_teacher_fullset", "value": 2.5815, - "source": "interscript/interscript-ml docs/RESULTS.md#ara-diac-small-1.0" + "source": "interscript/interscript-models docs/RESULTS.md#ara-diac-small-1.0" }, { "name": "der_student_fullset", "value": 8.259, - "source": "interscript/interscript-ml docs/RESULTS.md#ara-diac-small-1.0" + "source": "interscript/interscript-models docs/RESULTS.md#ara-diac-small-1.0" } ], "parity": { @@ -331,6 +341,7 @@ }, { "id": "heb-diac-1.1", + "kind": null, "task": "diacritization", "scripts": ["Hebr", "Hebr"], "precision": "fp32", @@ -338,12 +349,12 @@ "url": null, "parts": [ { - "url": "https://github.com/interscript/interscript-ml/releases/download/heb-diac-1.1/heb.zip.part-00", + "url": "https://github.com/interscript/interscript-models/releases/download/heb-diac-1.1/heb.zip.part-00", "sha256": "8dac0d62e9b0ac4b137412eaf3a542953f11575f12f66f76cb93d3f5d788fc1b", "size": 1500000000 }, { - "url": "https://github.com/interscript/interscript-ml/releases/download/heb-diac-1.1/heb.zip.part-01", + "url": "https://github.com/interscript/interscript-models/releases/download/heb-diac-1.1/heb.zip.part-01", "sha256": "230c5ab731ea5f001f802d60b9e9a328c10b43bc311a19ef1974f114384313ed", "size": 1280987905 } @@ -370,6 +381,7 @@ }, { "id": "ara-diac-1.0", + "kind": null, "task": "diacritization", "scripts": ["Arab", "Arab"], "precision": "fp32", @@ -377,12 +389,12 @@ "url": null, "parts": [ { - "url": "https://github.com/interscript/interscript-ml/releases/download/ara-diac-1.0/ara-diac-1.0-fp32.zip.part-00", + "url": "https://github.com/interscript/interscript-models/releases/download/ara-diac-1.0/ara-diac-1.0-fp32.zip.part-00", "sha256": "6282a9291fb45ac7fcc3abe79d33b5913e91f87dcff8b305bf1d57ed98a3ed0b", "size": 1500000000 }, { - "url": "https://github.com/interscript/interscript-ml/releases/download/ara-diac-1.0/ara-diac-1.0-fp32.zip.part-01", + "url": "https://github.com/interscript/interscript-models/releases/download/ara-diac-1.0/ara-diac-1.0-fp32.zip.part-01", "sha256": "7e7254a89ec75493871f6f61a993315097dad1bb0ba63c4584453c309e8cb906", "size": 1280311342 } @@ -409,11 +421,12 @@ }, { "id": "ara-diac-small-1.0-int8", + "kind": null, "task": "diacritization", "scripts": ["Arab", "Arab"], "precision": "int8", "filename": "ara-diac-small-1.0-int8.zip", - "url": "https://github.com/interscript/interscript-ml/releases/download/ara-diac-small-1.0/ara-diac-small-1.0-int8.zip", + "url": "https://github.com/interscript/interscript-models/releases/download/ara-diac-small-1.0/ara-diac-small-1.0-int8.zip", "parts": null, "sha256": "af247d837bee7e755b5287102cfb715c48dc90c2cf41ec1c41e7238e1a72b678", "size": 256959713, @@ -421,22 +434,22 @@ { "name": "der_teacher_greedy", "value": 1.3205, - "source": "interscript/interscript-ml docs/RESULTS.md#ara-diac-small-1.0" + "source": "interscript/interscript-models docs/RESULTS.md#ara-diac-small-1.0" }, { "name": "der_student_greedy", "value": 3.658, - "source": "interscript/interscript-ml docs/RESULTS.md#ara-diac-small-1.0" + "source": "interscript/interscript-models docs/RESULTS.md#ara-diac-small-1.0" }, { "name": "der_teacher_fullset", "value": 2.5815, - "source": "interscript/interscript-ml docs/RESULTS.md#ara-diac-small-1.0" + "source": "interscript/interscript-models docs/RESULTS.md#ara-diac-small-1.0" }, { "name": "der_student_fullset", "value": 8.259, - "source": "interscript/interscript-ml docs/RESULTS.md#ara-diac-small-1.0" + "source": "interscript/interscript-models docs/RESULTS.md#ara-diac-small-1.0" } ], "parity": { @@ -447,11 +460,12 @@ }, { "id": "ara-diac-small-2.0", + "kind": null, "task": "diacritization", "scripts": ["Arab", "Arab"], "precision": "fp32", "filename": "ara-diac-small-2.0-fp32.zip", - "url": "https://github.com/interscript/interscript-ml/releases/download/ara-diac-small-2.0/ara-diac-small-2.0-fp32.zip", + "url": "https://github.com/interscript/interscript-models/releases/download/ara-diac-small-2.0/ara-diac-small-2.0-fp32.zip", "parts": null, "sha256": "d9aa95d0e8fd0d7da80fd2f7dabedf0ac65f55ba984de565d252f5e14906d98c", "size": 1418516306, @@ -459,12 +473,12 @@ { "name": "der_teacher_fullset", "value": 2.289, - "source": "interscript/interscript-ml docs/RESULTS.md#run-006-r7-muon" + "source": "interscript/interscript-models docs/RESULTS.md#run-006-r7-muon" }, { "name": "der_student_fullset", "value": 5.0821, - "source": "interscript/interscript-ml docs/RESULTS.md#run-006-r7-muon" + "source": "interscript/interscript-models docs/RESULTS.md#run-006-r7-muon" } ], "parity": { @@ -475,11 +489,12 @@ }, { "id": "ara-diac-small-2.0-int8", + "kind": null, "task": "diacritization", "scripts": ["Arab", "Arab"], "precision": "int8", "filename": "ara-diac-small-2.0-int8.zip", - "url": "https://github.com/interscript/interscript-ml/releases/download/ara-diac-small-2.0/ara-diac-small-2.0-int8.zip", + "url": "https://github.com/interscript/interscript-models/releases/download/ara-diac-small-2.0/ara-diac-small-2.0-int8.zip", "parts": null, "sha256": "f4d07acdaf9c18cded97e15e0c5636e6cee90de2fa32c5a373a44c0cc912bf50", "size": 264043016, @@ -487,12 +502,12 @@ { "name": "der_teacher_fullset", "value": 2.289, - "source": "interscript/interscript-ml docs/RESULTS.md#run-006-r7-muon" + "source": "interscript/interscript-models docs/RESULTS.md#run-006-r7-muon" }, { "name": "der_student_fullset", "value": 4.8218, - "source": "interscript/interscript-ml docs/RESULTS.md#run-006-r7-muon" + "source": "interscript/interscript-models docs/RESULTS.md#run-006-r7-muon" } ], "parity": { @@ -503,6 +518,7 @@ }, { "id": "ara-diac-2.0", + "kind": null, "task": "diacritization", "scripts": ["Arab", "Arab"], "precision": "fp32", @@ -510,12 +526,12 @@ "url": null, "parts": [ { - "url": "https://github.com/interscript/interscript-ml/releases/download/ara-diac-2.0/ara-diac-2.0-fp32.zip.part-00", + "url": "https://github.com/interscript/interscript-models/releases/download/ara-diac-2.0/ara-diac-2.0-fp32.zip.part-00", "sha256": "28c272b5ce58031c356e75823eb581f12351a1675a52d8e4000f851c80d5e14d", "size": 1500000000 }, { - "url": "https://github.com/interscript/interscript-ml/releases/download/ara-diac-2.0/ara-diac-2.0-fp32.zip.part-01", + "url": "https://github.com/interscript/interscript-models/releases/download/ara-diac-2.0/ara-diac-2.0-fp32.zip.part-01", "sha256": "86016da35e14a8fc05d4fbf3b1ad9ef24c29f9f7893f1716e40bef4b491effa2", "size": 1280299051 } @@ -552,11 +568,12 @@ }, { "id": "ara-diac-2.0-int8", + "kind": null, "task": "diacritization", "scripts": ["Arab", "Arab"], "precision": "int8", "filename": "ara-diac-2.0-int8.zip", - "url": "https://github.com/interscript/interscript-ml/releases/download/ara-diac-2.0/ara-diac-2.0-int8.zip", + "url": "https://github.com/interscript/interscript-models/releases/download/ara-diac-2.0/ara-diac-2.0-int8.zip", "parts": null, "sha256": "7f1d55d43f97564ad62fc14a7e0cabe52c01d8298b747677a75900de748bb6c6", "size": 519207340, @@ -590,11 +607,12 @@ }, { "id": "ara-diac-small-2.1", + "kind": null, "task": "diacritization", "scripts": ["Arab", "Arab"], "precision": "fp32", "filename": "ara-diac-small-2.1-fp32.zip", - "url": "https://github.com/interscript/interscript-ml/releases/download/ara-diac-small-2.1/ara-diac-small-2.1-fp32.zip", + "url": "https://github.com/interscript/interscript-models/releases/download/ara-diac-small-2.1/ara-diac-small-2.1-fp32.zip", "parts": null, "sha256": "a69918de9f87664fd97cdab3bf68b655966f608211dbeadd7b6fb3f7b03e5ffb", "size": 1419756840, @@ -602,12 +620,12 @@ { "name": "der_teacher_fullset", "value": 2.289, - "source": "interscript/interscript-ml docs/RESULTS.md#run-006-r7-muon" + "source": "interscript/interscript-models docs/RESULTS.md#run-006-r7-muon" }, { "name": "der_student_fullset", "value": 4.5701, - "source": "interscript/interscript-ml docs/RESULTS.md#run-006-r7-muon" + "source": "interscript/interscript-models docs/RESULTS.md#run-006-r7-muon" } ], "parity": { @@ -618,11 +636,12 @@ }, { "id": "ara-diac-small-2.1-int8", + "kind": null, "task": "diacritization", "scripts": ["Arab", "Arab"], "precision": "int8", "filename": "ara-diac-small-2.1-int8.zip", - "url": "https://github.com/interscript/interscript-ml/releases/download/ara-diac-small-2.1/ara-diac-small-2.1-int8.zip", + "url": "https://github.com/interscript/interscript-models/releases/download/ara-diac-small-2.1/ara-diac-small-2.1-int8.zip", "parts": null, "sha256": "9b4e4aad2ab53a8cfbec432a78e27b409376627243aec3fd043f06574552a32b", "size": 267965657, @@ -630,12 +649,12 @@ { "name": "der_teacher_fullset", "value": 2.289, - "source": "interscript/interscript-ml docs/RESULTS.md#run-006-r7-muon" + "source": "interscript/interscript-models docs/RESULTS.md#run-006-r7-muon" }, { "name": "der_student_fullset", "value": 4.5701, - "source": "interscript/interscript-ml docs/RESULTS.md#run-006-r7-muon" + "source": "interscript/interscript-models docs/RESULTS.md#run-006-r7-muon" } ], "parity": { @@ -646,11 +665,12 @@ }, { "id": "ara-diac-layerdrop-1.0", + "kind": null, "task": "diacritization", "scripts": ["Arab", "Arab"], "precision": "fp32", "filename": "ara-diac-layerdrop-1.0-fp32.zip", - "url": "https://github.com/interscript/interscript-ml/releases/download/ara-diac-layerdrop-1.0/ara-diac-layerdrop-1.0-fp32.zip", + "url": "https://github.com/interscript/interscript-models/releases/download/ara-diac-layerdrop-1.0/ara-diac-layerdrop-1.0-fp32.zip", "parts": null, "sha256": "8f297bdc0e7eba5160c6ed8a2a437abbd399687fd14f2f2b640a4b4c639a3d16", "size": 1015218371, @@ -658,12 +678,12 @@ { "name": "der_teacher_fullset", "value": 2.2921, - "source": "interscript/interscript-ml docs/RESULTS.md#layerdrop-6ep" + "source": "interscript/interscript-models docs/RESULTS.md#layerdrop-6ep" }, { "name": "der_student_fullset", "value": 5.784, - "source": "interscript/interscript-ml docs/RESULTS.md#layerdrop-6ep" + "source": "interscript/interscript-models docs/RESULTS.md#layerdrop-6ep" } ], "parity": { @@ -674,11 +694,12 @@ }, { "id": "ara-diac-layerdrop-1.0-int8", + "kind": null, "task": "diacritization", "scripts": ["Arab", "Arab"], "precision": "int8", "filename": "ara-diac-layerdrop-1.0-int8.zip", - "url": "https://github.com/interscript/interscript-ml/releases/download/ara-diac-layerdrop-1.0/ara-diac-layerdrop-1.0-int8.zip", + "url": "https://github.com/interscript/interscript-models/releases/download/ara-diac-layerdrop-1.0/ara-diac-layerdrop-1.0-int8.zip", "parts": null, "sha256": "68dcc06cd8861e017c4d4912ce3e1fe057d78b79b0a7b6246ae8603c99930963", "size": 191603048, @@ -686,12 +707,12 @@ { "name": "der_teacher_fullset", "value": 2.2921, - "source": "interscript/interscript-ml docs/RESULTS.md#layerdrop-6ep" + "source": "interscript/interscript-models docs/RESULTS.md#layerdrop-6ep" }, { "name": "der_student_fullset", "value": 5.784, - "source": "interscript/interscript-ml docs/RESULTS.md#layerdrop-6ep" + "source": "interscript/interscript-models docs/RESULTS.md#layerdrop-6ep" } ], "parity": { @@ -702,11 +723,12 @@ }, { "id": "ara-diac-layerdrop-1.0-int4", + "kind": null, "task": "diacritization", "scripts": ["Arab", "Arab"], "precision": "int4", "filename": "ara-diac-layerdrop-1.0-int4.zip", - "url": "https://github.com/interscript/interscript-ml/releases/download/ara-diac-layerdrop-1.0/ara-diac-layerdrop-1.0-int4.zip", + "url": "https://github.com/interscript/interscript-models/releases/download/ara-diac-layerdrop-1.0/ara-diac-layerdrop-1.0-int4.zip", "parts": null, "sha256": "352e02105645a6e283e2c3602505bea0e90085b741666fd863e478e4835e7bc9", "size": 145195702, @@ -714,12 +736,12 @@ { "name": "der_teacher_fullset", "value": 2.2921, - "source": "interscript/interscript-ml docs/RESULTS.md#layerdrop-6ep" + "source": "interscript/interscript-models docs/RESULTS.md#layerdrop-6ep" }, { "name": "der_student_fullset", "value": 5.784, - "source": "interscript/interscript-ml docs/RESULTS.md#layerdrop-6ep" + "source": "interscript/interscript-models docs/RESULTS.md#layerdrop-6ep" } ], "parity": { @@ -730,11 +752,12 @@ }, { "id": "ara-diac-small-2.1-fp16", + "kind": null, "task": "diacritization", "scripts": ["Arab", "Arab"], "precision": "fp16", "filename": "ara-diac-small-2.1-fp16.zip", - "url": "https://github.com/interscript/interscript-ml/releases/download/ara-diac-small-2.1/ara-diac-small-2.1-fp16.zip", + "url": "https://github.com/interscript/interscript-models/releases/download/ara-diac-small-2.1/ara-diac-small-2.1-fp16.zip", "parts": null, "sha256": "395ee72137d1cdfa9cd53bf167ad0d2f5fe5cbf9a5b62f118e0c9e0fa63c972d", "size": 706351903, @@ -742,12 +765,12 @@ { "name": "der_teacher_fullset", "value": 2.289, - "source": "interscript/interscript-ml docs/RESULTS.md#run-006-r7-muon" + "source": "interscript/interscript-models docs/RESULTS.md#run-006-r7-muon" }, { "name": "der_student_fullset", "value": 4.5701, - "source": "interscript/interscript-ml docs/RESULTS.md#run-006-r7-muon" + "source": "interscript/interscript-models docs/RESULTS.md#run-006-r7-muon" } ], "parity": { @@ -755,6 +778,134 @@ "cer_delta": 0.0714 }, "license": "BSD-3-Clause" + }, + { + "id": "ara-diac-small-int8static-2.1", + "kind": null, + "task": "diacritization", + "scripts": ["Arab", "Arab"], + "precision": "int8", + "filename": "ara-diac-small-int8static-2.1-int8.zip", + "url": "https://github.com/interscript/interscript-models/releases/download/ara-diac-small-int8static-2.1/ara-diac-small-int8static-2.1-int8.zip", + "parts": null, + "sha256": "7836bfd5582ad054ee419bfbd45a2b4a87fe8dc91e1f849898428faefd0c95da", + "size": 515362212, + "metrics": [ + { + "name": "der_teacher_fullset", + "value": 2.289, + "source": "interscript/interscript-models docs/RESULTS.md#run-006-r7-muon" + }, + { + "name": "der_student_fullset", + "value": 4.5701, + "source": "interscript/interscript-models docs/RESULTS.md#run-006-r7-muon" + } + ], + "parity": { + "samples": 2480, + "cer_delta": 0.1038 + }, + "license": "BSD-3-Clause" + }, + { + "id": "ara-diac-plane-1.0", + "kind": "plane", + "task": "diacritization", + "scripts": ["Arab", "Arab"], + "precision": "int8", + "filename": "ara-diac-plane-1.0.zip", + "url": "https://github.com/interscript/interscript-models/releases/download/ara-diac-plane-1.0/ara-diac-plane-1.0.zip", + "parts": null, + "sha256": "1310204e36acc4fcbb988e9f9c46cf6422e7ea3860bd467fb469ea0ad4d39ef3", + "size": 219076965, + "metrics": [ + { + "name": "der_total_greedy", + "value": 3.5905, + "source": "interscript/interscript-models docs/RESULTS.md#run-018-plane-v2" + }, + { + "name": "der_morph_greedy", + "value": 2.2149, + "source": "interscript/interscript-models docs/RESULTS.md#run-018-plane-v2" + } + ], + "parity": null, + "license": "BSD-3-Clause" + }, + { + "id": "ara-diac-plane-large-1.0", + "kind": "plane", + "task": "diacritization", + "scripts": ["Arab", "Arab"], + "precision": "int8", + "filename": "ara-diac-plane-large-1.0.zip", + "url": "https://github.com/interscript/interscript-models/releases/download/ara-diac-plane-large-1.0/ara-diac-plane-large-1.0.zip", + "parts": null, + "sha256": "a66df2e12ddc935e8ab59836fd45486f17ac40b764c1619c03b2b0e191608a52", + "size": 866052488, + "metrics": [ + { + "name": "der_total_greedy", + "value": 2.7397, + "source": "interscript/interscript-models docs/RESULTS.md#run-019-epoch-2--27397-second-epoch-closes-a-third-of-the-gap-sota-gate-still-open-2026-10-05" + }, + { + "name": "der_morph_greedy", + "value": 1.6717, + "source": "interscript/interscript-models docs/RESULTS.md#run-019-epoch-2--27397-second-epoch-closes-a-third-of-the-gap-sota-gate-still-open-2026-10-05" + }, + { + "name": "wer_total_greedy", + "value": 8.6083, + "source": "interscript/interscript-models docs/RESULTS.md#run-019-epoch-2--27397-second-epoch-closes-a-third-of-the-gap-sota-gate-still-open-2026-10-05" + } + ], + "parity": null, + "license": "BSD-3-Clause" + }, + { + "id": "heb-diac-plane-1.0", + "kind": "plane", + "task": "diacritization", + "scripts": ["Hebr", "Hebr"], + "precision": "int8", + "filename": "heb-diac-plane-1.0.zip", + "url": "https://github.com/interscript/interscript-models/releases/download/heb-diac-plane-1.0/heb-diac-plane-1.0.zip", + "parts": null, + "sha256": "dd9ae811c60c38f28ead57bdefa022b281867401ee13e38563b990eaf7f81f95", + "size": 219322007, + "metrics": [ + { + "name": "der_total_greedy", + "value": 12.48, + "source": "interscript/interscript-models docs/RESULTS.md#run-021-heb-plane--1248-nakdimon-der-the-plane-family-generalizes-seq2seq-student-beaten-at-396pp-2026-10-07" + } + ], + "parity": null, + "license": "BSD-3-Clause" + }, + { + "id": "heb-diac-plane-2.0", + "kind": "plane", + "task": "diacritization", + "scripts": ["Hebr", "Hebr"], + "precision": "int8", + "filename": "heb-diac-plane-2.0.zip", + "url": "https://github.com/interscript/interscript-models/releases/download/heb-diac-plane-2.0/heb-diac-plane-2.0.zip", + "parts": null, + "sha256": "9c8f0432294ebae6abc847ddb79572c1ab979f4851f03d44a9bb6495ba767a9e", + "size": 415901226, + "metrics": [ + { + "name": "der_total_greedy", + "value": 8.18, + "source": "interscript/interscript-models docs/RESULTS.md#run-022-heb-plane-base--818-der-at-the-artifact-level-gate-cleared-hebrew--430pp-2026-10-07" + } + ], + "parity": null, + "license": "BSD-3-Clause" } ] } diff --git a/src/ml.ts b/src/ml.ts new file mode 100644 index 0000000..29fd97f --- /dev/null +++ b/src/ml.ts @@ -0,0 +1,132 @@ +/** + * WO15: the neural edge surface — kind-aware execution on the worker. + * + * Plane artifacts small enough for worker memory (budget below) run + * IN-WORKER: zip fetched from the Releases data channel (Cache API + * between calls), sha-verified, decoded by the interscript-ts plane + * runtime. Larger models answer 503 with a pointer to the + * client/runtime path — workerd caps isolate memory at 128 MB, and we + * say so rather than OOM. + */ + +import { createPlaneModel } from "interscript/ml" +import { getModel } from "./models.js" + +/** In-worker execution budget (bytes). 100 MB leaves headroom inside + * the 128 MB workerd isolate limit for the session + decode. */ +export const WORKER_MODEL_MAX_BYTES = 100 * 1024 * 1024 + +export class MlError extends Error { + constructor( + readonly status: number, + readonly code: string, + message: string, + ) { + super(message) + } +} + +export interface MlRequest { + readonly id: string + readonly text: string + readonly preserveDiacritics?: boolean +} + +/** A resolved artifact: from the index, or supplied directly (tests, + * self-hosters with mirrors). */ +export interface PlaneArtifact { + readonly url: string + readonly sha256: string + readonly size?: number | undefined +} + +interface ZipCache { + match(key: RequestInfo): Promise + put(key: RequestInfo, res: Response): Promise +} + +function cacheInterface(): ZipCache | undefined { + if (typeof caches === "undefined") return undefined + // workerd exposes caches.default; tests run without it (no cache) + return (caches as unknown as { default?: ZipCache }).default +} + +async function fetchZip(url: string, sha256: string): Promise { + const cache = cacheInterface() + const key = `https://ml.interscript.internal/zip/${sha256}` + const hit = await cache?.match(key) + if (hit) return new Uint8Array(await hit.arrayBuffer()) + const res = await fetch(url) + if (!res.ok) { + throw new MlError(502, "artifact_fetch_failed", `artifact fetch failed: ${res.status}`) + } + const bytes = new Uint8Array(await res.arrayBuffer()) + const digest = await crypto.subtle.digest("SHA-256", bytes.buffer as ArrayBuffer) + const got = Array.from(new Uint8Array(digest)) + .map((b) => b.toString(16).padStart(2, "0")) + .join("") + if (got !== sha256) { + throw new MlError(502, "artifact_sha_mismatch", `artifact sha256 mismatch: ${got}`) + } + await cache?.put(key, new Response(bytes)) + return bytes +} + +/** Errors that mean "not executable here, try the fallback path". */ +export function isEdgeIneligible(err: unknown): boolean { + return err instanceof MlError && err.status >= 500 +} + +export async function runModel(req: MlRequest): Promise { + const entry = getModel(req.id) + if (!entry) { + throw new MlError(404, "unknown_model", `unknown model id '${req.id}'`) + } + const kind = (entry as { kind?: string }).kind ?? "imf-v1" + if (kind !== "plane") { + throw new MlError( + 501, "kind_not_edge_executable", + `kind '${kind}' does not run in-worker yet; use the interscript runtime (py/ruby/ts)`, + ) + } + if (!entry.url || !entry.sha256) { + throw new MlError(503, "artifact_unresolvable", "model has no release artifact") + } + const artifact: PlaneArtifact = { + url: entry.url, sha256: entry.sha256, size: entry.size ?? undefined, + } + const args: Parameters[0] = { + artifact, text: req.text, + } + if (req.preserveDiacritics !== undefined) { + args.preserveDiacritics = req.preserveDiacritics + } + return executePlane(args) +} + +export interface ExecutePlaneArgs { + readonly artifact: PlaneArtifact + readonly text: string + preserveDiacritics?: boolean +} + +export async function executePlane(args: ExecutePlaneArgs): Promise { + const { artifact, text } = args + if ((artifact.size ?? 0) > WORKER_MODEL_MAX_BYTES) { + throw new MlError( + 503, "model_too_large_for_worker", + `artifact exceeds the in-worker budget (${WORKER_MODEL_MAX_BYTES} bytes); ` + + `workerd isolates cap at 128 MB. Run via the interscript runtime ` + + `(py/ruby/ts) or a self-hosted inference service.`, + ) + } + const zip = await fetchZip(artifact.url, artifact.sha256) + const model = await createPlaneModel(zip) + try { + return await model.transform(text, { + preserveDiacritics: args.preserveDiacritics ?? false, + }) + } finally { + await model.dispose() + } +} diff --git a/src/rest.ts b/src/rest.ts index 90993fe..43e2ecd 100644 --- a/src/rest.ts +++ b/src/rest.ts @@ -27,6 +27,7 @@ import { InputTooLongError, MapNotFoundError } from "./errors.js" import { INFER_TIMEOUT_MS, LIMITS } from "./limits.js" import { bundledSystemCodes } from "./engine.js" import { getModel, listModels, MODELS_INDEX_VERSION } from "./models.js" +import { isEdgeIneligible, MlError, runModel } from "./ml.js" import { OPENAPI } from "./openapi.js" import { docsPage } from "./docs.js" @@ -225,6 +226,27 @@ rest.post("/v1/infer", async (c) => { if (!getModel(model)) { return errorResponse(404, "model_not_found", `Couldn't locate ${model}`) } + // Edge-first (WO15): plane artifacts inside the worker memory budget + // execute IN-WORKER; anything else falls through to the proxy. + const preserve = (body as { preserve_diacritics?: boolean }).preserve_diacritics + try { + const req: { id: string; text: string; preserveDiacritics?: boolean } = { + id: model, text: input, + } + if (preserve !== undefined) req.preserveDiacritics = preserve + const output = await runModel(req) + return c.json({ model, input, output, executed: "in-worker" }) + } catch (err) { + if (!isEdgeIneligible(err)) { + if (err instanceof MlError && err.status === 404) { + return errorResponse(404, "model_not_found", `Couldn't locate ${model}`) + } + if (err instanceof MlError && err.status === 502) { + return errorResponse(502, err.code, err.message) + } + throw err + } + } const { ML_ENDPOINT, ML_TOKEN } = (c.env ?? {}) as Record if (!ML_ENDPOINT || !ML_TOKEN) { return errorResponse( diff --git a/src/types/interscript-ml.d.ts b/src/types/interscript-ml.d.ts new file mode 100644 index 0000000..94b312e --- /dev/null +++ b/src/types/interscript-ml.d.ts @@ -0,0 +1,17 @@ +/** + * Augmentation: the plane loader ships in the next interscript + * release (interscript-ts#99/#100). Until the dependency bumps, the + * worker declares the contract it consumes; vitest aliases + * "interscript/ml" to the local interscript-ts source so tests run + * the REAL implementation. + */ +declare module "interscript/ml" { + export interface PlaneTransformOptions { + preserveDiacritics?: boolean + } + export interface PlaneModelLike { + transform(input: string, opts?: PlaneTransformOptions): Promise + dispose(): Promise + } + export function createPlaneModel(zipBytes: Uint8Array): Promise +} diff --git a/test/fixtures/tiny-plane.zip b/test/fixtures/tiny-plane.zip new file mode 100644 index 0000000..cda0b93 Binary files /dev/null and b/test/fixtures/tiny-plane.zip differ diff --git a/test/ml.test.ts b/test/ml.test.ts new file mode 100644 index 0000000..b872742 --- /dev/null +++ b/test/ml.test.ts @@ -0,0 +1,111 @@ +/** + * WO15: the neural edge surface — in-worker plane execution. + * Runs the REAL plane runtime (vitest alias -> interscript-ts source) + * against a REAL local HTTP server serving the tiny-plane fixture. + */ + +import { createServer, type Server } from "node:http" +import { readFileSync } from "node:fs" +import { afterAll, beforeAll, describe, expect, it } from "vitest" +import { + executePlane, + isEdgeIneligible, + MlError, + runModel, + WORKER_MODEL_MAX_BYTES, +} from "../src/ml.js" +import { createHash } from "node:crypto" + +const FIXTURE = new URL("./fixtures/tiny-plane.zip", import.meta.url) +const bytes = new Uint8Array(readFileSync(FIXTURE)) +const sha = createHash("sha256").update(bytes).digest("hex") + +let server: Server +let port = 0 + +beforeAll(async () => { + server = createServer((req, res) => { + res.writeHead(200, { "content-type": "application/zip" }) + res.end(Buffer.from(bytes)) + }) + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)) + port = (server.address() as { port: number }).port +}) + +afterAll(async () => { + await new Promise((resolve) => server.close(() => resolve())) +}) + +const artifact = () => ({ url: `http://127.0.0.1:${port}/tiny-plane.zip`, sha256: sha }) + +describe("executePlane (real fetch -> verify -> session -> decode)", () => { + it("decodes the fixture through the full chain", async () => { + const out = await executePlane({ artifact: artifact(), text: "كتب" }) + expect(out).toBe("كُتُبُ") // MASK(4) -> 1 -> 2: damma on k=2 + }) + + it("preserve mode round-trips user diacritics", async () => { + const out = await executePlane({ + artifact: artifact(), text: "كُتُبُ", preserveDiacritics: true, + }) + expect(out).toBe("كُتُبُ") + }) + + it("rejects a sha mismatch loudly", async () => { + await expect( + executePlane({ artifact: { ...artifact(), sha256: "0".repeat(64) }, text: "كتب" }), + ).rejects.toThrow(/sha256 mismatch/) + }) + + it("refuses artifacts above the in-worker budget", async () => { + await expect( + executePlane({ + artifact: { ...artifact(), size: WORKER_MODEL_MAX_BYTES + 1 }, + text: "كتب", + }), + ).rejects.toThrow(/in-worker budget/) + }) +}) + +describe("runModel dispatch", () => { + it("404s on unknown ids", async () => { + await expect(runModel({ id: "nope", text: "x" })).rejects.toMatchObject({ + status: 404, + }) + }) + + it("marks server-side ineligibility for the proxy fallback", () => { + const err = new MlError(503, "model_too_large_for_worker", "big") + expect(isEdgeIneligible(err)).toBe(true) + expect(isEdgeIneligible(new MlError(404, "unknown_model", "x"))).toBe(false) + expect(isEdgeIneligible(new Error("x"))).toBe(false) + }) +}) + +describe("POST /v1/infer edge-first dispatch", () => { + it("404s unknown models (route contract)", async () => { + const { rest } = await import("../src/rest.js") + const res = await rest.request("/v1/infer", { + method: "POST", + body: JSON.stringify({ model: "does-not-exist", input: "x" }), + }) + expect(res.status).toBe(404) + expect(((await res.json()) as { error: { code: string } }).error.code).toBe( + "model_not_found", + ) + }) + + it("falls back to the proxy (503 unconfigured) for oversized plane artifacts", async () => { + const { rest } = await import("../src/rest.js") + // heb-diac-plane-2.0 is 416MB — over the in-worker budget, no + // ML_ENDPOINT configured in tests -> proxy path -> 503 + const res = await rest.request("/v1/infer", { + method: "POST", + body: JSON.stringify({ model: "heb-diac-plane-2.0", input: "שלום" }), + }) + expect(res.status).toBe(503) + expect(((await res.json()) as { error: { code: string } }).error.code).toBe( + "inference_unconfigured", + ) + }) +}) diff --git a/vitest.config.ts b/vitest.config.ts index f08f2fe..753e812 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -1,6 +1,17 @@ import { defineConfig } from "vitest/config" +import { fileURLToPath } from "node:url" + export default defineConfig({ + resolve: { + alias: { + // tests run the REAL plane runtime from the local interscript-ts + // checkout (the published package lags by one release) + "interscript/ml": fileURLToPath( + new URL("../interscript-ts/src/ml/index.ts", import.meta.url), + ), + }, + }, test: { exclude: ["**/node_modules/**", "**/dist/**", "vendor/**", "coverage/**"], },