diff --git a/README.md b/README.md index 2e7690d..f91ce2e 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ __The DVE offers__: - Format normalization to Parquet for a unified data representation - Data modelling and typecasting - Business-rule validations executed on supported backends such as Spark and DuckDB, with the option to add custom backends +- Validate and enforce referential integrity checks with minimal configuration - Deriving new fields and entities - Clear validation reporting, including summary insights and record-level error messages diff --git a/docs/user_guidance/entity_relationships.md b/docs/user_guidance/entity_relationships.md index 5029aa7..bf0f6c1 100644 --- a/docs/user_guidance/entity_relationships.md +++ b/docs/user_guidance/entity_relationships.md @@ -37,26 +37,27 @@ In order to perform these validations, how to link normalised entities needs to To allow the DVE to link between normalised assets, the following information should be provided (per linkable entity): -- parent_entity: the immediate parent of the entity -- join_fields: how to join the entity with its parent in dictionary form (parent_field_name: child_field_name) -- mandatory: whether the child entity is a mandatory field in the immediate parent +- `parent_entity`: the immediate parent of the entity +- `join_fields`: how to join the entity with its parent in dictionary form (parent_field_name: child_field_name) +- `is_root_entity`: indicates that the entity is a root node in a hierarchical model +- `mandatory`: whether the child entity is a mandatory field in the immediate parent There is also the functionality to customise errors related to either missing parent or group rejections: -- missing_parent_id_error_code: the error code to display if a record is rejected as it has no valid parent record -- missing_parent_id_error_message: the error message to display if a record is rejected as it hs no valid parent record -- no_valid_records_error_code: the error code to display if parent records are removed due to no valid children in a mandatory field -- no_valid_records_error_message: the error message to display if parent records are removed due to no valid children in a mandatory field +- `missing_parent_id_error_code`: the error code to display if a record is rejected as it has no valid parent record +- `missing_parent_id_error_message`: the error message to display if a record is rejected as it has no valid parent record +- `no_valid_records_error_code`: the error code to display if parent records are removed due to no valid children in a mandatory field +- `no_valid_records_error_message`: the error message to display if parent records are removed due to no valid children in a mandatory field +- `empty_entity_error_code`: the error code to display if a __mandatory__ entity has any records post filtering +- `empty_entity_error_message`: the error message to display if a __mandatory__ entity has any records post filtering !!! note - For root entities, you don't need to specify entity relationships - this will be inferred based on their absence. - But you may wish to so that error codes and messages can be customised. Ensure that for root entities the parent_entity - abd join_fields values are left blank. - + Specifying root entities is __optional__. Root entities will be inferred based on their absence. + You may wish to specify root entities so that error codes and messages can be customised (e.g. empty entity). + + When specifying root entities you should ensure that parent_entity and join_fields values are left blank. ## Entity Hierarchy Object - - The details provided in the entity_relationships section of the dischema are used to create an EntityHierarchy object. Please refer to [Advanced User Guidance: Entity Hierarchy](../advanced_guidance/package_documentation/entity_hierarchy.md). diff --git a/poetry.lock b/poetry.lock index 95befcd..18f789e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3587,25 +3587,22 @@ test = ["pytest", "pytest-cov"] [[package]] name = "zensical" -version = "0.0.46" +version = "0.0.63" description = "A modern static site generator built by the creators of Material for MkDocs" optional = false python-versions = ">=3.10" groups = ["docs"] files = [ - {file = "zensical-0.0.46-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:d91af81ab058c8693dfd75f2f77b4c73bcba4125681d1d276f38624291820bd2"}, - {file = "zensical-0.0.46-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:d9221264a9a87409900a47e29985607b0c9245dacb89077e87c8e16e31edc167"}, - {file = "zensical-0.0.46-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec43018d5343ca2e1d71aa352eeddd560fef504effd03025840a5a783abefa4f"}, - {file = "zensical-0.0.46-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:26e98fb8ab7ab50cdd20a73e2c7d4d9aae0b46cf2d8691e6bb22f9c261b8a60a"}, - {file = "zensical-0.0.46-cp310-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:46fe578f26963f8ee89567983e62737b6fadc9197d4742e1020b522e092d7baa"}, - {file = "zensical-0.0.46-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aef03fa186a5589148e10b62610500989c6b075a2c08e1554233adbf91b2a3dc"}, - {file = "zensical-0.0.46-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:bc7446cdf97a8dea390f20ed2bd6b030cddc1bd36a8ce113ea3efef6fa61c573"}, - {file = "zensical-0.0.46-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:bbee37801f1ed500f158dc0992c569282950f780ae353c37fe6969f99983d701"}, - {file = "zensical-0.0.46-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:9487c147c9cceb50c04d0ad70b024821a6eab1629dafd70ab6d1e86ec841e623"}, - {file = "zensical-0.0.46-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:f42a4683c762f026878d19ede4bcf7bfbb84dbecb5ad923949abb77806ed88a5"}, - {file = "zensical-0.0.46-cp310-abi3-win32.whl", hash = "sha256:85f018f2a7ee76a83915c87ddb12b58cf343fd6154081d33ac95b6751b011dd7"}, - {file = "zensical-0.0.46-cp310-abi3-win_amd64.whl", hash = "sha256:1543a693a160de60e86ca589592401b584670e7e12c5ae30e3c2ba76786f7ec3"}, - {file = "zensical-0.0.46.tar.gz", hash = "sha256:3ec21f4fb1e78cd7c0d6b07ae336b04770e27ba020dabc457b2790e5d34f1978"}, + {file = "zensical-0.0.63-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:069af2ee0254eaa7099faad908f0ae0017c151ef0160b21a039d08efaa61b38a"}, + {file = "zensical-0.0.63-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:6ec02283d946569f68fdd1ab4a1dd2de560c0ad540874d5ecd4a4f1e96bdbe1d"}, + {file = "zensical-0.0.63-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:af2711ff1397338c730cc9232f687066da7181751257f317877501b1b77a44ea"}, + {file = "zensical-0.0.63-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f1afb0e2819a588d956c1684c90e23fc0c7efc89e1ec141403a9fb41b75b1993"}, + {file = "zensical-0.0.63-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:925b3dd8bb6812b780f5e00828c42bd1eebd675b1eb6b4a58135926bb716bdde"}, + {file = "zensical-0.0.63-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6365bce465a7f6755533eaa80aa20c8a48d1b39fa7b12cd394eece3921e2c530"}, + {file = "zensical-0.0.63-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:63706419125f4c46461b322c8aaed1bbc3b917dd389ecf29601d713fc962bfb6"}, + {file = "zensical-0.0.63-cp310-abi3-win_amd64.whl", hash = "sha256:43e223b8d6ad1f08a772232926c123a000abd30ffe98d9db63a2731ec6275329"}, + {file = "zensical-0.0.63-cp310-abi3-win_arm64.whl", hash = "sha256:73c29ca1cd00243384fdc69414fac1c4367814203116acc9d198121b7a91012a"}, + {file = "zensical-0.0.63.tar.gz", hash = "sha256:95f68b494fa6a11a59065f7965d27acfca404dd867f81d31295b5ab7172f2f8a"}, ] [package.dependencies] @@ -3614,7 +3611,7 @@ deepmerge = ">=2.0" jinja2 = ">=3.1" markdown = ">=3.7" pygments = ">=2.20" -pymdown-extensions = ">=10.21.3" +pymdown-extensions = ">=11.0" pyyaml = ">=6.0.2" tomli = ">=2.4.0" @@ -3641,4 +3638,4 @@ type = ["pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "a2df40ebbf2383a36c3031ca7d04330dd8e60130b291cc136d25c22647e13716" +content-hash = "6d996888c9d149407fb885b454745bf763f9cba56404a29148ffe0b9fd57b0de" diff --git a/pyproject.toml b/pyproject.toml index ddfd701..5cac028 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -109,7 +109,7 @@ mkdocs = "1.6.1" mkdocstrings = { version = "1.0.3", extras = ["python"] } griffelib = "2.0.1" pymdown-extensions = "11.0.1" -zensical = "0.0.46" +zensical = "0.0.63" [tool.ruff] line-length = 100