release v0.10.0 (#144)

.mise.toml

@@ -1,4 +1,4 @@
 [tools]
 python="3.12"
 poetry="2.4.1"
-java="liberica-1.8.0"
+java="zulu-17.60.17"

.tool-versions

@@ -1,3 +1,3 @@
 python 3.12.12
 poetry 2.4.1
-java liberica-1.8.0
+java zulu-17.60.17

README.md

@@ -17,6 +17,7 @@ __The DVE offers__:
 - Format normalization to Parquet for a unified data representation
 - Data modelling and typecasting
 - Business-rule validations executed on supported backends such as Spark and DuckDB, with the option to add custom backends
+- Validate and enforce referential integrity checks with minimal configuration
 - Deriving new fields and entities
 - Clear validation reporting, including summary insights and record-level error messages
 
@@ -41,7 +42,8 @@ Below is a list of features that we would like to implement or have been request
 | Uplift to Python 3.11                                                           | 0.2.0             | Yes       |
 | Uplift Pyspark to 3.5                                                           | 0.8.0             | Yes       |
 | Allow DVE to run on Python 3.12+                                                | 0.8.0             | Yes       |
-| Upgrade to Pydantic 2.0                                                         | 0.9.0             | Yes        |
+| Upgrade to Pydantic 2.0                                                         | 0.9.0             | Yes       |
+| Upgrade DuckDB to v1.4                                                          | 0.10.0            | Yes       |
 | Uplift Pyspark to 4.0+                                                          | TBA               | No        |
 | Polars upgrade to v1+ | TBA | No |
 | DuckDB upgrade to v1.5+ | TBA | No |

docs/advanced_guidance/json_schemas/contract/components/base_entity.schema.json

@@ -26,8 +26,15 @@
             "items": {
                 "type": "string"
             }
+        },
+        "reader_additional_checks": {
+            "description": "A mapping of additional checks to perform on entities after initial read",
+            "type": "object",
+            "additionalProperties": {
+                "$ref": "reader_additional_checks.json"
         }
-    },
+    }
+},
 
     "required": ["fields"]
 }

docs/advanced_guidance/json_schemas/contract/components/reader_additional_checks.schema.json

@@ -0,0 +1,22 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "$id": "data-ingest:contract/components/reader_additional_checks.schema.json",
+    "title": "reader_additional_checks",
+    "description": "Additional checks to perform on initially read entities",
+    "type": "object",
+    "properties": {
+        "error_code": {
+            "description": "The code to be used for the additional check specified",
+            "type": "string"
+        },
+        "error_message": {
+            "description": "The message to be displayed for the additional check specified.",
+            "type": "string"
+        }
+    },
+    "required": [
+        "error_code",
+        "error_message"
+    ],
+    "additionalProperties": false
+}

docs/advanced_guidance/json_schemas/dataset.schema.json

@@ -10,6 +10,9 @@
         },
         "transformations": {
             "$ref": "transformations/transformations.schema.json"
+        },
+        "entity_relationships": {
+            "$ref": "entity_relationships.schema.json"
         }
     },
     "required": [

docs/advanced_guidance/json_schemas/entity_relationships.schema.json

@@ -0,0 +1,50 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "$id": "data-ingest:entity_relationships.schema.json",
+    "title": "entity_relationships",
+    "description": "Description of relationships to link normalised entities back to parent entities.",
+    "type": "object",
+    "patternProperties": {
+        "^[A-Za-z0-9_]+.$": {
+            "type": "object",
+            "properties": {
+                "parent_entity": {
+                    "type": "string"
+                },
+                "join_fields": {
+                    "type": "object",
+                    "additionalProperties": {
+                        "type": "string"
+                    }
+                },
+                "is_root_entity": {
+                    "type": "boolean"
+                },
+                "mandatory": {
+                    "type": "boolean"
+                },
+                "missing_parent_id_error_code": {
+                    "type": "string"
+                },
+                "missing_parent_id_error_message": {
+                    "type": "string"
+                },
+                "no_valid_records_error_code": {
+                    "type": "string"
+                },
+                "no_valid_records_error_message": {
+                    "type": "string"
+                },
+                "empty_entity_error_code": {
+                    "type": "string",
+                    "minLength": 1
+                },
+                "empty_entity_error_message": {
+                    "type": "string",
+                    "minLength": 1
+                }
+            },
+            "additionalProperties": false
+        }
+    }
+}

docs/advanced_guidance/package_documentation/entity_hierarchy.md

@@ -0,0 +1,5 @@
+::: dve.core_engine.configuration.v1.hierarchy
+    handler: python
+    options:
+        show_root_heading: true
+        heading_level: 2

docs/user_guidance/entity_relationships.md

@@ -0,0 +1,63 @@
+---
+title: Entity Relationships
+tags:
+    - Linkage
+    - Relationships
+    - Missing
+    - Parent
+    - Group
+    - Rejections
+---
+
+Sometimes a user may choose to use the file transformation stage to `normalise` a heavily nested dataset into separate entities during the initial reading of data. This would be done by specifying different entities in the dataset section of the contract configuration in the `dischema` file. This allows for easier interaction when customising errors in the data contract or writing transformations in the business rules.
+
+`Normalising` assets can lead to more complex validations being required. For example in the flights dataset:
+
+```mermaid
+erDiagram
+    COUNTRY ||--|{ AIRPORT : ""
+    AIRPORT ||--o{ FLIGHT : ""
+    FLIGHT ||--o{ PASSENGER : ""
+    AIRPORT ||--|{ STAFF_MEMBER : ""
+```
+
+### Missing Parent Records
+
+It could be that an airport record is deemed invalid and removed. Due to this, any flight records that linked to the now removed airport record are themselves invalid - a situation we refer to as a `missing_parent` issue, but are now existing in an entirely different entity.
+
+### No Valid Mandatory Records
+
+It could also be the case that staff records are a mandatory field for airport records. If all staff records for a particular airport record are removed during validation, this itself would invalidate the airport record - a situation we refer to as `no_valid_records` issue - but again the invalid airport record is in a different entity.
+
+### Dischema
+
+In order to perform these validations, how to link normalised entities needs to be provided. This can be specified in the `entity_relationships` section of the `dischema`.
+
+## Entity Relationships Content
+
+To allow the DVE to link between normalised assets, the following information should be provided (per linkable entity):
+
+- `parent_entity`: the immediate parent of the entity
+- `join_fields`: how to join the entity with its parent in dictionary form (parent_field_name: child_field_name)
+- `is_root_entity`: indicates that the entity is a root node in a hierarchical model
+- `mandatory`: whether the child entity is a mandatory field in the immediate parent
+
+There is also the functionality to customise errors related to either missing parent or group rejections:
+
+- `missing_parent_id_error_code`: the error code to display if a record is rejected as it has no valid parent record
+- `missing_parent_id_error_message`: the error message to display if a record is rejected as it has no valid parent record
+- `no_valid_records_error_code`: the error code to display if parent records are removed due to no valid children in a mandatory field
+- `no_valid_records_error_message`: the error message to display if parent records are removed due to no valid children in a mandatory field
+- `empty_entity_error_code`: the error code to display if a __mandatory__ entity has any records post filtering
+- `empty_entity_error_message`: the error message to display if a __mandatory__ entity has any records post filtering
+
+!!! note
+    Specifying root entities is __optional__. Root entities will be inferred based on their absence.
+    You may wish to specify root entities so that error codes and messages can be customised (e.g. empty entity).
+
+    When specifying root entities you should ensure that parent_entity and join_fields values are left blank.
+
+## Entity Hierarchy Object
+
+The details provided in the entity_relationships section of the dischema are used to create an EntityHierarchy object.
+Please refer to [Advanced User Guidance: Entity Hierarchy](../advanced_guidance/package_documentation/entity_hierarchy.md).

docs/user_guidance/getting_started.md

@@ -68,6 +68,8 @@ Within the example above, there are two parent keys - `schemas` and `datasets`.
 !!! note
     The "splitting" of entities is considerably more useful in situtations where you want to normalise/de-normalise your data. If you're unfamiliar with this concept, you can read more about it [here](https://en.wikipedia.org/wiki/Database_normalization). However, you should keep in mind potential performance impacts of doing this. If you have rules that requires fields from different entities, you will have to perform a `join` between the split entities to be able to perform the rule.
 
+To support with the application of more complex validation relating to parent and child records within normalised data, the [entity_relationships](entity_relationships.md) section of the `dischema` enables users to specify parent-child relationships and to customise error codes related to missing parent and group level validation issues. 
+
 For each dataset definition, you will need to provide a `reader_config` which describes how to load the data during the [File Transformation](file_transformation.md) stage. So, in the example above, we expect `movies` to come in as a `JSON` file. However, you can add more readers if you have the same data in different data formats (e.g. `csv`, `xml`, `json`). Regardless of what file format, the [File Transformation](file_transformation.md) stage will convert the submitted data into a "stringified" parquet format which is a requirement for the subsequent stages.
 
 To learn more about how you can construct your Data Contract please read [here](data_contract.md).

docs/user_guidance/install.md

@@ -78,11 +78,12 @@ Once you have installed the DVE you are almost ready to use it. To be able to ru
 
 ## DVE Version Compatability Matrix
 
-| DVE Version  | Python Version | DuckDB Version | Spark Version   | Pydantic Version |
-| ------------ | -------------- | -------------- | --------------- | ---------------- |
-| >=0.9.0      | >=3.10,<3.13   | 1.1.3          | >=3.5.0,<=3.5.5 | 2.13.4           | 
-| >=0.8.0      | >=3.10,<3.13   | 1.1.3          | 3.5.2           | 1.10.19          |
-| >=0.7.2      | >=3.10,<3.12   | 1.1.*          | 3.4.*           | 1.10.16          |
-| >=0.6        | >=3.10,<3.12   | 1.1.*          | 3.4.*           | 1.10.15          |
-| >=0.2,<0.6   | >=3.10,<3.12   | 1.1.0          | 3.4.4           | 1.10.15          |
-| 0.1          | >=3.7.2,<3.8   | 1.1.0          | 3.2.1           | 1.10.15          |
+| DVE Version  | Python Version   | DuckDB Version   | Spark Version   | Pydantic Version |
+| ------------ | --------------   | ---------------- | --------------- | ---------------- |
+| >=0.10.0     | >=3.10,<1.13     | __>=1.4,<1.4.5__ | >=3.5.0,<=3.5.5 | 2.13.4           |
+| >=0.9.0      | >=3.10,<3.13     | 1.1.3            | >=3.5.0,<=3.5.5 | __2.13.4__       |
+| >=0.8.0      | >=3.10,<3.13     | __1.1.3__        | __3.5.2__       | 1.10.19          |
+| >=0.7.2      | >=3.10,<3.12     | 1.1.*            | 3.4.*           | __1.10.16__      |
+| >=0.6        | >=3.10,<3.12     | __1.1.*__        | __3.4.*__       | 1.10.15          |
+| >=0.2,<0.6   | __>=3.10,<3.12__ | 1.1.0            | 3.4.4           | 1.10.15          |
+| 0.1          | >=3.7.2,<3.8     | 1.1.0            | 3.2.1           | 1.10.15          |