Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.rst
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ Fixed
conflict regardless of priority, matching schema validation; only ``role_extensions`` are resolved
by priority. This aligns the compiler with ADR 0031 and removes a path where a higher-priority file
could silently replace a role's base definition.
* Migration ``0007_coursescope`` no longer fails with MySQL error 3780 when the platform's course table has a collation other than ``utf8mb4_unicode_ci``, because on MySQL it gives ``course_overview_id`` the collation of the course column it references before creating the foreign key (`#593 <https://github.com/openedx/openedx-authz/issues/593>`_).

Changed
========
Expand Down
90 changes: 90 additions & 0 deletions src/openedx_authz/migrations/0007_coursescope.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,98 @@
# Generated by Django 4.2.24 on 2026-02-06 17:19

import django.db.models.deletion
from django.apps import apps as global_apps
from django.conf import settings
from django.db import migrations, models

MYSQL_VENDOR = "mysql"
TABLE_COLLATION_SQL = """
SELECT table_collation
FROM information_schema.tables
WHERE table_schema = DATABASE() AND table_name = %s
"""


def get_column_collation(connection, table_name, column_name):
"""Return the collation of a MySQL column.

Django's introspection returns no collation for a column that uses its table's default
collation, so that case reads the table's default collation instead.

Args:
connection: The MySQL database connection.
table_name: The table that holds the column, for example ``"course_overviews_courseoverview"``.
column_name: The column to read, for example ``"id"``.

Returns:
str | None: The column's collation, for example ``"utf8mb4_0900_ai_ci"``, or ``None`` when the
table or the column does not exist.
"""
with connection.cursor() as cursor:
if table_name not in connection.introspection.table_names(cursor):
return None
columns = {column.name: column for column in connection.introspection.get_table_description(cursor, table_name)}
if column_name not in columns:
return None
if columns[column_name].collation:
return columns[column_name].collation
cursor.execute(TABLE_COLLATION_SQL, [table_name])
return cursor.fetchone()[0]


def align_course_overview_collation(apps, schema_editor):
"""Give course_overview_id the collation of the CourseOverview column it references.

opaque-keys declares utf8mb4_unicode_ci for course key fields, and the foreign key copies
that collation to course_overview_id, while the platform's course table keeps the collation
the database had when the platform created it. MySQL only creates a foreign key between
string columns with the same collation, so on databases whose default is, for example,
utf8mb4_0900_ai_ci, creating the foreign key fails with error 3780. Django creates the
foreign key after every operation of this migration, so this step runs before it and the
new, empty column takes the referenced collation. Nothing changes when the database is not
MySQL, when the referenced table or column does not exist, or when the collations already match.
The step runs outside a transaction, because Django rejects DDL inside one on MySQL,
which cannot roll DDL back.

Args:
apps: The historical app registry of this migration.
schema_editor: The schema editor running this migration.

Returns:
None
"""
connection = schema_editor.connection
if connection.vendor != MYSQL_VENDOR:
return
course_overview_model = global_apps.get_model(settings.OPENEDX_AUTHZ_COURSE_OVERVIEW_MODEL)
course_overview_id_field = course_overview_model._meta.pk
course_scope_model = apps.get_model("openedx_authz", "CourseScope")
course_scope_field = course_scope_model._meta.get_field("course_overview")
course_scope_table = course_scope_model._meta.db_table

referenced_collation = get_column_collation(
connection, course_overview_model._meta.db_table, course_overview_id_field.column
)
if referenced_collation is None:
return
if referenced_collation == get_column_collation(connection, course_scope_table, course_scope_field.column):
return

null_sql = "NULL" if course_scope_field.null else "NOT NULL"
schema_editor.execute(
schema_editor.sql_alter_column
% {
"table": schema_editor.quote_name(course_scope_table),
"changes": schema_editor.sql_alter_column_type
% {
"column": schema_editor.quote_name(course_scope_field.column),
"type": f"{course_overview_id_field.rel_db_type(connection)} {null_sql}",
"collation": f" COLLATE {schema_editor.quote_name(referenced_collation)}",
"comment": "",
},
}
)


class Migration(migrations.Migration):
dependencies = [
Expand Down Expand Up @@ -41,4 +130,5 @@ class Migration(migrations.Migration):
},
bases=("openedx_authz.scope",),
),
migrations.RunPython(align_course_overview_collation, migrations.RunPython.noop, atomic=False),
]
156 changes: 156 additions & 0 deletions src/openedx_authz/tests/test_migrations.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
"""Unit Tests for openedx_authz migrations."""

from importlib import import_module
from types import SimpleNamespace
from unittest.mock import MagicMock, patch

from ddt import data, ddt, unpack
from django.apps import apps
from django.contrib.auth import get_user_model
from django.contrib.auth.models import Group
from django.core.management import CommandError, call_command
from django.db.migrations.loader import MigrationLoader
from django.test import TestCase
from opaque_keys.edx.django.models import CourseKeyField

Expand Down Expand Up @@ -87,6 +89,160 @@
empty_group_name = f"{OBJECT_PREFIX}empty_group"

schema_source_migration = import_module("openedx_authz.migrations.0012_move_builtin_schema_sources_to_applets")
course_scope_migration = import_module("openedx_authz.migrations.0007_coursescope")

APP_LABEL = "openedx_authz"
COURSE_SCOPE_TABLE = "openedx_authz_coursescope"


MYSQL_ALTER_COLUMN = "ALTER TABLE %(table)s %(changes)s"
MYSQL_ALTER_COLUMN_TYPE = "MODIFY %(column)s %(type)s%(collation)s%(comment)s"
MYSQL_DATA_TYPES = {"CharField": "varchar(%(max_length)s)"}


def build_mysql_schema_editor(collations_by_table, table_collation="utf8mb4_unicode_ci"):
"""Build a schema editor whose connection behaves like MySQL for the alignment step.

Args:
collations_by_table: The collation that introspection reports for each table's column, for
example ``{"openedx_authz_coursescope": "utf8mb4_unicode_ci"}``. ``None`` means the column
uses its table's default collation.
table_collation: The default collation of every table.

Returns:
MagicMock: A schema editor with a MySQL connection, its SQL templates and backtick quoting.
"""

def get_table_description(cursor, table_name): # pylint: disable=unused-argument
return [
SimpleNamespace(name=column_name, collation=collations_by_table[table_name])
for column_name in ("id", "course_overview_id")
]

mysql_connection = MagicMock(vendor="mysql", data_types=MYSQL_DATA_TYPES)
mysql_connection.introspection.table_names.return_value = list(collations_by_table)
mysql_connection.introspection.get_table_description.side_effect = get_table_description
mysql_connection.cursor.return_value.__enter__.return_value.fetchone.return_value = (table_collation,)
return MagicMock(
connection=mysql_connection,
sql_alter_column=MYSQL_ALTER_COLUMN,
sql_alter_column_type=MYSQL_ALTER_COLUMN_TYPE,
quote_name=lambda name: f"`{name}`",
)


class TestCourseScopeCollationAlignment(TestCase):
"""Migration 0007 gives course_overview_id the collation of the column it references on MySQL."""

@classmethod
def setUpClass(cls):
"""Load the app registry of migration 0007, where CourseScope still has course_overview."""
super().setUpClass()
cls.migration_apps = (
MigrationLoader(None, ignore_no_migrations=True).project_state((APP_LABEL, "0007_coursescope")).apps
)

def test_different_collation_aligns_the_new_column(self):
"""A referenced column with another collation makes the step change the new column to it.

Expected Result:
- One ALTER sets the referenced collation and keeps the column type and nullability.
"""
schema_editor = build_mysql_schema_editor(
{CourseOverview._meta.db_table: "utf8mb4_0900_ai_ci", COURSE_SCOPE_TABLE: "utf8mb4_unicode_ci"}
)

course_scope_migration.align_course_overview_collation(self.migration_apps, schema_editor)

schema_editor.execute.assert_called_once_with(
"ALTER TABLE `openedx_authz_coursescope` MODIFY `course_overview_id` varchar(255) NULL "
"COLLATE `utf8mb4_0900_ai_ci`"
)

def test_referenced_column_with_table_default_uses_the_table_collation(self):
"""A referenced column without its own collation takes its table's default collation.

Expected Result:
- The ALTER uses the table's default collation.
"""
schema_editor = build_mysql_schema_editor(
{CourseOverview._meta.db_table: None, COURSE_SCOPE_TABLE: "utf8mb4_unicode_ci"},
table_collation="utf8mb4_0900_ai_ci",
)

course_scope_migration.align_course_overview_collation(self.migration_apps, schema_editor)

self.assertIn("COLLATE `utf8mb4_0900_ai_ci`", schema_editor.execute.call_args[0][0])

def test_matching_collation_runs_no_alter(self):
"""Columns that already share a collation, as on Tutor-managed MySQL, are left as they are.

Expected Result:
- No ALTER runs.
"""
schema_editor = build_mysql_schema_editor(
{CourseOverview._meta.db_table: "utf8mb4_unicode_ci", COURSE_SCOPE_TABLE: "utf8mb4_unicode_ci"}
)

course_scope_migration.align_course_overview_collation(self.migration_apps, schema_editor)

schema_editor.execute.assert_not_called()

def test_step_reads_the_configured_course_overview_model(self):
"""The referenced table comes from the model named by OPENEDX_AUTHZ_COURSE_OVERVIEW_MODEL.

Expected Result:
- Introspection reads the stub CourseOverview table the test settings configure.
"""
schema_editor = build_mysql_schema_editor(
{CourseOverview._meta.db_table: "utf8mb4_unicode_ci", COURSE_SCOPE_TABLE: "utf8mb4_unicode_ci"}
)

course_scope_migration.align_course_overview_collation(self.migration_apps, schema_editor)

described_tables = [
call.args[1] for call in schema_editor.connection.introspection.get_table_description.call_args_list
]
self.assertEqual(described_tables, [CourseOverview._meta.db_table, COURSE_SCOPE_TABLE])

def test_missing_referenced_table_runs_no_alter(self):
"""When the CourseOverview table does not exist, the step leaves the new column as it is.

Expected Result:
- No ALTER runs, and Django reports any foreign key problem itself.
"""
schema_editor = build_mysql_schema_editor({COURSE_SCOPE_TABLE: "utf8mb4_unicode_ci"})

course_scope_migration.align_course_overview_collation(self.migration_apps, schema_editor)

schema_editor.execute.assert_not_called()

def test_step_runs_outside_a_transaction(self):
"""The alignment step is not wrapped in a transaction.

MySQL cannot roll DDL back, so Django raises TransactionManagementError when the ALTER
runs inside one, which the test settings on SQLite do not reproduce.

Expected Result:
- The RunPython operation of 0007 sets atomic to False.
"""
alignment_operation = course_scope_migration.Migration.operations[-1]

self.assertIs(alignment_operation.code, course_scope_migration.align_course_overview_collation)
self.assertIs(alignment_operation.atomic, False)

def test_other_database_vendor_runs_no_sql(self):
"""On databases other than MySQL the step does nothing.

Expected Result:
- No cursor is opened and no SQL runs.
"""
schema_editor = MagicMock(connection=MagicMock(vendor="sqlite"))

course_scope_migration.align_course_overview_collation(self.migration_apps, schema_editor)

schema_editor.connection.cursor.assert_not_called()
schema_editor.execute.assert_not_called()


class TestBuiltinSchemaSourceMigration(TestCase):
Expand Down
Loading