diff --git a/lms/djangoapps/instructor_task/tasks_helper/grades.py b/lms/djangoapps/instructor_task/tasks_helper/grades.py index 75084e340af7..881da3cd655f 100644 --- a/lms/djangoapps/instructor_task/tasks_helper/grades.py +++ b/lms/djangoapps/instructor_task/tasks_helper/grades.py @@ -207,7 +207,12 @@ def graded_scorable_blocks_header(self): headers in the final report. """ scorable_blocks_map = OrderedDict() - grading_context = grades_context.grading_context_for_course(self.course) + # Pass the already-loaded structure rather than using + # grading_context_for_course, which calls get_course_in_cache again. + # BlockStructureManager.get_collected() deserializes fresh from the cache + # backend on every call, so the convenience helper would make a large + # course pay that cost -- and its peak allocation -- a second time. + grading_context = grades_context.grading_context(self.course, self.course_structure) for assignment_type_name, subsection_infos in grading_context['all_graded_subsections_by_type'].items(): for subsection_index, subsection_info in enumerate(subsection_infos, start=1): for scorable_block in subsection_info['scored_descendants']: @@ -267,6 +272,23 @@ def __init__(self, context, users): self.verified_users = set(IDVerificationService.get_verified_user_ids(users)) +class _ProblemGradeBulkContext: + """ + Bulk-loads the per-learner data the problem grade report reads, so that a + batch costs a fixed number of queries instead of scaling with batch size. + + Deliberately narrower than _CourseGradeBulkContext: the problem report emits + no cohort, team, certificate or course-tag columns, so prefetching those + would be wasted work. It does read persisted course and subsection grades + (via course_grade.problem_scores, which walks every graded subsection) and + each learner's enrollment status. + """ + + def __init__(self, context, users): + prefetch_course_and_subsection_grades(context.course_id, users) + CourseEnrollment.bulk_fetch_enrollment_states(users, context.course_id) + + class _CourseGradeBulkContext: # pylint: disable=missing-class-docstring def __init__(self, context, users): self.certs = _CertificateBulkContext(context, users) @@ -440,6 +462,8 @@ class GradeReportBase: """ Base class for grade reports (ProblemGradeReport and CourseGradeReport). """ + # Batch size for chunking the list of enrollees in the course. + USER_BATCH_SIZE = 100 def __init__(self, context): self.context = context @@ -473,7 +497,7 @@ def _batch_users(self): """ Returns a generator of batches of users. """ - def grouper(iterable, chunk_size=100, fillvalue=None): + def grouper(iterable, chunk_size=self.USER_BATCH_SIZE, fillvalue=None): args = [iter(iterable)] * chunk_size return zip_longest(*args, fillvalue=fillvalue) @@ -535,8 +559,6 @@ class CourseGradeReport(GradeReportBase): """ Class to encapsulate functionality related to generating user/row had header data for Corse Grade Reports. """ - # Batch size for chunking the list of enrollees in the course. - USER_BATCH_SIZE = 100 @classmethod def generate(cls, _xblock_instance_args, _entry_id, course_id, _task_input, action_name): @@ -775,6 +797,8 @@ def _rows_for_users(self, users): """ Returns a list of rows for the given users for this report. """ + _ProblemGradeBulkContext(self.context, users) + success_rows, error_rows = [], [] for student, course_grade, error in CourseGradeFactory().iter( users, diff --git a/lms/djangoapps/instructor_task/tests/test_tasks_helper.py b/lms/djangoapps/instructor_task/tests/test_tasks_helper.py index 9d94f2d2d3bd..e65308194772 100644 --- a/lms/djangoapps/instructor_task/tests/test_tasks_helper.py +++ b/lms/djangoapps/instructor_task/tests/test_tasks_helper.py @@ -38,7 +38,11 @@ from lms.djangoapps.course_blocks.transformers import library_content from lms.djangoapps.courseware.models import StudentModule from lms.djangoapps.grades.course_data import CourseData -from lms.djangoapps.grades.models import PersistentCourseGrade, PersistentSubsectionGradeOverride +from lms.djangoapps.grades.models import ( + PersistentCourseGrade, + PersistentSubsectionGrade, + PersistentSubsectionGradeOverride, +) from lms.djangoapps.grades.subsection_grade import CreateSubsectionGrade from lms.djangoapps.grades.transformer import GradesTransformer from lms.djangoapps.instructor_analytics.basic import UNAVAILABLE, list_problem_responses @@ -897,6 +901,48 @@ def setUp(self): self.student_2 = self.create_student('üser_2') self.csv_header_row = ['Student ID', 'Email', 'Username', 'Enrollment Status', 'Grade'] + @override_settings(RATELIMIT_ENABLE=False) + def test_query_count_per_learner(self): + """ + Pin the report's query count at two cohort sizes, with every learner + holding persisted course and subsection grades. + + The report reads course_grade.problem_scores, which walks every graded + subsection, so the fixture has a graded problem that every learner has + answered. Without one the report has no problem columns and never reads + a subsection grade. + + The count is 8N + 16 (56 at 5 learners, 216 at 25), against 11N + 14 + (69, 289) before _ProblemGradeBulkContext prefetched the persisted + grades and enrollment states. What still scales is read per learner + while computing problem scores: courseware state, submission scores and + the course access checks. + """ + vertical = BlockFactory.create( + parent_location=self.problem_section.location, + category='vertical', + metadata={'graded': True}, + display_name='Problem Vertical' + ) + self.define_option_problem('Problem1', parent=vertical) + learners = [self.student_1, self.student_2] + for learner in learners: + self.submit_student_answer(learner.username, 'Problem1', ['Option 1']) + + for extra_learners, expected_queries in ((3, 56), (20, 216)): + for i in range(extra_learners): + learner = self.create_student(f'query_count_üser_{extra_learners}_{i}') + self.submit_student_answer(learner.username, 'Problem1', ['Option 1']) + learners.append(learner) + assert PersistentSubsectionGrade.objects.filter(course_id=self.course.id).count() == len(learners) + + bs_api.update_course_in_cache(self.course.id) + RequestCache.clear_all_namespaces() + + with patch('lms.djangoapps.instructor_task.tasks_helper.runner._get_current_task'): + with self.assertNumQueries(expected_queries, table_ignorelist=QUERY_COUNT_TABLE_IGNORELIST): + ProblemGradeReport.generate(None, None, self.course.id, {}, 'graded') + @patch('lms.djangoapps.instructor_task.tasks_helper.runner._get_current_task') @ddt.data(True, False) def test_no_problems(self, use_tempfile, _): # noqa: PT019