Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 28 additions & 4 deletions lms/djangoapps/instructor_task/tasks_helper/grades.py
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,12 @@ def graded_scorable_blocks_header(self):
headers in the final report.
"""
scorable_blocks_map = OrderedDict()
grading_context = grades_context.grading_context_for_course(self.course)
# Pass the already-loaded structure rather than using
# grading_context_for_course, which calls get_course_in_cache again.
# BlockStructureManager.get_collected() deserializes fresh from the cache
# backend on every call, so the convenience helper would make a large
# course pay that cost -- and its peak allocation -- a second time.
grading_context = grades_context.grading_context(self.course, self.course_structure)
for assignment_type_name, subsection_infos in grading_context['all_graded_subsections_by_type'].items():
for subsection_index, subsection_info in enumerate(subsection_infos, start=1):
for scorable_block in subsection_info['scored_descendants']:
Expand Down Expand Up @@ -267,6 +272,23 @@ def __init__(self, context, users):
self.verified_users = set(IDVerificationService.get_verified_user_ids(users))


class _ProblemGradeBulkContext:
"""
Bulk-loads the per-learner data the problem grade report reads, so that a
batch costs a fixed number of queries instead of scaling with batch size.

Deliberately narrower than _CourseGradeBulkContext: the problem report emits
no cohort, team, certificate or course-tag columns, so prefetching those
would be wasted work. It does read persisted course and subsection grades
(via course_grade.problem_scores, which walks every graded subsection) and
each learner's enrollment status.
"""

def __init__(self, context, users):
prefetch_course_and_subsection_grades(context.course_id, users)
CourseEnrollment.bulk_fetch_enrollment_states(users, context.course_id)


class _CourseGradeBulkContext: # pylint: disable=missing-class-docstring
def __init__(self, context, users):
self.certs = _CertificateBulkContext(context, users)
Expand Down Expand Up @@ -440,6 +462,8 @@ class GradeReportBase:
"""
Base class for grade reports (ProblemGradeReport and CourseGradeReport).
"""
# Batch size for chunking the list of enrollees in the course.
USER_BATCH_SIZE = 100

def __init__(self, context):
self.context = context
Expand Down Expand Up @@ -473,7 +497,7 @@ def _batch_users(self):
"""
Returns a generator of batches of users.
"""
def grouper(iterable, chunk_size=100, fillvalue=None):
def grouper(iterable, chunk_size=self.USER_BATCH_SIZE, fillvalue=None):
args = [iter(iterable)] * chunk_size
return zip_longest(*args, fillvalue=fillvalue)

Expand Down Expand Up @@ -535,8 +559,6 @@ class CourseGradeReport(GradeReportBase):
"""
Class to encapsulate functionality related to generating user/row had header data for Corse Grade Reports.
"""
# Batch size for chunking the list of enrollees in the course.
USER_BATCH_SIZE = 100

@classmethod
def generate(cls, _xblock_instance_args, _entry_id, course_id, _task_input, action_name):
Expand Down Expand Up @@ -775,6 +797,8 @@ def _rows_for_users(self, users):
"""
Returns a list of rows for the given users for this report.
"""
_ProblemGradeBulkContext(self.context, users)

success_rows, error_rows = [], []
for student, course_grade, error in CourseGradeFactory().iter(
users,
Expand Down
48 changes: 47 additions & 1 deletion lms/djangoapps/instructor_task/tests/test_tasks_helper.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,11 @@
from lms.djangoapps.course_blocks.transformers import library_content
from lms.djangoapps.courseware.models import StudentModule
from lms.djangoapps.grades.course_data import CourseData
from lms.djangoapps.grades.models import PersistentCourseGrade, PersistentSubsectionGradeOverride
from lms.djangoapps.grades.models import (
PersistentCourseGrade,
PersistentSubsectionGrade,
PersistentSubsectionGradeOverride,
)
from lms.djangoapps.grades.subsection_grade import CreateSubsectionGrade
from lms.djangoapps.grades.transformer import GradesTransformer
from lms.djangoapps.instructor_analytics.basic import UNAVAILABLE, list_problem_responses
Expand Down Expand Up @@ -897,6 +901,48 @@ def setUp(self):
self.student_2 = self.create_student('üser_2')
self.csv_header_row = ['Student ID', 'Email', 'Username', 'Enrollment Status', 'Grade']

@override_settings(RATELIMIT_ENABLE=False)
def test_query_count_per_learner(self):
"""
Pin the report's query count at two cohort sizes, with every learner
holding persisted course and subsection grades.

The report reads course_grade.problem_scores, which walks every graded
subsection, so the fixture has a graded problem that every learner has
answered. Without one the report has no problem columns and never reads
a subsection grade.

The count is 8N + 16 (56 at 5 learners, 216 at 25), against 11N + 14
(69, 289) before _ProblemGradeBulkContext prefetched the persisted
grades and enrollment states. What still scales is read per learner
while computing problem scores: courseware state, submission scores and
the course access checks.
"""
vertical = BlockFactory.create(
parent_location=self.problem_section.location,
category='vertical',
metadata={'graded': True},
display_name='Problem Vertical'
)
self.define_option_problem('Problem1', parent=vertical)
learners = [self.student_1, self.student_2]
for learner in learners:
self.submit_student_answer(learner.username, 'Problem1', ['Option 1'])

for extra_learners, expected_queries in ((3, 56), (20, 216)):
for i in range(extra_learners):
learner = self.create_student(f'query_count_üser_{extra_learners}_{i}')
self.submit_student_answer(learner.username, 'Problem1', ['Option 1'])
learners.append(learner)
assert PersistentSubsectionGrade.objects.filter(course_id=self.course.id).count() == len(learners)

bs_api.update_course_in_cache(self.course.id)
RequestCache.clear_all_namespaces()

with patch('lms.djangoapps.instructor_task.tasks_helper.runner._get_current_task'):
with self.assertNumQueries(expected_queries, table_ignorelist=QUERY_COUNT_TABLE_IGNORELIST):
ProblemGradeReport.generate(None, None, self.course.id, {}, 'graded')

@patch('lms.djangoapps.instructor_task.tasks_helper.runner._get_current_task')
@ddt.data(True, False)
def test_no_problems(self, use_tempfile, _): # noqa: PT019
Expand Down
Loading