From 05ba5d5473b42ec1831a99aaf50bdfb793fcd8a0 Mon Sep 17 00:00:00 2001 From: Tobias Macey Date: Fri, 31 Jul 2026 15:22:40 -0400 Subject: [PATCH 1/4] perf: stream report uploads instead of buffering the whole file in memory DjangoStorageReportStore.store() read its entire buffer into memory, re-encoded it, and copied it into a ContentFile before handing it to the storage backend: buff_contents = buff.read() # whole file into RAM if not isinstance(buff_contents, bytes): buff_contents = buff_contents.encode('utf-8') # second copy buff = ContentFile(buff_contents) # third self.storage.save(path, buff) That put peak memory at roughly 3x the report size at the final step, and it undid most of the benefit of TemporaryFileReportMixin: that mixin spills rows to a temp file specifically to avoid holding the report in memory, then the upload read the whole thing straight back in. The ReportStore docstring has anticipated this for some time -- "Should probably refactor later to create a ReportFile object that can simply be appended to for the sake of memory efficiency". A binary buffer is now handed to the backend as-is, so it streams: S3Boto3Storage uploads in parts, FileSystemStorage writes in chunks, and peak memory is one chunk rather than the whole report. Text buffers still work, and still have to be read in full because the backends require bytes. Both in-tree callers that pass text are unaffected in behaviour; the type is probed with a zero-length read rather than by inspecting mode attributes, which are not consistently present across the raw file objects, ContentFile and BytesIO instances that callers pass. Two callers move to binary so they take the streaming path: * TemporaryFileReportMixin now opens its temp files 'w+b' and writes through a TextIOWrapper, which is what makes the on-disk grade report actually bounded end to end rather than only during row generation. * store_rows() builds its CSV in a BytesIO by the same means. Both wrappers use newline='' per the csv module's contract -- the writer emits its own line terminators and must not have them translated again -- and are detached rather than closed, so the underlying file outlives the wrapper. Tests assert the streaming behaviourally rather than by inspecting call args: a streaming backend reads in bounded chunks, so an unbounded read() is the signature of the report being slurped into RAM. Added to ReportStoreTestMixin so they run against all three configurations, including the S3 stub. Round-trip tests cover the binary, text and store_rows paths; CSV bytes are unchanged. Completes the last of the five findings in the issue. Refs: https://github.com/openedx/openedx-platform/issues/38943 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Ki1NYMjSjVB4uz1gdgsjBh --- lms/djangoapps/instructor_task/models.py | 49 +++++++++++--- .../instructor_task/tasks_helper/grades.py | 28 +++++++- .../instructor_task/tests/test_models.py | 65 ++++++++++++++++++- 3 files changed, 129 insertions(+), 13 deletions(-) diff --git a/lms/djangoapps/instructor_task/models.py b/lms/djangoapps/instructor_task/models.py index c38282ca54c6..2bc44e2d8325 100644 --- a/lms/djangoapps/instructor_task/models.py +++ b/lms/djangoapps/instructor_task/models.py @@ -17,13 +17,14 @@ import json import logging import os.path +from io import BytesIO, TextIOWrapper from uuid import uuid4 from botocore.exceptions import ClientError from django.apps import apps from django.conf import settings from django.contrib.auth.models import User # pylint: disable=imported-auth-user -from django.core.files.base import ContentFile +from django.core.files.base import ContentFile, File from django.db import models, transaction from django.utils.translation import gettext as _ from model_utils.models import TimeStampedModel @@ -293,27 +294,57 @@ def store(self, course_id, filename, buff, parent_dir=''): Store the contents of `buff` in a directory determined by hashing `course_id`, and name the file `filename`. `buff` can be any file-like object, ready to be read from the beginning. + + A binary buffer is handed to the storage backend as-is, so the backend + streams it -- S3Boto3Storage uploads in parts and FileSystemStorage + writes in chunks -- and peak memory stays at one chunk rather than the + whole report. This matters for grade reports, which can run to hundreds + of megabytes on large courses; previously the entire file was read into + memory, re-encoded, and copied into a ContentFile before upload, so an + on-disk report still cost roughly 3x its size in RAM at the final step. + + A text buffer still works, but has to be read and encoded in full + because the storage backends require bytes. Callers handling + potentially large reports should pass a binary file. """ path = self.path_to(course_id, filename, parent_dir) + + if self._yields_bytes(buff): + self.storage.save(path, File(buff, name=filename)) + return + # See https://github.com/boto/boto/issues/2868 # Boto doesn't play nice with unicode in python3 - buff_contents = buff.read() - - if not isinstance(buff_contents, bytes): - buff_contents = buff_contents.encode('utf-8') + self.storage.save(path, ContentFile(buff.read().encode('utf-8'))) - buff = ContentFile(buff_contents) + @staticmethod + def _yields_bytes(buff): + """ + Return True if reading from `buff` produces bytes rather than str. - self.storage.save(path, buff) + Probes with a zero-length read so the buffer is left positioned exactly + where it was, rather than inspecting the type -- callers pass a mix of + raw file objects, ContentFile and BytesIO, and mode attributes are not + consistently present across them. + """ + try: + return isinstance(buff.read(0), bytes) + except (AttributeError, TypeError, ValueError): + return False def store_rows(self, course_id, filename, rows, parent_dir=''): """ Given a course_id, filename, and rows (each row is an iterable of strings), write the rows to the storage backend in csv format. """ - output_buffer = ContentFile('') - csvwriter = csv.writer(output_buffer) + output_buffer = BytesIO() + # newline='' per the csv module's contract; the writer emits its own + # line terminators and must not have them translated again. + text_wrapper = TextIOWrapper(output_buffer, encoding='utf-8', newline='', write_through=True) + csvwriter = csv.writer(text_wrapper) csvwriter.writerows(self._get_utf8_encoded_rows(rows)) + # Detach so closing the wrapper does not close the buffer underneath it. + text_wrapper.detach() output_buffer.seek(0) self.store(course_id, filename, output_buffer, parent_dir) diff --git a/lms/djangoapps/instructor_task/tasks_helper/grades.py b/lms/djangoapps/instructor_task/tasks_helper/grades.py index 9b6a764137db..f8a5f1c23191 100644 --- a/lms/djangoapps/instructor_task/tasks_helper/grades.py +++ b/lms/djangoapps/instructor_task/tasks_helper/grades.py @@ -7,6 +7,7 @@ import re from collections import OrderedDict, defaultdict from datetime import datetime +from io import TextIOWrapper from itertools import chain from tempfile import TemporaryFile from time import time @@ -351,7 +352,11 @@ def _generate(self): self.context.update_status('TemporaryFileReportMixin - 1: Starting grade report') batched_rows = self._batched_rows() - with TemporaryFile('r+') as success_file, TemporaryFile('r+') as error_file: + # Binary temp files, written through a text wrapper. The report store + # streams a binary buffer straight to the storage backend, where a text + # one has to be read into memory and encoded in full -- which would undo + # most of the benefit of spilling to disk in the first place. + with TemporaryFile('w+b') as success_file, TemporaryFile('w+b') as error_file: self.context.update_status('TemporaryFileReportMixin - 2: Compiling grades into temp files') has_errors = self.iter_and_write_batched_rows(batched_rows, success_file, error_file) @@ -360,13 +365,24 @@ def _generate(self): return self.context.update_status('TemporaryFileReportMixin - 4: Completed grades') + @staticmethod + def _csv_writer_for(binary_file): + """ + Return a csv.writer over a binary file, plus the wrapper to flush. + + newline='' is the csv module's documented requirement: the writer emits + its own line terminators and they must not be translated again. + """ + wrapper = TextIOWrapper(binary_file, encoding='utf-8', newline='', write_through=True) + return csv.writer(wrapper), wrapper + def iter_and_write_batched_rows(self, batched_rows, success_file, error_file): """ Iterate through batched rows, writing returned chunks to disk as we go. This should hopefully help us avoid out of memory errors. """ - success_writer = csv.writer(success_file) - error_writer = csv.writer(error_file) + success_writer, success_wrapper = self._csv_writer_for(success_file) + error_writer, error_wrapper = self._csv_writer_for(error_file) # Write headers success_writer.writerow(self._success_headers()) @@ -386,6 +402,12 @@ def iter_and_write_batched_rows(self, batched_rows, success_file, error_file): self.context.task_progress.attempted = succeeded + failed self.context.task_progress.total = self.context.task_progress.attempted + # Detach rather than close: the wrappers must flush their buffered text + # into the temp files before those are read back for upload, but the + # temp files themselves stay open and are closed by the caller. + success_wrapper.detach() + error_wrapper.detach() + return self.context.task_progress.failed > 0 def upload_temp_files(self, success_file, error_file, has_errors): diff --git a/lms/djangoapps/instructor_task/tests/test_models.py b/lms/djangoapps/instructor_task/tests/test_models.py index 204f4d1fac90..15b542d8a653 100644 --- a/lms/djangoapps/instructor_task/tests/test_models.py +++ b/lms/djangoapps/instructor_task/tests/test_models.py @@ -5,7 +5,7 @@ import copy import time -from io import StringIO +from io import BytesIO, StringIO import pytest from django.conf import settings @@ -51,6 +51,69 @@ def create_report_store(self): """ pass # pylint: disable=unnecessary-pass + def test_store_streams_binary_buffer(self): + """ + A binary buffer should reach the storage backend without first being + read into memory in its entirety. + + Asserted behaviourally rather than by inspecting call args: a streaming + backend reads in bounded chunks (File.chunks / upload_fileobj both pass + an explicit size), so an unbounded read() is the signature of the whole + report being slurped into RAM before upload. + """ + unbounded_reads = [] + + class _RecordingBytesIO(BytesIO): + """BytesIO that notes any read() not bounded by an explicit size.""" + def read(self, size=-1, /): + if size is None or size < 0: + unbounded_reads.append(size) + return super().read(size) + + report_store = self.create_report_store() # pylint: disable=assignment-from-no-return + payload = b'student_id,grade\n' + b'1,0.5\n' * 5000 + + report_store.store(self.course_id, 'streamed.csv', _RecordingBytesIO(payload)) + + assert unbounded_reads == [] + with report_store.storage.open(report_store.path_to(self.course_id, 'streamed.csv')) as stored: + assert stored.read() == payload + + def test_store_text_buffer_round_trips(self): + """ + Text buffers are still accepted, and are utf-8 encoded on the way out. + + Not every caller has a binary file to hand, so this path has to keep + working even though it cannot stream. + """ + report_store = self.create_report_store() # pylint: disable=assignment-from-no-return + contents = 'student_id,grade\n1,0.5\nüser,1.0\n' + + report_store.store(self.course_id, 'text.csv', StringIO(contents)) + + with report_store.storage.open(report_store.path_to(self.course_id, 'text.csv')) as stored: + assert stored.read() == contents.encode('utf-8') + + def test_store_rows_round_trips(self): + """ + store_rows() builds its CSV in a binary buffer, so it takes the + streaming path too. Verify the bytes on disk are unchanged by that. + """ + report_store = self.create_report_store() # pylint: disable=assignment-from-no-return + + report_store.store_rows( + self.course_id, + 'rows.csv', + [['student_id', 'grade'], [1, 0.5], ['üser', 'Not Attempted']], + ) + + with report_store.storage.open(report_store.path_to(self.course_id, 'rows.csv')) as stored: + assert stored.read().decode('utf-8').splitlines() == [ + 'student_id,grade', + '1,0.5', + 'üser,Not Attempted', + ] + def test_links_for_order(self): """ Test that ReportStore.links_for() returns file download links From 461fd37499d582547d83568f822afa53b8bbb9d9 Mon Sep 17 00:00:00 2001 From: Tobias Macey Date: Fri, 31 Jul 2026 16:17:57 -0400 Subject: [PATCH 2/4] style: satisfy pylint use-implicit-booleaness-not-comparison in report store test Upstream's pylint config flags C1803 on `unbounded_reads == []`. Switched to a truthiness check and added the recorded reads to the assertion message, so a failure names which unbounded read happened rather than just reporting False. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Ki1NYMjSjVB4uz1gdgsjBh --- lms/djangoapps/instructor_task/tests/test_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/djangoapps/instructor_task/tests/test_models.py b/lms/djangoapps/instructor_task/tests/test_models.py index 15b542d8a653..b9ea380f299b 100644 --- a/lms/djangoapps/instructor_task/tests/test_models.py +++ b/lms/djangoapps/instructor_task/tests/test_models.py @@ -75,7 +75,7 @@ def read(self, size=-1, /): report_store.store(self.course_id, 'streamed.csv', _RecordingBytesIO(payload)) - assert unbounded_reads == [] + assert not unbounded_reads, f'buffer was read without a size bound: {unbounded_reads}' with report_store.storage.open(report_store.path_to(self.course_id, 'streamed.csv')) as stored: assert stored.read() == payload From d7ea9661a092466333ec75cddf98702dbcb3c0a9 Mon Sep 17 00:00:00 2001 From: Tobias Macey Date: Mon, 9 Nov 2026 13:45:30 -0500 Subject: [PATCH 3/4] docs: name ORA submission zips as the large report in store() Our largest problem grade report is 85 MB, while ORA submission-file zips, which upload_ora2_submission_files builds in a TemporaryFile, reach several GB, so they are the case streaming the upload matters for. Claude-Session: https://claude.ai/code/session_01FGZebDghNhg4LQEvB7NWni --- lms/djangoapps/instructor_task/models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lms/djangoapps/instructor_task/models.py b/lms/djangoapps/instructor_task/models.py index 2bc44e2d8325..cd5ff3bd3177 100644 --- a/lms/djangoapps/instructor_task/models.py +++ b/lms/djangoapps/instructor_task/models.py @@ -298,10 +298,10 @@ def store(self, course_id, filename, buff, parent_dir=''): A binary buffer is handed to the storage backend as-is, so the backend streams it -- S3Boto3Storage uploads in parts and FileSystemStorage writes in chunks -- and peak memory stays at one chunk rather than the - whole report. This matters for grade reports, which can run to hundreds - of megabytes on large courses; previously the entire file was read into - memory, re-encoded, and copied into a ContentFile before upload, so an - on-disk report still cost roughly 3x its size in RAM at the final step. + whole report. This matters most for ORA submission-file exports, which + are built in a temporary file and can run to gigabytes; previously the + entire file was read into memory and copied into a ContentFile before + upload, so an on-disk report was held in RAM in full at the final step. A text buffer still works, but has to be read and encoded in full because the storage backends require bytes. Callers handling From db9ac2013216206be2721de3a866609fe1210891 Mon Sep 17 00:00:00 2001 From: Tobias Macey Date: Fri, 13 Nov 2026 13:48:31 -0500 Subject: [PATCH 4/4] test: run the S3 report store tests against S3Boto3Storage The test settings set GRADES_DOWNLOAD STORAGE_TYPE to localfs, and ReportStore.from_config checks STORAGE_TYPE before STORAGE_CLASS, so the S3 test case built a FileSystemStorage. MockS3Boto3Mixin also replaced S3Boto3Storage.save, so the S3 upload path was never exercised. The fixture now drops STORAGE_TYPE and replaces only the S3 resource, with an in-memory stub that consumes uploads in bounded chunks, so S3Boto3Storage.save and _save run for real. Claude-Session: https://claude.ai/code/session_01FGZebF8pmhpsgn5eqD2G83 --- .../instructor_task/tests/test_models.py | 113 ++++++++++++++++-- 1 file changed, 106 insertions(+), 7 deletions(-) diff --git a/lms/djangoapps/instructor_task/tests/test_models.py b/lms/djangoapps/instructor_task/tests/test_models.py index b9ea380f299b..d0001518c9c4 100644 --- a/lms/djangoapps/instructor_task/tests/test_models.py +++ b/lms/djangoapps/instructor_task/tests/test_models.py @@ -5,14 +5,18 @@ import copy import time +from datetime import UTC, datetime from io import BytesIO, StringIO +from types import SimpleNamespace +from unittest.mock import PropertyMock, patch import pytest +from botocore.exceptions import ClientError from django.conf import settings from django.test import SimpleTestCase, TestCase, override_settings from opaque_keys.edx.locator import CourseLocator +from storages.backends.s3boto3 import S3Boto3Storage -from common.test.utils import MockS3Boto3Mixin from lms.djangoapps.instructor_task.models import TASK_INPUT_LENGTH, InstructorTask, ReportStore from lms.djangoapps.instructor_task.tests.test_base import TestReportMixin @@ -159,24 +163,119 @@ def create_report_store(self): return ReportStore.from_config(config_name='GRADES_DOWNLOAD') -class DjangoStorageReportStoreS3TestCase(MockS3Boto3Mixin, ReportStoreTestMixin, TestReportMixin, SimpleTestCase): +class _FakeS3Object: """ - Test the DjangoStorageReportStore implementation using S3 stubs. + The parts of a boto3 S3 Object that S3Boto3Storage calls, backed by a dict. """ + + def __init__(self, s3, key): + self._s3 = s3 + self.key = key + self.content_encoding = None + + def upload_fileobj(self, fileobj, ExtraArgs=None, Config=None): # pylint: disable=invalid-name,unused-argument + """Consume the upload in bounded chunks, as s3transfer's part uploads do.""" + parts = [] + while chunk := fileobj.read(self._s3.CHUNK_SIZE): + parts.append(chunk) + self._s3.objects[self.key] = (b''.join(parts), datetime.now(UTC)) + + def load(self, **_kwargs): + if self.key not in self._s3.objects: + raise ClientError({'Error': {'Code': '404'}, 'ResponseMetadata': {'HTTPStatusCode': 404}}, 'HeadObject') + + def download_fileobj(self, fileobj, ExtraArgs=None, Config=None): # pylint: disable=invalid-name,unused-argument + fileobj.write(self._s3.objects[self.key][0]) + + @property + def content_length(self): + return len(self._s3.objects[self.key][0]) + + @property + def last_modified(self): + return self._s3.objects[self.key][1] + + +class _FakeS3: + """ + An in-memory stand-in for the boto3 S3 resource behind S3Boto3Storage. + + Only the network layer is replaced: S3Boto3Storage.save and _save run for + real, so the store's buffer goes through the same wrapping it gets in + production before it reaches upload_fileobj. + """ + CHUNK_SIZE = 64 * 1024 + + def __init__(self): + self.objects = {} + self.meta = SimpleNamespace(client=SimpleNamespace( + get_paginator=lambda _operation: SimpleNamespace(paginate=self._list_objects), + generate_presigned_url=self._presigned_url, + )) + + def Bucket(self, name): # pylint: disable=invalid-name + return SimpleNamespace(name=name, Object=lambda key: _FakeS3Object(self, key)) + + def _list_objects(self, Bucket, Delimiter, Prefix): # pylint: disable=invalid-name,unused-argument + return [{ + 'Contents': [ + {'Key': key} for key in sorted(self.objects) + if key.startswith(Prefix) and Delimiter not in key[len(Prefix):] + ], + }] + + @staticmethod + def _presigned_url(ClientMethod, Params, ExpiresIn, HttpMethod=None): # pylint: disable=invalid-name,unused-argument + return f"https://{Params['Bucket']}.s3.test/{Params['Key']}" + + +class DjangoStorageReportStoreS3TestCase(ReportStoreTestMixin, TestReportMixin, SimpleTestCase): + """ + Test the DjangoStorageReportStore implementation on S3Boto3Storage, with + only the S3 API replaced. + """ + def setUp(self): + super().setUp() + self.s3 = _FakeS3() + connection = patch('storages.backends.s3.S3Storage.connection', new_callable=PropertyMock, return_value=self.s3) + connection.start() + self.addCleanup(connection.stop) + def create_report_store(self): """ Create and return a DjangoStorageReportStore configured to use S3 for storage. + + The test settings' STORAGE_TYPE ('localfs') is dropped: from_config + checks it before STORAGE_CLASS, so leaving it in builds a + FileSystemStorage and none of these tests touch S3. """ test_settings = copy.deepcopy(settings.GRADES_DOWNLOAD) + del test_settings['STORAGE_TYPE'] test_settings['STORAGE_CLASS'] = 'storages.backends.s3boto3.S3Boto3Storage' test_settings['STORAGE_KWARGS'] = { - 'bucket': settings.GRADES_DOWNLOAD['BUCKET'], - 'location': settings.GRADES_DOWNLOAD['ROOT_PATH'], + 'bucket_name': settings.GRADES_DOWNLOAD['BUCKET'], + # S3 keys have no leading slash; ROOT_PATH is a local temp dir here. + 'location': settings.GRADES_DOWNLOAD['ROOT_PATH'].lstrip('/'), } with override_settings(GRADES_DOWNLOAD=test_settings): - self.mocked_connection.create_bucket(settings.GRADES_DOWNLOAD['STORAGE_KWARGS']['bucket']) - return ReportStore.from_config(config_name='GRADES_DOWNLOAD') + report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD') + assert isinstance(report_store.storage, S3Boto3Storage) + return report_store + + def test_store_uploads_to_s3(self): + """ + A binary report is uploaded as the object's whole body, in bounded + chunks, under the course's hashed path. + """ + report_store = self.create_report_store() + payload = b'student_id,grade\n' + b'1,0.5\n' * 50000 + + report_store.store(self.course_id, 'uploaded.csv', BytesIO(payload)) + + key = f"{report_store.storage.location}/{report_store.path_to(self.course_id, 'uploaded.csv')}" + assert self.s3.objects[key][0] == payload + assert len(payload) > 4 * _FakeS3.CHUNK_SIZE class TestS3ReportStorage(TestCase):