diff --git a/lms/djangoapps/instructor_task/models.py b/lms/djangoapps/instructor_task/models.py index c38282ca54c6..cd5ff3bd3177 100644 --- a/lms/djangoapps/instructor_task/models.py +++ b/lms/djangoapps/instructor_task/models.py @@ -17,13 +17,14 @@ import json import logging import os.path +from io import BytesIO, TextIOWrapper from uuid import uuid4 from botocore.exceptions import ClientError from django.apps import apps from django.conf import settings from django.contrib.auth.models import User # pylint: disable=imported-auth-user -from django.core.files.base import ContentFile +from django.core.files.base import ContentFile, File from django.db import models, transaction from django.utils.translation import gettext as _ from model_utils.models import TimeStampedModel @@ -293,27 +294,57 @@ def store(self, course_id, filename, buff, parent_dir=''): Store the contents of `buff` in a directory determined by hashing `course_id`, and name the file `filename`. `buff` can be any file-like object, ready to be read from the beginning. + + A binary buffer is handed to the storage backend as-is, so the backend + streams it -- S3Boto3Storage uploads in parts and FileSystemStorage + writes in chunks -- and peak memory stays at one chunk rather than the + whole report. This matters most for ORA submission-file exports, which + are built in a temporary file and can run to gigabytes; previously the + entire file was read into memory and copied into a ContentFile before + upload, so an on-disk report was held in RAM in full at the final step. + + A text buffer still works, but has to be read and encoded in full + because the storage backends require bytes. Callers handling + potentially large reports should pass a binary file. """ path = self.path_to(course_id, filename, parent_dir) + + if self._yields_bytes(buff): + self.storage.save(path, File(buff, name=filename)) + return + # See https://github.com/boto/boto/issues/2868 # Boto doesn't play nice with unicode in python3 - buff_contents = buff.read() - - if not isinstance(buff_contents, bytes): - buff_contents = buff_contents.encode('utf-8') + self.storage.save(path, ContentFile(buff.read().encode('utf-8'))) - buff = ContentFile(buff_contents) + @staticmethod + def _yields_bytes(buff): + """ + Return True if reading from `buff` produces bytes rather than str. - self.storage.save(path, buff) + Probes with a zero-length read so the buffer is left positioned exactly + where it was, rather than inspecting the type -- callers pass a mix of + raw file objects, ContentFile and BytesIO, and mode attributes are not + consistently present across them. + """ + try: + return isinstance(buff.read(0), bytes) + except (AttributeError, TypeError, ValueError): + return False def store_rows(self, course_id, filename, rows, parent_dir=''): """ Given a course_id, filename, and rows (each row is an iterable of strings), write the rows to the storage backend in csv format. """ - output_buffer = ContentFile('') - csvwriter = csv.writer(output_buffer) + output_buffer = BytesIO() + # newline='' per the csv module's contract; the writer emits its own + # line terminators and must not have them translated again. + text_wrapper = TextIOWrapper(output_buffer, encoding='utf-8', newline='', write_through=True) + csvwriter = csv.writer(text_wrapper) csvwriter.writerows(self._get_utf8_encoded_rows(rows)) + # Detach so closing the wrapper does not close the buffer underneath it. + text_wrapper.detach() output_buffer.seek(0) self.store(course_id, filename, output_buffer, parent_dir) diff --git a/lms/djangoapps/instructor_task/tasks_helper/grades.py b/lms/djangoapps/instructor_task/tasks_helper/grades.py index 4ac75d2fd185..75084e340af7 100644 --- a/lms/djangoapps/instructor_task/tasks_helper/grades.py +++ b/lms/djangoapps/instructor_task/tasks_helper/grades.py @@ -7,6 +7,7 @@ import re from collections import OrderedDict, defaultdict from datetime import datetime +from io import TextIOWrapper from itertools import chain from tempfile import TemporaryFile from time import time @@ -351,7 +352,11 @@ def _generate(self): self.context.update_status('TemporaryFileReportMixin - 1: Starting grade report') batched_rows = self._batched_rows() - with TemporaryFile('r+') as success_file, TemporaryFile('r+') as error_file: + # Binary temp files, written through a text wrapper. The report store + # streams a binary buffer straight to the storage backend, where a text + # one has to be read into memory and encoded in full -- which would undo + # most of the benefit of spilling to disk in the first place. + with TemporaryFile('w+b') as success_file, TemporaryFile('w+b') as error_file: self.context.update_status('TemporaryFileReportMixin - 2: Compiling grades into temp files') has_errors = self.iter_and_write_batched_rows(batched_rows, success_file, error_file) @@ -360,13 +365,24 @@ def _generate(self): return self.context.update_status('TemporaryFileReportMixin - 4: Completed grades') + @staticmethod + def _csv_writer_for(binary_file): + """ + Return a csv.writer over a binary file, plus the wrapper to flush. + + newline='' is the csv module's documented requirement: the writer emits + its own line terminators and they must not be translated again. + """ + wrapper = TextIOWrapper(binary_file, encoding='utf-8', newline='', write_through=True) + return csv.writer(wrapper), wrapper + def iter_and_write_batched_rows(self, batched_rows, success_file, error_file): """ Iterate through batched rows, writing returned chunks to disk as we go. This should hopefully help us avoid out of memory errors. """ - success_writer = csv.writer(success_file) - error_writer = csv.writer(error_file) + success_writer, success_wrapper = self._csv_writer_for(success_file) + error_writer, error_wrapper = self._csv_writer_for(error_file) # Write headers success_writer.writerow(self._success_headers()) @@ -386,6 +402,12 @@ def iter_and_write_batched_rows(self, batched_rows, success_file, error_file): self.context.task_progress.attempted = succeeded + failed self.context.task_progress.total = self.context.task_progress.attempted + # Detach rather than close: the wrappers must flush their buffered text + # into the temp files before those are read back for upload, but the + # temp files themselves stay open and are closed by the caller. + success_wrapper.detach() + error_wrapper.detach() + return self.context.task_progress.failed > 0 def upload_temp_files(self, success_file, error_file, has_errors): diff --git a/lms/djangoapps/instructor_task/tests/test_models.py b/lms/djangoapps/instructor_task/tests/test_models.py index 204f4d1fac90..d0001518c9c4 100644 --- a/lms/djangoapps/instructor_task/tests/test_models.py +++ b/lms/djangoapps/instructor_task/tests/test_models.py @@ -5,14 +5,18 @@ import copy import time -from io import StringIO +from datetime import UTC, datetime +from io import BytesIO, StringIO +from types import SimpleNamespace +from unittest.mock import PropertyMock, patch import pytest +from botocore.exceptions import ClientError from django.conf import settings from django.test import SimpleTestCase, TestCase, override_settings from opaque_keys.edx.locator import CourseLocator +from storages.backends.s3boto3 import S3Boto3Storage -from common.test.utils import MockS3Boto3Mixin from lms.djangoapps.instructor_task.models import TASK_INPUT_LENGTH, InstructorTask, ReportStore from lms.djangoapps.instructor_task.tests.test_base import TestReportMixin @@ -51,6 +55,69 @@ def create_report_store(self): """ pass # pylint: disable=unnecessary-pass + def test_store_streams_binary_buffer(self): + """ + A binary buffer should reach the storage backend without first being + read into memory in its entirety. + + Asserted behaviourally rather than by inspecting call args: a streaming + backend reads in bounded chunks (File.chunks / upload_fileobj both pass + an explicit size), so an unbounded read() is the signature of the whole + report being slurped into RAM before upload. + """ + unbounded_reads = [] + + class _RecordingBytesIO(BytesIO): + """BytesIO that notes any read() not bounded by an explicit size.""" + def read(self, size=-1, /): + if size is None or size < 0: + unbounded_reads.append(size) + return super().read(size) + + report_store = self.create_report_store() # pylint: disable=assignment-from-no-return + payload = b'student_id,grade\n' + b'1,0.5\n' * 5000 + + report_store.store(self.course_id, 'streamed.csv', _RecordingBytesIO(payload)) + + assert not unbounded_reads, f'buffer was read without a size bound: {unbounded_reads}' + with report_store.storage.open(report_store.path_to(self.course_id, 'streamed.csv')) as stored: + assert stored.read() == payload + + def test_store_text_buffer_round_trips(self): + """ + Text buffers are still accepted, and are utf-8 encoded on the way out. + + Not every caller has a binary file to hand, so this path has to keep + working even though it cannot stream. + """ + report_store = self.create_report_store() # pylint: disable=assignment-from-no-return + contents = 'student_id,grade\n1,0.5\nüser,1.0\n' + + report_store.store(self.course_id, 'text.csv', StringIO(contents)) + + with report_store.storage.open(report_store.path_to(self.course_id, 'text.csv')) as stored: + assert stored.read() == contents.encode('utf-8') + + def test_store_rows_round_trips(self): + """ + store_rows() builds its CSV in a binary buffer, so it takes the + streaming path too. Verify the bytes on disk are unchanged by that. + """ + report_store = self.create_report_store() # pylint: disable=assignment-from-no-return + + report_store.store_rows( + self.course_id, + 'rows.csv', + [['student_id', 'grade'], [1, 0.5], ['üser', 'Not Attempted']], + ) + + with report_store.storage.open(report_store.path_to(self.course_id, 'rows.csv')) as stored: + assert stored.read().decode('utf-8').splitlines() == [ + 'student_id,grade', + '1,0.5', + 'üser,Not Attempted', + ] + def test_links_for_order(self): """ Test that ReportStore.links_for() returns file download links @@ -96,24 +163,119 @@ def create_report_store(self): return ReportStore.from_config(config_name='GRADES_DOWNLOAD') -class DjangoStorageReportStoreS3TestCase(MockS3Boto3Mixin, ReportStoreTestMixin, TestReportMixin, SimpleTestCase): +class _FakeS3Object: """ - Test the DjangoStorageReportStore implementation using S3 stubs. + The parts of a boto3 S3 Object that S3Boto3Storage calls, backed by a dict. """ + + def __init__(self, s3, key): + self._s3 = s3 + self.key = key + self.content_encoding = None + + def upload_fileobj(self, fileobj, ExtraArgs=None, Config=None): # pylint: disable=invalid-name,unused-argument + """Consume the upload in bounded chunks, as s3transfer's part uploads do.""" + parts = [] + while chunk := fileobj.read(self._s3.CHUNK_SIZE): + parts.append(chunk) + self._s3.objects[self.key] = (b''.join(parts), datetime.now(UTC)) + + def load(self, **_kwargs): + if self.key not in self._s3.objects: + raise ClientError({'Error': {'Code': '404'}, 'ResponseMetadata': {'HTTPStatusCode': 404}}, 'HeadObject') + + def download_fileobj(self, fileobj, ExtraArgs=None, Config=None): # pylint: disable=invalid-name,unused-argument + fileobj.write(self._s3.objects[self.key][0]) + + @property + def content_length(self): + return len(self._s3.objects[self.key][0]) + + @property + def last_modified(self): + return self._s3.objects[self.key][1] + + +class _FakeS3: + """ + An in-memory stand-in for the boto3 S3 resource behind S3Boto3Storage. + + Only the network layer is replaced: S3Boto3Storage.save and _save run for + real, so the store's buffer goes through the same wrapping it gets in + production before it reaches upload_fileobj. + """ + CHUNK_SIZE = 64 * 1024 + + def __init__(self): + self.objects = {} + self.meta = SimpleNamespace(client=SimpleNamespace( + get_paginator=lambda _operation: SimpleNamespace(paginate=self._list_objects), + generate_presigned_url=self._presigned_url, + )) + + def Bucket(self, name): # pylint: disable=invalid-name + return SimpleNamespace(name=name, Object=lambda key: _FakeS3Object(self, key)) + + def _list_objects(self, Bucket, Delimiter, Prefix): # pylint: disable=invalid-name,unused-argument + return [{ + 'Contents': [ + {'Key': key} for key in sorted(self.objects) + if key.startswith(Prefix) and Delimiter not in key[len(Prefix):] + ], + }] + + @staticmethod + def _presigned_url(ClientMethod, Params, ExpiresIn, HttpMethod=None): # pylint: disable=invalid-name,unused-argument + return f"https://{Params['Bucket']}.s3.test/{Params['Key']}" + + +class DjangoStorageReportStoreS3TestCase(ReportStoreTestMixin, TestReportMixin, SimpleTestCase): + """ + Test the DjangoStorageReportStore implementation on S3Boto3Storage, with + only the S3 API replaced. + """ + def setUp(self): + super().setUp() + self.s3 = _FakeS3() + connection = patch('storages.backends.s3.S3Storage.connection', new_callable=PropertyMock, return_value=self.s3) + connection.start() + self.addCleanup(connection.stop) + def create_report_store(self): """ Create and return a DjangoStorageReportStore configured to use S3 for storage. + + The test settings' STORAGE_TYPE ('localfs') is dropped: from_config + checks it before STORAGE_CLASS, so leaving it in builds a + FileSystemStorage and none of these tests touch S3. """ test_settings = copy.deepcopy(settings.GRADES_DOWNLOAD) + del test_settings['STORAGE_TYPE'] test_settings['STORAGE_CLASS'] = 'storages.backends.s3boto3.S3Boto3Storage' test_settings['STORAGE_KWARGS'] = { - 'bucket': settings.GRADES_DOWNLOAD['BUCKET'], - 'location': settings.GRADES_DOWNLOAD['ROOT_PATH'], + 'bucket_name': settings.GRADES_DOWNLOAD['BUCKET'], + # S3 keys have no leading slash; ROOT_PATH is a local temp dir here. + 'location': settings.GRADES_DOWNLOAD['ROOT_PATH'].lstrip('/'), } with override_settings(GRADES_DOWNLOAD=test_settings): - self.mocked_connection.create_bucket(settings.GRADES_DOWNLOAD['STORAGE_KWARGS']['bucket']) - return ReportStore.from_config(config_name='GRADES_DOWNLOAD') + report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD') + assert isinstance(report_store.storage, S3Boto3Storage) + return report_store + + def test_store_uploads_to_s3(self): + """ + A binary report is uploaded as the object's whole body, in bounded + chunks, under the course's hashed path. + """ + report_store = self.create_report_store() + payload = b'student_id,grade\n' + b'1,0.5\n' * 50000 + + report_store.store(self.course_id, 'uploaded.csv', BytesIO(payload)) + + key = f"{report_store.storage.location}/{report_store.path_to(self.course_id, 'uploaded.csv')}" + assert self.s3.objects[key][0] == payload + assert len(payload) > 4 * _FakeS3.CHUNK_SIZE class TestS3ReportStorage(TestCase):