Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 40 additions & 9 deletions lms/djangoapps/instructor_task/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,14 @@
import json
import logging
import os.path
from io import BytesIO, TextIOWrapper
from uuid import uuid4

from botocore.exceptions import ClientError
from django.apps import apps
from django.conf import settings
from django.contrib.auth.models import User # pylint: disable=imported-auth-user
from django.core.files.base import ContentFile
from django.core.files.base import ContentFile, File
from django.db import models, transaction
from django.utils.translation import gettext as _
from model_utils.models import TimeStampedModel
Expand Down Expand Up @@ -293,27 +294,57 @@ def store(self, course_id, filename, buff, parent_dir=''):
Store the contents of `buff` in a directory determined by hashing
`course_id`, and name the file `filename`. `buff` can be any file-like
object, ready to be read from the beginning.

A binary buffer is handed to the storage backend as-is, so the backend
streams it -- S3Boto3Storage uploads in parts and FileSystemStorage
writes in chunks -- and peak memory stays at one chunk rather than the
whole report. This matters most for ORA submission-file exports, which
are built in a temporary file and can run to gigabytes; previously the
entire file was read into memory and copied into a ContentFile before
upload, so an on-disk report was held in RAM in full at the final step.

A text buffer still works, but has to be read and encoded in full
because the storage backends require bytes. Callers handling
potentially large reports should pass a binary file.
"""
path = self.path_to(course_id, filename, parent_dir)

if self._yields_bytes(buff):
self.storage.save(path, File(buff, name=filename))
return

# See https://github.com/boto/boto/issues/2868
# Boto doesn't play nice with unicode in python3
buff_contents = buff.read()

if not isinstance(buff_contents, bytes):
buff_contents = buff_contents.encode('utf-8')
self.storage.save(path, ContentFile(buff.read().encode('utf-8')))

buff = ContentFile(buff_contents)
@staticmethod
def _yields_bytes(buff):
"""
Return True if reading from `buff` produces bytes rather than str.

self.storage.save(path, buff)
Probes with a zero-length read so the buffer is left positioned exactly
where it was, rather than inspecting the type -- callers pass a mix of
raw file objects, ContentFile and BytesIO, and mode attributes are not
consistently present across them.
"""
try:
return isinstance(buff.read(0), bytes)
except (AttributeError, TypeError, ValueError):
return False

def store_rows(self, course_id, filename, rows, parent_dir=''):
"""
Given a course_id, filename, and rows (each row is an iterable of
strings), write the rows to the storage backend in csv format.
"""
output_buffer = ContentFile('')
csvwriter = csv.writer(output_buffer)
output_buffer = BytesIO()
# newline='' per the csv module's contract; the writer emits its own
# line terminators and must not have them translated again.
text_wrapper = TextIOWrapper(output_buffer, encoding='utf-8', newline='', write_through=True)
csvwriter = csv.writer(text_wrapper)
csvwriter.writerows(self._get_utf8_encoded_rows(rows))
# Detach so closing the wrapper does not close the buffer underneath it.
text_wrapper.detach()
output_buffer.seek(0)
self.store(course_id, filename, output_buffer, parent_dir)

Expand Down
28 changes: 25 additions & 3 deletions lms/djangoapps/instructor_task/tasks_helper/grades.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import re
from collections import OrderedDict, defaultdict
from datetime import datetime
from io import TextIOWrapper
from itertools import chain
from tempfile import TemporaryFile
from time import time
Expand Down Expand Up @@ -351,7 +352,11 @@ def _generate(self):
self.context.update_status('TemporaryFileReportMixin - 1: Starting grade report')
batched_rows = self._batched_rows()

with TemporaryFile('r+') as success_file, TemporaryFile('r+') as error_file:
# Binary temp files, written through a text wrapper. The report store
# streams a binary buffer straight to the storage backend, where a text
# one has to be read into memory and encoded in full -- which would undo
# most of the benefit of spilling to disk in the first place.
with TemporaryFile('w+b') as success_file, TemporaryFile('w+b') as error_file:
self.context.update_status('TemporaryFileReportMixin - 2: Compiling grades into temp files')
has_errors = self.iter_and_write_batched_rows(batched_rows, success_file, error_file)

Expand All @@ -360,13 +365,24 @@ def _generate(self):

return self.context.update_status('TemporaryFileReportMixin - 4: Completed grades')

@staticmethod
def _csv_writer_for(binary_file):
"""
Return a csv.writer over a binary file, plus the wrapper to flush.

newline='' is the csv module's documented requirement: the writer emits
its own line terminators and they must not be translated again.
"""
wrapper = TextIOWrapper(binary_file, encoding='utf-8', newline='', write_through=True)
return csv.writer(wrapper), wrapper

def iter_and_write_batched_rows(self, batched_rows, success_file, error_file):
"""
Iterate through batched rows, writing returned chunks to disk as we go.
This should hopefully help us avoid out of memory errors.
"""
success_writer = csv.writer(success_file)
error_writer = csv.writer(error_file)
success_writer, success_wrapper = self._csv_writer_for(success_file)
error_writer, error_wrapper = self._csv_writer_for(error_file)

# Write headers
success_writer.writerow(self._success_headers())
Expand All @@ -386,6 +402,12 @@ def iter_and_write_batched_rows(self, batched_rows, success_file, error_file):
self.context.task_progress.attempted = succeeded + failed
self.context.task_progress.total = self.context.task_progress.attempted

# Detach rather than close: the wrappers must flush their buffered text
# into the temp files before those are read back for upload, but the
# temp files themselves stay open and are closed by the caller.
success_wrapper.detach()
error_wrapper.detach()

return self.context.task_progress.failed > 0

def upload_temp_files(self, success_file, error_file, has_errors):
Expand Down
178 changes: 170 additions & 8 deletions lms/djangoapps/instructor_task/tests/test_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,18 @@

import copy
import time
from io import StringIO
from datetime import UTC, datetime
from io import BytesIO, StringIO
from types import SimpleNamespace
from unittest.mock import PropertyMock, patch

import pytest
from botocore.exceptions import ClientError
from django.conf import settings
from django.test import SimpleTestCase, TestCase, override_settings
from opaque_keys.edx.locator import CourseLocator
from storages.backends.s3boto3 import S3Boto3Storage

from common.test.utils import MockS3Boto3Mixin
from lms.djangoapps.instructor_task.models import TASK_INPUT_LENGTH, InstructorTask, ReportStore
from lms.djangoapps.instructor_task.tests.test_base import TestReportMixin

Expand Down Expand Up @@ -51,6 +55,69 @@ def create_report_store(self):
"""
pass # pylint: disable=unnecessary-pass

def test_store_streams_binary_buffer(self):
"""
A binary buffer should reach the storage backend without first being
read into memory in its entirety.

Asserted behaviourally rather than by inspecting call args: a streaming
backend reads in bounded chunks (File.chunks / upload_fileobj both pass
an explicit size), so an unbounded read() is the signature of the whole
report being slurped into RAM before upload.
"""
unbounded_reads = []

class _RecordingBytesIO(BytesIO):
"""BytesIO that notes any read() not bounded by an explicit size."""
def read(self, size=-1, /):
if size is None or size < 0:
unbounded_reads.append(size)
return super().read(size)

report_store = self.create_report_store() # pylint: disable=assignment-from-no-return
payload = b'student_id,grade\n' + b'1,0.5\n' * 5000

report_store.store(self.course_id, 'streamed.csv', _RecordingBytesIO(payload))

assert not unbounded_reads, f'buffer was read without a size bound: {unbounded_reads}'
Comment thread
blarghmatey marked this conversation as resolved.
Resolved
with report_store.storage.open(report_store.path_to(self.course_id, 'streamed.csv')) as stored:
assert stored.read() == payload

def test_store_text_buffer_round_trips(self):
"""
Text buffers are still accepted, and are utf-8 encoded on the way out.

Not every caller has a binary file to hand, so this path has to keep
working even though it cannot stream.
"""
report_store = self.create_report_store() # pylint: disable=assignment-from-no-return
contents = 'student_id,grade\n1,0.5\nüser,1.0\n'

report_store.store(self.course_id, 'text.csv', StringIO(contents))

with report_store.storage.open(report_store.path_to(self.course_id, 'text.csv')) as stored:
assert stored.read() == contents.encode('utf-8')

def test_store_rows_round_trips(self):
"""
store_rows() builds its CSV in a binary buffer, so it takes the
streaming path too. Verify the bytes on disk are unchanged by that.
"""
report_store = self.create_report_store() # pylint: disable=assignment-from-no-return

report_store.store_rows(
self.course_id,
'rows.csv',
[['student_id', 'grade'], [1, 0.5], ['üser', 'Not Attempted']],
)

with report_store.storage.open(report_store.path_to(self.course_id, 'rows.csv')) as stored:
assert stored.read().decode('utf-8').splitlines() == [
'student_id,grade',
'1,0.5',
'üser,Not Attempted',
]

def test_links_for_order(self):
"""
Test that ReportStore.links_for() returns file download links
Expand Down Expand Up @@ -96,24 +163,119 @@ def create_report_store(self):
return ReportStore.from_config(config_name='GRADES_DOWNLOAD')


class DjangoStorageReportStoreS3TestCase(MockS3Boto3Mixin, ReportStoreTestMixin, TestReportMixin, SimpleTestCase):
class _FakeS3Object:
"""
Test the DjangoStorageReportStore implementation using S3 stubs.
The parts of a boto3 S3 Object that S3Boto3Storage calls, backed by a dict.
"""

def __init__(self, s3, key):
self._s3 = s3
self.key = key
self.content_encoding = None

def upload_fileobj(self, fileobj, ExtraArgs=None, Config=None): # pylint: disable=invalid-name,unused-argument
"""Consume the upload in bounded chunks, as s3transfer's part uploads do."""
parts = []
while chunk := fileobj.read(self._s3.CHUNK_SIZE):
parts.append(chunk)
self._s3.objects[self.key] = (b''.join(parts), datetime.now(UTC))

def load(self, **_kwargs):
if self.key not in self._s3.objects:
raise ClientError({'Error': {'Code': '404'}, 'ResponseMetadata': {'HTTPStatusCode': 404}}, 'HeadObject')

def download_fileobj(self, fileobj, ExtraArgs=None, Config=None): # pylint: disable=invalid-name,unused-argument
fileobj.write(self._s3.objects[self.key][0])

@property
def content_length(self):
return len(self._s3.objects[self.key][0])

@property
def last_modified(self):
return self._s3.objects[self.key][1]


class _FakeS3:
"""
An in-memory stand-in for the boto3 S3 resource behind S3Boto3Storage.

Only the network layer is replaced: S3Boto3Storage.save and _save run for
real, so the store's buffer goes through the same wrapping it gets in
production before it reaches upload_fileobj.
"""
CHUNK_SIZE = 64 * 1024

def __init__(self):
self.objects = {}
self.meta = SimpleNamespace(client=SimpleNamespace(
get_paginator=lambda _operation: SimpleNamespace(paginate=self._list_objects),
generate_presigned_url=self._presigned_url,
))

def Bucket(self, name): # pylint: disable=invalid-name
return SimpleNamespace(name=name, Object=lambda key: _FakeS3Object(self, key))

def _list_objects(self, Bucket, Delimiter, Prefix): # pylint: disable=invalid-name,unused-argument
return [{
'Contents': [
{'Key': key} for key in sorted(self.objects)
if key.startswith(Prefix) and Delimiter not in key[len(Prefix):]
],
}]

@staticmethod
def _presigned_url(ClientMethod, Params, ExpiresIn, HttpMethod=None): # pylint: disable=invalid-name,unused-argument
return f"https://{Params['Bucket']}.s3.test/{Params['Key']}"


class DjangoStorageReportStoreS3TestCase(ReportStoreTestMixin, TestReportMixin, SimpleTestCase):
"""
Test the DjangoStorageReportStore implementation on S3Boto3Storage, with
only the S3 API replaced.
"""
def setUp(self):
super().setUp()
self.s3 = _FakeS3()
connection = patch('storages.backends.s3.S3Storage.connection', new_callable=PropertyMock, return_value=self.s3)
connection.start()
self.addCleanup(connection.stop)

def create_report_store(self):
"""
Create and return a DjangoStorageReportStore configured to use S3 for
storage.

The test settings' STORAGE_TYPE ('localfs') is dropped: from_config
checks it before STORAGE_CLASS, so leaving it in builds a
FileSystemStorage and none of these tests touch S3.
"""
test_settings = copy.deepcopy(settings.GRADES_DOWNLOAD)
del test_settings['STORAGE_TYPE']
test_settings['STORAGE_CLASS'] = 'storages.backends.s3boto3.S3Boto3Storage'
test_settings['STORAGE_KWARGS'] = {
'bucket': settings.GRADES_DOWNLOAD['BUCKET'],
'location': settings.GRADES_DOWNLOAD['ROOT_PATH'],
'bucket_name': settings.GRADES_DOWNLOAD['BUCKET'],
# S3 keys have no leading slash; ROOT_PATH is a local temp dir here.
'location': settings.GRADES_DOWNLOAD['ROOT_PATH'].lstrip('/'),
}
with override_settings(GRADES_DOWNLOAD=test_settings):
self.mocked_connection.create_bucket(settings.GRADES_DOWNLOAD['STORAGE_KWARGS']['bucket'])
return ReportStore.from_config(config_name='GRADES_DOWNLOAD')
report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD')
assert isinstance(report_store.storage, S3Boto3Storage)
return report_store

def test_store_uploads_to_s3(self):
"""
A binary report is uploaded as the object's whole body, in bounded
chunks, under the course's hashed path.
"""
report_store = self.create_report_store()
payload = b'student_id,grade\n' + b'1,0.5\n' * 50000

report_store.store(self.course_id, 'uploaded.csv', BytesIO(payload))

key = f"{report_store.storage.location}/{report_store.path_to(self.course_id, 'uploaded.csv')}"
assert self.s3.objects[key][0] == payload
assert len(payload) > 4 * _FakeS3.CHUNK_SIZE


class TestS3ReportStorage(TestCase):
Expand Down
Loading