Repository navigation
feat: standardize Course Videos API into authoring v1 #39102
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
05360ae
a973122
e5971c5
a1c46d5
2155ce9
f74beac
c55fdec
2072c84
1d945f3
800d10b
a6393f7
b72ca7a
a46d8cd
e07db66
e2c27d9
be52ded
1b54e40
50db3fc
7f95c34
3340920
36a2b03
c110f0b
4410ab3
d1ee91f
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| """ | ||
| Mark video uploads that never finished as failed. | ||
| """ | ||
| import logging | ||
| from datetime import datetime, timedelta | ||
|
|
||
| from django.core.management.base import BaseCommand | ||
| from edxval.models import Video | ||
| from pytz import UTC | ||
|
|
||
| from cms.djangoapps.contentstore.video_storage_handlers import MAX_UPLOAD_HOURS, convert_video_status | ||
|
|
||
| log = logging.getLogger(__name__) | ||
|
|
||
|
|
||
| class Command(BaseCommand): | ||
| """ | ||
| Mark every video still in the `upload` status after MAX_UPLOAD_HOURS as `upload_failed`. | ||
|
|
||
| Studio's video list does the same for the videos of the course it shows; this command does it | ||
| for all of them, without anyone having to open that list. | ||
|
|
||
| Example: | ||
| ./manage.py cms fail_stuck_video_uploads --dry-run | ||
| ./manage.py cms fail_stuck_video_uploads | ||
| """ | ||
| help = f'Mark videos still uploading after {MAX_UPLOAD_HOURS} hours as failed.' | ||
|
|
||
| def add_arguments(self, parser): | ||
| parser.add_argument( | ||
| '--dry-run', | ||
| action='store_true', | ||
| help='Report the stuck uploads without changing them.', | ||
| ) | ||
|
|
||
| def handle(self, *args, **options): | ||
| cutoff = datetime.now(UTC) - timedelta(hours=MAX_UPLOAD_HOURS) | ||
| stuck = list( | ||
| Video.objects.filter(status='upload', created__lt=cutoff) | ||
| .values('edx_video_id', 'status', 'created') | ||
| ) | ||
| for video in stuck: | ||
| if options['dry_run']: | ||
| log.info('Video [%s] is still in upload since [%s]', video['edx_video_id'], video['created']) | ||
| else: | ||
| convert_video_status(video) | ||
| log.info( | ||
| '%s %d video upload(s) stuck for more than %d hours', | ||
| 'Found' if options['dry_run'] else 'Marked as failed', | ||
| len(stuck), | ||
| MAX_UPLOAD_HOURS, | ||
| ) | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,53 @@ | ||
| """ | ||
| Tests for the `fail_stuck_video_uploads` management command. | ||
| """ | ||
| from datetime import datetime, timedelta | ||
|
|
||
| import pytz | ||
| from django.core.management import call_command | ||
| from django.test import TestCase | ||
| from edxval.api import create_video | ||
| from edxval.models import Video | ||
|
|
||
|
|
||
| class FailStuckVideoUploadsTest(TestCase): | ||
| """ | ||
| Tests for the `fail_stuck_video_uploads` management command. | ||
| """ | ||
|
|
||
| def create_video(self, edx_video_id, status, age): | ||
| """Create a video in `status` whose upload started `age` ago.""" | ||
| create_video({ | ||
| 'edx_video_id': edx_video_id, | ||
| 'client_video_id': f'{edx_video_id}.mp4', | ||
| 'duration': 42.0, | ||
| 'status': status, | ||
| 'courses': ['course-v1:edX+DemoX+Demo_Course'], | ||
| 'encoded_videos': [], | ||
| }) | ||
| Video.objects.filter(edx_video_id=edx_video_id).update(created=datetime.now(pytz.utc) - age) | ||
|
|
||
| def statuses(self): | ||
| return dict(Video.objects.values_list('edx_video_id', 'status')) | ||
|
|
||
| def setUp(self): | ||
| super().setUp() | ||
| self.create_video('stuck', 'upload', timedelta(hours=25)) | ||
| self.create_video('uploading', 'upload', timedelta(hours=23)) | ||
| self.create_video('old-and-ready', 'file_complete', timedelta(hours=48)) | ||
|
|
||
| def test_marks_only_uploads_older_than_the_upload_window_as_failed(self): | ||
| call_command('fail_stuck_video_uploads') | ||
|
|
||
| assert self.statuses() == { | ||
| 'stuck': 'upload_failed', | ||
| 'uploading': 'upload', | ||
| 'old-and-ready': 'file_complete', | ||
| } | ||
|
|
||
| def test_dry_run_changes_nothing(self): | ||
| before = self.statuses() | ||
|
|
||
| call_command('fail_stuck_video_uploads', '--dry-run') | ||
|
|
||
| assert self.statuses() == before |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,204 @@ | ||
| """API Serializers for course video uploads.""" | ||
|
|
||
| from edx_rest_framework_extensions.shaping import MinimalViewMixin | ||
| from rest_framework import serializers | ||
| from rest_framework.settings import api_settings | ||
|
|
||
| from cms.djangoapps.contentstore.rest_api.serializers.common import StrictSerializer | ||
| from cms.djangoapps.contentstore.rest_api.v1.video_uploads_service import VIDEO_ORDERING_CHOICES | ||
| from cms.djangoapps.contentstore.video_storage_handlers import VIDEO_SUPPORTED_FILE_FORMATS | ||
|
|
||
| SUPPORTED_UPLOAD_CONTENT_TYPES = sorted(set(VIDEO_SUPPORTED_FILE_FORMATS.values())) | ||
|
|
||
| NON_FIELD_ERRORS_KEY = api_settings.NON_FIELD_ERRORS_KEY | ||
|
|
||
| #: The fields one video is reduced to under the minimal response preset. | ||
| MINIMAL_VIDEO_FIELDS = ("edx_video_id", "client_video_id", "status", "created", "duration") | ||
|
|
||
|
|
||
| def _flat_messages(detail, path=""): | ||
| """ | ||
| Return validation ``detail`` as a map of field path to messages. | ||
|
|
||
| Errors reported for the entries of a list of objects are nested one or two | ||
| levels below the field they belong to. Flattening them to | ||
| ``{"files[0].content_type": ["..."]}`` gives a client the message of every | ||
| invalid field without walking the containers it is reported inside. | ||
| """ | ||
| if isinstance(detail, dict): | ||
| flat = {} | ||
| for field, value in detail.items(): | ||
| nested = path if field == NON_FIELD_ERRORS_KEY and path else _join(path, field) | ||
| flat.update(_flat_messages(value, nested)) | ||
| return flat | ||
| if isinstance(detail, list) and any(isinstance(entry, (dict, list)) for entry in detail): | ||
| flat = {} | ||
| for index, entry in enumerate(detail): | ||
| if entry: | ||
| flat.update(_flat_messages(entry, f"{path}[{index}]")) | ||
| return flat | ||
| messages = detail if isinstance(detail, list) else [detail] | ||
| return {path or NON_FIELD_ERRORS_KEY: [str(message) for message in messages]} | ||
|
|
||
|
|
||
| def _join(path, field): | ||
| """Return the path of ``field`` inside ``path``.""" | ||
| return f"{path}.{field}" if path else str(field) | ||
|
|
||
|
|
||
| class VideoUploadFileSerializer(StrictSerializer): | ||
| """One requested upload slot: the file the caller intends to PUT.""" | ||
|
|
||
| file_name = serializers.CharField( | ||
| help_text="Name of the video file being uploaded. ASCII characters only.", | ||
| ) | ||
| content_type = serializers.ChoiceField( | ||
| choices=SUPPORTED_UPLOAD_CONTENT_TYPES, | ||
| help_text="MIME type of the video file. Only these types can be stored.", | ||
| ) | ||
|
|
||
| def validate_file_name(self, value): | ||
| """Reject names the storage backend cannot carry in its object metadata.""" | ||
| try: | ||
| value.encode("ascii") | ||
| except UnicodeEncodeError as error: | ||
| raise serializers.ValidationError( | ||
| "The file name must contain only ASCII characters." | ||
| ) from error | ||
| return value | ||
|
|
||
|
|
||
| class VideoUploadRequestSerializer(StrictSerializer): | ||
| """Request body for creating upload slots.""" | ||
|
|
||
| files = VideoUploadFileSerializer( | ||
| many=True, | ||
| allow_empty=False, | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. kept the 400 and declared it, since a request for zero upload slots is a client mistake. 36a2b03 adds a test that runs both versions (v0: 200 {"files": []}, v1: 400), and the row is in the differences table. But we can accept the empty list instead if required
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Verified against c110f0b: fixed as described. Resolving. |
||
| help_text="Files to create upload slots for. One slot is returned per entry, in order.", | ||
| ) | ||
|
|
||
| def run_validation(self, data=serializers.empty): | ||
| """Validate the body, reporting every invalid field as a list of messages.""" | ||
| try: | ||
| return super().run_validation(data) | ||
| except serializers.ValidationError as error: | ||
| raise serializers.ValidationError(_flat_messages(error.detail)) from error | ||
|
|
||
|
|
||
| class VideoUploadLinkSerializer(serializers.Serializer): # pylint: disable=abstract-method | ||
| """One created upload slot.""" | ||
|
|
||
| file_name = serializers.CharField( | ||
| help_text="Name of the video file, echoed from the request.", | ||
| ) | ||
| upload_url = serializers.CharField( | ||
| help_text="Short-lived pre-signed URL to PUT the video file to. Expires after 24 hours.", | ||
| ) | ||
| edx_video_id = serializers.CharField( | ||
| help_text="Identifier assigned to the video. Use it to address the video afterwards.", | ||
| ) | ||
|
|
||
|
|
||
| class VideoUploadResponseSerializer(serializers.Serializer): # pylint: disable=abstract-method | ||
| """Response body for creating upload slots.""" | ||
|
|
||
| files = VideoUploadLinkSerializer( | ||
| many=True, | ||
| help_text="Created upload slots, one per requested file, in request order.", | ||
| ) | ||
|
|
||
|
|
||
| class CourseVideoSerializer(serializers.Serializer): # pylint: disable=abstract-method | ||
| """One video asset attached to a course.""" | ||
|
|
||
| edx_video_id = serializers.CharField( | ||
| help_text="Identifier of the video.", | ||
| ) | ||
| client_video_id = serializers.CharField( | ||
| allow_blank=True, | ||
| help_text="Original file name the video was uploaded under.", | ||
| ) | ||
| created = serializers.DateTimeField( | ||
| allow_null=True, | ||
| help_text="Time the video record was created.", | ||
| ) | ||
| duration = serializers.FloatField( | ||
| allow_null=True, | ||
| help_text="Length of the video in seconds; 0 until the video has been processed.", | ||
| ) | ||
| status = serializers.CharField( | ||
| help_text=( | ||
| "Processing state of the video, in stable English: for example Uploading, " | ||
| "In Progress, Ready, Failed, YouTube Duplicate." | ||
| ), | ||
| ) | ||
| error_description = serializers.CharField( | ||
| allow_null=True, | ||
| allow_blank=True, | ||
| help_text="Details of the processing failure, when the video failed to process.", | ||
| ) | ||
| course_video_image_url = serializers.CharField( | ||
| allow_null=True, | ||
| help_text="Thumbnail image URL for this video in this course, or null when none is set.", | ||
| ) | ||
| download_link = serializers.CharField( | ||
| allow_blank=True, | ||
| help_text="URL of the desktop MP4 encoding, or an empty string when it is not ready.", | ||
| ) | ||
| file_size = serializers.IntegerField( | ||
| help_text="Size of the desktop MP4 encoding in bytes; 0 when it is not ready.", | ||
| ) | ||
| transcripts = serializers.ListField( | ||
| child=serializers.CharField(), | ||
| help_text="Language codes the video has transcripts for.", | ||
| ) | ||
| transcription_status = serializers.CharField( | ||
| allow_blank=True, | ||
| help_text=( | ||
| "Transcription state for courses on the current video workflow, in stable English; " | ||
| "empty for courses still using a course video upload token." | ||
| ), | ||
| ) | ||
| transcript_urls = serializers.DictField( | ||
| child=serializers.CharField(), | ||
| help_text="Transcript download URL keyed by language code.", | ||
| ) | ||
|
|
||
|
|
||
| class CourseVideoMinimalSerializer(CourseVideoSerializer): | ||
| """One video asset, reduced to the fields that identify it.""" | ||
|
|
||
| def get_fields(self): | ||
| """Return only the fields the minimal preset keeps.""" | ||
| fields = super().get_fields() | ||
| return {name: field for name, field in fields.items() if name in MINIMAL_VIDEO_FIELDS} | ||
|
|
||
|
|
||
| class CourseVideoQuerySerializer(serializers.Serializer): # pylint: disable=abstract-method | ||
| """Query parameters accepted by every course video address.""" | ||
|
|
||
| view = serializers.ChoiceField( | ||
| choices=[MinimalViewMixin.minimal_view_value], | ||
| required=False, | ||
| help_text="Response preset to return. Omit it for the full representation.", | ||
| ) | ||
|
|
||
|
|
||
| class CourseVideoListQuerySerializer(CourseVideoQuerySerializer): | ||
| """Query parameters accepted by the course video collection.""" | ||
|
|
||
| ordering = serializers.ChoiceField( | ||
| choices=VIDEO_ORDERING_CHOICES, | ||
| required=False, | ||
| help_text="Field to sort by. Prefix the field name with '-' to sort descending.", | ||
| ) | ||
| page = serializers.IntegerField( | ||
| required=False, | ||
| min_value=1, | ||
| help_text="Number of the page to return.", | ||
| ) | ||
| page_size = serializers.IntegerField( | ||
| required=False, | ||
| min_value=1, | ||
| help_text="Number of videos per page; values above the maximum are reduced to it.", | ||
| ) | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Before this merges, add an entry for this command on the Willow operator notes page, https://openedx.atlassian.net/wiki/spaces/COMM/pages/6476169217/Willow+-+Operator+Dev+Notes, so operators know it exists and can schedule it before the legacy video list stops doing the sweep.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@feanil I don’t have write access to this page. Could you grant me access so I can make the required additions?
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@Abdul-Muqadim-Arbisoft if you create an account on the Open edX confluence wiki, you sholud be able to edit it if you're logged in. Did you try that and it didn't work?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@feanil Tried it again, dont have the edit access, also sent an edit request
