diff --git a/cms/djangoapps/contentstore/management/commands/empty_asset_trashcan.py b/cms/djangoapps/contentstore/management/commands/empty_asset_trashcan.py index 61198c365d89..8a0321edda3a 100644 --- a/cms/djangoapps/contentstore/management/commands/empty_asset_trashcan.py +++ b/cms/djangoapps/contentstore/management/commands/empty_asset_trashcan.py @@ -1,6 +1,5 @@ """Script to Empty the trashcan""" - from django.core.management.base import BaseCommand from opaque_keys.edx.keys import CourseKey @@ -12,9 +11,13 @@ class Command(BaseCommand): """ - Command to Empty the trashcan. + Command to Empty the trashcan. """ - help = 'Empty the trashcan. Can pass an optional course_id to limit the damage.' + + help = ( + "Empty the asset trashcan, optionally just for one course.\n" + + "⚠️ This command is deprecated and will be removed in the future." + ) def add_arguments(self, parser): parser.add_argument('course_id', diff --git a/cms/djangoapps/contentstore/management/commands/restore_asset_from_trashcan.py b/cms/djangoapps/contentstore/management/commands/restore_asset_from_trashcan.py index d1f4cb685ce6..52394e5f6836 100644 --- a/cms/djangoapps/contentstore/management/commands/restore_asset_from_trashcan.py +++ b/cms/djangoapps/contentstore/management/commands/restore_asset_from_trashcan.py @@ -1,6 +1,5 @@ """Management command to restore assets from trash""" - from django.core.management.base import BaseCommand from xmodule.contentstore.utils import restore_asset_from_trashcan @@ -8,7 +7,9 @@ class Command(BaseCommand): """Command class to handle asset restore""" - help = '''Restore a deleted asset from the trashcan back to it's original course''' + + help = """Restore a deleted asset from the trashcan back to it's original course. +⚠️ This command is deprecated and will be removed in the future.""" def add_arguments(self, parser): parser.add_argument('location') diff --git a/cms/djangoapps/contentstore/management/commands/tests/test_export_olx.py b/cms/djangoapps/contentstore/management/commands/tests/test_export_olx.py index e8c001a2469f..3902238e93d4 100644 --- a/cms/djangoapps/contentstore/management/commands/tests/test_export_olx.py +++ b/cms/djangoapps/contentstore/management/commands/tests/test_export_olx.py @@ -71,6 +71,7 @@ def check_export_file(self, tar_file, course_key, with_tags=False): self.assertIn(f"{dirname}/about", names) # noqa: PT009 self.assertIn(f"{dirname}/about/overview.html", names) # noqa: PT009 self.assertIn(f"{dirname}/assets/assets.xml", names) # noqa: PT009 + assert tar_file.extractfile(f"{dirname}/assets/assets.xml").read() == b'' self.assertIn(f"{dirname}/policies", names) # noqa: PT009 if with_tags: self.assertIn(f"{dirname}/tags.csv", names) # noqa: PT009 diff --git a/cms/djangoapps/contentstore/views/tests/test_assets.py b/cms/djangoapps/contentstore/views/tests/test_assets.py index 1df352b70893..354d4193d76f 100644 --- a/cms/djangoapps/contentstore/views/tests/test_assets.py +++ b/cms/djangoapps/contentstore/views/tests/test_assets.py @@ -25,7 +25,6 @@ from common.djangoapps.static_replace import replace_static_urls from common.djangoapps.student.tests.factories import UserFactory from openedx.core.djangoapps.authz.tests.mixins import CourseAuthzTestMixin -from xmodule.assetstore import AssetMetadata # pylint: disable=wrong-import-order from xmodule.contentstore.content import StaticContent # pylint: disable=wrong-import-order from xmodule.contentstore.django import contentstore # pylint: disable=wrong-import-order from xmodule.modulestore import ModuleStoreEnum # pylint: disable=wrong-import-order @@ -225,8 +224,7 @@ def test_mocked_filtered_response(self, mock_get_all_content_for_course): """ Test the ajax asset interfaces """ - asset_key = self.course.id.make_asset_key( - AssetMetadata.GENERAL_ASSET_TYPE, 'test.jpg') + asset_key = self.course.id.make_asset_key('asset', 'test.jpg') upload_date = datetime(2015, 1, 12, 10, 30, tzinfo=UTC) thumbnail_location = [ 'c4x', 'edX', 'toy', 'thumbnail', 'test_thumb.jpg', None] diff --git a/common/djangoapps/static_replace/test/test_static_replace.py b/common/djangoapps/static_replace/test/test_static_replace.py index 1e3f0d25b8a9..897b5936f59b 100644 --- a/common/djangoapps/static_replace/test/test_static_replace.py +++ b/common/djangoapps/static_replace/test/test_static_replace.py @@ -22,7 +22,6 @@ ) from common.djangoapps.static_replace.services import ReplaceURLService from common.djangoapps.static_replace.wrapper import replace_urls_wrapper -from xmodule.assetstore.assetmgr import AssetManager # pylint: disable=wrong-import-order from xmodule.contentstore.content import StaticContent # pylint: disable=wrong-import-order from xmodule.contentstore.django import contentstore # pylint: disable=wrong-import-order from xmodule.exceptions import NotFoundError # pylint: disable=wrong-import-order @@ -326,7 +325,7 @@ def get_content_digest_for_asset_path(cls, prefix, path): asset_key = StaticContent.get_asset_key_from_path(cls.courses[prefix].id, relative_path) try: - content = AssetManager.find(asset_key, as_stream=True) + content = contentstore().find(asset_key, as_stream=True) return content.content_digest except (ItemNotFoundError, NotFoundError): return None diff --git a/openedx/core/djangoapps/content/course_overviews/tests/test_course_overviews.py b/openedx/core/djangoapps/content/course_overviews/tests/test_course_overviews.py index 95c9e2df965c..e8822e4c964a 100644 --- a/openedx/core/djangoapps/content/course_overviews/tests/test_course_overviews.py +++ b/openedx/core/djangoapps/content/course_overviews/tests/test_course_overviews.py @@ -26,7 +26,6 @@ from openedx.core.djangoapps.models.course_details import CourseDetails from openedx.core.djangolib.testing.utils import CacheIsolationTestCase from openedx.core.lib.courses import course_image_url -from xmodule.assetstore.assetmgr import AssetManager # pylint: disable=wrong-import-order from xmodule.contentstore.content import StaticContent # pylint: disable=wrong-import-order from xmodule.contentstore.django import contentstore # pylint: disable=wrong-import-order from xmodule.course_block import ( # pylint: disable=wrong-import-order @@ -1071,7 +1070,7 @@ def test_happy_path(self, create_after_overview): # Now make sure our thumbnails are of the sizes we expect... for image_url, expected_size in [(image_urls['small'], config.small), (image_urls['large'], config.large)]: image_key = StaticContent.get_location_from_path(image_url) - image_content = AssetManager.find(image_key) + image_content = contentstore().find(image_key) image = Image.open(BytesIO(image_content.data)) assert image.size == expected_size @@ -1123,7 +1122,7 @@ def test_different_resolutions(self, src_dimensions): for image_url, target in [(image_urls['small'], config.small), (image_urls['large'], config.large)]: image_key = StaticContent.get_location_from_path(image_url) - image_content = AssetManager.find(image_key) + image_content = contentstore().find(image_key) image = Image.open(BytesIO(image_content.data)) # Naming convention for thumbnail diff --git a/openedx/core/djangoapps/contentserver/test/test_contentserver.py b/openedx/core/djangoapps/contentserver/test/test_contentserver.py index 2a436703ff7c..6002b045b9f9 100644 --- a/openedx/core/djangoapps/contentserver/test/test_contentserver.py +++ b/openedx/core/djangoapps/contentserver/test/test_contentserver.py @@ -22,7 +22,6 @@ from common.djangoapps.student.models import CourseEnrollment from common.djangoapps.student.tests.factories import AdminFactory, UserFactory from openedx.core.djangoapps.waffle_utils.models import WaffleFlagCourseOverrideModel -from xmodule.assetstore.assetmgr import AssetManager from xmodule.contentstore.content import VERSIONED_ASSETS_PREFIX, StaticContent from xmodule.contentstore.django import contentstore from xmodule.modulestore.django import modulestore @@ -47,7 +46,7 @@ def get_versioned_asset_url(asset_path): """ try: locator = StaticContent.get_location_from_path(asset_path) - content = AssetManager.find(locator, as_stream=True) + content = contentstore().find(locator, as_stream=True) return StaticContent.add_version_to_asset_path(asset_path, content.content_digest) except (InvalidKeyError, ItemNotFoundError): pass @@ -61,7 +60,7 @@ def get_old_style_versioned_asset_url(asset_path): """ try: locator = StaticContent.get_location_from_path(asset_path) - content = AssetManager.find(locator, as_stream=True) + content = contentstore().find(locator, as_stream=True) return f'{VERSIONED_ASSETS_PREFIX}/{content.content_digest}{asset_path}' except (InvalidKeyError, ItemNotFoundError): pass diff --git a/openedx/core/djangoapps/contentserver/views.py b/openedx/core/djangoapps/contentserver/views.py index 461d4a7bb1b9..80aaf47b2060 100644 --- a/openedx/core/djangoapps/contentserver/views.py +++ b/openedx/core/djangoapps/contentserver/views.py @@ -25,8 +25,8 @@ from common.djangoapps.student.models import CourseEnrollment from openedx.core.djangoapps.header_control import force_header_for_response from openedx.core.djangoapps.waffle_utils import CourseWaffleFlag -from xmodule.assetstore.assetmgr import AssetManager from xmodule.contentstore.content import XASSET_LOCATION_TAG, StaticContent +from xmodule.contentstore.django import contentstore from xmodule.exceptions import NotFoundError from xmodule.modulestore import InvalidLocationError from xmodule.modulestore.exceptions import ItemNotFoundError @@ -141,7 +141,7 @@ def process_request(request): if request.META.get('HTTP_RANGE'): # If we have a StaticContent, get a StaticContentStream. Can't manipulate the bytes otherwise. if isinstance(content, StaticContent): - content = AssetManager.find(loc, as_stream=True) + content = contentstore().find(loc, as_stream=True) header_value = request.META['HTTP_RANGE'] try: @@ -367,7 +367,7 @@ def load_asset_from_location(location): content = get_cached_content(location) if content is None: # Not in cache, so just try and load it from the asset manager. - content = AssetManager.find(location, as_stream=True) + content = contentstore().find(location, as_stream=True) # Now that we fetched it, let's go ahead and try to cache it. We cap this at 1MB # because it's the default for memcached and also we don't want to do too much diff --git a/openedx/core/lib/courses.py b/openedx/core/lib/courses.py index eb879d5453c9..086f186b06db 100644 --- a/openedx/core/lib/courses.py +++ b/openedx/core/lib/courses.py @@ -9,7 +9,6 @@ from opaque_keys.edx.locator import CourseKey from organizations.models import Organization -from xmodule.assetstore.assetmgr import AssetManager from xmodule.contentstore.content import StaticContent from xmodule.contentstore.django import contentstore from xmodule.modulestore.django import modulestore @@ -51,7 +50,7 @@ def create_course_image_thumbnail(course, dimensions): - dimensions is a tuple of (width, height) """ course_image_asset_key = StaticContent.compute_location(course.id, course.course_image) - course_image = AssetManager.find(course_image_asset_key) # a StaticContent obj + course_image = contentstore().find(course_image_asset_key) # a StaticContent obj _content, thumb_loc = contentstore().generate_thumbnail(course_image, dimensions=dimensions) diff --git a/openedx/core/lib/xblock_serializer/utils.py b/openedx/core/lib/xblock_serializer/utils.py index 89d60926691c..5649833cfd7b 100644 --- a/openedx/core/lib/xblock_serializer/utils.py +++ b/openedx/core/lib/xblock_serializer/utils.py @@ -13,8 +13,8 @@ from opaque_keys.edx.keys import AssetKey, CourseKey from common.djangoapps.static_replace import replace_static_urls -from xmodule.assetstore.assetmgr import AssetManager from xmodule.contentstore.content import StaticContent +from xmodule.contentstore.django import contentstore from xmodule.exceptions import NotFoundError from xmodule.modulestore.exceptions import ItemNotFoundError from xmodule.util.sandboxing import course_code_library_asset_name @@ -32,7 +32,7 @@ def get_asset_content_from_path(course_key, asset_path): """ try: asset_key = StaticContent.get_asset_key_from_path(course_key, asset_path) - return AssetManager.find(asset_key) + return contentstore().find(asset_key) except (ItemNotFoundError, NotFoundError): return None @@ -108,7 +108,7 @@ def get_python_lib_zip_if_using(olx: str, course_id: CourseKey) -> StaticFile | python_lib_filename = course_code_library_asset_name() asset_key = StaticContent.get_asset_key_from_path(course_id, python_lib_filename) # Now, it seems like this capa problem uses python_lib.zip - but does it exist in the course? - if AssetManager.find(asset_key, throw_on_not_found=False): + if contentstore().find(asset_key, throw_on_not_found=False): url = '/' + str(StaticContent.compute_location(course_id, python_lib_filename)) return StaticFile(name=python_lib_filename, url=url, data=None) return None @@ -141,7 +141,7 @@ def get_js_input_files_if_using(olx: str, course_id: CourseKey) -> [StaticFile]: if html_file_fullpath: html_filename = html_file_fullpath.split('/')[-1] asset_key = StaticContent.get_asset_key_from_path(course_id, html_filename) - html_file_content = AssetManager.find(asset_key, throw_on_not_found=False) + html_file_content = contentstore().find(asset_key, throw_on_not_found=False) if html_file_content: static_assets = _extract_static_assets(str(html_file_content.data)) for static_asset in static_assets: diff --git a/xmodule/assetstore/__init__.py b/xmodule/assetstore/__init__.py index 14708d8f02eb..b422dd0bc964 100644 --- a/xmodule/assetstore/__init__.py +++ b/xmodule/assetstore/__init__.py @@ -1,5 +1,8 @@ """ Classes representing asset metadata. + +Deprecated: asset metadata in the modulestore is never populated by Studio, and this whole module will be removed. +Use ``xmodule.contentstore.django.contentstore()`` to access course assets instead. """ @@ -11,7 +14,15 @@ from lxml import etree from opaque_keys.edx.keys import AssetKey, CourseKey # noqa: F401 +from xmodule.util.deprecation import deprecated + +DEPRECATION_MESSAGE = ( + "xmodule.assetstore is deprecated and will be removed. " + "Use xmodule.contentstore.django.contentstore() to access course assets instead." +) + +@deprecated(DEPRECATION_MESSAGE) class AssetMetadata: """ Stores the metadata associated with a particular course asset. The asset metadata gets stored @@ -229,6 +240,7 @@ def add_all_assets_as_xml(node, assets): asset.to_xml(asset_node) +@deprecated(DEPRECATION_MESSAGE) class CourseAssetsFromStorage: """ Wrapper class for asset metadata lists returned from modulestore storage. diff --git a/xmodule/assetstore/assetmgr.py b/xmodule/assetstore/assetmgr.py index 4f50730d9d42..dfb4a329c42e 100644 --- a/xmodule/assetstore/assetmgr.py +++ b/xmodule/assetstore/assetmgr.py @@ -1,26 +1,27 @@ """ -Asset Manager +Very old Asset Manager API. -Interface allowing course asset saving/retrieving. -Handles: - - saving asset in the BlobStore -and- saving asset metadata in course modulestore. - - retrieving asset metadata from course modulestore -and- returning URL to asset -or- asset bytes. - -Phase 1: Checks to see if an asset's metadata can be found in the course's modulestore. - If not found, fails over to access the asset from the contentstore. - At first, the asset metadata will never be found, since saving isn't implemented yet. -Note: Hotfix (PLAT-734) No asset calls find_asset_metadata, and directly accesses from contentstore. +Has not done anything other than wrap ``contentstore().find(...)`` since 2015. +Deprecated: use ``xmodule.contentstore.django.contentstore().find()`` instead. """ from xmodule.contentstore.django import contentstore +from xmodule.util.deprecation import deprecated + +DEPRECATION_MESSAGE = ( + "AssetManager is deprecated and will be removed. " + "Use xmodule.contentstore.django.contentstore().find() instead." +) +@deprecated(DEPRECATION_MESSAGE) class AssetManager: """ Manager for saving/loading course assets. """ @staticmethod + @deprecated(DEPRECATION_MESSAGE) # The class decorator only warns on instantiation, so also warn here. def find(asset_key, throw_on_not_found=True, as_stream=False): """ Finds course asset in the deprecated contentstore. diff --git a/xmodule/assetstore/tests/test_asset_xml.py b/xmodule/assetstore/tests/test_asset_xml.py index 5b08ffed5c70..8c02701f277e 100644 --- a/xmodule/assetstore/tests/test_asset_xml.py +++ b/xmodule/assetstore/tests/test_asset_xml.py @@ -5,6 +5,7 @@ import unittest +import pytest from lxml import etree from opaque_keys.edx.locator import CourseLocator from path import Path as path @@ -12,6 +13,9 @@ from xmodule.assetstore import AssetMetadata from xmodule.modulestore.tests.test_assetstore import AssetStoreTestData +# These tests exercise the deprecated AssetMetadata class directly. +pytestmark = pytest.mark.filterwarnings("ignore:xmodule.assetstore is deprecated:DeprecationWarning") + class TestAssetXml(unittest.TestCase): """ diff --git a/xmodule/contentstore/content.py b/xmodule/contentstore/content.py index 871ebf63d29f..08a6e0abc034 100644 --- a/xmodule/contentstore/content.py +++ b/xmodule/contentstore/content.py @@ -12,7 +12,7 @@ from opaque_keys.edx.locator import AssetLocator, LibraryLocatorV2 from PIL import Image -from xmodule.assetstore.assetmgr import AssetManager +from xmodule.contentstore.django import contentstore from xmodule.exceptions import NotFoundError from xmodule.modulestore.exceptions import ItemNotFoundError @@ -247,7 +247,7 @@ def get_canonicalized_asset_path(course_key, path, base_url, excluded_exts, enco serve_from_cdn = False content_digest = None try: - content = AssetManager.find(asset_key, as_stream=True) + content = contentstore().find(asset_key, as_stream=True) serve_from_cdn = not getattr(content, "locked", True) content_digest = getattr(content, "content_digest", None) except (ItemNotFoundError, NotFoundError): diff --git a/xmodule/contentstore/utils.py b/xmodule/contentstore/utils.py index 0a114045f9f9..8879d5e2e667 100644 --- a/xmodule/contentstore/utils.py +++ b/xmodule/contentstore/utils.py @@ -1,10 +1,12 @@ # pylint: disable=missing-module-docstring from xmodule.contentstore.content import StaticContent +from xmodule.util.deprecation import deprecated from .django import contentstore +@deprecated("trashcan functionality is deprecated.") # https://github.com/openedx/openedx-platform/issues/39260 def empty_asset_trashcan(course_locs): ''' This method will hard delete all assets (optionally within a course_id) from the trashcan @@ -25,6 +27,7 @@ def empty_asset_trashcan(course_locs): store.delete(asset['asset_key']) +@deprecated("trashcan functionality is deprecated.") # https://github.com/openedx/openedx-platform/issues/39260 def restore_asset_from_trashcan(location): ''' This method will restore an asset which got soft deleted and put back in the original course diff --git a/xmodule/modulestore/__init__.py b/xmodule/modulestore/__init__.py index 79496eb55e16..dd97e3897bf0 100644 --- a/xmodule/modulestore/__init__.py +++ b/xmodule/modulestore/__init__.py @@ -27,6 +27,7 @@ from openedx.core.lib.json_utils import EdxJSONEncoder # noqa: F401 from xmodule.assetstore import AssetMetadata from xmodule.errortracker import make_error_tracker +from xmodule.util.deprecation import deprecated from xmodule.util.misc import get_library_or_course_attribute from .exceptions import InsufficientSpecificationError, InvalidLocationError # noqa: F401 @@ -585,9 +586,17 @@ def insert_or_update(self, asset_md): self.add(metadata_to_insert) +ASSET_METADATA_DEPRECATION_MESSAGE = ( + "Asset metadata in the modulestore is deprecated and will be removed. It is never populated by Studio; " + "use xmodule.contentstore.django.contentstore() to access course assets instead." +) + + class ModuleStoreAssetBase: """ The methods for accessing assets and their metadata + + The asset metadata methods are deprecated; use ``contentstore()`` to access course assets instead. """ def _find_course_asset(self, asset_key): """ @@ -608,6 +617,7 @@ def _find_course_asset(self, asset_key): return course_assets, idx + @deprecated(ASSET_METADATA_DEPRECATION_MESSAGE) def find_asset_metadata(self, asset_key, **kwargs): """ Find the metadata for a particular course asset. @@ -627,6 +637,7 @@ def find_asset_metadata(self, asset_key, **kwargs): mdata.from_storable(all_assets[asset_idx]) return mdata + @deprecated(ASSET_METADATA_DEPRECATION_MESSAGE) def get_all_asset_metadata(self, course_key, asset_type, start=0, maxresults=-1, sort=None, **kwargs): # pylint: disable=unused-argument """ Returns a list of asset metadata for all assets of the given asset_type in the course. @@ -725,6 +736,7 @@ def _save_assets_by_type(self, course_key, asset_metadata_list, course_assets, u all_assets.insert_or_update(asset_md) return assets_by_type + @deprecated(ASSET_METADATA_DEPRECATION_MESSAGE) def save_asset_metadata(self, asset_metadata, user_id, import_only): """ Saves the asset metadata for a particular course's asset. @@ -739,6 +751,7 @@ def save_asset_metadata(self, asset_metadata, user_id, import_only): """ raise NotImplementedError() + @deprecated(ASSET_METADATA_DEPRECATION_MESSAGE) def save_asset_metadata_list(self, asset_metadata_list, user_id, import_only): """ Saves a list of asset metadata for a particular course's asset. @@ -753,18 +766,21 @@ def save_asset_metadata_list(self, asset_metadata_list, user_id, import_only): """ raise NotImplementedError() + @deprecated(ASSET_METADATA_DEPRECATION_MESSAGE) def set_asset_metadata_attrs(self, asset_key, attrs, user_id): """ Base method to over-ride in modulestore. """ raise NotImplementedError() + @deprecated(ASSET_METADATA_DEPRECATION_MESSAGE) def delete_asset_metadata(self, asset_key, user_id): """ Base method to over-ride in modulestore. """ raise NotImplementedError() + @deprecated(ASSET_METADATA_DEPRECATION_MESSAGE) def set_asset_metadata_attr(self, asset_key, attr, value, user_id): """ Add/set the given attr on the asset at the given location. Value can be any type which pymongo accepts. @@ -781,6 +797,7 @@ def set_asset_metadata_attr(self, asset_key, attr, value, user_id): """ return self.set_asset_metadata_attrs(asset_key, {attr: value}, user_id) + @deprecated(ASSET_METADATA_DEPRECATION_MESSAGE) def copy_all_asset_metadata(self, source_course_key, dest_course_key, user_id): """ Copy all the course assets from source_course_key to dest_course_key. diff --git a/xmodule/modulestore/mixed.py b/xmodule/modulestore/mixed.py index 2a3a04699a79..0e3b2ea79730 100644 --- a/xmodule/modulestore/mixed.py +++ b/xmodule/modulestore/mixed.py @@ -23,8 +23,9 @@ ) from xmodule.assetstore import AssetMetadata +from xmodule.util.deprecation import deprecated -from . import XMODULE_FIELDS_WITH_USAGE_KEYS, ModuleStoreWriteBase +from . import ASSET_METADATA_DEPRECATION_MESSAGE, XMODULE_FIELDS_WITH_USAGE_KEYS, ModuleStoreWriteBase from .draft_and_published import ModuleStoreDraftAndPublished from .exceptions import DuplicateCourseError, ItemNotFoundError @@ -455,6 +456,7 @@ def delete_course(self, course_key, user_id): # pylint: disable=arguments-diffe store = self._get_modulestore_for_courselike(course_key) return store.delete_course(course_key, user_id) + @deprecated(ASSET_METADATA_DEPRECATION_MESSAGE) def save_asset_metadata(self, asset_metadata, user_id, import_only=False): """ Saves the asset metadata for a particular course's asset. @@ -470,6 +472,7 @@ def save_asset_metadata(self, asset_metadata, user_id, import_only=False): store = self._get_modulestore_for_courselike(asset_metadata.asset_id.course_key) return store.save_asset_metadata(asset_metadata, user_id, import_only) + @deprecated(ASSET_METADATA_DEPRECATION_MESSAGE) def save_asset_metadata_list(self, asset_metadata_list, user_id, import_only=False): """ Saves the asset metadata for each asset in a list of asset metadata. @@ -488,6 +491,7 @@ def save_asset_metadata_list(self, asset_metadata_list, user_id, import_only=Fal store = self._get_modulestore_for_courselike(asset_metadata_list[0].asset_id.course_key) return store.save_asset_metadata_list(asset_metadata_list, user_id, import_only) + @deprecated(ASSET_METADATA_DEPRECATION_MESSAGE) @strip_key def find_asset_metadata(self, asset_key, **kwargs): """ @@ -502,6 +506,7 @@ def find_asset_metadata(self, asset_key, **kwargs): store = self._get_modulestore_for_courselike(asset_key.course_key) return store.find_asset_metadata(asset_key, **kwargs) + @deprecated(ASSET_METADATA_DEPRECATION_MESSAGE) @strip_key def get_all_asset_metadata(self, course_key, asset_type, start=0, maxresults=-1, sort=None, **kwargs): """ @@ -524,6 +529,7 @@ def get_all_asset_metadata(self, course_key, asset_type, start=0, maxresults=-1, store = self._get_modulestore_for_courselike(course_key) return store.get_all_asset_metadata(course_key, asset_type, start, maxresults, sort, **kwargs) + @deprecated(ASSET_METADATA_DEPRECATION_MESSAGE) def delete_asset_metadata(self, asset_key, user_id): """ Deletes a single asset's metadata. @@ -538,6 +544,7 @@ def delete_asset_metadata(self, asset_key, user_id): store = self._get_modulestore_for_courselike(asset_key.course_key) return store.delete_asset_metadata(asset_key, user_id) + @deprecated(ASSET_METADATA_DEPRECATION_MESSAGE) def copy_all_asset_metadata(self, source_course_key, dest_course_key, user_id): """ Copy all the course assets from source_course_key to dest_course_key. @@ -563,6 +570,7 @@ def copy_all_asset_metadata(self, source_course_key, dest_course_key, user_id): # Courses in the same modulestore can be handled by the modulestore itself. source_store.copy_all_asset_metadata(source_course_key, dest_course_key, user_id) + @deprecated(ASSET_METADATA_DEPRECATION_MESSAGE) def set_asset_metadata_attr(self, asset_key, attr, value, user_id): """ Add/set the given attr on the asset at the given location. Value can be any type which pymongo accepts. @@ -580,6 +588,7 @@ def set_asset_metadata_attr(self, asset_key, attr, value, user_id): store = self._get_modulestore_for_courselike(asset_key.course_key) return store.set_asset_metadata_attrs(asset_key, {attr: value}, user_id) + @deprecated(ASSET_METADATA_DEPRECATION_MESSAGE) def set_asset_metadata_attrs(self, asset_key, attr_dict, user_id): # pylint: disable=arguments-differ """ Add/set the given dict of attrs on the asset at the given location. Value can be any type which pymongo accepts. diff --git a/xmodule/modulestore/mongo/base.py b/xmodule/modulestore/mongo/base.py index 4f8a5dbf5671..d552c5a375ae 100644 --- a/xmodule/modulestore/mongo/base.py +++ b/xmodule/modulestore/mongo/base.py @@ -1479,14 +1479,10 @@ def delete_all_asset_metadata(self, course_key, user_id): # pylint: disable=unu Arguments: course_key (CourseKey): course_identifier """ - # Using the course_id, find the course asset metadata document. # A single document exists per course to store the course asset metadata. - try: - course_assets = self._find_course_assets(course_key) - self.asset_collection.delete_many({'_id': course_assets.doc_id}) - except ItemNotFoundError: - # When deleting asset metadata, if a course's asset metadata is not present, no big deal. - pass + course_key = self.fill_in_run(course_key) + if course_key.run is not None: + self.asset_collection.delete_many({'course_id': str(course_key)}) def heartbeat(self): """ diff --git a/xmodule/modulestore/tests/django_utils.py b/xmodule/modulestore/tests/django_utils.py index d4a2030ac496..73aa885402c7 100644 --- a/xmodule/modulestore/tests/django_utils.py +++ b/xmodule/modulestore/tests/django_utils.py @@ -155,6 +155,7 @@ def contentstore_config(): }, # allow for additional options that can be keyed on a name, e.g. 'trashcan' 'ADDITIONAL_OPTIONS': { + # Deprecated - https://github.com/openedx/openedx-platform/issues/39260 'trashcan': { 'bucket': 'trash_fs' } diff --git a/xmodule/modulestore/tests/test_assetstore.py b/xmodule/modulestore/tests/test_assetstore.py index 207f1ba4f558..f8612997bc7c 100644 --- a/xmodule/modulestore/tests/test_assetstore.py +++ b/xmodule/modulestore/tests/test_assetstore.py @@ -6,6 +6,8 @@ import unittest from datetime import datetime, timedelta +from shutil import rmtree +from tempfile import mkdtemp from zoneinfo import ZoneInfo import ddt @@ -19,7 +21,20 @@ from xmodule.modulestore import IncorrectlySortedList, ModuleStoreEnum, SortedAssetList from xmodule.modulestore.exceptions import ItemNotFoundError from xmodule.modulestore.tests.factories import CourseFactory -from xmodule.modulestore.tests.utils import SPLIT_MODULESTORE_SETUP, MixedModulestoreBuilder, XmlModulestoreBuilder +from xmodule.modulestore.tests.utils import ( + SPLIT_MODULESTORE_SETUP, + TEST_DATA_DIR, + MixedModulestoreBuilder, + XmlModulestoreBuilder, +) +from xmodule.modulestore.xml_exporter import export_course_to_xml +from xmodule.modulestore.xml_importer import import_course_from_xml + +# These tests exercise the deprecated asset metadata APIs directly. +pytestmark = [ + pytest.mark.filterwarnings("ignore:Asset metadata in the modulestore is deprecated:DeprecationWarning"), + pytest.mark.filterwarnings("ignore:xmodule.assetstore is deprecated:DeprecationWarning"), +] class AssetStoreTestData: @@ -570,6 +585,58 @@ def test_delete_all_different_type(self, storebuilder): assert len(store.get_all_asset_metadata(course.id, 'different')) == 1 + @ddt.data(SPLIT_MODULESTORE_SETUP) + def test_asset_metadata_api_is_deprecated(self, storebuilder): + """ + The modulestore asset metadata APIs and AssetMetadata emit DeprecationWarnings. + """ + with storebuilder.build() as (__, store): + course = CourseFactory.create(modulestore=store) + asset_key = course.id.make_asset_key('asset', 'pic1.jpg') + with self.assertWarnsRegex(DeprecationWarning, 'Asset metadata in the modulestore is deprecated'): + store.get_all_asset_metadata(course.id, 'asset') + with self.assertWarnsRegex(DeprecationWarning, 'Asset metadata in the modulestore is deprecated'): + store.find_asset_metadata(asset_key) + with self.assertWarnsRegex(DeprecationWarning, 'xmodule.assetstore is deprecated'): + AssetMetadata(asset_key) + + @ddt.data(SPLIT_MODULESTORE_SETUP) + def test_import_ignores_asset_metadata(self, storebuilder): + """ + Importing a course whose assets/assets.xml lists assets does not store + any asset metadata. + + We deliberately ignore it because assetstore is deprecated and + assets.xml has been unused for years. + """ + with storebuilder.build() as (contentstore, store): + courses = import_course_from_xml( + store, ModuleStoreEnum.UserID.test, TEST_DATA_DIR, source_dirs=['manual-testing-complete'], + static_content_store=contentstore, create_if_not_present=True, raise_on_failure=True, + ) + assert len(courses) == 1 + assert store.get_all_asset_metadata(courses[0].id, None) == [] + + @ddt.data(SPLIT_MODULESTORE_SETUP) + def test_export_writes_empty_asset_metadata(self, storebuilder): + """ + Exporting a course always writes an empty assets/assets.xml. + + We do this just for backwards compatibility, but the assets.xml file + hasn't been used in any meaningful way in years. + """ + with storebuilder.build() as (contentstore, store): + course = CourseFactory.create(modulestore=store) + self.setup_assets(course.id, None, store) + assert len(store.get_all_asset_metadata(course.id, 'asset')) == 2 + + export_dir = mkdtemp() + self.addCleanup(rmtree, export_dir) + export_course_to_xml(store, contentstore, course.id, export_dir, 'exported_course') + + with open(f'{export_dir}/exported_course/assets/assets.xml', 'rb') as assets_xml: + assert assets_xml.read() == b'' + @ddt.data('XML_MODULESTORE_BUILDER', 'MIXED_MODULESTORE_BUILDER') def test_xml_not_yet_implemented(self, storebuilderName): """ diff --git a/xmodule/modulestore/tests/test_cross_modulestore_import_export.py b/xmodule/modulestore/tests/test_cross_modulestore_import_export.py index e70057b447a9..ee6108a3c6ff 100644 --- a/xmodule/modulestore/tests/test_cross_modulestore_import_export.py +++ b/xmodule/modulestore/tests/test_cross_modulestore_import_export.py @@ -146,13 +146,6 @@ def test_round_trip( dest_course_key, ) - self.assertAssetsMetadataEqual( - source_store, - source_course_key, - dest_store, - dest_course_key, - ) - def test_split_course_export_import(self): # Construct the contentstore for storing the first import with MongoContentstoreBuilder().build() as source_content: diff --git a/xmodule/modulestore/xml_exporter.py b/xmodule/modulestore/xml_exporter.py index c5ed9502217a..c2e2e71032d4 100644 --- a/xmodule/modulestore/xml_exporter.py +++ b/xmodule/modulestore/xml_exporter.py @@ -15,7 +15,6 @@ from xblock.fields import Reference, ReferenceList, ReferenceValueDict, Scope from openedx.core.djangoapps.content_tagging.api import export_tags_in_csv_file, get_object_tag_counts -from xmodule.assetstore import AssetMetadata from xmodule.contentstore.content import StaticContent from xmodule.exceptions import NotFoundError from xmodule.modulestore import LIBRARY_ROOT, EdxJSONEncoder, ModuleStoreEnum @@ -204,19 +203,14 @@ def process_root(self, root, export_fs): lxml.etree.ElementTree(root).write(course_xml, encoding='utf-8') def process_extra(self, root, courselike, root_courselike_dir, xml_centric_courselike_key, export_fs): - # Export the modulestore's asset metadata. + # Asset metadata in the modulestore is deprecated and is no longer exported, but we still write an empty + # assets/assets.xml file (which is all that it ever contained in practice) for backwards compatibility. set_custom_attribute("export_asset_started", str(courselike)) - asset_dir = root_courselike_dir + '/' + AssetMetadata.EXPORTED_ASSET_DIR + '/' + asset_dir = root_courselike_dir + '/assets/' if not os.path.isdir(asset_dir): os.makedirs(asset_dir) - asset_root = lxml.etree.Element(AssetMetadata.ALL_ASSETS_XML_TAG) - course_assets = self.modulestore.get_all_asset_metadata(self.courselike_key, None) - for asset_md in course_assets: - # All asset types are exported using the "asset" tag - but their asset type is specified in each asset key. - asset = lxml.etree.SubElement(asset_root, AssetMetadata.ASSET_XML_TAG) - asset_md.to_xml(asset) - with OSFS(asset_dir).open(AssetMetadata.EXPORTED_ASSET_FILENAME, 'wb') as asset_xml_file: - lxml.etree.ElementTree(asset_root).write(asset_xml_file, encoding='utf-8') + with OSFS(asset_dir).open('assets.xml', 'wb') as asset_xml_file: + asset_xml_file.write(b'') # export the static assets set_custom_attribute("export_static_assets_started", str(courselike)) diff --git a/xmodule/modulestore/xml_importer.py b/xmodule/modulestore/xml_importer.py index 237d9e0eff15..735c17c87e40 100644 --- a/xmodule/modulestore/xml_importer.py +++ b/xmodule/modulestore/xml_importer.py @@ -32,7 +32,6 @@ import xblock from django.core.exceptions import ObjectDoesNotExist from django.utils.translation import gettext as _ -from lxml import etree from opaque_keys.edx.keys import UsageKey from opaque_keys.edx.locator import LibraryLocator from openedx_events.content_authoring.data import CourseData @@ -44,7 +43,6 @@ from common.djangoapps.util.monitoring import monitor_import_failure from openedx.core.djangoapps.content_tagging.api import import_course_tags_from_csv -from xmodule.assetstore import AssetMetadata from xmodule.contentstore.content import StaticContent from xmodule.errortracker import make_error_tracker from xmodule.modulestore import ModuleStoreEnum @@ -360,54 +358,6 @@ def import_static(self, data_path, dest_id): content_subdir=simport, verbose=self.verbose ) - def import_asset_metadata(self, data_dir, course_id): - """ - Read in assets XML file, parse it, and add all asset metadata to the modulestore. - """ - asset_dir = path(data_dir) / AssetMetadata.EXPORTED_ASSET_DIR - assets_filename = AssetMetadata.EXPORTED_ASSET_FILENAME - asset_xml_file = asset_dir / assets_filename - - def make_asset_id(course_id, asset_xml): - """ - Construct an asset ID out of a complete asset XML section. - """ - asset_type = None - asset_name = None - for child in asset_xml.iterchildren(): - if child.tag == AssetMetadata.ASSET_TYPE_ATTR: - asset_type = child.text - elif child.tag == AssetMetadata.ASSET_BASENAME_ATTR: - asset_name = child.text - return course_id.make_asset_key(asset_type, asset_name) - - all_assets = [] - try: - xml_data = etree.parse(asset_xml_file).getroot() - assert xml_data.tag == AssetMetadata.ALL_ASSETS_XML_TAG - for asset in xml_data.iterchildren(): - if asset.tag == AssetMetadata.ASSET_XML_TAG: - # Construct the asset key. - asset_key = make_asset_id(course_id, asset) - asset_md = AssetMetadata(asset_key) - asset_md.from_xml(asset) - all_assets.append(asset_md) - except OSError: - # file does not exist. - logging.info(f'Course import {course_id}: No {assets_filename} file present.') - return - except Exception as exc: # pylint: disable=W0703 - if self.raise_on_failure: # pylint: disable=no-else-raise - monitor_import_failure(course_id, 'Updating', exception=exc) - logging.exception(f'Course import {course_id}: Error while parsing {assets_filename}.') - raise ErrorReadingFileException(assets_filename) # pylint: disable=raise-missing-from # noqa: B904 - else: - return - - # Now add all asset metadata to the modulestore. - if len(all_assets) > 0: - self.store.save_asset_metadata_list(all_assets, all_assets[0].edited_by, import_only=True) - def import_courselike(self, runtime, courselike_key, dest_id, source_courselike): """ Import the base block @@ -575,8 +525,7 @@ def run_imports(self): # Import all static pieces. self.import_static(data_path, dest_id) - # Import asset metadata stored in XML. - self.import_asset_metadata(data_path, dest_id) + # Note: assets/assets.xml (deprecated asset metadata) is intentionally ignored. # Import all children self.import_children(source_courselike, courselike, courselike_key, dest_id) diff --git a/xmodule/tests/__init__.py b/xmodule/tests/__init__.py index f0de6e2a5506..ded0dfc43db7 100644 --- a/xmodule/tests/__init__.py +++ b/xmodule/tests/__init__.py @@ -23,7 +23,6 @@ from openedx.core.djangoapps.discussions.services import DiscussionConfigService from openedx.core.djangoapps.video_config.services import VideoConfigService from openedx.core.lib.cache_utils import CacheService -from xmodule.assetstore import AssetMetadata from xmodule.contentstore.django import contentstore from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.draft_and_published import ModuleStoreDraftAndPublished @@ -534,22 +533,3 @@ def assertAssetsEqual(self, expected_store, expected_course_key, actual_store, a expected_thumbs = expected_store.get_all_content_thumbnails_for_course(expected_course_key) actual_thumbs = actual_store.get_all_content_thumbnails_for_course(actual_course_key) self._assertAssetsEqual(expected_course_key, expected_thumbs, actual_course_key, actual_thumbs) - - def assertAssetsMetadataEqual(self, expected_modulestore, expected_course_key, actual_modulestore, actual_course_key): # pylint: disable=line-too-long - """ - Assert that the modulestore asset metdata for the ``expected_course_key`` and the ``actual_course_key`` - are equivalent. - """ - expected_course_assets = expected_modulestore.get_all_asset_metadata( - expected_course_key, None, sort=('displayname', ModuleStoreEnum.SortOrder.descending) - ) - actual_course_assets = actual_modulestore.get_all_asset_metadata( - actual_course_key, None, sort=('displayname', ModuleStoreEnum.SortOrder.descending) - ) - assert len(expected_course_assets) == len(actual_course_assets) - for idx, expected_val in enumerate(expected_course_assets): - for attr in AssetMetadata.ATTRS_ALLOWED_TO_UPDATE: - if attr in ('edited_on',): - # edited_on is updated upon import. - continue - assert getattr(expected_val, attr) == getattr(actual_course_assets[idx], attr) diff --git a/xmodule/util/deprecation.py b/xmodule/util/deprecation.py new file mode 100644 index 000000000000..f55ebe702e24 --- /dev/null +++ b/xmodule/util/deprecation.py @@ -0,0 +1,12 @@ +""" +Helpers for marking xmodule APIs as deprecated. +""" + +import sys + +if sys.version_info >= (3, 13): + from warnings import deprecated +else: + from typing_extensions import deprecated # TODO: remove once we require Python 3.13+ + +__all__ = ['deprecated']