[WIP] Re-implementing Restore to use Pydantic

docs/decisions/0025-backup-restore.rst

@@ -0,0 +1,46 @@
+25. Learning Package Serialization and Validation Approach
+==========================================================
+
+Context
+-------
+
+Content Libraries map 1:1 to LearningPackages and these need to be imported and exported as file archives. Initial support for this was released in Ulmo, but we wanted to revisit it to make it more robust during the Verawood timeline. This is part of that effort.
+
+* Flexibility of Structure
+* Standardization of validation (JSON Schema)
+* Justify ZIP
+* Justify TOML
+* Max 100,000 items.
+* Use of fsspec as abstraction
+
+Phases
+
+Archive → Filesystem → Learning Package Doc + Resources → Input Models → LearningPackage
+
+
+Decision
+--------
+
+Some key points:
+
+1. We intentionally separate input and output formats because the output format
+   will change over time, but the various input formats must continue to be
+   supported. We don't inherit from one from the other because we don't *want*
+   those changes to be automatically propogated--that breaks compatibility.
+2. We assemble into giant JSON in order to simplify validation and allow for
+   more flexibility in structural representation. There's the archive layer and
+   then the logical layer and then serialization into the database.
+
+
+Archive -> Model (validation) + Resources -> Database
+
+
+
+Consequences
+------------
+
+
+
+Rejected alternatives
+---------------------
+

requirements/base.in

@@ -15,3 +15,7 @@ rules<4.0                 # Django extension for rules-based authorization check
 tomlkit                   # Parses and writes TOML configuration files
 
 edx-organizations         # Implemented the "Organization" model that CatalogCourse/CourseRun are keyed to
+
+fsspec                    # Used by openedx_content's backup_restore to abstract zip access
+
+pydantic[email]           # Used by openedx_content's backup_restore for input validation

requirements/base.txt

@@ -6,6 +6,8 @@
 #
 amqp==5.3.1
     # via kombu
+annotated-types==0.7.0
+    # via pydantic
 asgiref==3.11.1
     # via django
 attrs==26.1.0
@@ -67,7 +69,9 @@ djangorestframework==3.17.1
     #   edx-drf-extensions
     #   edx-organizations
 dnspython==2.8.0
-    # via pymongo
+    # via
+    #   email-validator
+    #   pymongo
 drf-jwt==1.19.2
     # via edx-drf-extensions
 edx-django-utils==8.0.1
@@ -82,8 +86,14 @@ edx-opaque-keys==4.0.0
     #   edx-organizations
 edx-organizations==8.0.0
     # via -r requirements/base.in
+email-validator==2.3.0
+    # via pydantic
+fsspec==2026.3.0
+    # via -r requirements/base.in
 idna==3.11
-    # via requests
+    # via
+    #   email-validator
+    #   requests
 kombu==5.6.2
     # via celery
 packaging==26.0
@@ -96,6 +106,10 @@ psutil==7.2.2
     # via edx-django-utils
 pycparser==3.0
     # via cffi
+pydantic[email]==2.13.3
+    # via -r requirements/base.in
+pydantic-core==2.46.3
+    # via pydantic
 pyjwt[crypto]==2.12.1
     # via
     #   drf-jwt
@@ -123,7 +137,13 @@ stevedore==5.7.0
 tomlkit==0.14.0
     # via -r requirements/base.in
 typing-extensions==4.15.0
-    # via edx-opaque-keys
+    # via
+    #   edx-opaque-keys
+    #   pydantic
+    #   pydantic-core
+    #   typing-inspection
+typing-inspection==0.4.2
+    # via pydantic
 tzdata==2026.1
     # via kombu
 tzlocal==5.3.1

requirements/dev.txt

@@ -8,6 +8,10 @@ amqp==5.3.1
     # via
     #   -r requirements/quality.txt
     #   kombu
+annotated-types==0.7.0
+    # via
+    #   -r requirements/quality.txt
+    #   pydantic
 asgiref==3.11.1
     # via
     #   -r requirements/quality.txt
@@ -164,6 +168,7 @@ djangorestframework-stubs==3.16.9
 dnspython==2.8.0
     # via
     #   -r requirements/quality.txt
+    #   email-validator
     #   pymongo
 docutils==0.22.4
     # via
@@ -192,6 +197,10 @@ edx-opaque-keys==4.0.0
     #   edx-organizations
 edx-organizations==8.0.0
     # via -r requirements/quality.txt
+email-validator==2.3.0
+    # via
+    #   -r requirements/quality.txt
+    #   pydantic
 filelock==3.25.2
     # via
     #   -r requirements/ci.txt
@@ -200,6 +209,8 @@ filelock==3.25.2
     #   virtualenv
 freezegun==1.5.5
     # via -r requirements/quality.txt
+fsspec==2026.3.0
+    # via -r requirements/quality.txt
 grimp==3.14
     # via
     #   -r requirements/quality.txt
@@ -211,6 +222,7 @@ id==1.6.1
 idna==3.11
     # via
     #   -r requirements/quality.txt
+    #   email-validator
     #   requests
 import-linter==2.11
     # via -r requirements/quality.txt
@@ -353,6 +365,12 @@ pycparser==3.0
     # via
     #   -r requirements/quality.txt
     #   cffi
+pydantic[email]==2.13.3
+    # via -r requirements/quality.txt
+pydantic-core==2.46.3
+    # via
+    #   -r requirements/quality.txt
+    #   pydantic
 pydocstyle==6.3.0
     # via -r requirements/quality.txt
 pygments==2.20.0
@@ -516,6 +534,13 @@ typing-extensions==4.15.0
     #   grimp
     #   import-linter
     #   mypy
+    #   pydantic
+    #   pydantic-core
+    #   typing-inspection
+typing-inspection==0.4.2
+    # via
+    #   -r requirements/quality.txt
+    #   pydantic
 tzdata==2026.1
     # via
     #   -r requirements/quality.txt

requirements/doc.txt

@@ -12,6 +12,10 @@ amqp==5.3.1
     # via
     #   -r requirements/test.txt
     #   kombu
+annotated-types==0.7.0
+    # via
+    #   -r requirements/test.txt
+    #   pydantic
 asgiref==3.11.1
     # via
     #   -r requirements/test.txt
@@ -132,6 +136,7 @@ djangorestframework-stubs==3.16.9
 dnspython==2.8.0
     # via
     #   -r requirements/test.txt
+    #   email-validator
     #   pymongo
 doc8==2.0.0
     # via -r requirements/doc.in
@@ -161,15 +166,22 @@ edx-opaque-keys==4.0.0
     #   edx-organizations
 edx-organizations==8.0.0
     # via -r requirements/test.txt
+email-validator==2.3.0
+    # via
+    #   -r requirements/test.txt
+    #   pydantic
 freezegun==1.5.5
     # via -r requirements/test.txt
+fsspec==2026.3.0
+    # via -r requirements/test.txt
 grimp==3.14
     # via
     #   -r requirements/test.txt
     #   import-linter
 idna==3.11
     # via
     #   -r requirements/test.txt
+    #   email-validator
     #   requests
 imagesize==2.0.0
     # via sphinx
@@ -249,6 +261,12 @@ pycparser==3.0
     # via
     #   -r requirements/test.txt
     #   cffi
+pydantic[email]==2.13.3
+    # via -r requirements/test.txt
+pydantic-core==2.46.3
+    # via
+    #   -r requirements/test.txt
+    #   pydantic
 pydata-sphinx-theme==0.16.1
     # via sphinx-book-theme
 pygments==2.20.0
@@ -381,7 +399,14 @@ typing-extensions==4.15.0
     #   grimp
     #   import-linter
     #   mypy
+    #   pydantic
+    #   pydantic-core
     #   pydata-sphinx-theme
+    #   typing-inspection
+typing-inspection==0.4.2
+    # via
+    #   -r requirements/test.txt
+    #   pydantic
 tzdata==2026.1
     # via
     #   -r requirements/test.txt

requirements/quality.txt

@@ -8,6 +8,10 @@ amqp==5.3.1
     # via
     #   -r requirements/test.txt
     #   kombu
+annotated-types==0.7.0
+    # via
+    #   -r requirements/test.txt
+    #   pydantic
 asgiref==3.11.1
     # via
     #   -r requirements/test.txt
@@ -134,6 +138,7 @@ djangorestframework-stubs==3.16.9
 dnspython==2.8.0
     # via
     #   -r requirements/test.txt
+    #   email-validator
     #   pymongo
 docutils==0.22.4
     # via readme-renderer
@@ -158,8 +163,14 @@ edx-opaque-keys==4.0.0
     #   edx-organizations
 edx-organizations==8.0.0
     # via -r requirements/test.txt
+email-validator==2.3.0
+    # via
+    #   -r requirements/test.txt
+    #   pydantic
 freezegun==1.5.5
     # via -r requirements/test.txt
+fsspec==2026.3.0
+    # via -r requirements/test.txt
 grimp==3.14
     # via
     #   -r requirements/test.txt
@@ -169,6 +180,7 @@ id==1.6.1
 idna==3.11
     # via
     #   -r requirements/test.txt
+    #   email-validator
     #   requests
 import-linter==2.11
     # via -r requirements/test.txt
@@ -269,6 +281,12 @@ pycparser==3.0
     # via
     #   -r requirements/test.txt
     #   cffi
+pydantic[email]==2.13.3
+    # via -r requirements/test.txt
+pydantic-core==2.46.3
+    # via
+    #   -r requirements/test.txt
+    #   pydantic
 pydocstyle==6.3.0
     # via -r requirements/quality.in
 pygments==2.20.0
@@ -394,6 +412,13 @@ typing-extensions==4.15.0
     #   grimp
     #   import-linter
     #   mypy
+    #   pydantic
+    #   pydantic-core
+    #   typing-inspection
+typing-inspection==0.4.2
+    # via
+    #   -r requirements/test.txt
+    #   pydantic
 tzdata==2026.1
     # via
     #   -r requirements/test.txt

requirements/test.txt

@@ -8,6 +8,10 @@ amqp==5.3.1
     # via
     #   -r requirements/base.txt
     #   kombu
+annotated-types==0.7.0
+    # via
+    #   -r requirements/base.txt
+    #   pydantic
 asgiref==3.11.1
     # via
     #   -r requirements/base.txt
@@ -118,6 +122,7 @@ djangorestframework-stubs==3.16.9
 dnspython==2.8.0
     # via
     #   -r requirements/base.txt
+    #   email-validator
     #   pymongo
 drf-jwt==1.19.2
     # via
@@ -138,13 +143,20 @@ edx-opaque-keys==4.0.0
     #   edx-organizations
 edx-organizations==8.0.0
     # via -r requirements/base.txt
+email-validator==2.3.0
+    # via
+    #   -r requirements/base.txt
+    #   pydantic
 freezegun==1.5.5
     # via -r requirements/test.in
+fsspec==2026.3.0
+    # via -r requirements/base.txt
 grimp==3.14
     # via import-linter
 idna==3.11
     # via
     #   -r requirements/base.txt
+    #   email-validator
     #   requests
 import-linter==2.11
     # via -r requirements/test.in
@@ -199,6 +211,12 @@ pycparser==3.0
     # via
     #   -r requirements/base.txt
     #   cffi
+pydantic[email]==2.13.3
+    # via -r requirements/base.txt
+pydantic-core==2.46.3
+    # via
+    #   -r requirements/base.txt
+    #   pydantic
 pygments==2.20.0
     # via
     #   pytest
@@ -279,6 +297,13 @@ typing-extensions==4.15.0
     #   grimp
     #   import-linter
     #   mypy
+    #   pydantic
+    #   pydantic-core
+    #   typing-inspection
+typing-inspection==0.4.2
+    # via
+    #   -r requirements/base.txt
+    #   pydantic
 tzdata==2026.1
     # via
     #   -r requirements/base.txt