Support PEtab SciML problems and enable linting

doc/conf.py

@@ -9,6 +9,8 @@
 import sys
 import warnings
 
+from pydantic import BaseModel
+
 # -- Path setup --------------------------------------------------------------
 
 # If extensions (or modules to document with autodoc) are in another directory,
@@ -126,6 +128,9 @@ def skip_some_objects(app, what, name, obj, skip, options):
     """Exclude some objects from the documentation"""
     if getattr(obj, "__module__", None) == "collections":
         return True
+    # Napoleon + Pydantic v2 bug: BaseModel itself triggers __getattr__ error
+    if obj is BaseModel:
+        return True
 
 
 def setup(app):

petab/v2/C.py

@@ -258,6 +258,8 @@
 MAPPING_FILES = "mapping_files"
 #: Extensions key in the YAML file
 EXTENSIONS = "extensions"
+#: PEtab SciML extension
+EXT_ID_SCIML = "sciml"
 
 
 # MAPPING

petab/v2/core.py

@@ -19,6 +19,7 @@
     Annotated,
     Any,
     Generic,
+    Literal,
     Self,
     TypeVar,
     get_args,
@@ -308,6 +309,22 @@ def __iadd__(self, other: T) -> BaseTable[T]:
         return self
 
 
+# SciML extension classes — imported after BaseTable is defined to avoid
+# circular imports (sciml.py does not import from core.py).
+from .extensions.sciml import (  # noqa: E402
+    HybridizationTable,
+    SciMLConfig,
+    SciMLExt,
+)
+
+
+class ProblemExtensions:
+    """Runtime extension state attached to a :class:`Problem`."""
+
+    def __init__(self, sciml: SciMLExt = None):
+        self.sciml: SciMLExt = sciml or SciMLExt()
+
+
 class Observable(BaseModel):
     """Observable definition."""
 
@@ -318,9 +335,9 @@ class Observable(BaseModel):
     #: Observable name.
     name: str | None = Field(alias=C.OBSERVABLE_NAME, default=None)
     #: Observable formula.
-    formula: sp.Basic | None = Field(alias=C.OBSERVABLE_FORMULA, default=None)
+    formula: sp.Basic = Field(alias=C.OBSERVABLE_FORMULA)
     #: Noise formula.
-    noise_formula: sp.Basic | None = Field(alias=C.NOISE_FORMULA, default=None)
+    noise_formula: sp.Basic = Field(alias=C.NOISE_FORMULA)
     #: Noise distribution.
     noise_distribution: NoiseDistribution = Field(
         alias=C.NOISE_DISTRIBUTION, default=NoiseDistribution.NORMAL
@@ -926,7 +943,8 @@ class Parameter(BaseModel):
     )
     #: Nominal value.
     nominal_value: Annotated[
-        float | None, BeforeValidator(_convert_nan_to_none)
+        # PEtab SciML supports arrays via "array" nominal values
+        float | Literal["array"] | None, BeforeValidator(_convert_nan_to_none)
     ] = Field(alias=C.NOMINAL_VALUE, default=None)
     #: Is the parameter to be estimated?
     estimate: bool = Field(alias=C.ESTIMATE, default=True)
@@ -1133,22 +1151,33 @@ def __init__(
         measurement_tables: list[MeasurementTable] = None,
         parameter_tables: list[ParameterTable] = None,
         mapping_tables: list[MappingTable] = None,
+        extensions: ProblemExtensions = None,
         config: ProblemConfig = None,
     ):
-        from ..v2.lint import default_validation_tasks
+        from ..v2.lint import default_validation_tasks, sciml_validation_tasks
 
         self.config = config
         self.models: list[Model] = models or []
-        self.validation_tasks: list[ValidationTask] = (
-            default_validation_tasks.copy()
-        )
+        if (
+            config
+            and config.extensions
+            and config.extensions.get(C.EXT_ID_SCIML)
+        ):
+            self.validation_tasks: list[ValidationTask] = (
+                sciml_validation_tasks.copy()
+            )
+        else:
+            self.validation_tasks: list[ValidationTask] = (
+                default_validation_tasks.copy()
+            )
 
         self.observable_tables = observable_tables or [ObservableTable()]
         self.condition_tables = condition_tables or [ConditionTable()]
         self.experiment_tables = experiment_tables or [ExperimentTable()]
         self.measurement_tables = measurement_tables or [MeasurementTable()]
         self.mapping_tables = mapping_tables or [MappingTable()]
         self.parameter_tables = parameter_tables or [ParameterTable()]
+        self.extensions = extensions or ProblemExtensions()
 
     def __repr__(self):
         return f"<{self.__class__.__name__} id={self.id!r}>"
@@ -1321,6 +1350,45 @@ def from_yaml(
             else None
         )
 
+        extensions = ProblemExtensions()
+        if config.extensions and config.extensions.get(C.EXT_ID_SCIML):
+            from petab_sciml import ArrayDataStandard, NNModel, NNModelStandard
+
+            # Neural network classes are constructed via pytorch for now to get
+            # the proper inputs
+            neural_networks = [
+                NNModel.from_pytorch_module(
+                    NNModelStandard.load_data(
+                        _generate_path(
+                            file_path=nn_config.location,
+                            base_path=base_path,
+                        )
+                    ).to_pytorch_module(),
+                    nn_model_id=nn_id,
+                )
+                for nn_id, nn_config in (
+                    config.extensions[C.EXT_ID_SCIML].neural_networks or {}
+                ).items()
+            ]
+
+            hybridization_tables = [
+                HybridizationTable.from_tsv(f, base_path)
+                for f in config.extensions[C.EXT_ID_SCIML].hybridization_files
+            ]
+
+            array_data_files = [
+                ArrayDataStandard.load_data(_generate_path(f, base_path))
+                for f in config.extensions[C.EXT_ID_SCIML].array_files
+            ]
+
+            extensions = ProblemExtensions(
+                sciml=SciMLExt(
+                    neural_networks=neural_networks,
+                    hybridization_tables=hybridization_tables,
+                    array_data_files=array_data_files,
+                )
+            )
+
         return Problem(
             config=config,
             models=models,
@@ -1330,6 +1398,7 @@ def from_yaml(
             measurement_tables=measurement_tables,
             parameter_tables=parameter_tables,
             mapping_tables=mapping_tables,
+            extensions=extensions,
         )
 
     @staticmethod
@@ -1940,14 +2009,21 @@ def validate(
 
         validation_results = ValidationResultList()
 
-        if self.config and self.config.extensions:
-            extensions = ",".join(self.config.extensions.keys())
+        supported_extensions = {C.EXT_ID_SCIML}
+        if (
+            self.config
+            and self.config.extensions
+            and (self.config.extensions.keys() - supported_extensions)
+        ):
+            extensions_without_support = ",".join(
+                self.config.extensions.keys() - supported_extensions
+            )
             validation_results.append(
                 ValidationIssue(
                     ValidationIssueSeverity.WARNING,
-                    "Validation of PEtab extensions is not yet implemented, "
-                    "but the given problem uses the following extensions: "
-                    f"{extensions}",
+                    "The given problem uses the following extensions for "
+                    "which validation is not yet implemented: "
+                    f"{extensions_without_support}",
                 )
             )
 
@@ -2505,6 +2581,23 @@ class ProblemConfig(BaseModel):
         validate_assignment=True,
     )
 
+    @field_validator("extensions", mode="before")
+    @classmethod
+    def _parse_extensions(cls, v):
+        """Parse extensions dict and convert known extensions to their specific
+        config classes."""
+        if isinstance(v, dict):
+            parsed_extensions = {}
+            for ext_name, ext_config in v.items():
+                if ext_name == C.EXT_ID_SCIML:
+                    # Convert sciml extension to SciMLConfig
+                    parsed_extensions[ext_name] = SciMLConfig(**ext_config)
+                else:
+                    # Keep other extensions as ExtensionConfig
+                    parsed_extensions[ext_name] = ExtensionConfig(**ext_config)
+            return parsed_extensions
+        return v
+
     # convert parameter_file to list
     @field_validator(
         "parameter_files",
@@ -2542,12 +2635,22 @@ def to_yaml(self, filename: str | Path):
 
         for model_id in data.get("model_files", {}):
             data["model_files"][model_id][C.MODEL_LOCATION] = str(
-                data["model_files"][model_id]["location"]
+                data["model_files"][model_id][C.MODEL_LOCATION]
             )
         if data["id"] is None:
             # The schema requires a valid id or no id field at all.
             del data["id"]
 
+        for ext_id, d_ext in data[C.EXTENSIONS].items():
+            if ext_id == C.EXT_ID_SCIML:
+                # convert Paths to strings
+                for key in ("array_files", "hybridization_files"):
+                    d_ext[key] = list(map(str, d_ext[key]))
+                for nn in d_ext["neural_networks"]:
+                    d_ext["neural_networks"][nn][C.MODEL_LOCATION] = str(
+                        d_ext["neural_networks"][nn][C.MODEL_LOCATION]
+                    )
+
         write_yaml(data, filename)
 
     @property