Added support for FileField and ImageField from sqlalchemy-file

python-ellar · eadwinCode · Jun 10, 2024 · Jun 8, 2024 · Jun 8, 2024 · Jun 8, 2024
commit d3b00b89d900a7e99c8d30dbdbd95145019c471b
diff --git a/Makefile b/Makefile
@@ -11,6 +11,7 @@ clean: ## Removing cached python compiled files
 	find . -name __pycache__  | xargs  rm -rfv
 	find . -name .pytest_cache  | xargs  rm -rfv
 	find . -name .ruff_cache  | xargs  rm -rfv
+	find . -name .mypy_cache  | xargs  rm -rfv
 
 install: ## Install dependencies
 	pip install -r requirements.txt

diff --git a/ellar_sql/constant.py b/ellar_sql/constant.py
@@ -13,6 +13,7 @@
     "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
     "pk": "pk_%(table_name)s",
 }
+DEFAULT_STORAGE_PLACEHOLDER = "DEFAULT_STORAGE_PLACEHOLDER".lower()
 
 
 class DeclarativeBasePlaceHolder(sa_orm.DeclarativeBase):

diff --git a/ellar_sql/model/typeDecorator/__init__.py b/ellar_sql/model/typeDecorator/__init__.py
@@ -1,7 +1,10 @@
+from .file import File, FileField, ImageField
 from .guid import GUID
 from .ipaddress import GenericIP
 
 __all__ = [
     "GUID",
     "GenericIP",
+    "FileField",
+    "ImageField",
 ]
diff --git a/ellar_sql/model/typeDecorator/file/__init__.py b/ellar_sql/model/typeDecorator/file/__init__.py
@@ -0,0 +1,20 @@
+from .file import File
+from .file_tracker import ModifiedFileFieldSessionTracker
+from .processors import Processor, ThumbnailGenerator
+from .types import FileField, ImageField
+from .validators import ContentTypeValidator, ImageValidator, SizeValidator, Validator
+
+__all__ = [
+    "ImageValidator",
+    "SizeValidator",
+    "Validator",
+    "ContentTypeValidator",
+    "File",
+    "FileField",
+    "ImageField",
+    "Processor",
+    "ThumbnailGenerator",
+]
+
+
+ModifiedFileFieldSessionTracker.setup()
diff --git a/ellar_sql/model/typeDecorator/file/exceptions.py b/ellar_sql/model/typeDecorator/file/exceptions.py
@@ -0,0 +1,6 @@
+from sqlalchemy_file.exceptions import ContentTypeValidationError  # noqa
+from sqlalchemy_file.exceptions import InvalidImageError  # noqa
+from sqlalchemy_file.exceptions import DimensionValidationError  # noqa
+from sqlalchemy_file.exceptions import AspectRatioValidationError  # noqa
+from sqlalchemy_file.exceptions import SizeValidationError  # noqa
+from sqlalchemy_file.exceptions import ValidationError  # noqa
diff --git a/ellar_sql/model/typeDecorator/file/file.py b/ellar_sql/model/typeDecorator/file/file.py
@@ -0,0 +1,125 @@
+import typing as t
+import uuid
+import warnings
+from datetime import datetime
+
+from ellar.app import current_injector
+from ellar_storage import StorageService, StoredFile
+from sqlalchemy_file.file import File as BaseFile
+
+from ellar_sql.constant import DEFAULT_STORAGE_PLACEHOLDER
+
+
+class File(BaseFile):
+    """Takes a file as content and uploads it to the appropriate storage
+    according to the attached Column and file information into the
+    database as JSON.
+
+    Default attributes provided for all ``File`` include:
+
+    Attributes:
+        filename (str):  This is the name of the uploaded file
+        file_id:   This is the generated UUID for the uploaded file
+        upload_storage:   Name of the storage used to save the uploaded file
+        path:            This is a  combination of `upload_storage` and `file_id` separated by
+                        `/`. This will be use later to retrieve the file
+        content_type:   This is the content type of the uploaded file
+        uploaded_at (datetime):    This is the upload date in ISO format
+        url (str):            CDN url of the uploaded file
+        file:           Only available for saved content, internally call
+                      [StorageManager.get_file()][sqlalchemy_file.storage.StorageManager.get_file]
+                      on path and return an instance of `StoredFile`
+    """
+
+    def __init__(
+        self,
+        content: t.Any = None,
+        filename: t.Optional[str] = None,
+        content_type: t.Optional[str] = None,
+        content_path: t.Optional[str] = None,
+        **kwargs: t.Dict[str, t.Any],
+    ) -> None:
+        super().__init__(
+            content=content,
+            filename=filename,
+            content_path=content_path,
+            content_type=content_type,
+            **kwargs,
+        )
+
+    def save_to_storage(self, upload_storage: t.Optional[str] = None) -> None:
+        """Save current file into provided `upload_storage`."""
+        storage_service = current_injector.get(StorageService)
+        valid_upload_storage = storage_service.get_container(
+            upload_storage
+            if not upload_storage == DEFAULT_STORAGE_PLACEHOLDER
+            else None
+        ).name
+
+        extra = self.get("extra", {})
+        extra.update({"content_type": self.content_type})
+
+        metadata = self.get("metadata", None)
+        if metadata is not None:
+            warnings.warn(
+                'metadata attribute is deprecated. Use extra={"meta_data": ...} instead',
+                DeprecationWarning,
+                stacklevel=1,
+            )
+            extra.update({"meta_data": metadata})
+
+        if extra.get("meta_data", None) is None:
+            extra["meta_data"] = {}
+
+        extra["meta_data"].update(
+            {"filename": self.filename, "content_type": self.content_type}
+        )
+        stored_file = self.store_content(
+            self.original_content,
+            valid_upload_storage,
+            extra=extra,
+            headers=self.get("headers", None),
+            content_path=self.content_path,
+        )
+        self["file_id"] = stored_file.name
+        self["upload_storage"] = valid_upload_storage
+        self["uploaded_at"] = datetime.utcnow().isoformat()
+        self["path"] = f"{valid_upload_storage}/{stored_file.name}"
+        self["url"] = stored_file.get_cdn_url()
+        self["saved"] = True
+
+    def store_content(  # type:ignore[override]
+        self,
+        content: t.Any,
+        upload_storage: t.Optional[str] = None,
+        name: t.Optional[str] = None,
+        metadata: t.Optional[t.Dict[str, t.Any]] = None,
+        extra: t.Optional[t.Dict[str, t.Any]] = None,
+        headers: t.Optional[t.Dict[str, str]] = None,
+        content_path: t.Optional[str] = None,
+    ) -> StoredFile:
+        """Store content into provided `upload_storage`
+        with additional `metadata`. Can be used by processors
+        to store additional files.
+        """
+        name = name or str(uuid.uuid4())
+        storage_service = current_injector.get(StorageService)
+
+        stored_file = storage_service.save_content(
+            name=name,
+            content=content,
+            upload_storage=upload_storage,
+            metadata=metadata,
+            extra=extra,
+            headers=headers,
+            content_path=content_path,
+        )
+        self["files"].append(f"{upload_storage}/{name}")
+        return stored_file
+
+    @property
+    def file(self) -> StoredFile:  # type:ignore[override]
+        if self.get("saved", False):
+            storage_service = current_injector.get(StorageService)
+            return storage_service.get(self["path"])
+        raise RuntimeError("Only available for saved file")
diff --git a/ellar_sql/model/typeDecorator/file/file_tracker.py b/ellar_sql/model/typeDecorator/file/file_tracker.py
@@ -0,0 +1,41 @@
+import typing as t
+
+from ellar.app import current_injector
+from ellar_storage import StorageService
+from sqlalchemy import event, orm
+from sqlalchemy_file.types import FileFieldSessionTracker
+
+
+class ModifiedFileFieldSessionTracker(FileFieldSessionTracker):
+    @classmethod
+    def delete_files(cls, paths: t.Set[str], ctx: str) -> None:
+        if len(paths) == 0:
+            return
+
+        storage_service = current_injector.get(StorageService)
+
+        for path in paths:
+            storage_service.delete(path)
+
+    @classmethod
+    def unsubscribe_defaults(cls) -> None:
+        event.remove(
+            orm.Mapper, "mapper_configured", FileFieldSessionTracker._mapper_configured
+        )
+        event.remove(
+            orm.Mapper, "after_configured", FileFieldSessionTracker._after_configured
+        )
+        event.remove(orm.Session, "after_commit", FileFieldSessionTracker._after_commit)
+        event.remove(
+            orm.Session,
+            "after_soft_rollback",
+            FileFieldSessionTracker._after_soft_rollback,
+        )
+
+    @classmethod
+    def setup(cls) -> None:
+        cls.unsubscribe_defaults()
+        event.listen(orm.Mapper, "mapper_configured", cls._mapper_configured)
+        event.listen(orm.Mapper, "after_configured", cls._after_configured)
+        event.listen(orm.Session, "after_commit", cls._after_commit)
+        event.listen(orm.Session, "after_soft_rollback", cls._after_soft_rollback)
diff --git a/ellar_sql/model/typeDecorator/file/processors.py b/ellar_sql/model/typeDecorator/file/processors.py
@@ -0,0 +1,2 @@
+from sqlalchemy_file.processors import Processor  # noqa
+from sqlalchemy_file.processors import ThumbnailGenerator  # noqa
diff --git a/ellar_sql/model/typeDecorator/file/types.py b/ellar_sql/model/typeDecorator/file/types.py
@@ -0,0 +1,99 @@
+import typing as t
+
+from ellar.pydantic.types import Validator
+from sqlalchemy_file.types import FileField as BaseFileField
+
+from ellar_sql.constant import DEFAULT_STORAGE_PLACEHOLDER
+
+from .file import File
+from .processors import Processor, ThumbnailGenerator
+from .validators import ImageValidator
+
+
+class FileField(BaseFileField):
+    def __init__(
+        self,
+        *args: t.Tuple[t.Any],
+        upload_storage: t.Optional[str] = None,
+        validators: t.Optional[t.List[Validator]] = None,
+        processors: t.Optional[t.List[Processor]] = None,
+        upload_type: t.Type[File] = File,
+        multiple: t.Optional[bool] = False,
+        extra: t.Optional[t.Dict[str, t.Any]] = None,
+        headers: t.Optional[t.Dict[str, str]] = None,
+        **kwargs: t.Dict[str, t.Any],
+    ) -> None:
+        """Parameters:
+        upload_storage: storage to use
+        validators: List of validators to apply
+        processors: List of validators to apply
+        upload_type: File class to use, could be
+        used to set custom File class
+        multiple: Use this to save multiple files
+        extra: Extra attributes (driver specific)
+        headers: Additional request headers,
+        such as CORS headers. For example:
+        headers = {'Access-Control-Allow-Origin': 'http://mozilla.com'}.
+        """
+        super().__init__(
+            *args,
+            processors=processors,
+            validators=validators,
+            headers=headers,
+            upload_type=upload_type,
+            multiple=multiple,
+            extra=extra,
+            **kwargs,  # type: ignore[arg-type]
+        )
+        self.upload_storage = upload_storage or DEFAULT_STORAGE_PLACEHOLDER
+
+
+class ImageField(FileField):
+    def __init__(
+        self,
+        *args: t.Tuple[t.Any],
+        upload_storage: t.Optional[str] = None,
+        thumbnail_size: t.Optional[t.Tuple[int, int]] = None,
+        image_validator: t.Optional[ImageValidator] = None,
+        validators: t.Optional[t.List[Validator]] = None,
+        processors: t.Optional[t.List[Processor]] = None,
+        upload_type: t.Type[File] = File,
+        multiple: t.Optional[bool] = False,
+        extra: t.Optional[t.Dict[str, str]] = None,
+        headers: t.Optional[t.Dict[str, str]] = None,
+        **kwargs: t.Dict[str, t.Any],
+    ) -> None:
+        """Parameters
+        upload_storage: storage to use
+        image_validator: ImageField use default image
+        validator, Use this property to customize it.
+        thumbnail_size: If set, a thumbnail will be generated
+        from original image using [ThumbnailGenerator]
+        [sqlalchemy_file.processors.ThumbnailGenerator]
+        validators: List of additional validators to apply
+        processors: List of validators to apply
+        upload_type: File class to use, could be
+        used to set custom File class
+        multiple: Use this to save multiple files
+        extra: Extra attributes (driver specific).
+        """
+        if validators is None:
+            validators = []
+        if image_validator is None:
+            image_validator = ImageValidator()
+        if thumbnail_size is not None:
+            if processors is None:
+                processors = []
+            processors.append(ThumbnailGenerator(thumbnail_size))
+        validators.append(image_validator)
+        super().__init__(
+            *args,
+            upload_storage=upload_storage,
+            validators=validators,
+            processors=processors,
+            upload_type=upload_type,
+            multiple=multiple,
+            extra=extra,
+            headers=headers,
+            **kwargs,
+        )
diff --git a/ellar_sql/model/typeDecorator/file/validators.py b/ellar_sql/model/typeDecorator/file/validators.py
@@ -0,0 +1,4 @@
+from sqlalchemy_file.validators import Validator  # noqa
+from sqlalchemy_file.validators import ImageValidator  # noqa
+from sqlalchemy_file.validators import SizeValidator  # noqa
+from sqlalchemy_file.validators import ContentTypeValidator  # noqa
diff --git a/pyproject.toml b/pyproject.toml
@@ -34,7 +34,9 @@ classifiers = [
 dependencies = [
     "ellar-cli >= 0.3.7",
     "sqlalchemy >= 2.0.23",
-    "alembic >= 1.10.0"
+    "alembic >= 1.10.0",
+    "ellar-storage >= 0.1.2",
+    "sqlalchemy-file >= 0.6.0",
 ]
 
 [project.optional-dependencies]

diff --git a/requirements-tests.txt b/requirements-tests.txt
@@ -5,6 +5,7 @@ ellar-cli >= 0.3.7
 factory-boy >= 3.3.0
 httpx
 mypy == 1.10.0
+Pillow >=9.4.0, <10.1.0
 pytest >= 7.1.3,< 9.0.0
 pytest-asyncio
 pytest-cov >= 2.12.0,< 6.0.0

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -1,7 +1,9 @@
+import os
 import typing as t
 
 import pytest
 from ellar.testing import Test
+from ellar_storage import Provider, StorageModule, get_driver
 
 from ellar_sql import EllarSQLModule, EllarSQLService, model
 from ellar_sql.model.database_binds import __model_database_metadata__
@@ -97,9 +99,24 @@ def _setup(**kwargs):
                 }
             },
         )
+        storage_config = kwargs.setdefault("config_module", {}).setdefault(
+            "STORAGE_CONFIG", {}
+        )
+        storage_config.setdefault(
+            "storages",
+            {
+                "test": {
+                    "driver": get_driver(Provider.LOCAL),
+                    "options": {"key": os.path.join(tmp_path, "media")},
+                }
+            },
+        )
         sql_module.setdefault("migration_options", {"directory": "migrations"})
         tm = Test.create_test_module(
-            modules=[EllarSQLModule.setup(root_path=str(tmp_path), **sql_module)],
+            modules=[
+                EllarSQLModule.setup(root_path=str(tmp_path), **sql_module),
+                StorageModule.register_setup(),
+            ],
             **kwargs,
         )
         return tm.create_application()
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		from sqlalchemy_file.processors import Processor # noqa
		from sqlalchemy_file.processors import ThumbnailGenerator # noqa