From 70a31338f78bb3f1338d1c8cae6acdacb611dd49 Mon Sep 17 00:00:00 2001 From: jo Date: Wed, 15 Jun 2022 15:59:18 +0200 Subject: [PATCH] feat(api): create bulk_import command - filter by allowed_extensions --- .../storage/management/__init__.py | 0 .../storage/management/commands/__init__.py | 0 .../management/commands/bulk_import.py | 162 ++++++++++++++++++ .../storage/tests/management/__init__.py | 0 .../tests/management/commands/__init__.py | 0 .../management/commands/test_bulk_import.py | 118 +++++++++++++ api/requirements.txt | 1 + api/setup.py | 2 + docs/admin-manual/library.md | 16 ++ 9 files changed, 299 insertions(+) create mode 100644 api/libretime_api/storage/management/__init__.py create mode 100644 api/libretime_api/storage/management/commands/__init__.py create mode 100644 api/libretime_api/storage/management/commands/bulk_import.py create mode 100644 api/libretime_api/storage/tests/management/__init__.py create mode 100644 api/libretime_api/storage/tests/management/commands/__init__.py create mode 100644 api/libretime_api/storage/tests/management/commands/test_bulk_import.py create mode 100644 docs/admin-manual/library.md diff --git a/api/libretime_api/storage/management/__init__.py b/api/libretime_api/storage/management/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/api/libretime_api/storage/management/commands/__init__.py b/api/libretime_api/storage/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/api/libretime_api/storage/management/commands/bulk_import.py b/api/libretime_api/storage/management/commands/bulk_import.py new file mode 100644 index 000000000..c197da98e --- /dev/null +++ b/api/libretime_api/storage/management/commands/bulk_import.py @@ -0,0 +1,162 @@ +import logging +from pathlib import Path +from typing import List, Optional + +import requests +from django.conf import settings +from django.core.management.base import BaseCommand, CommandParser +from libretime_shared.files import compute_md5 + +logger = logging.getLogger(__name__) + +DEFAULT_ALLOWED_EXTENSIONS = [ + ".flac", + ".m4a", + ".mp3", + ".ogg", + ".opus", + ".wav", +] + + +class Command(BaseCommand): + help = "Bulk file upload." + + def add_arguments(self, parser: CommandParser): + parser.add_argument( + "--path", + help="Path to the directory to scan.", + required=True, + ) + parser.add_argument( + "--track-type", + help="Track type for the new files.", + ) + parser.add_argument( + "--allowed-extensions", + help="Allowed file extensions.", + action="append", + default=DEFAULT_ALLOWED_EXTENSIONS, + ) + parser.add_argument( + "--delete-after-upload", + help="Delete file if upload succeeded.", + action="store_true", + ) + parser.add_argument( + "--delete-if-exists", + help="Delete file if it already exists.", + action="store_true", + ) + + def handle(self, *args, **options): + url = settings.CONFIG.general.public_url + auth_key = settings.CONFIG.general.api_key + + delete_after_upload = options.get("delete_after_upload", False) + delete_if_exists = options.get("delete_if_exists", False) + + path = options.get("path") + track_type = options.get("track_type", None) + allowed_extensions = options.get("allowed_extensions") + + importer = Importer(url, auth_key, delete_after_upload, delete_if_exists) + importer.import_dir(Path(path).resolve(), track_type, allowed_extensions) + + +class Importer: + def __init__( + self, + url: str, + auth_key: str, + delete_after_upload: bool = False, + delete_if_exists: bool = False, + ) -> None: + self.url = url + self.auth_key = auth_key + + self.delete_after_upload = delete_after_upload + self.delete_if_exists = delete_if_exists + + def _check_file_md5(self, filepath: Path) -> bool: + from ...models import File + + file_md5 = compute_md5(filepath) + + return File.objects.filter(md5=file_md5).exists() + + def _upload_file(self, filepath: Path, track_type: Optional[str]) -> None: + try: + resp = requests.post( + f"{self.url}/rest/media", + auth=(self.auth_key, ""), + files=[ + ("file", (filepath.name, filepath.open("rb"))), + ], + timeout=30, + cookies={"tt_upload": track_type} if track_type is not None else {}, + ) + resp.raise_for_status() + + except requests.exceptions.HTTPError as exception: + raise RuntimeError(f"could not upload {filepath}") from exception + + def _delete_file(self, filepath: Path) -> None: + logger.info(f"deleting {filepath}") + filepath.unlink() + + def _handle_file(self, filepath: Path, track_type: Optional[str]) -> None: + logger.debug(f"handling file {filepath}") + + if not filepath.is_file(): + raise ValueError(f"provided path {filepath} is not a file") + + if self._check_file_md5(filepath): + logger.info(f"found similar md5sum, ignoring {filepath}") + if self.delete_if_exists: + self._delete_file(filepath) + return + + self._upload_file(filepath, track_type) + + if self.delete_after_upload: + self._delete_file(filepath) + + def _walk_dir( + self, + path: Path, + track_type: Optional[str], + allowed_extensions: List[str], + ) -> None: + if not path.is_dir(): + raise ValueError(f"provided path {path} is not a directory") + + for sub_path in path.iterdir(): + if sub_path.is_dir(): + self._walk_dir(sub_path, track_type, allowed_extensions) + continue + + if sub_path.suffix.lower() not in allowed_extensions: + continue + + self._handle_file(sub_path.resolve(), track_type) + + def _check_track_type(self, track_type: str) -> bool: + from ...models import TrackType + + return TrackType.objects.filter(code=track_type).exists() + + def import_dir( + self, + path: Path, + track_type: Optional[str], + allowed_extensions: List[str], + ) -> None: + if track_type is not None and not self._check_track_type(track_type): + raise ValueError(f"provided track type {track_type} does not exist") + + allowed_extensions = [ + (x if x.startswith(".") else "." + x) for x in allowed_extensions + ] + + self._walk_dir(path, track_type, allowed_extensions) diff --git a/api/libretime_api/storage/tests/management/__init__.py b/api/libretime_api/storage/tests/management/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/api/libretime_api/storage/tests/management/commands/__init__.py b/api/libretime_api/storage/tests/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/api/libretime_api/storage/tests/management/commands/test_bulk_import.py b/api/libretime_api/storage/tests/management/commands/test_bulk_import.py new file mode 100644 index 000000000..e752cf010 --- /dev/null +++ b/api/libretime_api/storage/tests/management/commands/test_bulk_import.py @@ -0,0 +1,118 @@ +# pylint: disable=protected-access + +from pathlib import Path +from shutil import copy +from typing import Tuple +from unittest.mock import MagicMock + +import pytest +from model_bakery import baker +from requests_mock import Mocker + +from ....._fixtures import AUDIO_FILENAME, fixture_path +from ....management.commands.bulk_import import Importer + +FAKE_URL = "https://somehost.com" + + +@pytest.fixture(name="import_paths") +def _import_paths(tmp_path: Path): + sub_dir = tmp_path / "dir1/dir2" + sub_dir.mkdir(parents=True) + + test_file = sub_dir / AUDIO_FILENAME + copy(fixture_path / AUDIO_FILENAME, test_file) + + return (tmp_path, test_file) + + +@pytest.fixture(name="track_type") +def _track_type(): + return baker.make( + "storage.TrackType", + code="MUS", + type_name="Music", + description="Description", + ) + + +@pytest.fixture(name="importer") +def _importer(requests_mock: Mocker): + requests_mock.post(f"{FAKE_URL}/rest/media", status_code=200) + + obj = Importer(FAKE_URL, "auth") + obj._handle_file = MagicMock(wraps=obj._handle_file) + obj._upload_file = MagicMock(wraps=obj._upload_file) + obj._delete_file = MagicMock(wraps=obj._delete_file) + + yield obj + + +def test_importer( + db, + import_paths: Tuple[Path, Path], + importer: Importer, + track_type, +): + importer.import_dir(import_paths[0], track_type.code, [".mp3"]) + + importer._handle_file.assert_called_with(import_paths[1], track_type.code) + importer._upload_file.assert_called_with(import_paths[1], track_type.code) + importer._delete_file.assert_not_called() + + +def test_importer_and_delete( + db, + import_paths: Tuple[Path, Path], + importer: Importer, + track_type, +): + importer.delete_after_upload = True + importer.import_dir(import_paths[0], track_type.code, [".mp3"]) + + importer._handle_file.assert_called_with(import_paths[1], track_type.code) + importer._upload_file.assert_called_with(import_paths[1], track_type.code) + importer._delete_file.assert_called_with(import_paths[1]) + + +def test_importer_existing_file( + db, + import_paths: Tuple[Path, Path], + importer: Importer, + track_type, +): + baker.make("storage.File", md5="46305a7cf42ee53976c88d337e47e940") + + importer.import_dir(import_paths[0], track_type.code, [".mp3"]) + + importer._handle_file.assert_called_with(import_paths[1], track_type.code) + importer._upload_file.assert_not_called() + importer._delete_file.assert_not_called() + + +def test_importer_existing_file_and_delete( + db, + import_paths: Tuple[Path, Path], + importer: Importer, + track_type, +): + baker.make("storage.File", md5="46305a7cf42ee53976c88d337e47e940") + + importer.delete_if_exists = True + importer.import_dir(import_paths[0], track_type.code, [".mp3"]) + + importer._handle_file.assert_called_with(import_paths[1], track_type.code) + importer._upload_file.assert_not_called() + importer._delete_file.assert_called_with(import_paths[1]) + + +def test_importer_missing_track_type( + db, + import_paths: Tuple[Path, Path], + importer: Importer, +): + with pytest.raises( + ValueError, + match="provided track type MISSING does not exist", + ): + importer.import_dir(import_paths[0], "MISSING", [".mp3"]) diff --git a/api/requirements.txt b/api/requirements.txt index 8fb78b937..9ef904775 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -7,3 +7,4 @@ djangorestframework~=3.13.1 drf-spectacular~=0.22.1 markdown psycopg2 +requests diff --git a/api/setup.py b/api/setup.py index 85f1f26cb..ebfd0fd85 100644 --- a/api/setup.py +++ b/api/setup.py @@ -38,6 +38,7 @@ setup( "django-filter~=21.1.0", "drf-spectacular~=0.22.1", "markdown", + "requests", ], extras_require={ "prod": [ @@ -50,6 +51,7 @@ setup( "psycopg2-binary", "pylint-django", "pytest-django", + "requests-mock", f"libretime-shared @ file://localhost/{here.parent / 'shared'}#egg=libretime_shared", ], }, diff --git a/docs/admin-manual/library.md b/docs/admin-manual/library.md new file mode 100644 index 000000000..288a9e853 --- /dev/null +++ b/docs/admin-manual/library.md @@ -0,0 +1,16 @@ +--- +title: Library management +sidebar_position: 30 +--- + +This page describe the available options to manage the LibreTime library. + +## Files bulk import + +To scan a directory and import the files into the library, you can use the following command: + +```bash +sudo -u www-data libretime-api bulk_import --path PATH_THE_DIRECTORY_TO_SCAN +``` + +See the command usage to get available options.