From e21abf1bf1f2f5c197641d5aed2c5addce576a70 Mon Sep 17 00:00:00 2001 From: Lucas Bickel Date: Wed, 30 Dec 2020 13:17:50 +0000 Subject: [PATCH] refactor(airtime_analyzer): rename to libretime-analyzer and make entrypoint pythonic --- airtime_mvc/application/models/StoredFile.php | 8 +- .../application/services/MediaService.php | 4 +- airtime_mvc/build/airtime-setup/load.php | 2 +- install | 4 +- python_apps/airtime_analyzer/README.md | 102 ++++++++++++++++++ python_apps/airtime_analyzer/README.rst | 95 ---------------- .../cli.py} | 12 +-- .../systemd/libretime-analyzer.service | 2 +- .../install/sysvinit/libretime-analyzer | 6 +- .../install/upstart/libretime-analyzer.conf | 4 +- .../airtime_analyzer/requirements-dev.txt | 3 + python_apps/airtime_analyzer/setup.py | 79 +++++--------- .../tests/metadata_analyzer_tests.py | 2 +- .../tests/test_data/unparsable.txt | 1 + travis/install.sh | 1 + 15 files changed, 155 insertions(+), 170 deletions(-) create mode 100644 python_apps/airtime_analyzer/README.md delete mode 100644 python_apps/airtime_analyzer/README.rst rename python_apps/airtime_analyzer/{bin/airtime_analyzer => airtime_analyzer/cli.py} (91%) create mode 100644 python_apps/airtime_analyzer/requirements-dev.txt create mode 100644 python_apps/airtime_analyzer/tests/test_data/unparsable.txt diff --git a/airtime_mvc/application/models/StoredFile.php b/airtime_mvc/application/models/StoredFile.php index e0d005391..1f0b55570 100644 --- a/airtime_mvc/application/models/StoredFile.php +++ b/airtime_mvc/application/models/StoredFile.php @@ -916,14 +916,14 @@ SQL; * on the local disk (like /tmp) over to Airtime's "stor" directory, * which is where all ingested music/media live. * - * This is done in PHP here on the web server rather than in airtime_analyzer because - * the airtime_analyzer might be running on a different physical computer than the web server, + * This is done in PHP here on the web server rather than in libretime-analyzer because + * the libretime-analyzer might be running on a different physical computer than the web server, * and it probably won't have access to the web server's /tmp folder. The stor/organize directory - * is, however, both accessible to the machines running airtime_analyzer and the web server + * is, however, both accessible to the machines running libretime-analyzer and the web server * on Airtime Pro. * * The file is actually copied to "stor/organize", which is a staging directory where files go - * before they're processed by airtime_analyzer, which then moves them to "stor/imported" in the final + * before they're processed by libretime-analyzer, which then moves them to "stor/imported" in the final * step. * * @param string $tempFilePath diff --git a/airtime_mvc/application/services/MediaService.php b/airtime_mvc/application/services/MediaService.php index 5ddad86bb..dc6a99a01 100644 --- a/airtime_mvc/application/services/MediaService.php +++ b/airtime_mvc/application/services/MediaService.php @@ -33,10 +33,10 @@ class Application_Service_MediaService } //Copy the temporary file over to the "organize" folder so that it's off our webserver - //and accessible by airtime_analyzer which could be running on a different machine. + //and accessible by libretime-analyzer which could be running on a different machine. $newTempFilePath = Application_Model_StoredFile::moveFileToStor($filePath, $originalFilename, $copyFile); - //Dispatch a message to airtime_analyzer through RabbitMQ, + //Dispatch a message to libretime-analyzer through RabbitMQ, //notifying it that there's a new upload to process! $storageBackend = new ProxyStorageBackend($CC_CONFIG["current_backend"]); Application_Model_RabbitMq::SendMessageToAnalyzer($newTempFilePath, diff --git a/airtime_mvc/build/airtime-setup/load.php b/airtime_mvc/build/airtime-setup/load.php index 31d20e0d6..46c210546 100644 --- a/airtime_mvc/build/airtime-setup/load.php +++ b/airtime_mvc/build/airtime-setup/load.php @@ -113,7 +113,7 @@ function checkRMQConnection() { * @return boolean true if airtime-analyzer is running */ function checkAnalyzerService() { - exec("pgrep -f airtime_analyzer", $out, $status); + exec("pgrep -f libretime-analyzer", $out, $status); if (($out > 0) && $status == 0) { return posix_kill(rtrim($out[0]), 0); } diff --git a/install b/install index ecb7ca8af..eb272b19b 100755 --- a/install +++ b/install @@ -1014,8 +1014,8 @@ loudCmd "usermod -G ${web_user} -a celery" systemInitInstall libretime-celery verbose "...Done" -verbose "\n * Installing airtime_analyzer..." -loudCmd "$python_bin ${AIRTIMEROOT}/python_apps/airtime_analyzer/setup.py install --install-scripts=/usr/bin --no-init-script" +verbose "\n * Installing libretime-analyzer..." +loudCmd "$python_bin ${AIRTIMEROOT}/python_apps/airtime_analyzer/setup.py install --install-scripts=/usr/bin" systemInitInstall libretime-analyzer $web_user verbose "...Done" diff --git a/python_apps/airtime_analyzer/README.md b/python_apps/airtime_analyzer/README.md new file mode 100644 index 000000000..b08d527d4 --- /dev/null +++ b/python_apps/airtime_analyzer/README.md @@ -0,0 +1,102 @@ +# libretime-analyzer + +libretime-analyzer is a daemon that processes LibreTime file uploads as background jobs. + +It performs metadata extraction using Mutagen and moves uploads into LibreTime's +music library directory (stor/imported). + +libretime-analyzer uses process isolation to make it resilient to crashes and runs in +a multi-tenant environment with no modifications. + +## Installation + +```bash +python setup.py install +``` + +You will need to allow the "airtime" RabbitMQ user to access all exchanges and queues within the /airtime vhost: + +```bash +rabbitmqctl set_permissions -p /airtime airtime .\* .\* .\* +``` + +## Usage + +This program must run as a user with permissions to write to your Airtime music library +directory. For standard Airtime installations, run it as the www-data user: + +```bash +sudo -u www-data libretime-analyzer --daemon +``` + +Or during development, add the --debug flag for more verbose output: + +```bash +sudo -u www-data libretime-analyzer --debug +``` + +To print usage instructions, run: + +```bash +libretime-analyzer --help +``` + +This application can be run as a daemon by running: + +```bash +libretime-analyzer -d +``` + +# Developers + +For development, you want to install libretime-analyzer system-wide but with everything symlinked back to the source +directory for convenience. This is super easy to do, just run: + +```bash +pip install -r requirements-dev.txt +pip install --editable . +``` + +To send an test message to libretime-analyzer, you can use the message_sender.php script in the tools directory. +For example, run: + +```bash +php tools/message_sender.php '{ "tmp_file_path" : "foo.mp3", "final_directory" : ".", "callback_url" : "http://airtime.localhost/rest/media/1", "api_key" : "YOUR_API_KEY" }' + +php tools/message_sender.php '{"tmp_file_path":"foo.mp3", "import_directory":"/srv/airtime/stor/imported/1","original_filename":"foo.mp3","callback_url": "http://airtime.localhost/rest/media/1", "api_key":"YOUR_API_KEY"}' +``` + +## Logging + +By default, logs are saved to: + +``` +/var/log/airtime/airtime_analyzer.log +``` + +This application takes care of rotating logs for you. + +## Unit Tests + +To run the unit tests, execute: + +```bash +nosetests +``` + +If you care about seeing console output (stdout), like when you're debugging or developing +a test, run: + +```bash +nosetests -s +``` + +To run the unit tests and generate a code coverage report, run: + +```bash +nosetests --with-coverage --cover-package=airtime_analyzer +``` + +## Running in a Multi-Tenant Environment + +## History and Design Motivation diff --git a/python_apps/airtime_analyzer/README.rst b/python_apps/airtime_analyzer/README.rst deleted file mode 100644 index 8ae0bdcac..000000000 --- a/python_apps/airtime_analyzer/README.rst +++ /dev/null @@ -1,95 +0,0 @@ -airtime_analyzer -========== - -airtime_analyzer is a daemon that processes Airtime file uploads as background jobs. -It performs metadata extraction using Mutagen and moves uploads into Airtime's -music library directory (stor/imported). - -airtime_analyzer uses process isolation to make it resilient to crashes and runs in -a multi-tenant environment with no modifications. - -Installation -========== - - $ sudo python setup.py install - -You will need to allow the "airtime" RabbitMQ user to access all exchanges and queues within the /airtime vhost: - - sudo rabbitmqctl set_permissions -p /airtime airtime .\* .\* .\* - - -Usage -========== - -This program must run as a user with permissions to write to your Airtime music library -directory. For standard Airtime installations, run it as the www-data user: - - $ sudo -u www-data airtime_analyzer --daemon - -Or during development, add the --debug flag for more verbose output: - - $ sudo -u www-data airtime_analyzer --debug - -To print usage instructions, run: - - $ airtime_analyzer --help - -This application can be run as a daemon by running: - - $ airtime_analyzer -d - -Other runtime flags can be listed by running: - - $ airtime_analyzer --help - - -Developers -========== - -For development, you want to install airtime_analyzer system-wide but with everything symlinked back to the source -directory for convenience. This is super easy to do, just run: - - $ sudo python setup.py develop - -To send an test message to airtime_analyzer, you can use the message_sender.php script in the tools directory. -For example, run: - - $ php tools/message_sender.php '{ "tmp_file_path" : "foo.mp3", "final_directory" : ".", "callback_url" : "http://airtime.localhost/rest/media/1", "api_key" : "YOUR_API_KEY" }' - - $ php tools/message_sender.php '{"tmp_file_path":"foo.mp3", "import_directory":"/srv/airtime/stor/imported/1","original_filename":"foo.mp3","callback_url": "http://airtime.localhost/rest/media/1", "api_key":"YOUR_API_KEY"}' - -Logging -========= - -By default, logs are saved to: - - /var/log/airtime/airtime_analyzer.log - -This application takes care of rotating logs for you. - - -Unit Tests -========== - -To run the unit tests, execute: - - $ nosetests - -If you care about seeing console output (stdout), like when you're debugging or developing -a test, run: - - $ nosetests -s - -To run the unit tests and generate a code coverage report, run: - - $ nosetests --with-coverage --cover-package=airtime_analyzer - - -Running in a Multi-Tenant Environment -=========== - - -History and Design Motivation -=========== - - diff --git a/python_apps/airtime_analyzer/bin/airtime_analyzer b/python_apps/airtime_analyzer/airtime_analyzer/cli.py similarity index 91% rename from python_apps/airtime_analyzer/bin/airtime_analyzer rename to python_apps/airtime_analyzer/airtime_analyzer/cli.py index 7b8ca69bb..4de4e260f 100755 --- a/python_apps/airtime_analyzer/bin/airtime_analyzer +++ b/python_apps/airtime_analyzer/airtime_analyzer/cli.py @@ -1,7 +1,6 @@ -#!/usr/bin/env python -"""Runs the airtime_analyzer application. """ - +Main CLI entrypoint for the libretime-analyzer app. +""" import daemon import argparse @@ -13,9 +12,9 @@ LIBRETIME_CONF_DIR = os.getenv('LIBRETIME_CONF_DIR', '/etc/airtime') DEFAULT_RMQ_CONFIG_PATH = os.path.join(LIBRETIME_CONF_DIR, 'airtime.conf') DEFAULT_HTTP_RETRY_PATH = '/tmp/airtime_analyzer_http_retries' -def run(): +def main(): '''Entry-point for this application''' - print("Airtime Analyzer {}".format(VERSION)) + print("LibreTime Analyzer {}".format(VERSION)) parser = argparse.ArgumentParser() parser.add_argument("-d", "--daemon", help="run as a daemon", action="store_true") parser.add_argument("--debug", help="log full debugging output", action="store_true") @@ -42,4 +41,5 @@ def run(): http_retry_queue_path=http_retry_queue_path, debug=args.debug) -run() +if __name__ == "__main__": + main() diff --git a/python_apps/airtime_analyzer/install/systemd/libretime-analyzer.service b/python_apps/airtime_analyzer/install/systemd/libretime-analyzer.service index c8983ff48..d646343ed 100644 --- a/python_apps/airtime_analyzer/install/systemd/libretime-analyzer.service +++ b/python_apps/airtime_analyzer/install/systemd/libretime-analyzer.service @@ -2,7 +2,7 @@ Description=LibreTime Media Analyzer Service [Service] -ExecStart=/usr/bin/airtime_analyzer +ExecStart=/usr/bin/libretime-analyzer User=libretime-analyzer Group=libretime-analyzer Restart=always diff --git a/python_apps/airtime_analyzer/install/sysvinit/libretime-analyzer b/python_apps/airtime_analyzer/install/sysvinit/libretime-analyzer index e341e4d0c..1b1c706e0 100755 --- a/python_apps/airtime_analyzer/install/sysvinit/libretime-analyzer +++ b/python_apps/airtime_analyzer/install/sysvinit/libretime-analyzer @@ -1,17 +1,17 @@ #!/bin/bash ### BEGIN INIT INFO -# Provides: airtime_analyzer +# Provides: libretime-analyzer # Required-Start: $local_fs $remote_fs $network $syslog $all # Required-Stop: $local_fs $remote_fs $network $syslog # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 -# Short-Description: Manage airtime_analyzer daemon +# Short-Description: Manage libretime-analyzer daemon ### END INIT INFO USERID=www-data GROUPID=www-data -NAME=airtime_analyzer +NAME=libretime-analyzer DAEMON=/usr/bin/$NAME PIDFILE=/var/run/$NAME.pid diff --git a/python_apps/airtime_analyzer/install/upstart/libretime-analyzer.conf b/python_apps/airtime_analyzer/install/upstart/libretime-analyzer.conf index eeeb45797..488c06d99 100644 --- a/python_apps/airtime_analyzer/install/upstart/libretime-analyzer.conf +++ b/python_apps/airtime_analyzer/install/upstart/libretime-analyzer.conf @@ -15,10 +15,10 @@ env LANG='en_US.UTF-8' env LC_ALL='en_US.UTF-8' #script -# airtime_analyzer +# libretime-analyzer #end script -exec airtime_analyzer +exec libretime-analyzer diff --git a/python_apps/airtime_analyzer/requirements-dev.txt b/python_apps/airtime_analyzer/requirements-dev.txt new file mode 100644 index 000000000..fb54268fa --- /dev/null +++ b/python_apps/airtime_analyzer/requirements-dev.txt @@ -0,0 +1,3 @@ +coverage +mock +nose diff --git a/python_apps/airtime_analyzer/setup.py b/python_apps/airtime_analyzer/setup.py index 7645fa9ac..8ca460894 100644 --- a/python_apps/airtime_analyzer/setup.py +++ b/python_apps/airtime_analyzer/setup.py @@ -1,58 +1,31 @@ -from __future__ import print_function from setuptools import setup -from subprocess import call -import sys import os # Change directory since setuptools uses relative paths -script_path = os.path.dirname(os.path.realpath(__file__)) -print(script_path) -os.chdir(script_path) +os.chdir(os.path.dirname(os.path.realpath(__file__))) -# Allows us to avoid installing the upstart init script when deploying airtime_analyzer -# on Airtime Pro: -if '--no-init-script' in sys.argv: - data_files = [] - sys.argv.remove('--no-init-script') # super hax -else: - data_files = [('/etc/init', ['install/upstart/airtime_analyzer.conf']), - ('/etc/init.d', ['install/sysvinit/airtime_analyzer'])] - print(data_files) - -setup(name='airtime_analyzer', - version='0.1', - description='Airtime Analyzer Worker and File Importer', - url='http://github.com/sourcefabric/Airtime', - author='Albert Santoni', - author_email='albert.santoni@sourcefabric.org', - license='MIT', - packages=['airtime_analyzer'], - scripts=['bin/airtime_analyzer'], - install_requires=[ - 'mutagen==1.42.0', - 'pika~=1.1.0', - 'file-magic', - 'nose', - 'coverage', - 'mock', - 'python-daemon', - 'requests>=2.7.0', - 'rgain3==1.0.0', - 'pycairo==1.19.1', - 'PyGObject<=3.36.1', - # These next 3 are required for requests to support SSL with SNI. Learned this the hard way... - # What sucks is that GCC is required to pip install these. - #'ndg-httpsclient', - #'pyasn1', - #'pyopenssl' - ], - zip_safe=False, - data_files=data_files) - -# Remind users to reload the initctl config so that "service start airtime_analyzer" works -if data_files: - print("Remember to reload the initctl configuration") - print("Run \"sudo initctl reload-configuration; sudo service airtime_analyzer restart\" now.") - print("Or on Ubuntu Xenial (16.04)") - print("Remember to reload the systemd configuration") - print("Run \"sudo systemctl daemon-reload; sudo service airtime_analyzer restart\" now.") +setup( + name="libretime-analyzer", + version="0.1", + description="Libretime Analyzer Worker and File Importer", + url="https://libretime.org", + author="LibreTime Contributors", + license="AGPLv3", + packages=["airtime_analyzer"], + entry_points={ + "console_scripts": [ + "libretime-analyzer=airtime_analyzer.cli:main", + ] + }, + install_requires=[ + "mutagen==1.42.0", + "pika~=1.1.0", + "file-magic", + "python-daemon", + "requests>=2.7.0", + "rgain3==1.0.0", + "pycairo==1.19.1", + "PyGObject<=3.36.1", + ], + zip_safe=False, +) diff --git a/python_apps/airtime_analyzer/tests/metadata_analyzer_tests.py b/python_apps/airtime_analyzer/tests/metadata_analyzer_tests.py index c92b82db6..e8e38f395 100644 --- a/python_apps/airtime_analyzer/tests/metadata_analyzer_tests.py +++ b/python_apps/airtime_analyzer/tests/metadata_analyzer_tests.py @@ -162,4 +162,4 @@ def test_mp3_bad_channels(): #Mutagen doesn't extract comments from mp3s it seems def test_unparsable_file(): - MetadataAnalyzer.analyze('README.rst', dict()) + MetadataAnalyzer.analyze('tests/test_data/unparsable.txt', dict()) diff --git a/python_apps/airtime_analyzer/tests/test_data/unparsable.txt b/python_apps/airtime_analyzer/tests/test_data/unparsable.txt new file mode 100644 index 000000000..073bdfe28 --- /dev/null +++ b/python_apps/airtime_analyzer/tests/test_data/unparsable.txt @@ -0,0 +1 @@ +test-file diff --git a/travis/install.sh b/travis/install.sh index edcaaffb3..dacafbf7f 100755 --- a/travis/install.sh +++ b/travis/install.sh @@ -7,6 +7,7 @@ if [[ -n "$TRAVIS_PHP_VERSION" ]]; then fi if [[ -n "$TRAVIS_PYTHON_VERSION" ]]; then pushd python_apps/airtime_analyzer + pip3 install -r requirements-dev.txt pip3 install -e . popd