Congegni/sintonia
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Fix trailing whitespaces in files

Browse source
This commit is contained in:
jo 2021-05-27 16:33:20 +02:00
parent 1af089219f
commit d8195f0fd8
46 changed files with 240 additions and 241 deletions
Download patch file Download diff file Expand all files Collapse all files

30
python_apps/airtime_analyzer/airtime_analyzer/message_listener.py
View file

@ -19,36 +19,36 @@ QUEUE = "airtime-uploads"
""" A message listener class that waits for messages from Airtime through RabbitMQ
notifying us about new uploads.
This is probably the most important class in this application. It connects
to RabbitMQ (or an AMQP server) and listens for messages that notify us
when a user uploads a new file to Airtime, either through the web interface
or via FTP (on Airtime Pro). When we get a notification, we spawn a child
process that extracts the uploaded audio file's metadata and moves it into
Airtime's music library directory. Lastly, the extracted metadata is
Airtime's music library directory. Lastly, the extracted metadata is
reported back to the Airtime web application.
There's a couple of Very Important technical details and constraints that you
need to know if you're going to work on this code:
1) airtime_analyzer is designed so it doesn't have to run on the same
1) airtime_analyzer is designed so it doesn't have to run on the same
computer as the web server. It just needs access to your Airtime library
folder (stor).
folder (stor).
2) airtime_analyzer is multi-tenant - One process can be used for many
Airtime instances. It's designed NOT to know about whether it's running
in a single tenant or multi-tenant environment. All the information it
in a single tenant or multi-tenant environment. All the information it
needs to import a file into an Airtime instance is passed in via those
RabbitMQ messages.
3) We're using a "topic exchange" for the new upload notification RabbitMQ
messages. This means if we run several airtime_analyzer processes on
messages. This means if we run several airtime_analyzer processes on
different computers, RabbitMQ will do round-robin dispatching of the
file notification. This is cheap, easy load balancing and
redundancy for us. You can even run multiple airtime_analyzer processes
on one machine if you want.
4) We run the actual work (metadata analysis and file moving) in a separate
child process so that if it crashes, we can stop RabbitMQ from resending
the file notification message to another airtime_analyzer process (NACK),
which would otherwise cause cascading failure. We also do this so that we
the file notification message to another airtime_analyzer process (NACK),
which would otherwise cause cascading failure. We also do this so that we
can report the problem file to the Airtime web interface ("import failed").
So that is a quick overview of the design constraints for this application, and
@ -164,10 +164,10 @@ class MessageListener:
api_key = ""
file_prefix = ""
""" Spin up a worker process. We use the multiprocessing module and multiprocessing.Queue
""" Spin up a worker process. We use the multiprocessing module and multiprocessing.Queue
to pass objects between the processes so that if the analyzer process crashes, it does not
take down the rest of the daemon and we NACK that message so that it doesn't get
propagated to other airtime_analyzer daemons (eg. running on other servers).
take down the rest of the daemon and we NACK that message so that it doesn't get
propagated to other airtime_analyzer daemons (eg. running on other servers).
We avoid cascading failure this way.
"""
try:
@ -208,8 +208,8 @@ class MessageListener:
except Exception as e:
logging.exception(e)
""" If ANY exception happens while processing a file, we're going to NACK to the
messaging server and tell it to remove the message from the queue.
""" If ANY exception happens while processing a file, we're going to NACK to the
messaging server and tell it to remove the message from the queue.
(NACK is a negative acknowledgement. We could use ACK instead, but this might come
in handy in the future.)
Exceptions in this context are unexpected, unhandled errors. We try to recover

8
python_apps/airtime_analyzer/airtime_analyzer/status_reporter.py
View file

@ -216,7 +216,7 @@ class StatusReporter:
# r = requests.Request(method='PUT', url=callback_url, data=put_payload,
# auth=requests.auth.HTTPBasicAuth(api_key, ''))
"""
r = requests.Request(method='PUT', url=callback_url, data=put_payload,
r = requests.Request(method='PUT', url=callback_url, data=put_payload,
auth=requests.auth.HTTPBasicAuth(api_key, ''))
StatusReporter._send_http_request(r)
@ -235,11 +235,11 @@ class StatusReporter:
StatusReporter._ipc_queue.put(r.prepare())
"""
"""
"""
# Encode the audio metadata as json and post it back to the callback_url
put_payload = json.dumps(audio_metadata)
logging.debug("sending http put with payload: " + put_payload)
r = requests.put(callback_url, data=put_payload,
r = requests.put(callback_url, data=put_payload,
auth=requests.auth.HTTPBasicAuth(api_key, ''),
timeout=StatusReporter._HTTP_REQUEST_TIMEOUT)
logging.debug("HTTP request returned status: " + str(r.status_code))
@ -266,7 +266,7 @@ class StatusReporter:
put_payload = json.dumps(audio_metadata)
# logging.debug("sending http put with payload: " + put_payload)
"""
r = requests.put(callback_url, data=put_payload,
r = requests.put(callback_url, data=put_payload,
auth=requests.auth.HTTPBasicAuth(api_key, ''),
timeout=StatusReporter._HTTP_REQUEST_TIMEOUT)
"""

16
python_apps/airtime_analyzer/tools/ftp-upload-hook.sh
View file

@ -16,27 +16,27 @@ post_file() {
mv "${file_path}" "${stripped_file_path}"
file_path="${stripped_file_path}"
filename="${file_path##*/}"
airtime_conf_path=/etc/airtime/airtime.conf
#instance_path will look like 1/1384, for example
http_path=$(grep base_url ${airtime_conf_path} | awk '{print $3;}' )
http_port=$(grep base_port ${airtime_conf_path} | awk '{print $3;}' )
http_port=$(grep base_port ${airtime_conf_path} | awk '{print $3;}' )
#post request url - http://bananas.airtime.pro/rest/media, for example
url=http://
url+=$http_path
url+=:
url+=$http_port
url+=/rest/media
api_key=$(grep api_key ${airtime_conf_path} | awk '{print $3;}' )
api_key=$(grep api_key ${airtime_conf_path} | awk '{print $3;}' )
# -f is needed to make curl fail if there's an HTTP error code
# -L is needed to follow redirects! (just in case)
until curl -fL --max-time 30 $url -u $api_key":" -X POST -F "file=@${file_path}"
until curl -fL --max-time 30 $url -u $api_key":" -X POST -F "file=@${file_path}"
do
retry_count=$[$retry_count+1]
if [ $retry_count -ge $max_retry ]; then

8
python_apps/airtime_analyzer/tools/message_sender.php
View file

@ -21,8 +21,8 @@ if ($argc <= 1)
exit();
}
$message = $argv[1];
$message = $argv[1];
$connection = new AMQPConnection(HOST, PORT, USER, PASS, VHOST);
if (!isset($connection))
{
@ -31,10 +31,10 @@ if (!isset($connection))
}
$channel = $connection->channel();
// declare/create the queue
$channel->queue_declare($queue, false, true, false, false);
// declare/create the exchange as a topic exchange.
$channel->exchange_declare($exchange, $exchangeType, false, true, false);

2
python_apps/airtime_analyzer/tools/test-hook-script.sh
View file

@ -3,7 +3,7 @@
post_file() {
file_path=${1}
filename="${file_path##*/}"
#kill process after 30 minutes (360*5=30 minutes)
max_retry=10
retry_count=0