Congegni
/
sintonia
7
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

more pypo fixes

This commit is contained in:
Kyle Robbertze 2020-01-23 12:37:49 +02:00
parent 44a0cb50e1
commit 6ebb1fd555
14 changed files with 132 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
installer/lib/requirements-debian-buster.apt
View File

@ -4,6 +4,8 @@ curl
ecasound
flac
git
gstreamer1.0-plugins-bad
gstreamer1.0-plugins-good
gstreamer1.0-plugins-ugly
icecast2
lame

2
installer/lib/requirements-debian-stretch.apt
View File

@ -4,6 +4,8 @@ curl
ecasound
flac
git
gstreamer1.0-plugins-bad
gstreamer1.0-plugins-good
gstreamer1.0-plugins-ugly
icecast2
lame

2
installer/lib/requirements-ubuntu-bionic.apt
View File

@ -4,6 +4,8 @@ coreutils
curl
ecasound
flac
gstreamer1.0-plugins-bad
gstreamer1.0-plugins-good
gstreamer1.0-plugins-ugly
icecast2
lame

2
installer/lib/requirements-ubuntu-xenial.apt
View File

@ -4,6 +4,8 @@ coreutils
curl
ecasound
flac
gstreamer1.0-plugins-bad
gstreamer1.0-plugins-good
gstreamer1.0-plugins-ugly
icecast2
lame

2
installer/lib/requirements-ubuntu-xenial_docker_minimal.apt
View File

@ -4,6 +4,8 @@ coreutils
curl
ecasound
flac
gstreamer1.0-plugins-bad
gstreamer1.0-plugins-good
gstreamer1.0-plugins-ugly
lame
libao-ocaml

43
python_apps/api_clients/api_clients/api_client.py
View File

@ -117,7 +117,8 @@ class ApiRequest(object):
def __call__(self,_post_data=None, **kwargs):
final_url = self.url.params(**kwargs).url()
if _post_data is not None: _post_data = urllib.parse.urlencode(_post_data)
if _post_data is not None:
_post_data = urllib.parse.urlencode(_post_data).encode('utf-8')
self.logger.debug(final_url)
try:
req = urllib.request.Request(final_url, _post_data)
@ -131,8 +132,7 @@ class ApiRequest(object):
self.logger.error('HTTP request to %s timed out', final_url)
raise
except Exception as e:
#self.logger.error('Exception: %s', e)
#self.logger.error("traceback: %s", traceback.format_exc())
#self.logger.exception(e)
raise
try:
@ -142,8 +142,7 @@ class ApiRequest(object):
else:
raise InvalidContentType()
except Exception:
#self.logger.error(response)
#self.logger.error("traceback: %s", traceback.format_exc())
#self.logger.exception(e)
raise
def req(self, *args, **kwargs):
@ -182,8 +181,10 @@ class RequestProvider(object):
def __contains__(self, request) : return request in self.requests
def __getattr__(self, attr):
if attr in self: return self.requests[attr]
else: return super(RequestProvider, self).__getattribute__(attr)
if attr in self:
return self.requests[attr]
else:
return super(RequestProvider, self).__getattribute__(attr)
class AirtimeApiClient(object):
@ -197,8 +198,7 @@ class AirtimeApiClient(object):
self.config.update(api_config)
self.services = RequestProvider(self.config)
except Exception as e:
self.logger.error('Error loading config file: %s', config_path)
self.logger.error("traceback: %s", traceback.format_exc())
self.logger.exception('Error loading config file: %s', config_path)
sys.exit(1)
def __get_airtime_version(self):
@ -239,7 +239,7 @@ class AirtimeApiClient(object):
try:
self.services.notify_liquidsoap_started()
except Exception as e:
self.logger.error(str(e))
self.logger.exception(e)
def notify_media_item_start_playing(self, media_id):
""" This is a callback from liquidsoap, we use this to notify
@ -248,14 +248,14 @@ class AirtimeApiClient(object):
try:
return self.services.update_start_playing_url(media_id=media_id)
except Exception as e:
self.logger.error(str(e))
self.logger.exception(e)
return None
def get_shows_to_record(self):
try:
return self.services.show_schedule_url()
except Exception as e:
self.logger.error(str(e))
self.logger.exception(e)
return None
def upload_recorded_show(self, files, show_id):
@ -307,8 +307,7 @@ class AirtimeApiClient(object):
logger.error("Server is down: %s", e.args)
logger.error("traceback: %s", traceback.format_exc())
except Exception as e:
logger.error("Exception: %s", e)
logger.error("traceback: %s", traceback.format_exc())
self.logger.exception(e)
#wait some time before next retry
time.sleep(retries_wait)
@ -320,7 +319,7 @@ class AirtimeApiClient(object):
return self.services.check_live_stream_auth(
username=username, password=password, djtype=dj_type)
except Exception as e:
self.logger.error(str(e))
self.logger.exception(e)
return {}
def construct_url(self,config_action_key):
@ -468,17 +467,14 @@ class AirtimeApiClient(object):
stream_id=stream_id,
boot_time=time).retry(5)
except Exception as e:
#TODO
logger.error("Exception: %s", e)
self.logger.exception(e)
def notify_source_status(self, sourcename, status):
try:
logger = self.logger
return self.services.update_source_status.req(sourcename=sourcename,
status=status).retry(5)
except Exception as e:
#TODO
logger.error("Exception: %s", e)
self.logger.exception(e)
def get_bootstrap_info(self):
""" Retrieve infomations needed on bootstrap time """
@ -494,7 +490,7 @@ class AirtimeApiClient(object):
try:
return self.services.get_files_without_replay_gain(dir_id=dir_id)
except Exception as e:
self.logger.error(str(e))
self.logger.exception(e)
return []
def get_files_without_silan_value(self):
@ -506,7 +502,7 @@ class AirtimeApiClient(object):
try:
return self.services.get_files_without_silan_value()
except Exception as e:
self.logger.error(str(e))
self.logger.exception(e)
return []
def update_replay_gain_values(self, pairs):
@ -549,8 +545,7 @@ class AirtimeApiClient(object):
response = self.services.update_stream_setting_table(_post_data={'data': json.dumps(data)})
return response
except Exception as e:
#TODO
self.logger.error(str(e))
self.logger.exception(e)
def update_metadata_on_tunein(self):
self.services.update_metadata_on_tunein()

11
python_apps/pypo/liquidsoap/__main__.py
View File

@ -1,12 +1,11 @@
""" Runs Airtime liquidsoap
"""
import argparse
import os
from . import generate_liquidsoap_cfg
import logging
import subprocess
from pypo import pure
PYPO_HOME = '/var/tmp/airtime/pypo/'
@ -16,16 +15,16 @@ def run():
parser = argparse.ArgumentParser()
parser.add_argument("-d", "--debug", help="run in debug mode", action="store_true")
args = parser.parse_args()
os.environ["HOME"] = PYPO_HOME
if args.debug:
logging.basicConfig(level=getattr(logging, 'DEBUG', None))
generate_liquidsoap_cfg.run()
''' check liquidsoap version if less than 1.3 use legacy liquidsoap script '''
liquidsoap_version=subprocess.check_output("liquidsoap --version", shell=True)
if "1.1.1" not in liquidsoap_version:
liquidsoap_version = subprocess.check_output("liquidsoap --version", shell=True, text=True)
if pure.version_cmp(liquidsoap_version, "1.3") < 0:
script_path = os.path.join(os.path.dirname(__file__), 'ls_script.liq')
else:
script_path = os.path.join(os.path.dirname(__file__), 'ls_script_legacy.liq')

8
python_apps/pypo/liquidsoap/generate_liquidsoap_cfg.py
View File

@ -28,21 +28,21 @@ def generate_liquidsoap_config(ss):
except: #Everything else is a string
str_buffer = "%s = \"%s\"\n" % (key, value)
fh.write(str_buffer.encode('utf-8'))
fh.write(str_buffer)
# ignore squashes unused variable errors from Liquidsoap
fh.write(("ignore(%s)\n" % key).encode('utf-8'))
fh.write("ignore(%s)\n" % key)
auth_path = os.path.dirname(os.path.realpath(__file__))
fh.write('log_file = "/var/log/airtime/pypo-liquidsoap/<script>.log"\n')
fh.write('auth_path = "%s/liquidsoap_auth.py"\n' % auth_path)
fh.close()
def run():
logging.basicConfig(format='%(message)s')
attempts = 0
max_attempts = 10
successful = False
while not successful:
try:
ac = AirtimeApiClient(logging.getLogger())

8
python_apps/pypo/pypo/__main__.py
View File

@ -155,11 +155,11 @@ def liquidsoap_get_info(telnet_lock, host, port, logger):
telnet_lock.acquire()
tn = telnetlib.Telnet(host, port)
msg = "version\n"
tn.write(msg)
tn.write("exit\n")
response = tn.read_all()
tn.write(msg.encode("utf-8"))
tn.write("exit\n".encode("utf-8"))
response = tn.read_all().decode("utf-8")
except Exception as e:
logger.error(str(e))
logger.error(e)
return None
finally:
telnet_lock.release()

12
python_apps/pypo/pypo/pure.py
View File

@ -1,10 +1,14 @@
import re
from packaging.version import Version, parse
def version_cmp(version1, version2):
def normalize(v):
return [int(x) for x in re.sub(r'(\.0+)*$','', v).split(".")]
return cmp(normalize(version1), normalize(version2))
version1 = parse(version1)
version2 = parse(version2)
if version1 > version2:
return 1
if version1 == version2:
return 0
return -1
def date_interval_to_seconds(interval):
"""

45
python_apps/pypo/pypo/pypofetch.py
View File

@ -10,9 +10,8 @@ import copy
import subprocess
import signal
from datetime import datetime
import traceback
from . import pure
import mimetypes
from . import pure
from queue import Empty
from threading import Thread, Timer
from subprocess import Popen, PIPE
@ -121,10 +120,7 @@ class PypoFetch(Thread):
self.listener_timeout = 0
self.logger.info("New timeout: %s" % self.listener_timeout)
except Exception as e:
top = traceback.format_exc()
self.logger.error('Exception: %s', e)
self.logger.error("traceback: %s", top)
self.logger.error("Exception in handling Message Handler message: %s", e)
self.logger.exception("Exception in handling Message Handler message")
def switch_source_temp(self, sourcename, status):
@ -152,8 +148,7 @@ class PypoFetch(Thread):
try:
info = self.api_client.get_bootstrap_info()
except Exception as e:
self.logger.error('Unable to get bootstrap info.. Exiting pypo...')
self.logger.error(str(e))
self.logger.exception('Unable to get bootstrap info.. Exiting pypo...')
self.logger.debug('info:%s', info)
commands = []
@ -199,7 +194,7 @@ class PypoFetch(Thread):
time.sleep(0.5)
except Exception as e:
self.logger.error(e)
self.logger.exception(e)
finally:
if self.telnet_lock.locked():
self.telnet_lock.release()
@ -238,7 +233,7 @@ class PypoFetch(Thread):
output = tn.read_all()
except Exception as e:
self.logger.error(str(e))
self.logger.exception(e)
finally:
self.telnet_lock.release()
@ -272,7 +267,7 @@ class PypoFetch(Thread):
tn.write('exit\n')
tn.read_all()
except Exception as e:
self.logger.error("Exception %s", e)
self.logger.exception(e)
finally:
self.telnet_lock.release()
@ -289,7 +284,7 @@ class PypoFetch(Thread):
tn.write('exit\n')
tn.read_all()
except Exception as e:
self.logger.error("Exception %s", e)
self.logger.exception(e)
finally:
self.telnet_lock.release()
@ -307,11 +302,11 @@ class PypoFetch(Thread):
tn.write('exit\n')
tn.read_all()
except Exception as e:
self.logger.error(str(e))
self.logger.exception(e)
finally:
self.telnet_lock.release()
except Exception as e:
self.logger.error("Exception %s", e)
self.logger.exception(e)
"""
Process the schedule
@ -344,7 +339,7 @@ class PypoFetch(Thread):
media_item = media[key]
if (media_item['type'] == 'file'):
fileExt = self.sanity_check_media_item(media_item)
dst = os.path.join(download_dir, media_item['id'] + fileExt)
dst = os.path.join(download_dir, "{}{}".format(media_item['id'], fileExt))
media_item['dst'] = dst
media_item['file_ready'] = False
media_filtered[key] = media_item
@ -358,7 +353,7 @@ class PypoFetch(Thread):
self.media_prepare_queue.put(copy.copy(media_filtered))
except Exception as e:
self.logger.error(e)
self.logger.exception(e)
# Send the data to pypo-push
self.logger.debug("Pushing to pypo-push")
@ -369,7 +364,7 @@ class PypoFetch(Thread):
try:
self.cache_cleanup(media)
except Exception as e:
self.logger.error(e)
self.logger.exception(e)
#do basic validation of file parameters. Useful for debugging
#purposes
@ -411,7 +406,9 @@ class PypoFetch(Thread):
for mkey in media:
media_item = media[mkey]
if media_item['type'] == 'file':
scheduled_file_set.add(media_item["id"] + media_item["file_ext"])
if "file_ext" not in media_item.keys():
media_item["file_ext"] = mimetypes.guess_extension(media_item['metadata']['mime'], strict=False)
scheduled_file_set.add("{}{}".format(media_item["id"], media_item["file_ext"]))
expired_files = cached_file_set - scheduled_file_set
@ -430,8 +427,7 @@ class PypoFetch(Thread):
else:
self.logger.info("File '%s' not removed. Still busy!" % path)
except Exception as e:
self.logger.error("Problem removing file '%s'" % f)
self.logger.error(traceback.format_exc())
self.logger.exception("Problem removing file '%s'" % f)
def manual_schedule_fetch(self):
success, self.schedule_data = self.api_client.get_schedule()
@ -501,18 +497,13 @@ class PypoFetch(Thread):
self.logger.info("Queue timeout. Fetching schedule manually")
manual_fetch_needed = True
except Exception as e:
top = traceback.format_exc()
self.logger.error('Exception: %s', e)
self.logger.error("traceback: %s", top)
self.logger.exception(e)
try:
if manual_fetch_needed:
self.persistent_manual_schedule_fetch(max_attempts=5)
except Exception as e:
top = traceback.format_exc()
self.logger.error('Failed to manually fetch the schedule.')
self.logger.error('Exception: %s', e)
self.logger.error("traceback: %s", top)
self.logger.exception('Failed to manually fetch the schedule.')
loops += 1

41
python_apps/pypo/pypo/pypoliquidsoap.py
View File

@ -97,26 +97,28 @@ class PypoLiquidsoap():
def verify_correct_present_media(self, scheduled_now):
#verify whether Liquidsoap is currently playing the correct files.
#if we find an item that Liquidsoap is not playing, then push it
#into one of Liquidsoap's queues. If Liquidsoap is already playing
#it do nothing. If Liquidsoap is playing a track that isn't in
#currently_playing then stop it.
"""
verify whether Liquidsoap is currently playing the correct files.
if we find an item that Liquidsoap is not playing, then push it
into one of Liquidsoap's queues. If Liquidsoap is already playing
it do nothing. If Liquidsoap is playing a track that isn't in
currently_playing then stop it.
#Check for Liquidsoap media we should source.skip
#get liquidsoap items for each queue. Since each queue can only have one
#item, we should have a max of 8 items.
Check for Liquidsoap media we should source.skip
get liquidsoap items for each queue. Since each queue can only have one
item, we should have a max of 8 items.
#2013-03-21-22-56-00_0: {
#id: 1,
#type: "stream_output_start",
#row_id: 41,
#uri: "http://stream2.radioblackout.org:80/blackout.ogg",
#start: "2013-03-21-22-56-00",
#end: "2013-03-21-23-26-00",
#show_name: "Untitled Show",
#independent_event: true
#},
2013-03-21-22-56-00_0: {
id: 1,
type: "stream_output_start",
row_id: 41,
uri: "http://stream2.radioblackout.org:80/blackout.ogg",
start: "2013-03-21-22-56-00",
end: "2013-03-21-23-26-00",
show_name: "Untitled Show",
independent_event: true
},
"""
try:
scheduled_now_files = \
@ -125,7 +127,7 @@ class PypoLiquidsoap():
scheduled_now_webstream = \
[x for x in scheduled_now if x["type"] == eventtypes.STREAM_OUTPUT_START]
schedule_ids = [x["row_id"] for x in scheduled_now_files]
schedule_ids = set([x["row_id"] for x in scheduled_now_files])
row_id_map = {}
liq_queue_ids = set()
@ -155,7 +157,6 @@ class PypoLiquidsoap():
to_be_removed.add(i["row_id"])
to_be_added.add(i["row_id"])
to_be_removed.update(liq_queue_ids - schedule_ids)
to_be_added.update(schedule_ids - liq_queue_ids)

82
python_apps/pypo/pypo/telnetliquidsoap.py
View File

@ -52,8 +52,8 @@ class TelnetLiquidsoap:
return True
tn = self.__connect()
msg = '%s.queue\nexit\n' % queue_id
tn.write(msg)
output = tn.read_all().splitlines()
tn.write(msg.encode('utf-8'))
output = tn.read_all().decode('utf-8').splitlines()
if len(output) == 3:
return len(output[0]) == 0
else:
@ -68,10 +68,10 @@ class TelnetLiquidsoap:
for i in self.queues:
msg = 'queues.%s_skip\n' % i
self.logger.debug(msg)
tn.write(msg)
tn.write("exit\n")
self.logger.debug(tn.read_all())
tn.write(msg.encode('utf-8'))
tn.write("exit\n".encode('utf-8'))
self.logger.debug(tn.read_all().decode('utf-8'))
except Exception:
raise
finally:
@ -85,10 +85,10 @@ class TelnetLiquidsoap:
msg = 'queues.%s_skip\n' % queue_id
self.logger.debug(msg)
tn.write(msg)
tn.write("exit\n")
self.logger.debug(tn.read_all())
tn.write(msg.encode('utf-8'))
tn.write("exit\n".encode('utf-8'))
self.logger.debug(tn.read_all().decode('utf-8'))
except Exception:
raise
finally:
@ -105,17 +105,17 @@ class TelnetLiquidsoap:
tn = self.__connect()
annotation = create_liquidsoap_annotation(media_item)
msg = '%s.push %s\n' % (queue_id, annotation.encode('utf-8'))
msg = '%s.push %s\n' % (queue_id, annotation)
self.logger.debug(msg)
tn.write(msg)
tn.write(msg.encode('utf-8'))
show_name = media_item['show_name']
msg = 'vars.show_name %s\n' % show_name.encode('utf-8')
tn.write(msg)
msg = 'vars.show_name %s\n' % show_name
tn.write(msg.encode('utf-8'))
self.logger.debug(msg)
tn.write("exit\n")
self.logger.debug(tn.read_all())
tn.write("exit\n".encode('utf-8'))
self.logger.debug(tn.read_all().decode('utf-8'))
except Exception:
raise
finally:
@ -131,14 +131,14 @@ class TelnetLiquidsoap:
msg = 'http.stop\n'
self.logger.debug(msg)
tn.write(msg)
tn.write(msg.encode('utf-8'))
msg = 'dynamic_source.id -1\n'
self.logger.debug(msg)
tn.write(msg)
tn.write(msg.encode('utf-8'))
tn.write("exit\n")
self.logger.debug(tn.read_all())
tn.write("exit\n".encode('utf-8'))
self.logger.debug(tn.read_all().decode('utf-8'))
except Exception as e:
self.logger.error(str(e))
@ -154,10 +154,10 @@ class TelnetLiquidsoap:
msg = 'dynamic_source.output_stop\n'
self.logger.debug(msg)
tn.write(msg)
tn.write(msg.encode('utf-8'))
tn.write("exit\n")
self.logger.debug(tn.read_all())
tn.write("exit\n".encode('utf-8'))
self.logger.debug(tn.read_all().decode('utf-8'))
except Exception as e:
self.logger.error(str(e))
@ -172,21 +172,21 @@ class TelnetLiquidsoap:
#TODO: DO we need this?
msg = 'streams.scheduled_play_start\n'
tn.write(msg)
tn.write(msg.encode('utf-8'))
msg = 'dynamic_source.output_start\n'
self.logger.debug(msg)
tn.write(msg)
tn.write(msg.encode('utf-8'))
tn.write("exit\n")
self.logger.debug(tn.read_all())
tn.write("exit\n".encode('utf-8'))
self.logger.debug(tn.read_all().decode('utf-8'))
self.current_prebuffering_stream_id = None
except Exception as e:
self.logger.error(str(e))
finally:
self.telnet_lock.release()
@ls_timeout
def start_web_stream_buffer(self, media_item):
try:
@ -195,14 +195,14 @@ class TelnetLiquidsoap:
msg = 'dynamic_source.id %s\n' % media_item['row_id']
self.logger.debug(msg)
tn.write(msg)
tn.write(msg.encode('utf-8'))
msg = 'http.restart %s\n' % media_item['uri'].encode('latin-1')
self.logger.debug(msg)
tn.write(msg)
tn.write(msg.encode('utf-8'))
tn.write("exit\n")
self.logger.debug(tn.read_all())
tn.write("exit\n".encode('utf-8'))
self.logger.debug(tn.read_all().decode('utf-8'))
self.current_prebuffering_stream_id = media_item['row_id']
except Exception as e:
@ -218,10 +218,10 @@ class TelnetLiquidsoap:
msg = 'dynamic_source.get_id\n'
self.logger.debug(msg)
tn.write(msg)
tn.write(msg.encode('utf-8'))
tn.write("exit\n")
stream_id = tn.read_all().splitlines()[0]
tn.write("exit\n".encode('utf-8'))
stream_id = tn.read_all().decode('utf-8').splitlines()[0]
self.logger.debug("stream_id: %s" % stream_id)
return stream_id
@ -243,9 +243,9 @@ class TelnetLiquidsoap:
self.telnet_lock.acquire()
tn = telnetlib.Telnet(self.ls_host, self.ls_port)
self.logger.info(command)
tn.write(command)
tn.write('exit\n')
tn.read_all()
tn.write(command.encode('utf-8'))
tn.write('exit\n'.encode('utf-8'))
tn.read_all().decode('utf-8')
except Exception as e:
self.logger.error(traceback.format_exc())
finally:
@ -259,10 +259,10 @@ class TelnetLiquidsoap:
tn = telnetlib.Telnet(self.ls_host, self.ls_port)
for i in commands:
self.logger.info(i)
tn.write(i)
tn.write(i.encode('utf-8'))
tn.write('exit\n')
tn.read_all()
tn.write('exit\n'.encode('utf-8'))
tn.read_all().decode('utf-8')
except Exception as e:
self.logger.error(str(e))
finally:

3
python_apps/pypo/setup.py
View File

@ -59,7 +59,8 @@ setup(name='airtime-playout',
'pyinotify',
'pytz',
'requests',
'defusedxml'
'defusedxml',
'packaging',
],
zip_safe=False,
data_files=data_files)