CC-5990, CC-5991 - Python cleanup, removed need for /usr/lib/airtime

python_apps/api_clients/setup.py

@@ -0,0 +1,33 @@
+from setuptools import setup
+from subprocess import call
+import sys
+import os
+
+script_path = os.path.dirname(os.path.realpath(__file__))
+print script_path
+os.chdir(script_path)
+
+setup(name='api_clients',
+      version='1.0',
+      description='Airtime API Client',
+      url='http://github.com/sourcefabric/Airtime',
+      author='sourcefabric',
+      license='AGPLv3',
+      packages=['api_clients'],
+      scripts=[],
+      install_requires=[
+#           'amqplib',
+#           'anyjson',
+#           'argparse',
+        'configobj'
+#           'docopt',
+#           'kombu',
+#           'mutagen',
+#           'poster',
+#           'PyDispatcher',
+#           'pyinotify',
+#           'pytz',
+#           'wsgiref'
+      ],
+      zip_safe=False,
+      data_files=[])

python_apps/media-monitor/bin/airtime-media-monitor

@@ -2,8 +2,6 @@
 
 exec 2>&1
 
-cd /usr/lib/airtime/media-monitor/
-
 set +e
 cat /etc/default/locale | grep -i "LANG=.*UTF-\?8"
 set -e
@@ -12,8 +10,6 @@ if [ "$?" != "0" ]; then
     exit 1
 fi
 
-export PYTHONPATH=/usr/lib/airtime/:/usr/lib/airtime/media-monitor/airtimefilemonitor/:\
-/usr/lib/airtime/media-monitor/mm2/:/usr/lib/airtime/std_err_override/:/usr/lib/airtime/api_clients:$PYTHONPATH
 export LC_ALL=`cat /etc/default/locale | grep "LANG=" | cut -d= -f2 | tr -d "\n\""`
 
-exec python /usr/lib/airtime/media-monitor/media_monitor.py > /var/log/airtime/media-monitor/py-interpreter.log 2>&1
+exec python -m media_monitor > /var/log/airtime/media-monitor/py-interpreter.log 2>&1

python_apps/media-monitor/media_monitor/__main__.py

@@ -1,10 +1,11 @@
 import logging
 import time
 import sys
+import os
 import mm2.mm2 as mm2
 from std_err_override import LogWriter
 
 global_cfg = '/etc/airtime/airtime.conf'
-logging_cfg = '/usr/lib/airtime/media-monitor/logging.cfg'
+logging_cfg = os.path.join(os.path.dirname(__file__), 'logging.cfg')
 
 mm2.main( global_cfg, logging_cfg )

python_apps/media-monitor/mm2/media/metadata/definitions.py

@@ -1,8 +1,8 @@
 # -*- coding: utf-8 -*-
-import media.metadata.process as md
+import process as md
 import re
 from os.path import normpath
-from media.monitor.pure import format_length, file_md5, is_airtime_recorded, \
+from ..monitor.pure import format_length, file_md5, is_airtime_recorded, \
     no_extension_basename
 
 defs_loaded = False

python_apps/media-monitor/mm2/media/metadata/process.py

@@ -1,11 +1,11 @@
 # -*- coding: utf-8 -*-
-from contextlib import contextmanager
-from media.monitor.pure import truncate_to_value, truncate_to_length, toposort
-from os.path import normpath
-from media.monitor.exceptions import BadSongFile
-from media.monitor.log import Loggable
-import media.monitor.pure as mmp
-from collections     import namedtuple
+from contextlib             import contextmanager
+from ..monitor.pure         import truncate_to_value, truncate_to_length, toposort
+from os.path                import normpath
+from ..monitor.exceptions   import BadSongFile
+from ..monitor.log          import Loggable
+from ..monitor              import pure as mmp
+from collections            import namedtuple
 import mutagen
 import subprocess
 import json

python_apps/media-monitor/mm2/media/monitor/airtime.py

@@ -1,22 +1,21 @@
 # -*- coding: utf-8 -*-
-from kombu.messaging  import Exchange, Queue, Consumer
-from kombu.connection import BrokerConnection
-from kombu.simple import SimpleQueue
-from os.path          import normpath
+from kombu.messaging    import Exchange, Queue, Consumer
+from kombu.connection   import BrokerConnection
+from kombu.simple       import SimpleQueue
+from os.path            import normpath
 
 import json
 import os
 import copy
 import time
 
-from media.monitor.exceptions import BadSongFile, InvalidMetadataElement
-from media.monitor.metadata   import Metadata
-from media.monitor.log        import Loggable
-from media.monitor.syncdb     import AirtimeDB
-from media.monitor.exceptions import DirectoryIsNotListed
-from media.monitor.bootstrap  import Bootstrapper
+from exceptions         import BadSongFile, InvalidMetadataElement, DirectoryIsNotListed
+from metadata           import Metadata
+from log                import Loggable
+from syncdb             import AirtimeDB
+from bootstrap          import Bootstrapper
 
-from media.saas.thread import apc, user
+from ..saas.thread      import apc, user
 
 class AirtimeNotifier(Loggable):
     """

python_apps/media-monitor/mm2/media/monitor/bootstrap.py

@@ -1,9 +1,9 @@
 import os
-from pydispatch           import dispatcher
-from media.monitor.events import NewFile, DeleteFile, ModifyFile
-from media.monitor.log    import Loggable
-from media.saas.thread    import getsig
-import media.monitor.pure as mmp
+from pydispatch       import dispatcher
+from events           import NewFile, DeleteFile, ModifyFile
+from log              import Loggable
+from ..saas.thread    import getsig
+import pure as mmp
 
 class Bootstrapper(Loggable):
     """

python_apps/media-monitor/mm2/media/monitor/config.py

@@ -3,8 +3,8 @@ import os
 import copy
 from configobj import ConfigObj
 
-from media.monitor.exceptions import NoConfigFile, ConfigAccessViolation
-import media.monitor.pure as mmp
+from exceptions import NoConfigFile, ConfigAccessViolation
+import pure as mmp
 
 class MMConfig(object):
     def __init__(self, path):

python_apps/media-monitor/mm2/media/monitor/eventcontractor.py

@@ -1,5 +1,5 @@
-from media.monitor.log    import Loggable
-from media.monitor.events import DeleteFile
+from log    import Loggable
+from events import DeleteFile
 
 class EventContractor(Loggable):
     def __init__(self):

python_apps/media-monitor/mm2/media/monitor/eventdrainer.py

@@ -1,7 +1,7 @@
 import socket
 import time
-from media.monitor.log     import Loggable
-from media.monitor.toucher import RepeatTimer
+from log     import Loggable
+from toucher import RepeatTimer
 from amqplib.client_0_8.exceptions import AMQPConnectionException
 
 class EventDrainer(Loggable):

python_apps/media-monitor/mm2/media/monitor/events.py

@@ -2,12 +2,12 @@
 import os
 import abc
 import re
-import media.monitor.pure   as mmp
-from media.monitor.pure       import LazyProperty
-from media.monitor.metadata   import Metadata
-from media.monitor.log        import Loggable
-from media.monitor.exceptions import BadSongFile
-from media.saas.thread        import getsig, user
+import pure         as mmp
+from pure           import LazyProperty
+from metadata       import Metadata
+from log            import Loggable
+from exceptions     import BadSongFile
+from ..saas.thread  import getsig, user
 
 class PathChannel(object):
     """ Simple struct to hold a 'signal' string and a related 'path'.

python_apps/media-monitor/mm2/media/monitor/handler.py

@@ -2,9 +2,9 @@
 from pydispatch import dispatcher
 import abc
 
-from media.monitor.log import Loggable
-from media.saas.thread import getsig
-import media.monitor.pure as mmp
+from log import Loggable
+from ..saas.thread import getsig
+import pure as mmp
 
 # Defines the handle interface
 class Handles(object):

python_apps/media-monitor/mm2/media/monitor/listeners.py

@@ -3,13 +3,13 @@ import pyinotify
 from pydispatch import dispatcher
 from functools import wraps
 
-import media.monitor.pure as mmp
-from media.monitor.pure import IncludeOnly
-from media.monitor.events import OrganizeFile, NewFile, MoveFile, DeleteFile, \
+import pure as mmp
+from pure import IncludeOnly
+from events import OrganizeFile, NewFile, MoveFile, DeleteFile, \
                                  DeleteDir, MoveDir,\
                                  DeleteDirWatch
-from media.monitor.log import Loggable
-from media.saas.thread import getsig, user
+from log import Loggable
+from ..saas.thread import getsig, user
 # Note: Because of the way classes that inherit from pyinotify.ProcessEvent
 # interact with constructors. you should only instantiate objects from them
 # using keyword arguments. For example:

python_apps/media-monitor/mm2/media/monitor/log.py

@@ -1,7 +1,7 @@
 import logging
 import abc
 import traceback
-from media.monitor.pure import LazyProperty
+from pure import LazyProperty
 
 appname = 'root'
 

python_apps/media-monitor/mm2/media/monitor/manager.py

@@ -1,16 +1,16 @@
 import pyinotify
 import time
 import os
-from pydispatch import dispatcher
+from pydispatch     import dispatcher
 
-from os.path                 import normpath
-from media.monitor.events    import PathChannel
-from media.monitor.log       import Loggable
-from media.monitor.listeners import StoreWatchListener, OrganizeListener
-from media.monitor.handler   import ProblemFileHandler
-from media.monitor.organizer import Organizer
-from media.saas.thread       import InstanceInheritingThread, getsig
-import media.monitor.pure as mmp
+from os.path        import normpath
+from events         import PathChannel
+from log            import Loggable
+from listeners      import StoreWatchListener, OrganizeListener
+from handler        import ProblemFileHandler
+from organizer      import Organizer
+from ..saas.thread  import InstanceInheritingThread, getsig
+import pure         as mmp
 
 
 class ManagerTimeout(InstanceInheritingThread,Loggable):

python_apps/media-monitor/mm2/media/monitor/metadata.py

@@ -2,17 +2,17 @@
 import mutagen
 import os
 import copy
-from mutagen.easymp4 import EasyMP4KeyError
-from mutagen.easyid3 import EasyID3KeyError
+from mutagen.easymp4    import EasyMP4KeyError
+from mutagen.easyid3    import EasyID3KeyError
 
-from media.monitor.exceptions import BadSongFile, InvalidMetadataElement
-from media.monitor.log        import Loggable
-from media.monitor.pure       import format_length
-import media.monitor.pure as mmp
+from exceptions         import BadSongFile, InvalidMetadataElement
+from log                import Loggable
+from pure               import format_length
+import pure             as mmp
 
 # emf related stuff
-from media.metadata.process import global_reader
-import media.metadata.definitions as defs
+from ..metadata.process import global_reader
+from ..metadata         import definitions as defs
 defs.load_definitions()
 
 """

python_apps/media-monitor/mm2/media/monitor/organizer.py

@@ -1,12 +1,12 @@
 # -*- coding: utf-8 -*-
-import media.monitor.pure   as mmp
-from media.monitor.handler    import ReportHandler
-from media.monitor.log        import Loggable
-from media.monitor.exceptions import BadSongFile
-from media.monitor.events     import OrganizeFile
-from pydispatch               import dispatcher
-from os.path                  import dirname
-from media.saas.thread        import getsig, user
+import pure         as mmp
+from handler        import ReportHandler
+from log            import Loggable
+from exceptions     import BadSongFile
+from events         import OrganizeFile
+from pydispatch     import dispatcher
+from os.path        import dirname
+from ..saas.thread  import getsig, user
 import os.path
 
 class Organizer(ReportHandler,Loggable):

python_apps/media-monitor/mm2/media/monitor/owners.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from media.monitor.log import Loggable
+from log import Loggable
 
 class Owner(Loggable):
     def __init__(self):

python_apps/media-monitor/mm2/media/monitor/pure.py

@@ -21,7 +21,7 @@ try: from functools import reduce
 except: pass
 from configobj import ConfigObj
 
-from media.monitor.exceptions import FailedToSetLocale, FailedToCreateDir
+from exceptions import FailedToSetLocale, FailedToCreateDir
 
 supported_extensions = [u"mp3", u"ogg", u"oga", u"flac", u"wav",
                         u'm4a', u'mp4', 'opus']

python_apps/media-monitor/mm2/media/monitor/request.py

@@ -1,8 +1,8 @@
 # -*- coding: utf-8 -*-
 
-from media.monitor.exceptions import BadSongFile
-from media.monitor.log        import Loggable
-from media.saas.thread import apc, InstanceInheritingThread
+from exceptions import BadSongFile
+from log        import Loggable
+from ..saas.thread import apc, InstanceInheritingThread
 
 class ThreadedRequestSync(InstanceInheritingThread, Loggable):
     def __init__(self, rs):

python_apps/media-monitor/mm2/media/monitor/syncdb.py

@@ -1,10 +1,10 @@
 # -*- coding: utf-8 -*-
 import os
-from media.monitor.log        import Loggable
-from media.monitor.exceptions import NoDirectoryInAirtime
-from media.saas.thread        import user
-from os.path                  import normpath, join
-import media.monitor.pure as mmp
+from log            import Loggable
+from exceptions     import NoDirectoryInAirtime
+from ..saas.thread  import user
+from os.path        import normpath, join
+import pure         as mmp
 
 class AirtimeDB(Loggable):
     def __init__(self, apc, reload_now=True):

python_apps/media-monitor/mm2/media/monitor/toucher.py

@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
-import media.monitor.pure as mmp
+import pure         as mmp
 import os
-from media.monitor.log        import Loggable
-from media.monitor.exceptions import CouldNotCreateIndexFile
-from media.saas.thread        import InstanceInheritingThread
+from log            import Loggable
+from exceptions     import CouldNotCreateIndexFile
+from ..saas.thread  import InstanceInheritingThread
 
 class Toucher(Loggable):
     """

python_apps/media-monitor/mm2/media/monitor/watchersyncer.py

@@ -2,13 +2,13 @@
 import time
 import copy
 
-from media.monitor.handler         import ReportHandler
-from media.monitor.log             import Loggable
-from media.monitor.exceptions      import BadSongFile
-from media.monitor.eventcontractor import EventContractor
-from media.monitor.events          import EventProxy
-from media.monitor.request         import ThreadedRequestSync, RequestSync
-from media.saas.thread             import InstanceInheritingThread, getsig
+from handler         import ReportHandler
+from log             import Loggable
+from exceptions      import BadSongFile
+from eventcontractor import EventContractor
+from events          import EventProxy
+from request         import ThreadedRequestSync, RequestSync
+from ..saas.thread   import InstanceInheritingThread, getsig
 
 class TimeoutWatcher(InstanceInheritingThread,Loggable):
     """

python_apps/media-monitor/mm2/media/saas/airtimeinstance.py

@@ -1,12 +1,12 @@
 import os
 from os.path import join, basename, dirname
 
-from media.monitor.exceptions import NoConfigFile
-from media.monitor.pure import LazyProperty
-from media.monitor.config import MMConfig
-from media.monitor.owners import Owner
-from media.monitor.events import EventRegistry
-from media.monitor.listeners import FileMediator
+from ..monitor.exceptions import NoConfigFile
+from ..monitor.pure import LazyProperty
+from ..monitor.config import MMConfig
+from ..monitor.owners import Owner
+from ..monitor.events import EventRegistry
+from ..monitor.listeners import FileMediator
 from api_clients.api_client import AirtimeApiClient
 
 # poor man's phantom types...

python_apps/media-monitor/mm2/media/saas/launcher.py

@@ -2,21 +2,21 @@ import os, sys
 import logging
 import logging.config
 
-import media.monitor.pure          as mmp
+from ..monitor               import pure as mmp
 
-from media.monitor.exceptions    import FailedToObtainLocale, FailedToSetLocale
-from media.monitor.log           import get_logger, setup_logging
+from ..monitor.exceptions    import FailedToObtainLocale, FailedToSetLocale
+from ..monitor.log           import get_logger, setup_logging
 from std_err_override            import LogWriter
-from media.saas.thread           import InstanceThread, user, apc, getsig
-from media.monitor.log           import Loggable
-from media.monitor.exceptions    import CouldNotCreateIndexFile
-from media.monitor.toucher       import ToucherThread
-from media.monitor.airtime       import AirtimeNotifier, AirtimeMessageReceiver
-from media.monitor.watchersyncer import WatchSyncer
-from media.monitor.eventdrainer  import EventDrainer
-from media.monitor.manager       import Manager
-from media.monitor.syncdb        import AirtimeDB
-from media.saas.airtimeinstance  import AirtimeInstance
+from ..saas.thread           import InstanceThread, user, apc, getsig
+from ..monitor.log           import Loggable
+from ..monitor.exceptions    import CouldNotCreateIndexFile
+from ..monitor.toucher       import ToucherThread
+from ..monitor.airtime       import AirtimeNotifier, AirtimeMessageReceiver
+from ..monitor.watchersyncer import WatchSyncer
+from ..monitor.eventdrainer  import EventDrainer
+from ..monitor.manager       import Manager
+from ..monitor.syncdb        import AirtimeDB
+from airtimeinstance  import AirtimeInstance
 
 class MM2(InstanceThread, Loggable):
 

python_apps/media-monitor/setup.py

@@ -34,7 +34,11 @@ setup(name='airtime-media-monitor',
       url='http://github.com/sourcefabric/Airtime',
       author='sourcefabric',
       license='AGPLv3',
-      packages=['media-monitor', 'media-monitor2'],
+      packages=['media_monitor', 'mm2', 'mm2.configs', 
+                'mm2.media', 'mm2.media.monitor', 
+                'mm2.media.metadata', 'mm2.media.saas'
+                ],
+      package_data={'': ['*.cfg']},
       scripts=['bin/airtime-media-monitor'],
       install_requires=[
           'amqplib',

python_apps/pypo/bin/airtime-liquidsoap

@@ -1,38 +1,3 @@
 #!/bin/bash -e
 
-debug="f"
-
-showhelp () {
-    echo "Usage: airtime-liquidsoap [options]
---help|-h                         Displays usage information.
---debug|-d                        Print error messages to console" 
-}
-
-set -- $(getopt -l help,debug "hd" "$@")
-while [ $# -gt 0 ]
-do
-    case "$1" in
-    (-h|--help) showhelp; exit 0;;
-    (-d|--debug) debug="t";;
-    
-    (--) shift; break;;
-    (-*) echo "$0: error - unrecognized option $1" 1>&2; exit 1;;
-    (*)  break;;
-    esac
-    shift
-done
-
-export HOME="/var/tmp/airtime/pypo/"
-if [ "$debug" = "t" ]; then
-    ls_path="/usr/bin/airtime-liquidsoap --verbose -f --debug"
-else
-    ls_path="/usr/bin/airtime-liquidsoap --verbose -f"
-fi
-
-export PYTHONPATH=/usr/lib/airtime/:/usr/lib/airtime/pypo/bin/:/usr/lib/airtime/std_err_override/:/usr/lib/airtime/api_clients:$PYTHONPATH
-ls_param="/usr/lib/airtime/pypo/bin/liquidsoap_scripts/ls_script.liq"
-
-cd /usr/lib/airtime/pypo/bin/liquidsoap_scripts
-python generate_liquidsoap_cfg.py
-
-exec ${ls_path} ${ls_param} 2>&1
+exec python -m liquidsoap 2>&1

python_apps/pypo/bin/airtime-playout

@@ -9,8 +9,7 @@ if [ "$?" != "0" ]; then
 fi
 
 export HOME="/var/tmp/airtime/pypo/"
-export PYTHONPATH=/usr/lib/airtime/:/usr/lib/airtime/pypo/bin/:/usr/lib/airtime/std_err_override/:/usr/lib/airtime/api_clients:$PYTHONPATH
 export LC_ALL=`cat /etc/default/locale | grep "LANG=" | cut -d= -f2 | tr -d "\n\""`
 export TERM=xterm
 
-exec python /usr/lib/airtime/pypo/bin/pypocli.py > /var/log/airtime/pypo/py-interpreter.log 2>&1
+exec python -m pypo > /var/log/airtime/pypo/py-interpreter.log 2>&1

python_apps/pypo/bin/pyponotify

@@ -1,3 +1,4 @@
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 import traceback
 

python_apps/pypo/liquidsoap/__main__.py

@@ -0,0 +1,27 @@
+""" Runs Airtime liquidsoap
+"""
+
+import argparse
+import os
+import generate_liquidsoap_cfg
+
+PYPO_HOME = '/var/tmp/airtime/pypo/'
+
+def run():
+    '''Entry-point for this application'''
+    print "Airtime Liquidsoap"
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-d", "--debug", help="run in debug mode", action="store_true")
+    args = parser.parse_args()
+    
+    os.environ["HOME"] = PYPO_HOME
+    
+    generate_liquidsoap_cfg.run()
+    script_path = os.path.join(os.path.dirname(__file__), 'ls_script.liq')
+    
+    if args.debug:
+        os.execl('/usr/bin/liquidsoap', 'airtime-liquidsoap', script_path, '--verbose', '-f', '--debug')
+    else:
+        os.execl('/usr/bin/liquidsoap', 'airtime-liquidsoap', script_path, '--verbose', '-f')
+
+run()

python_apps/pypo/liquidsoap/generate_liquidsoap_cfg.py

@@ -25,24 +25,25 @@ def generate_liquidsoap_config(ss):
 
     fh.write('log_file = "/var/log/airtime/pypo-liquidsoap/<script>.log"\n')
     fh.close()
-
-logging.basicConfig(format='%(message)s')
-attempts = 0
-max_attempts = 10
-successful = False
-
-while not successful:
-    try:
-        ac = AirtimeApiClient(logging.getLogger())
-        ss = ac.get_stream_setting()
-        generate_liquidsoap_config(ss)
-        successful = True
-    except Exception, e:
-        if attempts == max_attempts:
-            print "Unable to connect to the Airtime server."
-            logging.error(str(e))
-            logging.error("traceback: %s", traceback.format_exc())
-            sys.exit(1)
-        else:
-            time.sleep(3)
-    attempts += 1
+    
+def run():
+    logging.basicConfig(format='%(message)s')
+    attempts = 0
+    max_attempts = 10
+    successful = False
+    
+    while not successful:
+        try:
+            ac = AirtimeApiClient(logging.getLogger())
+            ss = ac.get_stream_setting()
+            generate_liquidsoap_config(ss)
+            successful = True
+        except Exception, e:
+            if attempts == max_attempts:
+                print "Unable to connect to the Airtime server."
+                logging.error(str(e))
+                logging.error("traceback: %s", traceback.format_exc())
+                sys.exit(1)
+            else:
+                time.sleep(3)
+        attempts += 1

python_apps/pypo/liquidsoap/ls_lib.liq

@@ -1,5 +1,5 @@
 def notify(m)
-  command = "/usr/lib/airtime/pypo/bin/liquidsoap_scripts/notify.sh --media-id=#{m['schedule_table_id']} &"
+  command = "timeout --signal=KILL 45 python pyponotify --media-id=#{m['schedule_table_id']} &"
   log(command)
   system(command)
 end
@@ -15,7 +15,7 @@ def notify_stream(m)
   #if a string has a single apostrophe in it, let's comment it out by ending the string before right before it
   #escaping the apostrophe, and then starting a new string right after it. This is why we use 3 apostrophes.
   json_str = string.replace(pattern="'",(fun (s) -> "'\''"), json_str)
-  command = "/usr/lib/airtime/pypo/bin/liquidsoap_scripts/notify.sh --webstream='#{json_str}' --media-id=#{!current_dyn_id} &"
+  command = "timeout --signal=KILL 45 python pyponotify --webstream='#{json_str}' --media-id=#{!current_dyn_id} &"
   
   if !current_dyn_id != "-1" then
     log(command)
@@ -93,14 +93,14 @@ def output_to(output_type, type, bitrate, host, port, pass, mount_point, url, de
     source = ref s
     def on_error(msg)
         connected := "false"
-        command = "/usr/lib/airtime/pypo/bin/liquidsoap_scripts/notify.sh --error='#{msg}' --stream-id=#{stream} --time=#{!time} &"
+        command = "timeout --signal=KILL 45 python pyponotify --error='#{msg}' --stream-id=#{stream} --time=#{!time} &"
         system(command)
         log(command)
         5.
     end
     def on_connect()
         connected := "true"
-        command = "/usr/lib/airtime/pypo/bin/liquidsoap_scripts/notify.sh --connect --stream-id=#{stream} --time=#{!time} &"
+        command = "timeout --signal=KILL 45 python pyponotify --connect --stream-id=#{stream} --time=#{!time} &"
         system(command)
         log(command)
     end