CC-2016: Rearrange python scripts for reusability

-moved files

python_apps/pypo/scripts/old_files/README

@@ -0,0 +1,23 @@
+This directory contains scripts not directly related to pypo
+These mainly are things related to liquidsoap & playout
+
+I added those scripts here to have them at hand for
+development and also to update/share them via svn
+
+
+scripts here:
+
+- ls_run.sh
+  wrapper to run liquid soap. makes sure that the "current" playlist is
+  filled with silence
+  
+- ls_script.liq (called by ls_run.sh)
+  the main liquidsoap control-script
+
+- ls_cue.liq (included by ls_script.liq)
+  contains a custom protocol that registers the cue-in/out values from the playlist script
+  
+- cue_file.py (called by ls_cue.liq)
+  a wrapper that does the actual cutting. 
+  it is called with: path_to_file[path] cue_in[ss.ms] cue_out[ss.ms] and does the mp3 cutting (with mp3cut)
+  returns a temporary file that can be used by ls (make sure to set "TEMP_DIR" in script) 

python_apps/pypo/scripts/old_files/cue_file.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+
+"""
+cue script that gets called by liquidsoap if a file in the playlist
+gives orders to cue (in/out). eg:
+cue_file:cue_in=90.0,cue_out=110.0:annotate:***
+
+cue_in is number of seconds from the beginning of the file.
+cue_out is number of seconds from the end of the file.
+
+params:  path_to_file, cue_in [float, seconds], cue_out [float, seconds]
+returns: path to the cued temp-file
+
+examples:
+calling: ./cue_file.py /storage/pypo/cache/2010-06-25-15-05-00/35.mp3 10 120.095
+returns: /tmp/lstf_UwDKcEngvF
+
+In this example, the first 10 seconds and last 120.095 seconds are cut off. The 
+middle part of the file is returned.
+
+One thing to mention here:
+The way pypo (ab)uses liquidsoap can bring in some unwanted effects. liquidsoap 
+is built in a way that it tries to collect the needed files to playout in advance.
+we 'force' liquidsoap to immediately start playing a newly loaded list, so ls has
+no time to prepare the files. If a file is played without cues, this does not affect
+the playout too much. My testing on a lame VM added a delay of +/- 10ms.
+
+If the first file in  a playlist is cued, the "mp3cut" command takes time to execute.
+On the same VM this takes an additional 200ms for an average size mp3-file.
+So the playout will start a bit delayed. This should not be a too big issue, but
+think about this behaviour if you eg access the files via network (nas) as the reading
+of files could take some time as well.
+
+So maybe we should think about a different implementation. One way would be to do the
+cueing during playlist preparation, so all the files would be pre-cut when they are 
+passed to liquidsoap. 
+Additionally this would allow to run an unpathed version of ls.
+
+"""
+
+import sys
+import shutil
+import random
+import string
+import time
+from datetime import timedelta
+import os
+
+from mutagen.mp3 import MP3
+from mutagen.oggvorbis import OggVorbis
+
+TEMP_DIR = '/tmp/';
+
+
+sys.stderr.write('\n** starting mp3/ogg cutter **\n\n')
+
+try: src = sys.argv[1]
+except Exception, e: 
+    sys.stderr.write('No file given. Exiting...\n')
+    sys.exit()
+
+try: cue_in = float(sys.argv[2])
+except Exception, e: 
+    cue_in = float(0)
+    pass
+
+try: cue_out = float(sys.argv[3])
+except Exception, e:
+    cue_out = float(0)
+    pass
+
+sys.stderr.write('in: %s - out: %s file: %s \n' % (cue_in, cue_out, src))
+dst = TEMP_DIR + 'lstf_' + "".join( [random.choice(string.letters) for i in xrange(10)] )
+#TODO, there is no checking whether this randomly generated file name already exists!
+
+
+# get length of track using mutagen. 
+#audio
+#command
+if src.lower().endswith('.mp3'):
+    audio = MP3(src)
+    dur = round(audio.info.length, 3)
+
+    sys.stderr.write('duration: ' + str(dur) + '\n')
+
+    cue_out = round(float(dur) - cue_out, 3)
+
+    str_cue_in = str(timedelta(seconds=cue_in)).replace(".", "+") # hh:mm:ss+mss, eg 00:00:20+000
+    str_cue_out = str(timedelta(seconds=cue_out)).replace(".", "+") #
+
+    """
+    now a bit a hackish part, don't know how to do this better...
+    need to cut the digits after the "+"
+
+    """
+    ts = str_cue_in.split("+")
+    try:
+        if len(ts[1]) == 6:
+            ts[1] = ts[1][0:3]
+            str_cue_in = "%s+%s" % (ts[0], ts[1])
+    except Exception, e:
+        pass
+
+    ts = str_cue_out.split("+")
+    try:
+        if len(ts[1]) == 6:
+            ts[1] = ts[1][0:3]
+            str_cue_out = "%s+%s" % (ts[0], ts[1])
+    except Exception, e:
+        pass
+
+    sys.stderr.write('in:       ' + str_cue_in + '\n')
+    sys.stderr.write('abs:      ' + str(str_cue_out) + '\n\n')
+
+    command = 'mp3cut -o %s -t %s-%s %s' % (dst, str_cue_in, str_cue_out, src)
+elif src.lower().endswith('.ogg'):
+    audio = OggVorbis(src)
+    dur = audio.info.length
+    sys.stderr.write('duration: ' + str(dur) + '\n')
+    
+    cue_out = float(dur) - cue_out
+    
+    #convert input format of ss.mmm to milliseconds and to string<
+    str_cue_in = str(int(round(cue_in*1000)))
+    
+    #convert input format of ss.mmm to milliseconds and to string
+    str_cue_out = str(int(round(cue_out*1000)))
+    
+    command = 'oggCut -s %s -e %s %s %s' % (str_cue_in, str_cue_out, src, dst)
+else:
+    sys.stderr.write('File name with invalid extension. Exiting...\n')
+    sys.exit()
+
+
+
+sys.stderr.write(command + '\n\n\n')
+os.system(command + ' > /dev/null 2>&1')
+
+print dst + "\n";

python_apps/pypo/scripts/old_files/include_daypart.liq

@@ -0,0 +1,70 @@
+
+#########################################
+# A/B queue-setup daypart
+#########################################
+
+# a/b queue setup
+daypart_q0 = request.queue(conservative=true,length=600.,id="daypart_q0")
+daypart_q1 = request.queue(conservative=true,length=600.,id="daypart_q1")
+
+daypart_q0 = audio_to_stereo(daypart_q0)
+daypart_q1 = audio_to_stereo(daypart_q1)
+
+daypart_active = ref 0
+daypart_queue = ref 1
+daypart_q0_enabled = ref false
+daypart_q1_enabled = ref false
+
+# push function, enqueues file in inactive queue (does not start automatically)
+def daypart_push(s)
+  list.hd(server.execute("daypart_q#{!daypart_queue}.push #{s}"))
+  print('push to #{!daypart_queue} - #{s}')
+  "Done"
+end
+
+
+# flips the queues
+def daypart_flip()
+
+  # set a/b-queue corresponding to active, see fallback below
+  if !daypart_active==1 then daypart_q0_enabled:=true else daypart_q0_enabled:=false end
+  if !daypart_active==0 then daypart_q1_enabled:=true else daypart_q1_enabled:=false end
+
+  # get playing (active) queue and flush it  
+  l = list.hd(server.execute("daypart_q#{!daypart_active}.queue"))
+  l = string.split(separator=" ",l)
+  list.iter(fun (rid) -> ignore(server.execute("daypart_q#{!daypart_active}.ignore #{rid}")), l)
+  
+  # skip the playing item
+  # source.skip(if !daypart_active==0 then daypart_q0 else daypart_q1 end)
+
+  # flip variables
+  daypart_active := 1-!daypart_active
+  daypart_queue := 1-!daypart_active
+
+  "Done"
+end
+
+
+# print status
+def daypart_status()
+  print('daypart_active: #{!daypart_active}')
+  print('daypart_queue : #{!daypart_queue}')
+  "Done"
+end
+
+# register for telnet access
+server.register(namespace="daypart","push", daypart_push)
+server.register(namespace="daypart","flip", fun (_) -> daypart_flip())
+server.register(namespace="daypart","status", fun (_) -> daypart_status())
+
+
+# activate / deactivate queues, needed for fallback to work
+daypart_q0 = switch(track_sensitive=true, [({!daypart_q0_enabled},daypart_q0)])
+daypart_q1 = switch(track_sensitive=true, [({!daypart_q1_enabled},daypart_q1)])
+daypart_q_holder = fallback(track_sensitive=true, [daypart_q0, daypart_q1])
+
+
+# finally the resulting daypart source
+daypart = fallback(track_sensitive=false, [daypart_q_holder, default])
+

python_apps/pypo/scripts/old_files/include_dynamic_vars.liq

@@ -0,0 +1,17 @@
+#######################################################################
+# Dynamic variables
+#######################################################################
+
+playlist_type = ref '0'
+pypo_data = ref '0'
+
+def set_playlist_type(s)
+  playlist_type := s
+end
+
+def set_pypo_data(s)
+  pypo_data := s
+end
+
+server.register(namespace="vars", "playlist_type", fun (s) ->  begin set_playlist_type(s) "Done!" end)
+server.register(namespace="vars", "pypo_data", fun (s) ->  begin set_pypo_data(s) "Done!" end)

python_apps/pypo/scripts/old_files/include_live_in.liq

@@ -0,0 +1,20 @@
+#######################################################################
+# Live input - From external icecast server
+#######################################################################
+live_in = input.http(id="live_in",autostart=false,buffer=.1, max=12.,couchcaster_list)
+live_in = buffer(id="buffer_live_in",buffer=.1,fallible=true,live_in)
+live_in = mksafe(live_in)
+
+live_active = ref false
+def live_switch(i)
+  print(i)
+  if i=='1' then live_active:=true else live_active:=false end
+  print(live_active)
+  "Done"
+end
+server.register(namespace="live","active", live_switch)
+
+live = switch(track_sensitive=false, [({!live_active},live_in)])
+
+to_live_s = to_live(jingles_cc)
+to_scheduler_s = to_scheduler()

python_apps/pypo/scripts/old_files/include_notify.liq

@@ -0,0 +1,22 @@
+########################################
+# call pypo api gateway
+########################################
+
+def notify(m)
+
+  if !playlist_type=='5' then 
+    #print('livesession')
+    system("./notify.sh --data='#{!pypo_data}' --media-id=#{m['media_id']} --export-source=scheduler")
+  end
+   
+  if !playlist_type=='6' then
+    #print('couchcaster')
+    system("./notify.sh --data='#{!pypo_data}' --media-id=#{m['media_id']} --export-source=scheduler")
+  end 
+    
+  if !playlist_type=='0' or !playlist_type=='1' or !playlist_type=='2' or !playlist_type=='3' or !playlist_type=='4' then
+    #print('include_notify.liq: notify on playlist')
+    system("./notify.sh --data='#{!pypo_data}' --media-id=#{m['media_id']}")
+  end
+  
+end

python_apps/pypo/scripts/old_files/include_scheduler.liq

@@ -0,0 +1,78 @@
+
+#########################################
+# A/B queue-setup Scheduler
+#########################################
+
+# a/b queue setup
+scheduler_q0 = request.queue(conservative=true,length=600.,id="scheduler_q0")
+scheduler_q1 = request.queue(conservative=true,length=600.,id="scheduler_q1")
+
+scheduler_q0 = audio_to_stereo(scheduler_q0)
+scheduler_q1 = audio_to_stereo(scheduler_q1)
+
+scheduler_active = ref 0
+scheduler_queue = ref 1
+scheduler_q0_enabled = ref false
+scheduler_q1_enabled = ref false
+
+# push function, enqueues file in inactive queue (does not start automatically)
+def scheduler_push(s)
+  list.hd(server.execute("scheduler_q#{!scheduler_queue}.push #{s}"))
+  print('push to #{!scheduler_queue} - #{s}')
+  "Done"
+end
+
+
+# flips the queues
+def scheduler_flip()
+
+  # set a/b-queue corresponding to active, see fallback below
+  if !scheduler_active==1 then scheduler_q0_enabled:=true else scheduler_q0_enabled:=false end
+  if !scheduler_active==0 then scheduler_q1_enabled:=true else scheduler_q1_enabled:=false end
+
+  # get playing (active) queue and flush it  
+  l = list.hd(server.execute("scheduler_q#{!scheduler_active}.queue"))
+  l = string.split(separator=" ",l)
+  list.iter(fun (rid) -> ignore(server.execute("scheduler_q#{!scheduler_active}.ignore #{rid}")), l)
+  
+  # skip the playing item
+  source.skip(if !scheduler_active==0 then scheduler_q0 else scheduler_q1 end)
+
+  # flip variables
+  scheduler_active := 1-!scheduler_active
+  scheduler_queue := 1-!scheduler_active
+
+
+  "Done"
+end
+
+
+# print status
+def scheduler_status()
+  print('scheduler_active: #{!scheduler_active}')
+  print('scheduler_queue : #{!scheduler_queue}')
+  print('pypo_data: #{!pypo_data}')
+
+  #print('user_id: #{!user_id}')
+  #print('playlist_id: #{!playlist_id}')
+  #print('transmission_id: #{!transmission_id}')  
+  #print('playlist_type: #{!playlist_type}')  
+  
+  "Done"
+end
+
+# register for telnet access
+server.register(namespace="scheduler","push", scheduler_push)
+server.register(namespace="scheduler","flip", fun (_) -> scheduler_flip())
+server.register(namespace="scheduler","status", fun (_) -> scheduler_status())
+
+
+# activate / deactivate queues, needed for fallback to work
+scheduler_q0 = switch(track_sensitive=true, [({!scheduler_q0_enabled},scheduler_q0)])
+scheduler_q1 = switch(track_sensitive=true, [({!scheduler_q1_enabled},scheduler_q1)])
+scheduler_q_holder = fallback(track_sensitive=true, [scheduler_q0, scheduler_q1])
+
+
+# finally the resulting scheduler source
+scheduler = fallback(track_sensitive=false, [scheduler_q_holder, default])
+

python_apps/pypo/scripts/old_files/library.liq

@@ -0,0 +1,37 @@
+# Define a transition that fades out the
+# old source, adds a single, and then 
+# plays the new source
+def to_live(jingle,old,new) = 
+  # Fade out old source
+  old = fade.final(old)
+  # Supperpose the jingle
+  s = add([jingle,old])
+  # Compose this in sequence with
+  # the new source
+  sequence([s,new])
+end
+
+def to_scheduler(old,new) = 
+  # We skip the file
+  # currently in new
+  # in order to being with
+  # a fresh file
+  # source.skip(new) 
+  sequence([old,new])
+end
+
+# A transition when switching back to files:
+def to_file(old,new) =
+  # We skip the file
+  # currently in new
+  # in order to being with
+  # a fresh file
+  # source.skip(new) 
+  sequence([old,new])
+end
+
+
+def dp_to_scheduler(old,new) = 
+  old = fade.final(type='log',duration=2.1,old)
+  sequence([old,new])
+end

python_apps/pypo/scripts/old_files/log_run.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+DATE=$(date '+%Y-%m-%d')
+CI_LOG=/var/log/obp/ci/log-$DATE.php
+
+clear
+echo 
+echo "##############################"
+echo "# STARTING PYPO MULTI-LOG    #"
+echo "##############################"
+sleep 1 
+clear
+
+# split
+multitail -s 2 -cS pyml /var/log/obp/pypo/debug.log \
+-cS pyml /var/log/obp/pypo/error.log \
+-l "tail -f -n 50 $CI_LOG | grep API"  \
+/var/log/obp/ls/ls_script.log 

python_apps/pypo/scripts/old_files/ls_config.liq.dist

@@ -0,0 +1,44 @@
+###########################################
+# liquidsoap config file                  #
+###########################################
+
+# This config assumes that there are 
+# two instances of LS running
+# the "scheduler" & the "fallback" instance
+
+
+###########################################
+# general settings                        #
+###########################################
+
+log_file = "/var/log/pypo/<script>.log"
+log_level = 5
+
+# archive directory
+archive_dir = "/opt/pypo/archive/"
+
+# list pointing to the current couchcaster mountpoint 
+couchcaster_list = "http://stage.openbroadcast.ch/mod/ml/api/pypo/current_couchcaster"
+
+
+
+###########################################
+# stream settings                         #
+###########################################
+icecast_host = "stream.domain.com"
+icecast_port = 8000
+icecast_pass = "hackme"
+
+# mountpoints
+mount_scheduler = "pypo_scheduler.mp3"
+mount_fallback = "pypo_fallback.mp3"
+mount_final = "pypo_final.mp3"
+
+# mount intra is used for scheduler >>> fallback stream
+mount_intra = "pypo_intra"
+
+# intra-LS streaming (no icecast here)
+intra_host = "pypo-fallback.my-playout-server.xyz.net"
+intra_port = 9000
+intra_pass = "hackmetoo"
+

python_apps/pypo/scripts/old_files/ls_cue.liq

@@ -0,0 +1,36 @@
+
+# Register the cut protocol
+def cue_protocol(arg,delay)
+	# The extraction program
+	# cut_file = "#{configure.libdir}/cut-file.py"
+	cue_script = "./cue_file.py"
+	# Parse args 
+	ret = string.extract(pattern="cue_in=(\d+)",arg)
+	start = 
+		if list.length(ret) == 0 then 
+			"0"
+		else
+			ret["1"]
+		end
+	ret = string.extract(pattern="cue_out=(\d+)",arg)
+	stop =
+		if list.length(ret) == 0 then
+			"0"
+		else
+			ret["1"]
+		end
+	ret = string.extract(pattern=":(.*)$",arg)
+	uri = 
+	   if list.length(ret) == 0 then
+			""
+		else
+			ret["1"]
+		end
+	x = get_process_lines("#{cue_script} #{quote(uri)} #{start} #{stop}")
+	if list.hd(x) != "" then
+		([list.hd(x)],[])
+	else
+		([uri],[])
+	end
+end
+add_post_processor("cue_file", temporary=true, cue_protocol)

python_apps/pypo/scripts/old_files/ls_run.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+# export home dir
+#export HOME=/home/pypo/
+
+# start liquidsoap with corresponding user & scrupt
+sudo -u pypo /usr/local/bin/liquidsoap ls_script.liq

python_apps/pypo/scripts/old_files/ls_script.liq

@@ -0,0 +1,106 @@
+######################################
+# main liquidsoap development script #
+######################################
+# author Jonas Ohrstrom <jonas@digris.ch>
+
+########################################
+# include configuration                #
+########################################
+
+%include "library/pervasives.liq"
+%include "ls_config.liq"
+%include "library.liq"
+%include "include_dynamic_vars.liq"
+%include "include_notify.liq"
+
+silence_threshold = -50.
+silence_time = 3.
+
+# log
+set("log.file.path",log_file)
+set("log.stdout", true)
+set("log.level",log_level)
+
+# telnet server
+set("server.telnet", true)
+
+######################################
+# some functions needed              #
+######################################
+def fcross(a,b) =
+    add(normalize=false,[b,a])
+end
+
+######################################
+# live recording functions
+######################################
+def live_start() =
+  log("got live source")
+  ignore(execute("archives.start"))
+end
+
+def live_stop() =
+  log("live source has gone")
+  ignore(execute("archives.stop"))
+end
+
+
+#######################################################################
+# File locations / sources
+#######################################################################
+silence = single("/opt/pypo/files/basic/silence.mp3")
+jingles_cc = playlist("/opt/pypo/files/jingles/jcc")
+fallback_airtime = playlist("/opt/pypo/files/basic/silence-playlist.lsp")
+fallback_airtime = audio_to_stereo(fallback_airtime)
+
+
+# default
+default = silence
+
+special = request.queue(id="special")
+
+
+#######################################################################
+# Includeing two A/B Queues, daypart & scheduler
+# this will give us the sources 'daypart' & 'scheduler' 
+#######################################################################
+%include "include_daypart.liq"
+%include "include_scheduler.liq"
+
+source = fallback(track_sensitive=false,transitions=[dp_to_scheduler],[strip_blank(threshold=silence_threshold,length=silence_time,scheduler),daypart])
+
+%include "include_live_in.liq"
+
+live = fallback(track_sensitive=false,[strip_blank(threshold=silence_threshold,length=silence_time,live),fallback_airtime])
+live = switch(track_sensitive=false, [({!live_active},live)])
+
+source = fallback(track_sensitive=false,transitions=[to_live_s, to_scheduler_s],[live, source]) 
+
+# handle the annotate fades
+faded = fade.in(type="log", fade.out(type="log", source))
+
+# add up with a crossfade function (defined above)
+source = cross(fcross,faded)
+
+# track start detection (for notifications)
+source = on_metadata(notify, source)
+#source = on_track(notify, source)
+
+# special to mix with final source
+source = smooth_add(normal=source,special=special)
+
+
+#####################################
+# Stream Output
+#####################################
+# finally the output | mp3
+#clock(id="clock_icecast",
+#  output.icecast(%mp3,
+#   host = icecast_host, port = icecast_port,
+#    password = icecast_pass, mount = mount_scheduler,
+#    fallible = true,
+#    restart = true,
+#    restart_delay = 5,
+#    buffer(source)))
+
+out(source)

python_apps/pypo/scripts/old_files/silence-playlist.lsp

@@ -0,0 +1,48 @@
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3
+/opt/pypo/files/basic/silence.mp3