Merge branch 'saas-dev-publishing' of github.com:sourcefabric/airtime into saas-dev-publishing

airtime_mvc/application/common/Billing.php

@@ -126,7 +126,8 @@ class Billing
             }
             else
             {
-                if ($product["status"] === "Active") {
+                if ($product["status"] === "Active" ||
+                    $product["status"] === "Suspended") {
                     $airtimeProduct = $product;
                     $subdomain = '';
 

airtime_mvc/application/configs/constants.php

@@ -42,6 +42,7 @@ define('DEFAULT_MICROTIME_FORMAT', 'Y-m-d H:i:s.u');
 define('DEFAULT_ICECAST_PORT', 8000);
 define('DEFAULT_ICECAST_PASS', 'hackme');
 define('DEFAULT_SHOW_COLOR', '76aca5');
+define('DEFAULT_INTERVAL_FORMAT', 'H:i:s.u');
 
 // Metadata Keys for files
 define('MDATA_KEY_FILEPATH'    , 'filepath');

airtime_mvc/application/controllers/ApiController.php

@@ -73,6 +73,7 @@ class ApiController extends Zend_Controller_Action
                 ->addActionContext('update-cue-values-by-silan'    , 'json')
                 ->addActionContext('get-usability-hint'            , 'json')
                 ->addActionContext('poll-celery'                   , 'json')
+                ->addActionContext('recalculate-schedule'          , 'json') //RKTN-260
                 ->initContext();
     }
 
@@ -1524,9 +1525,40 @@ class ApiController extends Zend_Controller_Action
         $streamData = Application_Model_StreamSetting::getEnabledStreamData();
 
         foreach ($streamData as $stream) {
-            $m3uFile .= "#EXTINF,".$stationName." - " . strtoupper($stream['codec']) . "\r\n";
+            $m3uFile .= "#EXTINF," . $stationName . " - " . strtoupper($stream['codec']) . "\r\n";
             $m3uFile .= $stream['url'] . "\r\n\r\n";
         }
         echo $m3uFile;
     }
+
+    public function recalculateScheduleAction()
+    {
+        $this->view->layout()->disableLayout();
+        $this->_helper->viewRenderer->setNoRender(true);
+
+        Zend_Session::start();
+
+        $scheduler = new Application_Model_Scheduler();
+        session_write_close();
+
+        $now = new DateTime("now", new DateTimeZone("UTC"));
+
+        $showInstances =  CcShowInstancesQuery::create()
+            ->filterByDbStarts($now, Criteria::GREATER_THAN)
+            //->filterByDbModifiedInstance(false)
+            ->orderByDbStarts()
+            ->find();
+            //->find($this->con);
+        $total = $showInstances->count();
+        $progress = 0;
+        foreach ($showInstances as $instance) {
+            echo(round(floatval($progress / $total)*100) . "% - " . $instance->getDbId() . "\n<br>");
+            flush();
+            ob_flush();
+            //while(@ob_end_clean());
+            $scheduler->removeGaps2($instance->getDbId());
+            $progress += 1;
+        }
+        echo("Recalculated $total shows.");
+    }
 }

airtime_mvc/application/controllers/BillingController.php

@@ -21,6 +21,12 @@ class BillingController extends Zend_Controller_Action {
 
     public function upgradeAction()
     {
+        //If you're not on a trial and you're suspended, we don't let you access the plans page and redirect you to the invoices
+        //page to force you to pay your bills first.
+        $isTrial = (Application_Model_Preference::GetPlanLevel() == 'trial');
+        if (!$isTrial && (Application_Model_Preference::getProvisioningStatus() == PROVISIONING_STATUS_SUSPENDED)) {
+            $this->_redirect('billing/invoices');
+        }
 
         Zend_Layout::getMvcInstance()->assign('parent_page', 'Billing');
 

airtime_mvc/application/layouts/scripts/layout.phtml

@@ -22,7 +22,9 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
 <?php echo $this->partial('partialviews/trialBox.phtml', array("is_trial"=>$this->isTrial(), "trial_remain"=> $this->trialRemaining())) ?>
 
     <div id="Panel" class="sticky">
-    <?php if($this->suspended) : ?>
+    <?php if ($this->suspended && $this->isTrial()) : ?>
+        <?php echo $this->partial('partialviews/suspendedtrial.phtml'); ?>
+    <?php elseif ($this->suspended && !$this->isTrial()) : ?>
         <?php echo $this->partial('partialviews/suspended.phtml'); ?>
     <?php else : ?>
 

airtime_mvc/application/models/RabbitMq.php

@@ -141,7 +141,7 @@ class Application_Model_RabbitMq
         $channel->exchange_declare($exchange, $exchangeType, false, true, $autoDeleteExchange);
         
         $msg = new AMQPMessage($jsonData, array('content_type' => 'text/plain'));
-        
+
         $channel->basic_publish($msg, $exchange);
         $channel->close();
         $conn->close();

airtime_mvc/application/models/Scheduler.php

@@ -385,6 +385,24 @@ class Application_Model_Scheduler
     	return $dt;
     }
 
+    private function findTimeDifference2($p_startDT, $p_endDT) {
+        $startEpoch = $p_startDT->format("U.u");
+        $endEpoch = $p_endDT->format("U.u");
+
+        //add two float numbers to 6 subsecond precision
+        //DateTime::createFromFormat("U.u") will have a problem if there is no decimal in the resulting number.
+        $newEpoch = bcsub($endEpoch, (string)$startEpoch, 6);
+
+        $dt = DateTime::createFromFormat("U.u", $newEpoch, new DateTimeZone("UTC"));
+
+        if ($dt === false) {
+            //PHP 5.3.2 problem
+            $dt = DateTime::createFromFormat("U", intval($newEpoch), new DateTimeZone("UTC"));
+        }
+
+        return $dt;
+    }
+
     /*
      * @param DateTime startDT in UTC
      * @param string duration
@@ -499,12 +517,70 @@ class Application_Model_Scheduler
 
         $itemStartDT = $instance->getDbStarts(null);
         foreach ($schedule as $item) {
+
+            $itemEndDT = $this->findEndTime($itemStartDT, $item->getDbClipLength());
+
+            $item->setDbStarts($itemStartDT)
+                ->setDbEnds($itemEndDT);
+
+            $itemStartDT = $itemEndDT;
+        }
+
+        $schedule->save($this->con);
+    }
+
+    /** Temporary hack to copy the track cue in, out, and length from the cc_files table to fix
+     *  incorrect track lengths (RKTN-260)
+     */
+    public function removeGaps2($showInstance, $exclude = null) {
+
+        $instance = CcShowInstancesQuery::create()->findPK($showInstance, $this->con);
+        if (is_null($instance)) {
+            throw new OutDatedScheduleException(_("The schedule you're viewing is out of date!"));
+        }
+
+        $itemStartDT = $instance->getDbStarts(null);
+
+        $schedule = CcScheduleQuery::create()
+            ->filterByDbInstanceId($showInstance)
+            ->filterByDbId($exclude, Criteria::NOT_IN)
+            ->orderByDbStarts()
+            ->find($this->con);
+
+        foreach ($schedule as $item) {
+
+            //START OF TIME RECALC HACK
+
+            //TODO: Copy the cue in, cue out, and track length from the cc_files table
+            $file = $item->getCcFiles($this->con);
+            if (!$file) {
+                continue;
+            }
+            $item->setDbCueIn($file->getDbCueIn());
+            $item->setDbCueOut($file->getDbCueOut());
+
+            $cueOut = new DateTime($file->getDbCueOut());
+            $cueIn = new DateTime($file->getDbCueIn());
+            $clipLength = $this->findTimeDifference2($cueIn, $cueOut);
+
+            //The clip length is supposed to be cue out - cue in:
+            //FIXME: How do we correctly do time arithmetic in PHP without losing the millseconds?
+            $item->setDbClipLength($clipLength->format(DEFAULT_INTERVAL_FORMAT));
+            $item->save($this->con);
+            //Ensure we don't get cached results
+            CcSchedulePeer::clearInstancePool();
+            //END OF TIME RECALC HACK
+
             $itemEndDT = $this->findEndTime($itemStartDT, $item->getDbClipLength());
             $item->setDbStarts($itemStartDT)
                 ->setDbEnds($itemEndDT)
                 ->save($this->con);
             $itemStartDT = $this->findTimeDifference($itemEndDT, $this->crossfadeDuration);
         }
+
+        $instance->updateDbTimeFilled($this->con); //FIXME: TIME RECALC HACK (Albert)
+
+        $schedule->save($this->con);
     }
 
     /**

airtime_mvc/application/modules/rest/controllers/MediaController.php

@@ -136,9 +136,8 @@ class Rest_MediaController extends Zend_Rest_Controller
                 ->appendBody("ERROR: Disk Quota reached.");
         }
         catch (Exception $e) {
-            $this->unknownErrorResponse();
+            $this->serviceUnavailableResponse();
-            Logging::error($e->getMessage());
+            Logging::error($e->getMessage() . "\n" . $e->getTraceAsString());
-            throw $e;
         }
     }
 
@@ -249,5 +248,12 @@ class Rest_MediaController extends Zend_Rest_Controller
         $resp->setHttpResponseCode(400);
         $resp->appendBody("An unknown error occurred.");
     }
+
+    private function serviceUnavailableResponse()
+    {
+        $resp = $this->getResponse();
+        $resp->setHttpResponseCode(400);
+        $resp->appendBody("An error occurred while processing your upload. Please try again in a few minutes.");
+    }
 }
 

airtime_mvc/application/views/scripts/billing/upgrade.phtml

@@ -265,10 +265,12 @@ echo($currentProduct["name"]);
 //echo Application_Model_Preference::GetPlanLevel();
 ?>
 </p>
+    <!--
     <div class="educational-discount">
         <h4>Are you a student or educator?</h4>
         <p>Sign up on an annual plan before the end of October to receive a special educational discount. <a href="https://www.airtime.pro/educational-discount">Find out more</a>.</p>
     </div>
+    -->
 
 <h3><?php echo(_pro('Choose a plan: ')); ?></h3>
  <form id="<?php echo $form->getId(); ?>" method="<?php echo $form->getMethod() ?>" action="<?php echo

airtime_mvc/application/views/scripts/partialviews/suspendedtrial.phtml

@@ -0,0 +1,9 @@
+<div class="suspension_notice">
+    <H2>Station Suspended</H2>
+    <p>
+        <?php echo(_pro(sprintf('Your free trial is now over. To restore your station, <a href="%s">please upgrade your plan now</a>.', '/billing/upgrade'))); ?>
+    </p>
+    <p>
+        <?php echo(_pro(sprintf('Your station will be <b>removed</b> within 7 days if you do not upgrade. If you believe this suspension was in error, <a href="%s">please contact support</a>.', 'https://sourcefabricberlin.zendesk.com/anonymous_requests/new'))); ?>
+    </p>
+</div>

python_apps/airtime_analyzer/airtime_analyzer/cuepoint_analyzer.py

@@ -28,12 +28,38 @@ class CuePointAnalyzer(Analyzer):
         try:
             results_json = subprocess.check_output(command, stderr=subprocess.STDOUT, close_fds=True)
             silan_results = json.loads(results_json)
-            metadata['length_seconds'] = float(silan_results['file duration'])
+
-            # Conver the length into a formatted time string
+            # Defensive coding against Silan wildly miscalculating the cue in and out times:
-            track_length = datetime.timedelta(seconds=metadata['length_seconds'])
+            silan_length_seconds = float(silan_results['file duration'])
-            metadata["length"] = str(track_length)
+            silan_cuein = format(silan_results['sound'][0][0], 'f')
-            metadata['cuein'] = format(silan_results['sound'][0][0], 'f')
+            silan_cueout = format(silan_results['sound'][0][1], 'f')
-            metadata['cueout'] = format(silan_results['sound'][0][1], 'f')
+
+            # Sanity check the results against any existing metadata passed to us (presumably extracted by Mutagen):
+            if 'length_seconds' in metadata:
+                # Silan has a rare bug where it can massively overestimate the length or cue out time sometimes.
+                if (silan_length_seconds - metadata['length_seconds'] > 3) or (float(silan_cueout) - metadata['length_seconds'] > 2):
+                    # Don't trust anything silan says then...
+                    raise Exception("Silan cue out {0} or length {1} differs too much from the Mutagen length {2}. Ignoring Silan values."
+                                    .format(silan_cueout, silan_length_seconds, metadata['length_seconds']))
+                # Don't allow silan to trim more than the greater of 3 seconds or 5% off the start of a track
+                if float(silan_cuein) > max(silan_length_seconds*0.05, 3):
+                    raise Exception("Silan cue in time {0} too big, ignoring.".format(silan_cuein))
+            else:
+                # Only use the Silan track length in the worst case, where Mutagen didn't give us one for some reason.
+                # (This is mostly to make the unit tests still pass.)
+                # Convert the length into a formatted time string.
+                metadata['length_seconds'] = silan_length_seconds #
+                track_length = datetime.timedelta(seconds=metadata['length_seconds'])
+                metadata["length"] = str(track_length)
+
+
+            ''' XXX: I've commented out the track_length stuff below because Mutagen seems more accurate than silan
+                     as of Mutagen version 1.31. We are always going to use Mutagen's length now because Silan's
+                     length can be off by a few seconds reasonably often.
+            '''
+
+            metadata['cuein'] = silan_cuein
+            metadata['cueout'] = silan_cueout
 
         except OSError as e: # silan was not found
             logging.warn("Failed to run: %s - %s. %s" % (command[0], e.strerror, "Do you have silan installed?"))

python_apps/airtime_analyzer/airtime_analyzer/metadata_analyzer.py

@@ -67,8 +67,11 @@ class MetadataAnalyzer(Analyzer):
             track_length = datetime.timedelta(seconds=info.length)
             metadata["length"] = str(track_length) #time.strftime("%H:%M:%S.%f", track_length)
             # Other fields for Airtime
-            metadata["cueout"] = metadata["length"] 
+            metadata["cueout"] = metadata["length"]
-      
+
+        # Set a default cue in time in seconds
+        metadata["cuein"] = 0.0;
+
         if hasattr(info, "bitrate"):
             metadata["bit_rate"] = info.bitrate
         

python_apps/airtime_analyzer/setup.py

@@ -27,7 +27,7 @@ setup(name='airtime_analyzer',
       packages=['airtime_analyzer'],
       scripts=['bin/airtime_analyzer'],
       install_requires=[
-          'mutagen',
+          'mutagen==1.31', # The Mutagen guys change stuff all the time that break our unit tests. Watch out for this.
           'pika',
           'daemon',
           'python-magic',

python_apps/airtime_analyzer/tests/metadata_analyzer_tests.py

@@ -24,7 +24,7 @@ def test_mp3_mono():
     metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-mono.mp3', dict())
     check_default_metadata(metadata)
     assert metadata['channels'] == 1
-    assert metadata['bit_rate'] == 64000
+    assert metadata['bit_rate'] == 64876
     assert abs(metadata['length_seconds'] - 3.9) < 0.1
     assert metadata['mime'] == 'audio/mp3' # Not unicode because MIMEs aren't.
     assert metadata['track_total'] == u'10' # MP3s can have a track_total
@@ -34,7 +34,7 @@ def test_mp3_jointstereo():
     metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-jointstereo.mp3', dict())
     check_default_metadata(metadata)
     assert metadata['channels'] == 2
-    assert metadata['bit_rate'] == 128000
+    assert metadata['bit_rate'] == 129757
     assert abs(metadata['length_seconds'] - 3.9) < 0.1
     assert metadata['mime'] == 'audio/mp3'
     assert metadata['track_total'] == u'10' # MP3s can have a track_total
@@ -43,7 +43,7 @@ def test_mp3_simplestereo():
     metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-simplestereo.mp3', dict())
     check_default_metadata(metadata)
     assert metadata['channels'] == 2
-    assert metadata['bit_rate'] == 128000
+    assert metadata['bit_rate'] == 129757
     assert abs(metadata['length_seconds'] - 3.9) < 0.1
     assert metadata['mime'] == 'audio/mp3'
     assert metadata['track_total'] == u'10' # MP3s can have a track_total
@@ -52,7 +52,7 @@ def test_mp3_dualmono():
     metadata = MetadataAnalyzer.analyze(u'tests/test_data/44100Hz-16bit-dualmono.mp3', dict())
     check_default_metadata(metadata)
     assert metadata['channels'] == 2
-    assert metadata['bit_rate'] == 128000
+    assert metadata['bit_rate'] == 129757
     assert abs(metadata['length_seconds'] - 3.9) < 0.1
     assert metadata['mime'] == 'audio/mp3'
     assert metadata['track_total'] == u'10' # MP3s can have a track_total
@@ -109,7 +109,7 @@ def test_mp3_utf8():
     assert metadata['genre'] == u'Я Б Г Д Ж Й'
     assert metadata['track_number'] == u'1'
     assert metadata['channels'] == 2
-    assert metadata['bit_rate'] == 128000
+    assert metadata['bit_rate'] == 129757
     assert abs(metadata['length_seconds'] - 3.9) < 0.1
     assert metadata['mime'] == 'audio/mp3'
     assert metadata['track_total'] == u'10' # MP3s can have a track_total
@@ -153,7 +153,7 @@ def test_mp3_bad_channels():
     metadata = MetadataAnalyzer.analyze(filename, dict())
     check_default_metadata(metadata)
     assert metadata['channels'] == 1
-    assert metadata['bit_rate'] == 64000
+    assert metadata['bit_rate'] == 64876
     assert abs(metadata['length_seconds'] - 3.9) < 0.1
     assert metadata['mime'] == 'audio/mp3' # Not unicode because MIMEs aren't.
     assert metadata['track_total'] == u'10' # MP3s can have a track_total

python_apps/pypo/pypo/listenerstat.py

@@ -1,6 +1,6 @@
 from threading import Thread
 import urllib2
-import xml.dom.minidom
+import defusedxml.minidom
 import base64
 from datetime import datetime
 import traceback
@@ -64,7 +64,7 @@ class ListenerStat(Thread):
         else:
             url = 'http://%(host)s:%(port)s/admin/stats.xml' % ip
             document = self.get_stream_server_xml(ip, url)
-        dom = xml.dom.minidom.parseString(document)
+        dom = defusedxml.minidom.parseString(document)
         sources = dom.getElementsByTagName("source")
 
         mount_stats = None
@@ -87,7 +87,7 @@ class ListenerStat(Thread):
     def get_shoutcast_stats(self, ip):
         url = 'http://%(host)s:%(port)s/admin.cgi?sid=1&mode=viewxml' % ip
         document = self.get_stream_server_xml(ip, url, is_shoutcast=True)
-        dom = xml.dom.minidom.parseString(document)
+        dom = defusedxml.parseString(document)
         current_listeners = dom.getElementsByTagName("CURRENTLISTENERS")
 
         timestamp = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")

python_apps/pypo/setup.py

@@ -58,7 +58,8 @@ setup(name='airtime-playout',
           'pyinotify',
           'pytz',
           'requests',
-          'wsgiref'
+          'wsgiref',
+          'defusedxml'
       ],
       zip_safe=False,
       data_files=data_files)