CC-5884: Modify Pypo -> Download files from cloud storage

airtime_mvc/application/configs/conf.php

@@ -30,6 +30,8 @@ class Config {
         // Name of the web server user
         $CC_CONFIG['webServerUser'] = $values['general']['web_server_user'];
         $CC_CONFIG['rabbitmq'] = $values['rabbitmq'];
+        
+        $CC_CONFIG['s3'] = $values['s3'];
 
         $CC_CONFIG['baseDir'] = $values['general']['base_dir'];
         $CC_CONFIG['baseUrl'] = $values['general']['base_url'];

airtime_mvc/application/models/Schedule.php

@@ -724,7 +724,7 @@ SQL;
         }
     }
 
-    private static function createFileScheduleEvent(&$data, $item, $media_id, $uri)
+    private static function createFileScheduleEvent(&$data, $item, $media_id, $uri, $filesize, $object_name, $isInCloud)
     {
         $start = self::AirtimeTimeToPypoTime($item["start"]);
         $end   = self::AirtimeTimeToPypoTime($item["end"]);
@@ -756,6 +756,9 @@ SQL;
             'show_name'         => $item["show_name"],
             'replay_gain'       => $replay_gain,
             'independent_event' => $independent_event,
+            'filesize'          => $filesize,
+            'object_name'       => $object_name,
+            'is_in_cloud'       => $isInCloud
         );
 
         if ($schedule_item['cue_in'] > $schedule_item['cue_out']) {
@@ -889,7 +892,10 @@ SQL;
                 $media_id = $item['file_id'];
                 $storedFile = Application_Model_StoredFile::RecallById($media_id);
                 $uri = $storedFile->getFilePath();
-                self::createFileScheduleEvent($data, $item, $media_id, $uri);
+                $object_name = $storedFile->getResourceId();
+                $filesize = $storedFile->getFileSize();
+                $isInCloud = $storedFile->isInCloud();
+                self::createFileScheduleEvent($data, $item, $media_id, $uri, $filesize, $object_name, $isInCloud);
             } 
             elseif (!is_null($item['stream_id'])) {
                 //row is type "webstream"

airtime_mvc/application/models/StoredFile.php

@@ -473,7 +473,7 @@ SQL;
 
         $mime = $this->_file->getDbMime();
 
-        if ($mime == "audio/ogg" || $mime == "application/ogg") {
+        if ($mime == "audio/ogg" || $mime == "application/ogg" || $mime == "audio/vorbis") {
             return "ogg";
         } elseif ($mime == "audio/mp3" || $mime == "audio/mpeg") {
             return "mp3";
@@ -495,15 +495,20 @@ SQL;
     {
         assert($this->_file);
         
-        $music_dir = Application_Model_MusicDir::getDirByPK($this->
+        if ($this->isInCloud()) {
-            _file->getDbDirectory());
+            return $this->getCloudUrl();
-        if (!$music_dir) {
+        } else {
-            throw new Exception("Invalid music_dir for file in database.");
+        
+            $music_dir = Application_Model_MusicDir::getDirByPK($this->
+                _file->getDbDirectory());
+            if (!$music_dir) {
+                throw new Exception("Invalid music_dir for file in database.");
+            }
+            $directory = $music_dir->getDirectory();
+            $filepath  = $this->_file->getDbFilepath();
+    
+            return Application_Common_OsPath::join($directory, $filepath);
         }
-        $directory = $music_dir->getDirectory();
-        $filepath  = $this->_file->getDbFilepath();
-
-        return Application_Common_OsPath::join($directory, $filepath);
     }
 
     /**
@@ -555,7 +560,37 @@ SQL;
     {
         return $baseUrl."api/get-media/file/".$this->getId().".".$this->getFileExtension();
     }
+    
+    public function isInCloud()
+    {
+        $location = CcMusicDirsQuery::create()->findPk($this->_file->getDbDirectory());
+        if ($location->getType() == "cloud") {
+            return true;
+        }
+        return false;
+    }
+    
+    public function getCloudUrl()
+    {
+        $CC_CONFIG = Config::getConfig();
+        return $CC_CONFIG["s3"]["host"]."/".$CC_CONFIG["s3"]["bucket"]."/" . urlencode($this->getResourceId());
+    }
+    
+    public function getResourceId()
+    {
+        return $this->_file->getDbResourceId();
+    }
 
+    public function getFileSize()
+    {
+        if ($this->isInCloud()) {
+            //TODO: error checking - 403 forbidden>
+            return strlen(file_get_contents($this->getCloudUrl()));
+        } else {
+            return filesize($this->getFilePath());
+        }
+    }
+    
     public static function Insert($md, $con)
     {
         // save some work by checking if filepath is given right away

airtime_mvc/application/modules/rest/controllers/MediaController.php

@@ -232,7 +232,7 @@ class Rest_MediaController extends Zend_Rest_Controller
             //Our RESTful API takes "full_path" as a field, which we then split and translate to match
             //our internal schema. Internally, file path is stored relative to a directory, with the directory
             //as a foreign key to cc_music_dirs.
-            if (isset($requestData["full_path"])) {
+            /*if (isset($requestData["full_path"])) {
                 $fileSizeBytes = filesize($requestData["full_path"]);
                 if ($fileSizeBytes === false)
                 {
@@ -254,20 +254,25 @@ class Rest_MediaController extends Zend_Rest_Controller
                     $file->setDbFilepath($filePathRelativeToStor);
                     $file->setDbDirectory(1); //1 corresponds to the default stor/imported directory.
                 }
-            } else if (isset($requestData["s3_object_name"])) {
+            }*/
+            
+            if (isset($requestData["s3_object_name"])) {
                 $cloud_cc_music_dir = CcMusicDirsQuery::create()
                     ->filterByType("cloud")
                     ->findOne();
                 $file->setDbDirectory($cloud_cc_music_dir->getId());
                 $file->setDbResourceId($requestData["s3_object_name"]);
+                
+                //Application_Model_Preference::updateDiskUsage($requestData["filesize"]);
             }
             
             $now  = new DateTime("now", new DateTimeZone("UTC"));
             $file->setDbMtime($now);
             $file->save();
             
-            /* $this->removeEmptySubFolders(
+            //get the filesize and update disk_usage
-                isset($_SERVER['AIRTIME_BASE']) ? $_SERVER['AIRTIME_BASE']."/srv/airtime/stor/organize/" : "/srv/airtime/stor/organize/"); */
+            $storedFile = Application_Model_StoredFile::RecallById($file->getDbId());
+            Application_Model_Preference::updateDiskUsage($storedFile->getFileSize());
             
             $this->getResponse()
                 ->setHttpResponseCode(200)

python_apps/pypo/pypofile.py

@@ -2,6 +2,7 @@
 
 from threading import Thread
 from Queue import Empty
+from cloud_storage_downloader import CloudStorageDownloader
 
 import logging
 import shutil
@@ -9,6 +10,7 @@ import os
 import sys
 import stat
 
+
 from std_err_override import LogWriter
 
 # configure logging
@@ -35,16 +37,21 @@ class PypoFile(Thread):
         """
         src = media_item['uri']
         dst = media_item['dst']
+        is_in_cloud = media_item['is_in_cloud']
 
         try:
-            src_size = os.path.getsize(src)
+            if is_in_cloud:
+                src_size = media_item['filesize']
+            else:
+                src_size = os.path.getsize(src)
         except Exception, e:
             self.logger.error("Could not get size of source file: %s", src)
             return
-
+        
         dst_exists = True
         try:
             dst_size = os.path.getsize(dst)
+            self.logger.debug(dst_size)
         except Exception, e:
             dst_exists = False
 
@@ -66,7 +73,11 @@ class PypoFile(Thread):
                 """
                 copy will overwrite dst if it already exists
                 """
-                shutil.copy(src, dst)
+                if is_in_cloud:
+                    csd = CloudStorageDownloader()
+                    csd.download_obj(dst, media_item['object_name'])
+                else:
+                    shutil.copy(src, dst)
 
                 #make file world readable
                 os.chmod(dst, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)