From c2411b6f4100249ac964b6dbb444efe6baefbf22 Mon Sep 17 00:00:00 2001 From: drigato Date: Thu, 10 Jul 2014 17:56:41 -0400 Subject: [PATCH] CC-5884: Modify Pypo -> Download files from cloud storage --- airtime_mvc/application/configs/conf.php | 2 + airtime_mvc/application/models/Schedule.php | 10 +++- airtime_mvc/application/models/StoredFile.php | 53 +++++++++++++++---- .../rest/controllers/MediaController.php | 13 +++-- python_apps/pypo/pypofile.py | 17 ++++-- 5 files changed, 77 insertions(+), 18 deletions(-) diff --git a/airtime_mvc/application/configs/conf.php b/airtime_mvc/application/configs/conf.php index 4063747c5..2c11755cf 100644 --- a/airtime_mvc/application/configs/conf.php +++ b/airtime_mvc/application/configs/conf.php @@ -30,6 +30,8 @@ class Config { // Name of the web server user $CC_CONFIG['webServerUser'] = $values['general']['web_server_user']; $CC_CONFIG['rabbitmq'] = $values['rabbitmq']; + + $CC_CONFIG['s3'] = $values['s3']; $CC_CONFIG['baseDir'] = $values['general']['base_dir']; $CC_CONFIG['baseUrl'] = $values['general']['base_url']; diff --git a/airtime_mvc/application/models/Schedule.php b/airtime_mvc/application/models/Schedule.php index 4b2cf044e..810852bbf 100644 --- a/airtime_mvc/application/models/Schedule.php +++ b/airtime_mvc/application/models/Schedule.php @@ -724,7 +724,7 @@ SQL; } } - private static function createFileScheduleEvent(&$data, $item, $media_id, $uri) + private static function createFileScheduleEvent(&$data, $item, $media_id, $uri, $filesize, $object_name, $isInCloud) { $start = self::AirtimeTimeToPypoTime($item["start"]); $end = self::AirtimeTimeToPypoTime($item["end"]); @@ -756,6 +756,9 @@ SQL; 'show_name' => $item["show_name"], 'replay_gain' => $replay_gain, 'independent_event' => $independent_event, + 'filesize' => $filesize, + 'object_name' => $object_name, + 'is_in_cloud' => $isInCloud ); if ($schedule_item['cue_in'] > $schedule_item['cue_out']) { @@ -889,7 +892,10 @@ SQL; $media_id = $item['file_id']; $storedFile = Application_Model_StoredFile::RecallById($media_id); $uri = $storedFile->getFilePath(); - self::createFileScheduleEvent($data, $item, $media_id, $uri); + $object_name = $storedFile->getResourceId(); + $filesize = $storedFile->getFileSize(); + $isInCloud = $storedFile->isInCloud(); + self::createFileScheduleEvent($data, $item, $media_id, $uri, $filesize, $object_name, $isInCloud); } elseif (!is_null($item['stream_id'])) { //row is type "webstream" diff --git a/airtime_mvc/application/models/StoredFile.php b/airtime_mvc/application/models/StoredFile.php index e30851e26..01ccdd0a6 100644 --- a/airtime_mvc/application/models/StoredFile.php +++ b/airtime_mvc/application/models/StoredFile.php @@ -473,7 +473,7 @@ SQL; $mime = $this->_file->getDbMime(); - if ($mime == "audio/ogg" || $mime == "application/ogg") { + if ($mime == "audio/ogg" || $mime == "application/ogg" || $mime == "audio/vorbis") { return "ogg"; } elseif ($mime == "audio/mp3" || $mime == "audio/mpeg") { return "mp3"; @@ -495,15 +495,20 @@ SQL; { assert($this->_file); - $music_dir = Application_Model_MusicDir::getDirByPK($this-> - _file->getDbDirectory()); - if (!$music_dir) { - throw new Exception("Invalid music_dir for file in database."); + if ($this->isInCloud()) { + return $this->getCloudUrl(); + } else { + + $music_dir = Application_Model_MusicDir::getDirByPK($this-> + _file->getDbDirectory()); + if (!$music_dir) { + throw new Exception("Invalid music_dir for file in database."); + } + $directory = $music_dir->getDirectory(); + $filepath = $this->_file->getDbFilepath(); + + return Application_Common_OsPath::join($directory, $filepath); } - $directory = $music_dir->getDirectory(); - $filepath = $this->_file->getDbFilepath(); - - return Application_Common_OsPath::join($directory, $filepath); } /** @@ -555,7 +560,37 @@ SQL; { return $baseUrl."api/get-media/file/".$this->getId().".".$this->getFileExtension(); } + + public function isInCloud() + { + $location = CcMusicDirsQuery::create()->findPk($this->_file->getDbDirectory()); + if ($location->getType() == "cloud") { + return true; + } + return false; + } + + public function getCloudUrl() + { + $CC_CONFIG = Config::getConfig(); + return $CC_CONFIG["s3"]["host"]."/".$CC_CONFIG["s3"]["bucket"]."/" . urlencode($this->getResourceId()); + } + + public function getResourceId() + { + return $this->_file->getDbResourceId(); + } + public function getFileSize() + { + if ($this->isInCloud()) { + //TODO: error checking - 403 forbidden> + return strlen(file_get_contents($this->getCloudUrl())); + } else { + return filesize($this->getFilePath()); + } + } + public static function Insert($md, $con) { // save some work by checking if filepath is given right away diff --git a/airtime_mvc/application/modules/rest/controllers/MediaController.php b/airtime_mvc/application/modules/rest/controllers/MediaController.php index 0489d086e..294677909 100644 --- a/airtime_mvc/application/modules/rest/controllers/MediaController.php +++ b/airtime_mvc/application/modules/rest/controllers/MediaController.php @@ -232,7 +232,7 @@ class Rest_MediaController extends Zend_Rest_Controller //Our RESTful API takes "full_path" as a field, which we then split and translate to match //our internal schema. Internally, file path is stored relative to a directory, with the directory //as a foreign key to cc_music_dirs. - if (isset($requestData["full_path"])) { + /*if (isset($requestData["full_path"])) { $fileSizeBytes = filesize($requestData["full_path"]); if ($fileSizeBytes === false) { @@ -254,20 +254,25 @@ class Rest_MediaController extends Zend_Rest_Controller $file->setDbFilepath($filePathRelativeToStor); $file->setDbDirectory(1); //1 corresponds to the default stor/imported directory. } - } else if (isset($requestData["s3_object_name"])) { + }*/ + + if (isset($requestData["s3_object_name"])) { $cloud_cc_music_dir = CcMusicDirsQuery::create() ->filterByType("cloud") ->findOne(); $file->setDbDirectory($cloud_cc_music_dir->getId()); $file->setDbResourceId($requestData["s3_object_name"]); + + //Application_Model_Preference::updateDiskUsage($requestData["filesize"]); } $now = new DateTime("now", new DateTimeZone("UTC")); $file->setDbMtime($now); $file->save(); - /* $this->removeEmptySubFolders( - isset($_SERVER['AIRTIME_BASE']) ? $_SERVER['AIRTIME_BASE']."/srv/airtime/stor/organize/" : "/srv/airtime/stor/organize/"); */ + //get the filesize and update disk_usage + $storedFile = Application_Model_StoredFile::RecallById($file->getDbId()); + Application_Model_Preference::updateDiskUsage($storedFile->getFileSize()); $this->getResponse() ->setHttpResponseCode(200) diff --git a/python_apps/pypo/pypofile.py b/python_apps/pypo/pypofile.py index c636e374c..4292adf62 100644 --- a/python_apps/pypo/pypofile.py +++ b/python_apps/pypo/pypofile.py @@ -2,6 +2,7 @@ from threading import Thread from Queue import Empty +from cloud_storage_downloader import CloudStorageDownloader import logging import shutil @@ -9,6 +10,7 @@ import os import sys import stat + from std_err_override import LogWriter # configure logging @@ -35,16 +37,21 @@ class PypoFile(Thread): """ src = media_item['uri'] dst = media_item['dst'] + is_in_cloud = media_item['is_in_cloud'] try: - src_size = os.path.getsize(src) + if is_in_cloud: + src_size = media_item['filesize'] + else: + src_size = os.path.getsize(src) except Exception, e: self.logger.error("Could not get size of source file: %s", src) return - + dst_exists = True try: dst_size = os.path.getsize(dst) + self.logger.debug(dst_size) except Exception, e: dst_exists = False @@ -66,7 +73,11 @@ class PypoFile(Thread): """ copy will overwrite dst if it already exists """ - shutil.copy(src, dst) + if is_in_cloud: + csd = CloudStorageDownloader() + csd.download_obj(dst, media_item['object_name']) + else: + shutil.copy(src, dst) #make file world readable os.chmod(dst, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)