diff --git a/app/Album.php b/app/Album.php index d71cc1fb..f72ee9ca 100644 --- a/app/Album.php +++ b/app/Album.php @@ -21,17 +21,19 @@ namespace Poniverse\Ponyfm; use Exception; +use Helpers; use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\SoftDeletes; -use Illuminate\Support\Facades\Auth; -use Illuminate\Support\Facades\Cache; -use Illuminate\Support\Facades\URL; +use Illuminate\Foundation\Bus\DispatchesJobs; +use Auth; +use Cache; +use Poniverse\Ponyfm\Traits\TrackCollection; +use URL; use Poniverse\Ponyfm\Traits\SlugTrait; -use Helpers; class Album extends Model { - use SoftDeletes, SlugTrait; + use SoftDeletes, SlugTrait, DispatchesJobs, TrackCollection; protected $dates = ['deleted_at']; @@ -81,12 +83,16 @@ class Album extends Model return $this->hasMany('Poniverse\Ponyfm\Track')->orderBy('track_number', 'asc'); } + public function trackFiles() { + return $this->hasManyThrough(TrackFile::class, Track::class, 'album_id', 'track_id'); + } + public function comments() { return $this->hasMany('Poniverse\Ponyfm\Comment')->orderBy('created_at', 'desc'); } - public static function mapPublicAlbumShow($album) + public static function mapPublicAlbumShow(Album $album) { $tracks = []; foreach ($album->tracks as $track) { @@ -99,7 +105,8 @@ class Album extends Model 'name' => $name, 'extension' => $format['extension'], 'url' => $album->getDownloadUrl($name), - 'size' => Helpers::formatBytes($album->getFilesize($name)) + 'size' => Helpers::formatBytes($album->getFilesize($name)), + 'isCacheable' => (in_array($name, Track::$CacheableFormats) ? true : false) ]; } @@ -131,7 +138,7 @@ class Album extends Model return $data; } - public static function mapPublicAlbumSummary($album) + public static function mapPublicAlbumSummary(Album $album) { $userData = [ 'stats' => [ @@ -145,24 +152,24 @@ class Album extends Model $userRow = $album->users[0]; $userData = [ 'stats' => [ - 'views' => (int) $userRow->view_count, - 'downloads' => (int) $userRow->download_count, + 'views' => (int)$userRow->view_count, + 'downloads' => (int)$userRow->download_count, ], - 'is_favourited' => (bool) $userRow->is_favourited + 'is_favourited' => (bool)$userRow->is_favourited ]; } return [ - 'id' => (int) $album->id, - 'track_count' => (int) $album->track_count, + 'id' => (int)$album->id, + 'track_count' => (int)$album->track_count, 'title' => $album->title, 'slug' => $album->slug, 'created_at' => $album->created_at, 'stats' => [ - 'views' => (int) $album->view_count, - 'downloads' => (int) $album->download_count, - 'comments' => (int) $album->comment_count, - 'favourites' => (int) $album->favourite_count + 'views' => (int)$album->view_count, + 'downloads' => (int)$album->download_count, + 'comments' => (int)$album->comment_count, + 'favourites' => (int)$album->favourite_count ], 'covers' => [ 'small' => $album->getCoverUrl(Image::SMALL), @@ -170,7 +177,7 @@ class Album extends Model ], 'url' => $album->url, 'user' => [ - 'id' => (int) $album->user->id, + 'id' => (int)$album->user->id, 'name' => $album->user->display_name, 'url' => $album->user->url, ], @@ -206,10 +213,18 @@ class Album extends Model return Cache::remember($this->getCacheKey('filesize-' . $format), 1440, function () use ($tracks, $format) { $size = 0; + foreach ($tracks as $track) { + /** @var $track Track */ + // Ensure that only downloadable tracks are added onto the file size if ($track->is_downloadable == 1) { - $size += $track->getFilesize($format); + try { + $size += $track->getFilesize($format); + + } catch (TrackFileNotFoundException $e) { + // do nothing - this track won't be included in the download + } } } @@ -255,6 +270,8 @@ class Album extends Model $index = 1; foreach ($tracks as $track) { + /** @var $track Track */ + $track->track_number = $index; $index++; $track->updateTags(); @@ -301,7 +318,9 @@ class Album extends Model continue; } + /** @var $track Track */ $track = Track::find($trackId); + if ($track->album_id != null && $track->album_id != $this->id) { $albumsToFix[] = $track->album; } @@ -316,6 +335,8 @@ class Album extends Model } foreach ($tracksToRemove as $track) { + /** @var $track Track */ + $track->album_id = null; $track->track_number = null; $track->updateTags(); @@ -323,6 +344,8 @@ class Album extends Model } foreach ($albumsToFix as $album) { + /** @var $album Album */ + $album->updateTrackNumbers(); } @@ -331,7 +354,7 @@ class Album extends Model } } - private function getCacheKey($key) + public function getCacheKey($key) { return 'album-' . $this->id . '-' . $key; } diff --git a/app/Commands/UploadTrackCommand.php b/app/Commands/UploadTrackCommand.php index 3432feb8..8f80a510 100644 --- a/app/Commands/UploadTrackCommand.php +++ b/app/Commands/UploadTrackCommand.php @@ -26,6 +26,8 @@ use AudioCache; use File; use Illuminate\Support\Facades\Log; use Illuminate\Support\Str; +use Symfony\Component\Process\Exception\ProcessFailedException; +use Symfony\Component\Process\Process; class UploadTrackCommand extends CommandBase { @@ -94,8 +96,6 @@ class UploadTrackCommand extends CommandBase $source = $trackFile->getPathname(); $index = 0; - $processes = []; - // Lossy uploads need to be identified and set as the master file // without being re-encoded. $audioObject = AudioCache::get($source); @@ -136,9 +136,16 @@ class UploadTrackCommand extends CommandBase $trackFile = new TrackFile(); $trackFile->is_master = $name === 'FLAC' ? true : false; $trackFile->format = $name; + + if (in_array($name, Track::$CacheableFormats) && $trackFile->is_master == false) { + $trackFile->is_cacheable = true; + } else { + $trackFile->is_cacheable = false; + } $track->trackFiles()->save($trackFile); - $target = $destination . '/' . $trackFile->getFilename(); //$track->getFilenameFor($name); + // Encode track file + $target = $trackFile->getFile(); $command = $format['command']; $command = str_replace('{$source}', '"' . $source . '"', $command); @@ -147,13 +154,16 @@ class UploadTrackCommand extends CommandBase Log::info('Encoding ' . $track->id . ' into ' . $target); $this->notify('Encoding ' . $name, $index / count(Track::$Formats) * 100); - $pipes = []; - $proc = proc_open($command, [0 => ['pipe', 'r'], 1 => ['pipe', 'w'], 2 => ['pipe', 'a']], $pipes); - $processes[] = $proc; - } + $process = new Process($command); + $process->mustRun(); - foreach ($processes as $proc) { - proc_close($proc); + // Update file size for track file + $trackFile->updateFilesize(); + + // Delete track file if it is cacheable + if ($trackFile->is_cacheable == true) { + File::delete($trackFile->getFile()); + } } $track->updateTags(); diff --git a/app/Console/Commands/ClearTrackCache.php b/app/Console/Commands/ClearTrackCache.php new file mode 100644 index 00000000..5c5feea1 --- /dev/null +++ b/app/Console/Commands/ClearTrackCache.php @@ -0,0 +1,111 @@ +. + */ + +namespace Poniverse\Ponyfm\Console\Commands; + +use Carbon\Carbon; +use File; +use Illuminate\Console\Command; +use Illuminate\Support\Facades\Cache; +use Poniverse\Ponyfm\TrackFile; + +class ClearTrackCache extends Command +{ + /** + * The name and signature of the console command. + * + * @var string + */ + protected $signature = 'track-cache:clear + {--tracks=expired : Clear only [expired] (default) or [all] cached tracks.} + {--force : Skip all prompts.}'; + + /** + * The console command description. + * + * @var string + */ + protected $description = 'Clears cached tracks. Defaults to expired tracks. Usage: php artisan track-cache:clear [--tracks=expired|all]'; + + /** + * Create a new command instance. + * + */ + public function __construct() + { + parent::__construct(); + } + + /** + * Execute the console command. + * + * @return mixed + */ + public function handle() + { + if ($this->option('tracks') === 'all') { + // Get all cacheable track files + $trackFiles = TrackFile::where('is_cacheable', true) + ->with('track.album') + ->get(); + } else { + // Get all expired track files + $trackFiles = TrackFile::where('is_cacheable', true) + ->where('expires_at', '<=', Carbon::now()) + ->with('track.album') + ->get(); + } + + // Delete above track files + if (count($trackFiles) === 0) { + $this->info('No tracks found. Exiting.'); + } else { + + if ($this->option('force') || $this->confirm(count($trackFiles) . ' cacheable track files found. Proceed to delete their files if they exist? [y|N]', false)) { + + $count = 0; + + foreach ($trackFiles as $trackFile) { + + // Set expiration to null (so can be re-cached upon request) + $trackFile->expires_at = null; + $trackFile->update(); + + // Delete file if exists + if (File::exists($trackFile->getFile())) { + $count++; + File::delete($trackFile->getFile()); + + $this->info('Deleted ' . $trackFile->getFile()); + } + + // Remove the cached file size for the album + Cache::forget($trackFile->track->album->getCacheKey('filesize-' . $trackFile->format)); + + } + $this->info($count . ' files deleted. Deletion complete. Exiting.'); + } else { + $this->info('Deletion cancelled. Exiting.'); + } + + } + } + +} \ No newline at end of file diff --git a/app/Console/Commands/RebuildFilesizes.php b/app/Console/Commands/RebuildFilesizes.php new file mode 100644 index 00000000..8b0e99ae --- /dev/null +++ b/app/Console/Commands/RebuildFilesizes.php @@ -0,0 +1,91 @@ +. + */ + +namespace Poniverse\Ponyfm\Console\Commands; + +use File; +use Illuminate\Console\Command; +use Poniverse\Ponyfm\TrackFile; + +class RebuildFilesizes extends Command +{ + /** + * The name and signature of the console command. + * + * @var string + */ + protected $signature = 'rebuild:filesizes + {--force : Skip all prompts.}'; + + /** + * The console command description. + * + * @var string + */ + protected $description = 'Rebuilds the filesize cache for each track file which currently exists on disk.'; + + /** + * Create a new command instance. + * + */ + public function __construct() + { + parent::__construct(); + } + + /** + * Execute the console command. + * + * @return mixed + */ + public function handle() + { + $this->info('This will only rebuild the cache for track files which exist on disk; non-existent files will be skipped.'); + + if ($this->option('force') || $this->confirm('Are you sure you want to rebuild the filesize cache? [y|N]', + false) + ) { + + TrackFile::chunk(200, function ($trackFiles) { + + $this->info('========== Start Chunk =========='); + + foreach ($trackFiles as $trackFile) { + /** @var TrackFile $trackFile */ + + if (File::exists($trackFile->getFile())) { + $size = $trackFile->updateFilesize(); + $this->info('ID ' . $trackFile->id . ' processed - ' . $size . ' bytes'); + } else { + $this->info('ID ' . $trackFile->id . ' skipped'); + } + } + + $this->info('=========== End Chunk ==========='); + + }); + + $this->info('Rebuild complete. Exiting.'); + + } else { + $this->info('Rebuild cancelled. Exiting.'); + } + } +} diff --git a/app/Console/Commands/RebuildTrackCache.php b/app/Console/Commands/RebuildTrackCache.php new file mode 100644 index 00000000..510236ba --- /dev/null +++ b/app/Console/Commands/RebuildTrackCache.php @@ -0,0 +1,258 @@ +. + */ + +namespace Poniverse\Ponyfm\Console\Commands; + +use File; +use Illuminate\Console\Command; +use Illuminate\Foundation\Bus\DispatchesJobs; +use Poniverse\Ponyfm\Jobs\EncodeTrackFile; +use Poniverse\Ponyfm\Track; +use Poniverse\Ponyfm\TrackFile; + +class RebuildTrackCache extends Command +{ + + use DispatchesJobs; + + /** + * The name and signature of the console command. + * + * @var string + */ + protected $signature = 'rebuild:track-cache + {--force : Skip all prompts.}'; + + /** + * The console command description. + * + * @var string + */ + protected $description = 'Rebuilds the track cache for when $CacheableFormats is changed. Deletes cacheable files and encodes missing files which are not cacheable.'; + + /** + * Create a new command instance. + * + */ + public function __construct() + { + parent::__construct(); + } + + /** + * Execute the console command. + * + * @return mixed + */ + public function handle() + { + $this->info('***'); + $this->info('If this is your first time running this command, it is *highly* recommended that you ensure the file sizes for all track files have been populated.'); + $this->info('***'); + + if ($this->option('force') || $this->confirm('Are you sure you want to delete all to-be-cached track files and encode missing non-cached track files?', + false) + ) { + + //========================================================================================================== + // Delete previously cached track files + //========================================================================================================== + + $this->output->newLine(1); + $this->info('========== Step 1/4 - Deleting previously cached track files. =========='); + + $count = 0; + + // Chunk track files which are cacheable and NOT master + TrackFile::where('is_cacheable', true) + ->where('is_master', false) + ->chunk(200, function ($trackFiles) use (&$count) { + // Delete chunked track files + foreach ($trackFiles as $trackFile) { + // Clear expiration so will be re-cached on next request + $trackFile->expires_at = null; + $trackFile->update(); + + // Delete files + if (File::exists($trackFile->getFile())) { + $count++; + File::delete($trackFile->getFile()); + $this->info('Deleted ' . $trackFile->getFile()); + } + } + + $this->info($count . ' track files deleted. Deletion complete. Continuing.'); + }); + + //========================================================================================================== + // Update the database entries for cacheable track files - non-cacheable to cacheable + //========================================================================================================== + + $this->output->newLine(3); + $this->info('========== Step 2/4 - Updating is_cacheable entries in database. =========='); + + $trackFileCount = 0; + $formats = []; + + // Find track files which are meant to be cacheable and NOT master, but currently not cacheable + TrackFile::where('is_cacheable', false) + ->whereIn('format', Track::$CacheableFormats) + ->where('is_master', false) + ->chunk(200, function ($trackFiles) use (&$trackFileCount, &$formats) { + $this->output->newLine(1); + $this->info('---------- Start Chunk ----------'); + + // Set above files to cacheable in the database + foreach ($trackFiles as $trackFile) { + $trackFileCount++; + + // Let user know which formats, previously not cached, were made cacheable + $formats[] = $trackFile->format; + + $trackFile->expires_at = null; + $trackFile->is_cacheable = true; + $trackFile->update(); + } + + $this->info('----------- End Chunk -----------'); + $this->output->newLine(1); + }); + + $this->info('Format(s) set from non-cacheable to cacheable: ' . implode(' ', array_unique($formats))); + $this->info($trackFileCount . ' non-cacheable track files set to cacheable.'); + + $this->output->newLine(2); + + //========================================================================================================== + // Update the database entries for cacheable track files - cacheable to non-cacheable + //========================================================================================================== + + $trackFileCount = 0; + $formats = []; + + // Chunk track files which are NOT meant to be cacheable, but currently cacheable + TrackFile::where('is_cacheable', true) + ->whereNotIn('format', Track::$CacheableFormats) + ->chunk(200, function ($trackFiles) use (&$trackFileCount, &$formats) { + $this->output->newLine(1); + $this->info('---------- Start Chunk ----------'); + + // Set chunked track files to non-cacheable in the database + foreach ($trackFiles as $trackFile) { + $trackFileCount++; + + // Let user know which formats, previously not cached, were made cacheable + $formats[] = $trackFile->format; + + $trackFile->expires_at = null; + $trackFile->is_cacheable = false; + $trackFile->update(); + } + + $this->info('----------- End Chunk -----------'); + $this->output->newLine(1); + $this->output->newLine(1); + }); + + + $this->info('Format(s) set from cacheable to non-cacheable: ' . implode(' ', array_unique($formats))); + $this->info($trackFileCount . ' cacheable track files set to non-cacheable.'); + + //========================================================================================================== + // Delete track files which have now been marked as cacheable + //========================================================================================================== + + $this->output->newLine(3); + $this->info('========== Step 3/4 - Deleting now-cacheable track files. =========='); + + $count = 0; + $trackFileCount = 0; + + // Find track files which are cacheable and NOT master + TrackFile::whereIn('format', Track::$CacheableFormats) + ->where('is_master', false) + ->chunk(200, function ($trackFiles) use (&$count, &$trackFileCount) { + $this->output->newLine(1); + $this->info('---------- Start Chunk ----------'); + + foreach ($trackFiles as $trackFile) { + $trackFileCount++; + + // Delete track files if track files exist; double-check that they are NOT master files + if (File::exists($trackFile->getFile()) && $trackFile->is_master == false) { + $count++; + + File::delete($trackFile->getFile()); + $this->info('Deleted ' . $trackFile->getFile()); + } + } + + $this->info('----------- End Chunk -----------'); + $this->output->newLine(1); + }); + + + $this->info(sprintf('%d track files deleted out of %d track files. Continuing.', $count, $trackFileCount)); + + //========================================================================================================== + // Encode missing (i.e., now non-cacheable) track files + //========================================================================================================== + + $this->output->newLine(3); + $this->info('========== Step 4/4 - Encoding missing track files. =========='); + + $count = 0; + + // Chunk non-cacheable track files + TrackFile::where('is_cacheable', false)->chunk(200, function ($trackFiles) use (&$count) { + $this->output->newLine(1); + $this->info('---------- Start Chunk ----------'); + + // Record the track files which do not exist (i.e., have not been encoded yet) + $emptyTrackFiles = []; + + foreach ($trackFiles as $trackFile) { + if (!File::exists($trackFile->getFile())) { + $count++; + $emptyTrackFiles[] = $trackFile; + } + } + + // Encode recorded track files + foreach ($emptyTrackFiles as $emptyTrackFile) { + $this->info("Started encoding track file ID {$emptyTrackFile->id}"); + $this->dispatch(new EncodeTrackFile($emptyTrackFile, false)); + } + + $this->info('----------- End Chunk -----------'); + $this->output->newLine(1); + }); + + + $this->info($count . ' track files encoded.'); + $this->output->newLine(1); + + $this->info('Rebuild complete. Exiting.'); + + } else { + $this->info('Rebuild cancelled. Exiting.'); + } + } +} diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 61b2ca07..ca89fb41 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -41,6 +41,9 @@ class Kernel extends ConsoleKernel \Poniverse\Ponyfm\Console\Commands\FixYearZeroLogs::class, \Poniverse\Ponyfm\Console\Commands\BootstrapLocalEnvironment::class, \Poniverse\Ponyfm\Console\Commands\PoniverseApiSetup::class, + \Poniverse\Ponyfm\Console\Commands\ClearTrackCache::class, + \Poniverse\Ponyfm\Console\Commands\RebuildTrackCache::class, + \Poniverse\Ponyfm\Console\Commands\RebuildFilesizes::class, ]; /** diff --git a/app/Exceptions.php b/app/Exceptions.php new file mode 100644 index 00000000..94afa01b --- /dev/null +++ b/app/Exceptions.php @@ -0,0 +1,14 @@ +findOrFail($id); + + } catch (ModelNotFoundException $e) { + return $this->notFound('Album not found!'); + } + + if (!in_array($format, Track::$CacheableFormats)) { + return $this->notFound('Format not found!'); + } + + $trackCount = $album->countDownloadableTracks($format); + $availableFilesCount = $album->countAvailableTrackFiles($format); + + if ($trackCount === $availableFilesCount) { + $url = $album->getDownloadUrl($format); + } else { + $album->encodeCacheableTrackFiles($format); + $url = null; + } + + return Response::json(['url' => $url], 200); + } + public function getIndex() { $page = 1; diff --git a/app/Http/Controllers/Api/Web/PlaylistsController.php b/app/Http/Controllers/Api/Web/PlaylistsController.php index 7ed44d91..aac920b3 100644 --- a/app/Http/Controllers/Api/Web/PlaylistsController.php +++ b/app/Http/Controllers/Api/Web/PlaylistsController.php @@ -20,6 +20,7 @@ namespace Poniverse\Ponyfm\Http\Controllers\Api\Web; +use Illuminate\Database\Eloquent\ModelNotFoundException; use Poniverse\Ponyfm\Commands\AddTrackToPlaylistCommand; use Poniverse\Ponyfm\Commands\CreatePlaylistCommand; use Poniverse\Ponyfm\Commands\DeletePlaylistCommand; @@ -31,6 +32,7 @@ use Poniverse\Ponyfm\ResourceLogItem; use Illuminate\Support\Facades\Auth; use Illuminate\Support\Facades\Input; use Illuminate\Support\Facades\Response; +use Poniverse\Ponyfm\Track; class PlaylistsController extends ApiControllerBase { @@ -110,6 +112,37 @@ class PlaylistsController extends ApiControllerBase return Response::json(Playlist::mapPublicPlaylistShow($playlist), 200); } + public function getCachedPlaylist($id, $format) + { + // Validation + try { + /** @var $playlist Playlist */ + $playlist = Playlist::with('tracks.trackFiles')->findOrFail($id); + } catch (ModelNotFoundException $e) { + return $this->notFound('Playlist not found!'); + } + + if ((!$playlist->is_public && !Auth::check()) || (!$playlist->is_public && ($playlist->user_id !== Auth::user()->id))) { + return $this->notFound('Playlist not found!'); + } + + if (!in_array($format, Track::$CacheableFormats)) { + return $this->notFound('Format not found!'); + } + + $trackCount = $playlist->countDownloadableTracks($format); + $availableFilesCount = $playlist->countAvailableTrackFiles($format); + + if ($trackCount === $availableFilesCount) { + $url = $playlist->getDownloadUrl($format); + } else { + $playlist->encodeCacheableTrackFiles($format); + $url = null; + } + + return Response::json(['url' => $url], 200); + } + public function getPinned() { $query = Playlist diff --git a/app/Http/Controllers/Api/Web/TracksController.php b/app/Http/Controllers/Api/Web/TracksController.php index 95580a79..81fefdce 100644 --- a/app/Http/Controllers/Api/Web/TracksController.php +++ b/app/Http/Controllers/Api/Web/TracksController.php @@ -20,10 +20,13 @@ namespace Poniverse\Ponyfm\Http\Controllers\Api\Web; +use Illuminate\Database\Eloquent\ModelNotFoundException; +use Illuminate\Support\Facades\File; use Poniverse\Ponyfm\Commands\DeleteTrackCommand; use Poniverse\Ponyfm\Commands\EditTrackCommand; use Poniverse\Ponyfm\Commands\UploadTrackCommand; use Poniverse\Ponyfm\Http\Controllers\ApiControllerBase; +use Poniverse\Ponyfm\Jobs\EncodeTrackFile; use Poniverse\Ponyfm\ResourceLogItem; use Poniverse\Ponyfm\Track; use Cover; @@ -70,6 +73,44 @@ class TracksController extends ApiControllerBase return Response::json(['track' => $returned_track], 200); } + public function getCachedTrack($id, $format) + { + // Validation + try { + $track = Track::findOrFail($id); + } catch (ModelNotFoundException $e) { + return $this->notFound('Track not found!'); + } + + if (!$track->canView(Auth::user())) + return $this->notFound('Track not found!'); + + if ($track->is_downloadable == false) + return $this->notFound('Track not found!'); + + if (!in_array($format, Track::$CacheableFormats)) { + return $this->notFound('Format not found!'); + } + + try { + $trackFile = $track->trackFiles()->where('format', $format)->firstOrFail(); + } catch (ModelNotFoundException $e) { + return $this->notFound('Track file not found!'); + } + + // Return URL or begin encoding + if ($trackFile->expires_at != null && File::exists($trackFile->getFile())) { + $url = $track->getUrlFor($format); + } elseif ($trackFile->is_in_progress === true) { + $url = null; + } else { + $this->dispatch(new EncodeTrackFile($trackFile, true)); + $url = null; + } + + return Response::json(['url' => $url], 200); + } + public function getIndex() { $page = 1; diff --git a/app/Http/routes.php b/app/Http/routes.php index 64bc0691..a4e4cd07 100644 --- a/app/Http/routes.php +++ b/app/Http/routes.php @@ -79,12 +79,15 @@ Route::group(['prefix' => 'api/web'], function() { Route::get('/tracks', 'Api\Web\TracksController@getIndex'); Route::get('/tracks/{id}', 'Api\Web\TracksController@getShow')->where('id', '\d+'); + Route::get('/tracks/cached/{id}/{format}', 'Api\Web\TracksController@getCachedTrack')->where(['id' => '\d+', 'format' => '.+']); Route::get('/albums', 'Api\Web\AlbumsController@getIndex'); Route::get('/albums/{id}', 'Api\Web\AlbumsController@getShow')->where('id', '\d+'); + Route::get('/albums/cached/{id}/{format}', 'Api\Web\AlbumsController@getCachedAlbum')->where(['id' => '\d+', 'format' => '.+']); Route::get('/playlists', 'Api\Web\PlaylistsController@getIndex'); Route::get('/playlists/{id}', 'Api\Web\PlaylistsController@getShow')->where('id', '\d+'); + Route::get('/playlists/cached/{id}/{format}', 'Api\Web\PlaylistsController@getCachedPlaylist')->where(['id' => '\d+', 'format' => '.+']); Route::get('/comments/{type}/{id}', 'Api\Web\CommentsController@getIndex')->where('id', '\d+'); diff --git a/app/Jobs/EncodeTrackFile.php b/app/Jobs/EncodeTrackFile.php new file mode 100644 index 00000000..2b135556 --- /dev/null +++ b/app/Jobs/EncodeTrackFile.php @@ -0,0 +1,129 @@ +. + */ + +namespace Poniverse\Ponyfm\Jobs; + +use Carbon\Carbon; +use Illuminate\Support\Facades\Config; +use Illuminate\Support\Facades\Log; +use OAuth2\Exception; +use Poniverse\Ponyfm\Jobs\Job; +use Illuminate\Queue\SerializesModels; +use Illuminate\Queue\InteractsWithQueue; +use Illuminate\Contracts\Bus\SelfHandling; +use Illuminate\Contracts\Queue\ShouldQueue; +use Poniverse\Ponyfm\Track; +use Poniverse\Ponyfm\TrackFile; +use Symfony\Component\Process\Exception\ProcessFailedException; +use Symfony\Component\Process\Process; + +class EncodeTrackFile extends Job implements SelfHandling, ShouldQueue +{ + use InteractsWithQueue, SerializesModels; + /** + * @var TrackFile + */ + private $trackFile; + /** + * @var + */ + private $isExpirable; + + /** + * Create a new job instance. + * @param TrackFile $trackFile + * @param $isExpirable + */ + public function __construct(TrackFile $trackFile, $isExpirable) + { + $this->trackFile = $trackFile; + $this->isExpirable = $isExpirable; + } + + /** + * Execute the job. + * + * @return void + */ + public function handle() + { + // Start the job + $this->trackFile->is_in_progress = true; + $this->trackFile->update(); + + // Use the track's master file as the source + $source = TrackFile::where('track_id', $this->trackFile->track_id) + ->where('is_master', true) + ->first() + ->getFile(); + + // Assign the target + $this->trackFile->track->ensureDirectoryExists(); + $target = $this->trackFile->getFile(); + + // Prepare the command + $format = Track::$Formats[$this->trackFile->format]; + $command = $format['command']; + $command = str_replace('{$source}', '"' . $source . '"', $command); + $command = str_replace('{$target}', '"' . $target . '"', $command); + + Log::info('Encoding track file ' . $this->trackFile->id . ' into ' . $target); + + // Start a synchronous process to encode the file + $process = new Process($command); + try { + $process->mustRun(); + } catch (ProcessFailedException $e) { + Log::error('An exception occured in the encoding process for track file ' . $this->trackFile->id . ' - ' . $e->getMessage()); + // Ensure queue fails + throw $e; + } finally { + Log::info($process->getOutput()); + } + + // Update the tags of the track + $this->trackFile->track->updateTags($this->trackFile->format); + + // Insert the expiration time for cached tracks + if ($this->isExpirable) { + $this->trackFile->expires_at = Carbon::now()->addMinutes(Config::get('ponyfm.track_file_cache_duration')); + $this->trackFile->update(); + } + + // Update file size + $this->trackFile->updateFilesize(); + + // Complete the job + $this->trackFile->is_in_progress = false; + $this->trackFile->update(); + } + + /** + * Handle a job failure. + * + * @return void + */ + public function failed() + { + $this->trackFile->is_in_progress = false; + $this->trackFile->expires_at = null; + $this->trackFile->update(); + } +} diff --git a/app/Playlist.php b/app/Playlist.php index f3621122..b30cc7b2 100644 --- a/app/Playlist.php +++ b/app/Playlist.php @@ -23,16 +23,18 @@ namespace Poniverse\Ponyfm; use Helpers; use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\SoftDeletes; -use Illuminate\Support\Facades\Auth; -use Illuminate\Support\Facades\Cache; -use Illuminate\Support\Facades\URL; +use Illuminate\Foundation\Bus\DispatchesJobs; +use Auth; +use Cache; +use Poniverse\Ponyfm\Traits\TrackCollection; +use URL; use Poniverse\Ponyfm\Traits\SlugTrait; class Playlist extends Model { - protected $table = 'playlists'; + use SoftDeletes, SlugTrait, DispatchesJobs, TrackCollection; - use SoftDeletes, SlugTrait; + protected $table = 'playlists'; protected $dates = ['deleted_at']; @@ -55,10 +57,12 @@ class Playlist extends Model return !$query; } - public static function mapPublicPlaylistShow($playlist) + public static function mapPublicPlaylistShow(Playlist $playlist) { $tracks = []; foreach ($playlist->tracks as $track) { + /** @var $track Track */ + $tracks[] = Track::mapPublicTrackSummary($track); } @@ -68,7 +72,8 @@ class Playlist extends Model 'name' => $name, 'extension' => $format['extension'], 'url' => $playlist->getDownloadUrl($name), - 'size' => Helpers::formatBytes($playlist->getFilesize($name)) + 'size' => Helpers::formatBytes($playlist->getFilesize($name)), + 'isCacheable' => (in_array($name, Track::$CacheableFormats) ? true : false) ]; } @@ -90,7 +95,7 @@ class Playlist extends Model return $data; } - public static function mapPublicPlaylistSummary($playlist) + public static function mapPublicPlaylistSummary(Playlist $playlist) { $userData = [ 'stats' => [ @@ -104,25 +109,25 @@ class Playlist extends Model $userRow = $playlist->users[0]; $userData = [ 'stats' => [ - 'views' => (int) $userRow->view_count, - 'downloads' => (int) $userRow->download_count, + 'views' => (int)$userRow->view_count, + 'downloads' => (int)$userRow->download_count, ], - 'is_favourited' => (bool) $userRow->is_favourited + 'is_favourited' => (bool)$userRow->is_favourited ]; } return [ - 'id' => (int) $playlist->id, + 'id' => (int)$playlist->id, 'track_count' => $playlist->track_count, 'title' => $playlist->title, 'slug' => $playlist->slug, 'created_at' => $playlist->created_at, - 'is_public' => (bool) $playlist->is_public, + 'is_public' => (bool)$playlist->is_public, 'stats' => [ - 'views' => (int) $playlist->view_count, - 'downloads' => (int) $playlist->download_count, - 'comments' => (int) $playlist->comment_count, - 'favourites' => (int) $playlist->favourite_count + 'views' => (int)$playlist->view_count, + 'downloads' => (int)$playlist->download_count, + 'comments' => (int)$playlist->comment_count, + 'favourites' => (int)$playlist->favourite_count ], 'covers' => [ 'small' => $playlist->getCoverUrl(Image::SMALL), @@ -130,7 +135,7 @@ class Playlist extends Model ], 'url' => $playlist->url, 'user' => [ - 'id' => (int) $playlist->user->id, + 'id' => (int)$playlist->user->id, 'name' => $playlist->user->display_name, 'url' => $playlist->user->url, ], @@ -151,6 +156,12 @@ class Playlist extends Model ->orderBy('position', 'asc'); } + public function trackFiles() + { + $trackIds = $this->tracks->lists('id'); + return TrackFile::whereIn('track_id', $trackIds); + } + public function users() { return $this->hasMany('Poniverse\Ponyfm\ResourceUser'); @@ -207,7 +218,17 @@ class Playlist extends Model return Cache::remember($this->getCacheKey('filesize-' . $format), 1440, function () use ($tracks, $format) { $size = 0; foreach ($tracks as $track) { - $size += $track->getFilesize($format); + /** @var $track Track */ + + // Ensure that only downloadable tracks are added onto the file size + if ($track->is_downloadable == 1) { + try { + $size += $track->getFilesize($format); + + } catch (TrackFileNotFoundException $e) { + // do nothing - this track won't be included in the download + } + } } return $size; diff --git a/app/Track.php b/app/Track.php index 6781c885..09604f20 100644 --- a/app/Track.php +++ b/app/Track.php @@ -20,7 +20,11 @@ namespace Poniverse\Ponyfm; -use Illuminate\Database\Query\Builder; +use Auth; +use Cache; +use Config; +use DB; +use Illuminate\Database\Eloquent\ModelNotFoundException; use Poniverse\Ponyfm\Traits\SlugTrait; use Exception; use External; @@ -28,13 +32,9 @@ use getid3_writetags; use Helpers; use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\SoftDeletes; -use Auth; -use Cache; -use Config; -use DB; +use Illuminate\Support\Str; use Log; use URL; -use Illuminate\Support\Str; class Track extends Model { @@ -94,9 +94,27 @@ class Track extends Model ], ]; + /** + * `TrackFiles` in these formats, with the exception of any master files, will + * be generated upon user request and kept around temporarily. + * + * After updating this array, run `php artisan rebuild:track-cache` to bring + * the track store into a consistent state. + * + * The strings in this array must match keys in the `Track::$Formats` array. + * + * @var array + */ + public static $CacheableFormats = [ + 'OGG Vorbis', + 'ALAC', + 'AAC' + ]; + public static function summary() { - return self::select('tracks.id', 'title', 'user_id', 'slug', 'is_vocal', 'is_explicit', 'created_at', 'published_at', + return self::select('tracks.id', 'title', 'user_id', 'slug', 'is_vocal', 'is_explicit', 'created_at', + 'published_at', 'duration', 'is_downloadable', 'genre_id', 'track_type_id', 'cover_id', 'album_id', 'comment_count', 'download_count', 'view_count', 'play_count', 'favourite_count'); } @@ -197,7 +215,7 @@ class Track extends Model return $processed; } - public static function mapPublicTrackShow($track) + public static function mapPublicTrackShow(Track $track) { $returnValue = self::mapPublicTrackSummary($track); $returnValue['description'] = $track->description; @@ -225,7 +243,8 @@ class Track extends Model 'name' => $trackFile->format, 'extension' => $trackFile->extension, 'url' => $trackFile->url, - 'size' => $trackFile->size + 'size' => $trackFile->size, + 'isCacheable' => (bool) $trackFile->is_cacheable ]; } @@ -243,7 +262,7 @@ class Track extends Model return $returnValue; } - public static function mapPublicTrackSummary($track) + public static function mapPublicTrackSummary(Track $track) { $userData = [ 'stats' => [ @@ -258,41 +277,41 @@ class Track extends Model $userRow = $track->users[0]; $userData = [ 'stats' => [ - 'views' => (int) $userRow->view_count, - 'plays' => (int) $userRow->play_count, + 'views' => (int)$userRow->view_count, + 'plays' => (int)$userRow->play_count, 'downloads' => $userRow->download_count, ], - 'is_favourited' => (bool) $userRow->is_favourited + 'is_favourited' => (bool)$userRow->is_favourited ]; } return [ - 'id' => (int) $track->id, + 'id' => (int)$track->id, 'title' => $track->title, 'user' => [ - 'id' => (int) $track->user->id, + 'id' => (int)$track->user->id, 'name' => $track->user->display_name, 'url' => $track->user->url ], 'stats' => [ - 'views' => (int) $track->view_count, - 'plays' => (int) $track->play_count, - 'downloads' => (int) $track->download_count, - 'comments' => (int) $track->comment_count, - 'favourites' => (int) $track->favourite_count + 'views' => (int)$track->view_count, + 'plays' => (int)$track->play_count, + 'downloads' => (int)$track->download_count, + 'comments' => (int)$track->comment_count, + 'favourites' => (int)$track->favourite_count ], 'url' => $track->url, 'slug' => $track->slug, - 'is_vocal' => (bool) $track->is_vocal, - 'is_explicit' => (bool) $track->is_explicit, - 'is_downloadable' => (bool) $track->is_downloadable, - 'is_published' => (bool) $track->isPublished(), + 'is_vocal' => (bool)$track->is_vocal, + 'is_explicit' => (bool)$track->is_explicit, + 'is_downloadable' => (bool)$track->is_downloadable, + 'is_published' => (bool)$track->isPublished(), 'published_at' => $track->published_at, 'duration' => $track->duration, 'genre' => $track->genre != null ? [ - 'id' => (int) $track->genre->id, + 'id' => (int)$track->genre->id, 'slug' => $track->genre->slug, 'name' => $track->genre->name ] : null, @@ -315,7 +334,7 @@ class Track extends Model ]; } - public static function mapPrivateTrackShow($track) + public static function mapPrivateTrackShow(Track $track) { $showSongs = []; foreach ($track->showSongs as $showSong) { @@ -336,7 +355,7 @@ class Track extends Model return $returnValue; } - public static function mapPrivateTrackSummary($track) + public static function mapPrivateTrackSummary(Track $track) { return [ 'id' => $track->id, @@ -420,18 +439,22 @@ class Track extends Model $this->updateHash(); } + /** + * Returns the size of this track's file in the given format. + * + * @param $formatName + * @return int filesize in bytes + * @throws TrackFileNotFoundException + */ public function getFilesize($formatName) { - return Cache::remember($this->getCacheKey('filesize-' . $formatName), 1440, function () use ($formatName) { - $file = $this->getFileFor($formatName); - $size = 0; + $trackFile = $this->trackFiles()->where('format', $formatName)->first(); - if (is_file($file)) { - $size = filesize($file); - } - - return $size; - }); + if ($trackFile) { + return (int) $trackFile->filesize; + } else { + throw new TrackFileNotFoundException(); + } } public function canView($user) @@ -569,11 +592,22 @@ class Track extends Model $this->hash = md5(Helpers::sanitizeInputForHashing($this->user->display_name) . ' - ' . Helpers::sanitizeInputForHashing($this->title)); } - public function updateTags() + public function updateTags($trackFileFormat = 'all') { - $this->trackFiles()->touch(); + if ($trackFileFormat === 'all') { + foreach ($this->trackFiles as $trackFile) { + $this->updateTagsForTrackFile($trackFile); + } + } else { + $trackFile = $this->trackFiles()->where('format', $trackFileFormat)->firstOrFail(); + $this->updateTagsForTrackFile($trackFile); + } + } - foreach ($this->trackFiles as $trackFile) { + private function updateTagsForTrackFile($trackFile) { + $trackFile->touch(); + + if (\File::exists($trackFile->getFile())) { $format = $trackFile->format; $data = self::$Formats[$format]; @@ -652,10 +686,12 @@ class Track extends Model if ($tagWriter->WriteTags()) { if (!empty($tagWriter->warnings)) { - Log::warning('Track #'.$this->id.': There were some warnings:
' . implode('

', $tagWriter->warnings)); + Log::warning('Track #' . $this->id . ': There were some warnings:
' . implode('

', + $tagWriter->warnings)); } } else { - Log::error('Track #' . $this->id . ': Failed to write tags!
' . implode('

', $tagWriter->errors)); + Log::error('Track #' . $this->id . ': Failed to write tags!
' . implode('

', + $tagWriter->errors)); } } diff --git a/app/TrackFile.php b/app/TrackFile.php index 1cccd287..63556f4e 100644 --- a/app/TrackFile.php +++ b/app/TrackFile.php @@ -22,15 +22,15 @@ namespace Poniverse\Ponyfm; use Helpers; use Illuminate\Database\Eloquent\Model; -use Illuminate\Support\Facades\App; -use Illuminate\Support\Facades\Cache; -use Illuminate\Support\Facades\URL; +use App; +use File; +use URL; class TrackFile extends Model { public function track() { - return $this->belongsTo('Poniverse\Ponyfm\Track'); + return $this->belongsTo('Poniverse\Ponyfm\Track')->withTrashed(); } /** @@ -84,9 +84,9 @@ class TrackFile extends Model return URL::to('/t' . $this->track_id . '/dl.' . $this->extension); } - public function getSizeAttribute($value) + public function getSizeAttribute() { - return Helpers::formatBytes($this->getFilesize($this->getFile())); + return Helpers::formatBytes($this->getFilesize()); } public function getFormat() @@ -96,16 +96,7 @@ class TrackFile extends Model protected function getFilesize() { - return Cache::remember($this->getCacheKey('filesize'), 1440, function () { - $file = $this->getFile(); - $size = 0; - - if (is_file($file)) { - $size = filesize($file); - } - - return $size; - }); + return $this->filesize; } public function getDirectory() @@ -134,4 +125,23 @@ class TrackFile extends Model { return 'track_file-' . $this->id . '-' . $key; } + + /** + * If this file exists, update its estimated filesize in the database. + * + * @return int $size + */ + public function updateFilesize() + { + $file = $this->getFile(); + + if (File::exists($file)) { + $size = File::size($file); + + $this->filesize = $size; + $this->update(); + } + + return $this->filesize; + } } diff --git a/app/Traits/TrackCollection.php b/app/Traits/TrackCollection.php new file mode 100644 index 00000000..2be06db0 --- /dev/null +++ b/app/Traits/TrackCollection.php @@ -0,0 +1,127 @@ +. + */ + +namespace Poniverse\Ponyfm\Traits; + + +use File; +use Illuminate\Database\Eloquent\Collection; +use Illuminate\Database\Eloquent\Relations\Relation; +use Poniverse\Ponyfm\Jobs\EncodeTrackFile; +use Poniverse\Ponyfm\TrackFile; + + +/** + * Class TrackCollection + * @package Poniverse\Ponyfm\Traits + * + * Contains common logic between albums and playlists. They share some functionality + * because they're both a form of downloadable track collection. + */ +trait TrackCollection +{ + /** + * This relation represents all tracks contained by the collection. + * + * @return Relation + */ + abstract public function tracks(); + + /** + * This relation represents all track files belonging to this collection's + * tracks. + * + * @return Relation + */ + abstract public function trackFiles(); + + + /** + * Returns the number of tracks that are available in the given format. + * + * @param string $format + * @return int the number of downloadable tracks in this collection + */ + public function countDownloadableTracks($format) { + return $this->downloadableTrackFiles($format)->count(); + } + + + /** + * Returns the number of currently-available track files (master files + + * currently cached files) for this collection in the given format. + * + * @param string $format + * @return int + */ + public function countAvailableTrackFiles($format) { + $trackFiles = $this->downloadableTrackFiles($format); + $availableCount = 0; + + foreach ($trackFiles as $trackFile) { + /** @var TrackFile $trackFile */ + + if ( + $trackFile->is_master || + ($trackFile->expires_at != null && File::exists($trackFile->getFile())) + ) { + $availableCount ++; + } + } + + return $availableCount; + } + + + /** + * Kicks off the encoding of any cacheable files in this collection that + * do not currently exist. + * + * @param $format + */ + public function encodeCacheableTrackFiles($format) { + $trackFiles = $this->downloadableTrackFiles($format); + + foreach ($trackFiles as $trackFile) { + /** @var TrackFile $trackFile */ + + if (!File::exists($trackFile->getFile()) && $trackFile->is_in_progress != true) { + $this->dispatch(new EncodeTrackFile($trackFile, true)); + } + } + } + + + /** + * Returns an Eloquent collection of downloadable TrackFiles for this {@link TrackCollection}. + * A {@link TrackFile} is considered downloadable if its associated {@link Track} is. + * + * @param $format + * @return Collection + */ + protected function downloadableTrackFiles($format) { + return $this->trackFiles()->with([ + 'track' => function ($query) { + $query->where('is_downloadable', true); + } + ])->where('format', $format)->get(); + } +} diff --git a/config/ponyfm.php b/config/ponyfm.php index add3e98f..1470dcd4 100644 --- a/config/ponyfm.php +++ b/config/ponyfm.php @@ -54,4 +54,15 @@ return [ 'use_powered_by_footer' => env('USE_POWERED_BY_FOOTER', true), + /* + |-------------------------------------------------------------------------- + | Cache Duration + |-------------------------------------------------------------------------- + | + | Duration in minutes for track files to be stored in cache. + | + */ + + 'track_file_cache_duration' => 1440, + ]; diff --git a/database/migrations/2015_10_26_192855_update_track_files_with_cache.php b/database/migrations/2015_10_26_192855_update_track_files_with_cache.php new file mode 100644 index 00000000..5352f319 --- /dev/null +++ b/database/migrations/2015_10_26_192855_update_track_files_with_cache.php @@ -0,0 +1,53 @@ +. + */ + +use Illuminate\Database\Schema\Blueprint; +use Illuminate\Database\Migrations\Migration; + +class UpdateTrackFilesWithCache extends Migration +{ + /** + * Run the migrations. + * + * @return void + */ + public function up() + { + Schema::table('track_files', function (Blueprint $table) { + $table->boolean('is_cacheable')->default(false)->index(); + $table->boolean('is_in_progress')->default(false); + $table->dateTime('expires_at')->nullable()->index(); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::table('track_files', function (Blueprint $table) { + $table->dropColumn('is_cacheable'); + $table->dropColumn('expires_at'); + $table->dropColumn('is_in_progress'); + }); + } +} diff --git a/database/migrations/2015_10_29_153827_update_track_files_with_filesize.php b/database/migrations/2015_10_29_153827_update_track_files_with_filesize.php new file mode 100644 index 00000000..70f80ef4 --- /dev/null +++ b/database/migrations/2015_10_29_153827_update_track_files_with_filesize.php @@ -0,0 +1,55 @@ +. + */ + +use Illuminate\Database\Migrations\Migration; +use Illuminate\Database\Schema\Blueprint; +use Illuminate\Support\Facades\Artisan; + +class UpdateTrackFilesWithFilesize extends Migration +{ + /** + * Run the migrations. + * + * @return void + */ + public function up() + { + Schema::table('track_files', function (Blueprint $table) { + $table->integer('filesize')->nullable()->unsigned(); + }); + + // Populate file sizes + Artisan::call('rebuild:filesizes', [ + '--force' => true, + ]); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::table('track_files', function (Blueprint $table) { + $table->dropColumn('filesize'); + }); + } +} diff --git a/public/images/loading.gif b/public/images/loading.gif new file mode 100644 index 00000000..724eacb5 Binary files /dev/null and b/public/images/loading.gif differ diff --git a/public/templates/albums/show.html b/public/templates/albums/show.html index ad2f7e06..704aea0e 100644 --- a/public/templates/albums/show.html +++ b/public/templates/albums/show.html @@ -1,11 +1,22 @@