PonyMirrors/Pony.fm
1
0
Fork 0
mirror of https://github.com/Poniverse/Pony.fm.git synced 2024-11-22 04:58:01 +01:00
Code Issues Projects Releases Packages Wiki Activity

#39: Implemented asynchronous encoding in uploads.

Browse source
This commit is contained in:
Peter Deltchev 2015-12-18 00:56:13 -08:00
parent 0375130ab6
commit 851dfff921
15 changed files with 319 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
app/Commands/CommandResponse.php
View file

@ -24,11 +24,25 @@ use Illuminate\Validation\Validator;
class CommandResponse
{
public static function fail($validator)
/**
* @var Validator
*/
private $_validator;
private $_response;
private $_didFail;
public static function fail($validatorOrMessages)
{
$response = new CommandResponse();
$response->_didFail = true;
$response->_validator = $validator;
if (is_array($validatorOrMessages)) {
$response->_messages = $validatorOrMessages;
$response->_validator = null;
} else {
$response->_validator = $validatorOrMessages;
}
return $response;
}
@ -42,10 +56,6 @@ class CommandResponse
return $cmdResponse;
}
private $_validator;
private $_response;
private $_didFail;
private function __construct()
{
}
@ -73,4 +83,14 @@ class CommandResponse
{
return $this->_validator;
}
public function getMessages()
{
if ($this->_validator !== null) {
return $this->_validator->messages()->getMessages();
} else {
return $this->_messages;
}
}
}

78
app/Commands/UploadTrackCommand.php
View file

@ -20,17 +20,22 @@
namespace Poniverse\Ponyfm\Commands;
use Config;
use Illuminate\Foundation\Bus\DispatchesJobs;
use Poniverse\Ponyfm\Exceptions\InvalidEncodeOptionsException;
use Poniverse\Ponyfm\Jobs\EncodeTrackFile;
use Poniverse\Ponyfm\Track;
use Poniverse\Ponyfm\TrackFile;
use AudioCache;
use File;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Str;
use Symfony\Component\Process\Exception\ProcessFailedException;
use Symfony\Component\Process\Process;
use Storage;
class UploadTrackCommand extends CommandBase
{
use DispatchesJobs;
private $_allowLossy;
private $_allowShortTrack;
private $_losslessFormats = [
@ -68,6 +73,21 @@ class UploadTrackCommand extends CommandBase
$trackFile = \Input::file('track');
$audio = \AudioCache::get($trackFile->getPathname());
$track = new Track();
$track->user_id = $user->id;
$track->title = pathinfo($trackFile->getClientOriginalName(), PATHINFO_FILENAME);
$track->duration = $audio->getDuration();
$track->is_listed = true;
$track->save();
$track->ensureDirectoryExists();
Storage::makeDirectory(Config::get('ponyfm.files_directory') . '/queued-tracks', 0755, false, true);
$trackFile = $trackFile->move(Config::get('ponyfm.files_directory').'/queued-tracks', $track->id);
$validator = \Validator::make(['track' => $trackFile], [
'track' =>
'required|'
@ -77,24 +97,13 @@ class UploadTrackCommand extends CommandBase
]);
if ($validator->fails()) {
$track->delete();
return CommandResponse::fail($validator);
}
$track = new Track();
try {
$track->user_id = $user->id;
$track->title = pathinfo($trackFile->getClientOriginalName(), PATHINFO_FILENAME);
$track->duration = $audio->getDuration();
$track->is_listed = true;
$track->save();
$destination = $track->getDirectory();
$track->ensureDirectoryExists();
$source = $trackFile->getPathname();
$index = 0;
// Lossy uploads need to be identified and set as the master file
// without being re-encoded.
@ -113,6 +122,7 @@ class UploadTrackCommand extends CommandBase
} else {
$validator->messages()->add('track', 'The track does not contain audio in a known lossy format.');
$track->delete();
return CommandResponse::fail($validator);
}
@ -126,6 +136,9 @@ class UploadTrackCommand extends CommandBase
File::copy($source, $trackFile->getFile());
}
$trackFiles = [];
foreach (Track::$Formats as $name => $format) {
// Don't bother with lossless transcodes of lossy uploads, and
// don't re-encode the lossy master.
@ -136,6 +149,7 @@ class UploadTrackCommand extends CommandBase
$trackFile = new TrackFile();
$trackFile->is_master = $name === 'FLAC' ? true : false;
$trackFile->format = $name;
$trackFile->status = TrackFile::STATUS_PROCESSING;
if (in_array($name, Track::$CacheableFormats) && $trackFile->is_master == false) {
$trackFile->is_cacheable = true;
@ -144,29 +158,21 @@ class UploadTrackCommand extends CommandBase
}
$track->trackFiles()->save($trackFile);
// Encode track file
$target = $trackFile->getFile();
$command = $format['command'];
$command = str_replace('{$source}', '"' . $source . '"', $command);
$command = str_replace('{$target}', '"' . $target . '"', $command);
Log::info('Encoding ' . $track->id . ' into ' . $target);
$this->notify('Encoding ' . $name, $index / count(Track::$Formats) * 100);
$process = new Process($command);
$process->mustRun();
// Update file size for track file
$trackFile->updateFilesize();
// Delete track file if it is cacheable
if ($trackFile->is_cacheable == true) {
File::delete($trackFile->getFile());
}
// All TrackFile records we need are synchronously created
// before kicking off the encode jobs in order to avoid a race
// condition with the "temporary" source file getting deleted.
$trackFiles[] = $trackFile;
}
$track->updateTags();
try {
foreach($trackFiles as $trackFile) {
$this->dispatch(new EncodeTrackFile($trackFile, false, true));
}
} catch (InvalidEncodeOptionsException $e) {
$track->delete();
return CommandResponse::fail(['track' => [$e->getMessage()]]);
}
} catch (\Exception $e) {
$track->delete();

2
app/Console/Commands/ImportMLPMA.php
View file

@ -365,7 +365,7 @@ class ImportMLPMA extends Command
$result = $upload->execute();
if ($result->didFail()) {
$this->error(json_encode($result->getValidator()->messages()->getMessages(), JSON_PRETTY_PRINT));
$this->error(json_encode($result->getMessages(), JSON_PRETTY_PRINT));
} else {
// Save metadata.

25
app/Exceptions/InvalidEncodeOptionsException.php Normal file
View file

@ -0,0 +1,25 @@
<?php
/**
* Pony.fm - A community for pony fan music.
* Copyright (C) 2015 Peter Deltchev
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
namespace Poniverse\Ponyfm\Exceptions;
use InvalidArgumentException;
class InvalidEncodeOptionsException extends InvalidArgumentException {}

29
app/Http/Controllers/Api/Web/TracksController.php
View file

@ -33,6 +33,7 @@ use Cover;
use Illuminate\Support\Facades\Auth;
use Illuminate\Support\Facades\Input;
use Illuminate\Support\Facades\Response;
use Poniverse\Ponyfm\TrackFile;
class TracksController extends ApiControllerBase
{
@ -40,7 +41,31 @@ class TracksController extends ApiControllerBase
{
session_write_close();
return $this->execute(new UploadTrackCommand());
try {
return $this->execute(new UploadTrackCommand());
} catch (\InvalidEncodeOptions $e) {
}
}
public function getUploadStatus($trackId)
{
// TODO: authorize this
$track = Track::findOrFail($trackId);
if ($track->status === Track::STATUS_PROCESSING){
return Response::json(['message' => 'Processing...'], 202);
} elseif ($track->status === Track::STATUS_COMPLETE) {
return Response::json(['message' => 'Processing complete!'], 201);
} else {
// something went wrong
return Response::json(['error' => 'Processing failed!'], 500);
}
}
public function postDelete($id)
@ -101,7 +126,7 @@ class TracksController extends ApiControllerBase
// Return URL or begin encoding
if ($trackFile->expires_at != null && File::exists($trackFile->getFile())) {
$url = $track->getUrlFor($format);
} elseif ($trackFile->is_in_progress === true) {
} elseif ($trackFile->status === TrackFile::STATUS_PROCESSING) {
$url = null;
} else {
$this->dispatch(new EncodeTrackFile($trackFile, true));

2
app/Http/Controllers/ApiControllerBase.php
View file

@ -35,7 +35,7 @@ abstract class ApiControllerBase extends Controller
if ($result->didFail()) {
return Response::json([
'message' => 'Validation failed',
'errors' => $result->getValidator()->messages()->getMessages()
'errors' => $result->getMessages()
], 400);
}

1
app/Http/routes.php
View file

@ -101,6 +101,7 @@ Route::group(['prefix' => 'api/web'], function() {
Route::group(['middleware' => 'auth'], function() {
Route::post('/tracks/upload', 'Api\Web\TracksController@postUpload');
Route::get('/tracks/{id}/upload-status', 'Api\Web\TracksController@getUploadStatus');
Route::post('/tracks/delete/{id}', 'Api\Web\TracksController@postDelete');
Route::post('/tracks/edit/{id}', 'Api\Web\TracksController@postEdit');

49
app/Jobs/EncodeTrackFile.php
View file

@ -2,6 +2,7 @@
/**
* Pony.fm - A community for pony fan music.
* Copyright (C) 2015 Peter Deltchev
* Copyright (C) 2015 Kelvin Zhang
*
* This program is free software: you can redistribute it and/or modify
@ -21,9 +22,11 @@
namespace Poniverse\Ponyfm\Jobs;
use Carbon\Carbon;
use File;
use Illuminate\Support\Facades\Config;
use Illuminate\Support\Facades\Log;
use OAuth2\Exception;
use Poniverse\Ponyfm\Exceptions\InvalidEncodeOptionsException;
use Poniverse\Ponyfm\Jobs\Job;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\InteractsWithQueue;
@ -45,16 +48,29 @@ class EncodeTrackFile extends Job implements SelfHandling, ShouldQueue
* @var
*/
private $isExpirable;
/**
* @var bool
*/
private $isForUpload;
/**
* Create a new job instance.
* @param TrackFile $trackFile
* @param $isExpirable
* @param bool $isExpirable
* @param bool $isForUpload indicates whether this encode job is for an upload
*/
public function __construct(TrackFile $trackFile, $isExpirable)
public function __construct(TrackFile $trackFile, $isExpirable, $isForUpload = false)
{
if(
(!$isForUpload && $trackFile->is_master) ||
($isForUpload && $trackFile->is_master && !$trackFile->getFormat()['is_lossless'])
) {
throw new InvalidEncodeOptionsException("Master files cannot be encoded unless we're generating a lossless master file during the upload process.");
}
$this->trackFile = $trackFile;
$this->isExpirable = $isExpirable;
$this->isForUpload = $isForUpload;
}
/**
@ -65,14 +81,19 @@ class EncodeTrackFile extends Job implements SelfHandling, ShouldQueue
public function handle()
{
// Start the job
$this->trackFile->is_in_progress = true;
$this->trackFile->status = TrackFile::STATUS_PROCESSING;
$this->trackFile->update();
// Use the track's master file as the source
$source = TrackFile::where('track_id', $this->trackFile->track_id)
->where('is_master', true)
->first()
->getFile();
if ($this->isForUpload) {
$source = $this->trackFile->track->getTemporarySourceFile();
} else {
$source = TrackFile::where('track_id', $this->trackFile->track_id)
->where('is_master', true)
->first()
->getFile();
}
// Assign the target
$this->trackFile->track->ensureDirectoryExists();
@ -111,8 +132,18 @@ class EncodeTrackFile extends Job implements SelfHandling, ShouldQueue
$this->trackFile->updateFilesize();
// Complete the job
$this->trackFile->is_in_progress = false;
$this->trackFile->status = TrackFile::STATUS_NOT_BEING_PROCESSED;
$this->trackFile->update();
if ($this->isForUpload) {
if (!$this->trackFile->is_master && $this->trackFile->is_cacheable) {
File::delete($this->trackFile->getFile());
}
if ($this->trackFile->track->status === Track::STATUS_COMPLETE) {
File::delete($this->trackFile->track->getTemporarySourceFile());
}
}
}
/**
@ -122,7 +153,7 @@ class EncodeTrackFile extends Job implements SelfHandling, ShouldQueue
*/
public function failed()
{
$this->trackFile->is_in_progress = false;
$this->trackFile->status = TrackFile::STATUS_PROCESSING_ERROR;
$this->trackFile->expires_at = null;
$this->trackFile->update();
}

45
app/Track.php
View file

@ -48,6 +48,12 @@ class Track extends Model
use RevisionableTrait;
// Used for the track's upload status.
const STATUS_COMPLETE = 0;
const STATUS_PROCESSING = 1;
const STATUS_ERROR = 2;
public static $Formats = [
'FLAC' => [
'index' => 0,
@ -579,6 +585,18 @@ class Track extends Model
return "{$this->getDirectory()}/{$this->id}.{$format['extension']}";
}
/**
* Returns the path to the "temporary" master file uploaded by the user.
* This file is used during the upload process to generate the actual master
* file stored by Pony.fm.
*
* @return string
*/
public function getTemporarySourceFile() {
return Config::get('ponyfm.files_directory') . '/queued-tracks/' . $this->id;
}
public function getUrlFor($format)
{
if (!isset(self::$Formats[$format])) {
@ -590,6 +608,33 @@ class Track extends Model
return URL::to('/t' . $this->id . '/dl.' . $format['extension']);
}
/**
* @return string one of the Track::STATUS_* values, indicating whether this track is currently being processed
*/
public function getStatusAttribute(){
return $this->trackFiles->reduce(function($carry, $trackFile){
if($trackFile->status === TrackFile::STATUS_PROCESSING_ERROR) {
return static::STATUS_ERROR;
} elseif (
$carry !== static::STATUS_ERROR &&
$trackFile->status === TrackFile::STATUS_PROCESSING) {
return static::STATUS_PROCESSING;
} elseif (
!in_array($carry, [static::STATUS_ERROR, static::STATUS_PROCESSING]) &&
$trackFile->status === TrackFile::STATUS_NOT_BEING_PROCESSED
) {
return static::STATUS_COMPLETE;
} else {
return $carry;
}
}, static::STATUS_COMPLETE);
}
public function updateHash()
{
$this->hash = md5(Helpers::sanitizeInputForHashing($this->user->display_name) . ' - ' . Helpers::sanitizeInputForHashing($this->title));

7
app/TrackFile.php
View file

@ -20,6 +20,7 @@
namespace Poniverse\Ponyfm;
use Config;
use Helpers;
use Illuminate\Database\Eloquent\Model;
use App;
@ -28,6 +29,12 @@ use URL;
class TrackFile extends Model
{
// used for the "status" property
const STATUS_NOT_BEING_PROCESSED = 0;
const STATUS_PROCESSING = 1;
const STATUS_PROCESSING_ERROR = 2;
public function track()
{
return $this->belongsTo('Poniverse\Ponyfm\Track')->withTrashed();

2
app/Traits/TrackCollection.php
View file

@ -103,7 +103,7 @@ trait TrackCollection
foreach ($trackFiles as $trackFile) {
/** @var TrackFile $trackFile */
if (!File::exists($trackFile->getFile()) && $trackFile->is_in_progress != true) {
if (!File::exists($trackFile->getFile()) && $trackFile->status == TrackFile::STATUS_NOT_BEING_PROCESSED) {
$this->dispatch(new EncodeTrackFile($trackFile, true));
}
}

3
database/migrations/2015_10_26_192855_update_track_files_with_cache.php
View file

@ -2,6 +2,7 @@
/**
* Pony.fm - A community for pony fan music.
* Copyright (C) 2015 Peter Deltchev
* Copyright (C) 2015 Kelvin Zhang
*
* This program is free software: you can redistribute it and/or modify
@ -32,7 +33,7 @@ class UpdateTrackFilesWithCache extends Migration
{
Schema::table('track_files', function (Blueprint $table) {
$table->boolean('is_cacheable')->default(false)->index();
$table->boolean('is_in_progress')->default(false);
$table->tinyInteger('is_in_progress')->default(false);
$table->dateTime('expires_at')->nullable()->index();
});
}

51
database/migrations/2015_12_18_025953_convert_track_file_in_progress_to_status.php Normal file
View file

@ -0,0 +1,51 @@
<?php
/**
* Pony.fm - A community for pony fan music.
* Copyright (C) 2015 Peter Deltchev
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Database\Migrations\Migration;
class ConvertTrackFileInProgressToStatus extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
//
Schema::table('track_files', function(Blueprint $table) {
$table->renameColumn('is_in_progress', 'status');
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
//
Schema::table('track_files', function (Blueprint $table) {
$table->renameColumn('status', 'is_in_progress');
});
}
}

8
public/templates/uploader/index.html
View file

@ -10,15 +10,17 @@
<p>Please note that you need to publish your tracks after uploading them before they will become available to the public.</p>
<ul class="uploads">
<li ng-repeat="upload in data.queue" ng-class="{'uploading': upload.isUploading, 'has-error': upload.error != null, 'is-processing': upload.isUploading && upload.error == null && upload.progress >= 100}" ng-animate="'upload-queue'">
<li ng-repeat="upload in data.queue" ng-class="{'uploading': upload.isUploading, 'has-error': upload.error != null, 'is-processing': upload.isProcessing || upload.progress >= 100}" ng-animate="'upload-queue'">
<p>
<span ng-show="!upload.success">
<strong ng-show="upload.isUploading && upload.error == null && upload.progress >= 100">Processing</strong>
<strong ng-show="upload.isUploading && upload.error == null && upload.progress < 100">Uploading</strong>
<strong ng-show="upload.isUploading && upload.error == null && upload.progress < 100">Uploading&hellip;</strong>
<strong ng-show="upload.isProcessing || (upload.isUploading && upload.progress >= 100)">Processing&hellip;</strong>
<strong ng-show="upload.error != null">Error</strong>
{{upload.name}} -
<strong ng-show="upload.error != null">{{upload.error}}</strong>
</span>
<span ng-show="upload.success">
<a href="/account/tracks/edit/{{upload.trackId}}" class="btn btn-small btn-primary">
Publish

51
resources/assets/scripts/app/services/upload.coffee
View file

@ -15,11 +15,39 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
angular.module('ponyfm').factory('upload', [
'$rootScope'
($rootScope) ->
'$rootScope', '$http', '$timeout'
($rootScope, $http, $timeout) ->
self =
queue: []
finishUploadWrapper: (upload)->
()->
self.finishUpload(upload)
# Polls for the upload's status
finishUpload: (upload) ->
# TODO: update upload status
$http.get("/api/web/tracks/#{upload.trackId}/upload-status").then(
# handle success or still-processing
(response)->
if response.status == 202
$timeout(self.finishUploadWrapper(upload), 5000)
else if response.status == 201
upload.isProcessing = false
upload.success = true
# handle error
,(response)->
upload.isProcessing = false
if response.headers['content-type'] == 'application/json'
upload.error = response.data.error
else
upload.error = 'There was an unknown error!'
)
upload: (files) ->
_.each files, (file) ->
upload =
@ -29,6 +57,8 @@ angular.module('ponyfm').factory('upload', [
size: file.size
index: self.queue.length
isUploading: true
isProcessing: false
trackId: null
success: false
error: null
@ -42,22 +72,31 @@ angular.module('ponyfm').factory('upload', [
upload.progress = e.loaded / upload.size * 100
$rootScope.$broadcast 'upload-progress', upload
# TODO: Implement polling here
# event listener
xhr.onload = -> $rootScope.$apply ->
upload.isUploading = false
if xhr.status != 200
upload.isProcessing = true
if xhr.status == 200
# kick off polling
upload.trackId = $.parseJSON(xhr.responseText).id
self.finishUpload(upload)
else
error =
if xhr.getResponseHeader('content-type') == 'application/json'
$.parseJSON(xhr.responseText).errors.track.join ', '
else
'There was an unknown error!'
upload.isProcessing = false
upload.error = error
$rootScope.$broadcast 'upload-error', [upload, error]
else
upload.success = true
upload.trackId = $.parseJSON(xhr.responseText).id
$rootScope.$broadcast 'upload-finished', upload
# send the track to the server
formData = new FormData();
formData.append('track', file);