#39: Implemented asynchronous encoding in uploads.

app/Commands/CommandResponse.php

@@ -24,11 +24,25 @@ use Illuminate\Validation\Validator;
 
 class CommandResponse
 {
-    public static function fail($validator)
+    /**
+     * @var Validator
+     */
+    private $_validator;
+    private $_response;
+    private $_didFail;
+
+    public static function fail($validatorOrMessages)
     {
         $response = new CommandResponse();
         $response->_didFail = true;
-        $response->_validator = $validator;
+
+        if (is_array($validatorOrMessages)) {
+            $response->_messages = $validatorOrMessages;
+            $response->_validator = null;
+
+        } else {
+            $response->_validator = $validatorOrMessages;
+        }
 
         return $response;
     }
@@ -42,10 +56,6 @@ class CommandResponse
         return $cmdResponse;
     }
 
-    private $_validator;
-    private $_response;
-    private $_didFail;
-
     private function __construct()
     {
     }
@@ -73,4 +83,14 @@ class CommandResponse
     {
         return $this->_validator;
     }
+
+    public function getMessages()
+    {
+        if ($this->_validator !== null) {
+            return $this->_validator->messages()->getMessages();
+
+        } else {
+            return $this->_messages;
+        }
+    }
 }

app/Commands/UploadTrackCommand.php

@@ -20,17 +20,22 @@
 
 namespace Poniverse\Ponyfm\Commands;
 
+use Config;
+use Illuminate\Foundation\Bus\DispatchesJobs;
+use Poniverse\Ponyfm\Exceptions\InvalidEncodeOptionsException;
+use Poniverse\Ponyfm\Jobs\EncodeTrackFile;
 use Poniverse\Ponyfm\Track;
 use Poniverse\Ponyfm\TrackFile;
 use AudioCache;
 use File;
-use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Str;
-use Symfony\Component\Process\Exception\ProcessFailedException;
+use Storage;
-use Symfony\Component\Process\Process;
 
 class UploadTrackCommand extends CommandBase
 {
+    use DispatchesJobs;
+
+
     private $_allowLossy;
     private $_allowShortTrack;
     private $_losslessFormats = [
@@ -68,6 +73,21 @@ class UploadTrackCommand extends CommandBase
         $trackFile = \Input::file('track');
         $audio = \AudioCache::get($trackFile->getPathname());
 
+
+        $track = new Track();
+        $track->user_id = $user->id;
+        $track->title = pathinfo($trackFile->getClientOriginalName(), PATHINFO_FILENAME);
+        $track->duration = $audio->getDuration();
+        $track->is_listed = true;
+
+        $track->save();
+        $track->ensureDirectoryExists();
+
+        Storage::makeDirectory(Config::get('ponyfm.files_directory') . '/queued-tracks', 0755, false, true);
+        $trackFile = $trackFile->move(Config::get('ponyfm.files_directory').'/queued-tracks', $track->id);
+
+
+
         $validator = \Validator::make(['track' => $trackFile], [
             'track' =>
                 'required|'
@@ -77,24 +97,13 @@ class UploadTrackCommand extends CommandBase
         ]);
 
         if ($validator->fails()) {
+            $track->delete();
             return CommandResponse::fail($validator);
         }
 
-        $track = new Track();
 
         try {
-            $track->user_id = $user->id;
-            $track->title = pathinfo($trackFile->getClientOriginalName(), PATHINFO_FILENAME);
-            $track->duration = $audio->getDuration();
-            $track->is_listed = true;
-
-            $track->save();
-
-            $destination = $track->getDirectory();
-            $track->ensureDirectoryExists();
-
             $source = $trackFile->getPathname();
-            $index = 0;
 
             // Lossy uploads need to be identified and set as the master file
             // without being re-encoded.
@@ -113,6 +122,7 @@ class UploadTrackCommand extends CommandBase
 
                 } else {
                     $validator->messages()->add('track', 'The track does not contain audio in a known lossy format.');
+                    $track->delete();
                     return CommandResponse::fail($validator);
                 }
 
@@ -126,6 +136,9 @@ class UploadTrackCommand extends CommandBase
                 File::copy($source, $trackFile->getFile());
             }
 
+
+            $trackFiles = [];
+
             foreach (Track::$Formats as $name => $format) {
                 // Don't bother with lossless transcodes of lossy uploads, and
                 // don't re-encode the lossy master.
@@ -136,6 +149,7 @@ class UploadTrackCommand extends CommandBase
                 $trackFile = new TrackFile();
                 $trackFile->is_master = $name === 'FLAC' ? true : false;
                 $trackFile->format = $name;
+                $trackFile->status = TrackFile::STATUS_PROCESSING;
 
                 if (in_array($name, Track::$CacheableFormats) && $trackFile->is_master == false) {
                     $trackFile->is_cacheable = true;
@@ -144,29 +158,21 @@ class UploadTrackCommand extends CommandBase
                 }
                 $track->trackFiles()->save($trackFile);
 
-                // Encode track file
+                // All TrackFile records we need are synchronously created
-                $target = $trackFile->getFile();
+                // before kicking off the encode jobs in order to avoid a race
-
+                // condition with the "temporary" source file getting deleted.
-                $command = $format['command'];
+                $trackFiles[] = $trackFile;
-                $command = str_replace('{$source}', '"' . $source . '"', $command);
-                $command = str_replace('{$target}', '"' . $target . '"', $command);
-
-                Log::info('Encoding ' . $track->id . ' into ' . $target);
-                $this->notify('Encoding ' . $name, $index / count(Track::$Formats) * 100);
-
-                $process = new Process($command);
-                $process->mustRun();
-
-                // Update file size for track file
-                $trackFile->updateFilesize();
-
-                // Delete track file if it is cacheable
-                if ($trackFile->is_cacheable == true) {
-                    File::delete($trackFile->getFile());
-                }
             }
 
-            $track->updateTags();
+            try {
+                foreach($trackFiles as $trackFile)  {
+                    $this->dispatch(new EncodeTrackFile($trackFile, false, true));
+                }
+
+            } catch (InvalidEncodeOptionsException $e) {
+                $track->delete();
+                return CommandResponse::fail(['track' => [$e->getMessage()]]);
+            }
 
         } catch (\Exception $e) {
             $track->delete();

app/Console/Commands/ImportMLPMA.php

@@ -365,7 +365,7 @@ class ImportMLPMA extends Command
             $result = $upload->execute();
 
             if ($result->didFail()) {
-                $this->error(json_encode($result->getValidator()->messages()->getMessages(), JSON_PRETTY_PRINT));
+                $this->error(json_encode($result->getMessages(), JSON_PRETTY_PRINT));
 
             } else {
                 // Save metadata.

app/Exceptions/InvalidEncodeOptionsException.php

@@ -0,0 +1,25 @@
+<?php
+
+/**
+ * Pony.fm - A community for pony fan music.
+ * Copyright (C) 2015 Peter Deltchev
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+namespace Poniverse\Ponyfm\Exceptions;
+
+use InvalidArgumentException;
+
+class InvalidEncodeOptionsException extends InvalidArgumentException {}

app/Http/Controllers/Api/Web/TracksController.php

@@ -33,6 +33,7 @@ use Cover;
 use Illuminate\Support\Facades\Auth;
 use Illuminate\Support\Facades\Input;
 use Illuminate\Support\Facades\Response;
+use Poniverse\Ponyfm\TrackFile;
 
 class TracksController extends ApiControllerBase
 {
@@ -40,7 +41,31 @@ class TracksController extends ApiControllerBase
     {
         session_write_close();
 
-        return $this->execute(new UploadTrackCommand());
+        try {
+            return $this->execute(new UploadTrackCommand());
+
+        } catch (\InvalidEncodeOptions $e) {
+
+        }
+    }
+
+    public function getUploadStatus($trackId)
+    {
+        // TODO: authorize this
+
+        $track = Track::findOrFail($trackId);
+
+        if ($track->status === Track::STATUS_PROCESSING){
+            return Response::json(['message' => 'Processing...'], 202);
+
+        } elseif ($track->status === Track::STATUS_COMPLETE) {
+            return Response::json(['message' => 'Processing complete!'], 201);
+
+        } else {
+            // something went wrong
+            return Response::json(['error' => 'Processing failed!'], 500);
+        }
+
     }
 
     public function postDelete($id)
@@ -101,7 +126,7 @@ class TracksController extends ApiControllerBase
         // Return URL or begin encoding
         if ($trackFile->expires_at != null && File::exists($trackFile->getFile())) {
             $url = $track->getUrlFor($format);
-        } elseif ($trackFile->is_in_progress === true) {
+        } elseif ($trackFile->status === TrackFile::STATUS_PROCESSING) {
             $url = null;
         } else {
             $this->dispatch(new EncodeTrackFile($trackFile, true));

app/Http/Controllers/ApiControllerBase.php

@@ -35,7 +35,7 @@ abstract class ApiControllerBase extends Controller
         if ($result->didFail()) {
             return Response::json([
                 'message' => 'Validation failed',
-                'errors' => $result->getValidator()->messages()->getMessages()
+                'errors' => $result->getMessages()
             ], 400);
         }
 

app/Http/routes.php

@@ -101,6 +101,7 @@ Route::group(['prefix' => 'api/web'], function() {
 
     Route::group(['middleware' => 'auth'], function() {
         Route::post('/tracks/upload', 'Api\Web\TracksController@postUpload');
+        Route::get('/tracks/{id}/upload-status', 'Api\Web\TracksController@getUploadStatus');
         Route::post('/tracks/delete/{id}', 'Api\Web\TracksController@postDelete');
         Route::post('/tracks/edit/{id}', 'Api\Web\TracksController@postEdit');
 

app/Jobs/EncodeTrackFile.php

@@ -2,6 +2,7 @@
 
 /**
  * Pony.fm - A community for pony fan music.
+ * Copyright (C) 2015 Peter Deltchev
  * Copyright (C) 2015 Kelvin Zhang
  *
  * This program is free software: you can redistribute it and/or modify
@@ -21,9 +22,11 @@
 namespace Poniverse\Ponyfm\Jobs;
 
 use Carbon\Carbon;
+use File;
 use Illuminate\Support\Facades\Config;
 use Illuminate\Support\Facades\Log;
 use OAuth2\Exception;
+use Poniverse\Ponyfm\Exceptions\InvalidEncodeOptionsException;
 use Poniverse\Ponyfm\Jobs\Job;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Queue\InteractsWithQueue;
@@ -45,16 +48,29 @@ class EncodeTrackFile extends Job implements SelfHandling, ShouldQueue
      * @var
      */
     private $isExpirable;
+    /**
+     * @var bool
+     */
+    private $isForUpload;
 
     /**
      * Create a new job instance.
      * @param TrackFile $trackFile
-     * @param $isExpirable
+     * @param bool $isExpirable
+     * @param bool $isForUpload indicates whether this encode job is for an upload
      */
-    public function __construct(TrackFile $trackFile, $isExpirable)
+    public function __construct(TrackFile $trackFile, $isExpirable, $isForUpload = false)
     {
+        if(
+            (!$isForUpload && $trackFile->is_master) ||
+            ($isForUpload && $trackFile->is_master && !$trackFile->getFormat()['is_lossless'])
+        ) {
+            throw new InvalidEncodeOptionsException("Master files cannot be encoded unless we're generating a lossless master file during the upload process.");
+        }
+
         $this->trackFile = $trackFile;
         $this->isExpirable = $isExpirable;
+        $this->isForUpload = $isForUpload;
     }
 
     /**
@@ -65,14 +81,19 @@ class EncodeTrackFile extends Job implements SelfHandling, ShouldQueue
     public function handle()
     {
         // Start the job
-        $this->trackFile->is_in_progress = true;
+        $this->trackFile->status = TrackFile::STATUS_PROCESSING;
         $this->trackFile->update();
 
         // Use the track's master file as the source
-        $source = TrackFile::where('track_id', $this->trackFile->track_id)
+        if ($this->isForUpload) {
-            ->where('is_master', true)
+            $source = $this->trackFile->track->getTemporarySourceFile();
-            ->first()
+
-            ->getFile();
+        } else {
+            $source = TrackFile::where('track_id', $this->trackFile->track_id)
+                ->where('is_master', true)
+                ->first()
+                ->getFile();
+        }
 
         // Assign the target
         $this->trackFile->track->ensureDirectoryExists();
@@ -111,8 +132,18 @@ class EncodeTrackFile extends Job implements SelfHandling, ShouldQueue
         $this->trackFile->updateFilesize();
 
         // Complete the job
-        $this->trackFile->is_in_progress = false;
+        $this->trackFile->status = TrackFile::STATUS_NOT_BEING_PROCESSED;
         $this->trackFile->update();
+
+        if ($this->isForUpload) {
+            if (!$this->trackFile->is_master && $this->trackFile->is_cacheable) {
+                File::delete($this->trackFile->getFile());
+            }
+
+            if ($this->trackFile->track->status === Track::STATUS_COMPLETE) {
+                File::delete($this->trackFile->track->getTemporarySourceFile());
+            }
+        }
     }
 
     /**
@@ -122,7 +153,7 @@ class EncodeTrackFile extends Job implements SelfHandling, ShouldQueue
      */
     public function failed()
     {
-        $this->trackFile->is_in_progress = false;
+        $this->trackFile->status = TrackFile::STATUS_PROCESSING_ERROR;
         $this->trackFile->expires_at = null;
         $this->trackFile->update();
     }

app/Track.php

@@ -48,6 +48,12 @@ class Track extends Model
 
     use RevisionableTrait;
 
+    // Used for the track's upload status.
+    const STATUS_COMPLETE = 0;
+    const STATUS_PROCESSING = 1;
+    const STATUS_ERROR = 2;
+
+
     public static $Formats = [
         'FLAC' => [
             'index' => 0,
@@ -579,6 +585,18 @@ class Track extends Model
         return "{$this->getDirectory()}/{$this->id}.{$format['extension']}";
     }
 
+    /**
+     * Returns the path to the "temporary" master file uploaded by the user.
+     * This file is used during the upload process to generate the actual master
+     * file stored by Pony.fm.
+     *
+     * @return string
+     */
+    public function getTemporarySourceFile() {
+        return Config::get('ponyfm.files_directory') . '/queued-tracks/' . $this->id;
+    }
+
+
     public function getUrlFor($format)
     {
         if (!isset(self::$Formats[$format])) {
@@ -590,6 +608,33 @@ class Track extends Model
         return URL::to('/t' . $this->id . '/dl.' . $format['extension']);
     }
 
+
+    /**
+     * @return string one of the Track::STATUS_* values, indicating whether this track is currently being processed
+     */
+    public function getStatusAttribute(){
+        return $this->trackFiles->reduce(function($carry, $trackFile){
+            if($trackFile->status === TrackFile::STATUS_PROCESSING_ERROR) {
+                return static::STATUS_ERROR;
+
+            } elseif (
+                $carry !== static::STATUS_ERROR &&
+                $trackFile->status === TrackFile::STATUS_PROCESSING) {
+                return static::STATUS_PROCESSING;
+
+            } elseif (
+                !in_array($carry, [static::STATUS_ERROR, static::STATUS_PROCESSING]) &&
+                $trackFile->status === TrackFile::STATUS_NOT_BEING_PROCESSED
+            ) {
+                return static::STATUS_COMPLETE;
+
+            } else {
+                return $carry;
+            }
+        }, static::STATUS_COMPLETE);
+    }
+
+
     public function updateHash()
     {
         $this->hash = md5(Helpers::sanitizeInputForHashing($this->user->display_name) . ' - ' . Helpers::sanitizeInputForHashing($this->title));

app/TrackFile.php

@@ -20,6 +20,7 @@
 
 namespace Poniverse\Ponyfm;
 
+use Config;
 use Helpers;
 use Illuminate\Database\Eloquent\Model;
 use App;
@@ -28,6 +29,12 @@ use URL;
 
 class TrackFile extends Model
 {
+    // used for the "status" property
+    const STATUS_NOT_BEING_PROCESSED = 0;
+    const STATUS_PROCESSING = 1;
+    const STATUS_PROCESSING_ERROR = 2;
+
+
     public function track()
     {
         return $this->belongsTo('Poniverse\Ponyfm\Track')->withTrashed();

app/Traits/TrackCollection.php

@@ -103,7 +103,7 @@ trait TrackCollection
         foreach ($trackFiles as $trackFile) {
             /** @var TrackFile $trackFile */
 
-            if (!File::exists($trackFile->getFile()) && $trackFile->is_in_progress != true) {
+            if (!File::exists($trackFile->getFile()) && $trackFile->status == TrackFile::STATUS_NOT_BEING_PROCESSED) {
                 $this->dispatch(new EncodeTrackFile($trackFile, true));
             }
         }

database/migrations/2015_10_26_192855_update_track_files_with_cache.php

@@ -2,6 +2,7 @@
 
 /**
  * Pony.fm - A community for pony fan music.
+ * Copyright (C) 2015 Peter Deltchev
  * Copyright (C) 2015 Kelvin Zhang
  *
  * This program is free software: you can redistribute it and/or modify
@@ -32,7 +33,7 @@ class UpdateTrackFilesWithCache extends Migration
     {
         Schema::table('track_files', function (Blueprint $table) {
             $table->boolean('is_cacheable')->default(false)->index();
-            $table->boolean('is_in_progress')->default(false);
+            $table->tinyInteger('is_in_progress')->default(false);
             $table->dateTime('expires_at')->nullable()->index();
         });
     }

database/migrations/2015_12_18_025953_convert_track_file_in_progress_to_status.php

@@ -0,0 +1,51 @@
+<?php
+
+/**
+ * Pony.fm - A community for pony fan music.
+ * Copyright (C) 2015 Peter Deltchev
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Database\Migrations\Migration;
+
+class ConvertTrackFileInProgressToStatus extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        //
+        Schema::table('track_files', function(Blueprint $table) {
+            $table->renameColumn('is_in_progress', 'status');
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        //
+        Schema::table('track_files', function (Blueprint $table) {
+            $table->renameColumn('status', 'is_in_progress');
+        });
+    }
+}

public/templates/uploader/index.html

@@ -10,15 +10,17 @@
     <p>Please note that you need to publish your tracks after uploading them before they will become available to the public.</p>
 
     <ul class="uploads">
-        <li ng-repeat="upload in data.queue" ng-class="{'uploading': upload.isUploading, 'has-error': upload.error != null, 'is-processing': upload.isUploading && upload.error == null && upload.progress >= 100}" ng-animate="'upload-queue'">
+        <li ng-repeat="upload in data.queue" ng-class="{'uploading': upload.isUploading, 'has-error': upload.error != null, 'is-processing': upload.isProcessing || upload.progress >= 100}" ng-animate="'upload-queue'">
             <p>
+
                 <span ng-show="!upload.success">
-                    <strong ng-show="upload.isUploading && upload.error == null && upload.progress >= 100">Processing</strong>
+                    <strong ng-show="upload.isUploading && upload.error == null && upload.progress < 100">Uploading&hellip;</strong>
-                    <strong ng-show="upload.isUploading && upload.error == null && upload.progress < 100">Uploading</strong>
+                    <strong ng-show="upload.isProcessing || (upload.isUploading && upload.progress >= 100)">Processing&hellip;</strong>
                     <strong ng-show="upload.error != null">Error</strong>
                     {{upload.name}} -
                     <strong ng-show="upload.error != null">{{upload.error}}</strong>
                 </span>
+
                 <span ng-show="upload.success">
                     <a href="/account/tracks/edit/{{upload.trackId}}" class="btn btn-small btn-primary">
                         Publish

resources/assets/scripts/app/services/upload.coffee

@@ -15,11 +15,39 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 angular.module('ponyfm').factory('upload', [
-    '$rootScope'
+    '$rootScope', '$http', '$timeout'
-    ($rootScope) ->
+    ($rootScope, $http, $timeout) ->
         self =
             queue: []
 
+            finishUploadWrapper: (upload)->
+                ()->
+                    self.finishUpload(upload)
+
+            # Polls for the upload's status
+            finishUpload: (upload) ->
+                # TODO: update upload status
+                $http.get("/api/web/tracks/#{upload.trackId}/upload-status").then(
+                    # handle success or still-processing
+                    (response)->
+                        if response.status == 202
+                            $timeout(self.finishUploadWrapper(upload), 5000)
+
+                        else if response.status == 201
+                            upload.isProcessing = false
+                            upload.success = true
+
+                    # handle error
+                    ,(response)->
+                        upload.isProcessing = false
+                        if response.headers['content-type'] == 'application/json'
+                            upload.error = response.data.error
+                        else
+                            upload.error = 'There was an unknown error!'
+            )
+
+
+
             upload: (files) ->
                 _.each files, (file) ->
                     upload =
@@ -29,6 +57,8 @@ angular.module('ponyfm').factory('upload', [
                         size: file.size
                         index: self.queue.length
                         isUploading: true
+                        isProcessing: false
+                        trackId: null
                         success: false
                         error: null
 
@@ -42,22 +72,31 @@ angular.module('ponyfm').factory('upload', [
                             upload.progress = e.loaded / upload.size * 100
                             $rootScope.$broadcast 'upload-progress', upload
 
+                    # TODO: Implement polling here
+                    # event listener
                     xhr.onload = -> $rootScope.$apply ->
                         upload.isUploading = false
-                        if xhr.status != 200
+                        upload.isProcessing = true
+
+                        if xhr.status == 200
+                            # kick off polling
+                            upload.trackId = $.parseJSON(xhr.responseText).id
+                            self.finishUpload(upload)
+
+                        else
                             error =
                                 if xhr.getResponseHeader('content-type') == 'application/json'
                                     $.parseJSON(xhr.responseText).errors.track.join ', '
                                 else
                                     'There was an unknown error!'
 
+                            upload.isProcessing = false
                             upload.error = error
                             $rootScope.$broadcast 'upload-error', [upload, error]
-                        else
-                            upload.success = true
-                            upload.trackId = $.parseJSON(xhr.responseText).id
 
                         $rootScope.$broadcast 'upload-finished', upload
+
+                    # send the track to the server
                     formData = new FormData();
                     formData.append('track', file);