#1: Reindexing now runs on its own queue + lots of code cleanup.

app/Console/Commands/RebuildSearchIndex.php

@@ -26,6 +26,7 @@ use Poniverse\Ponyfm\Models\Album;
 use Poniverse\Ponyfm\Models\Playlist;
 use Poniverse\Ponyfm\Models\Track;
 use Poniverse\Ponyfm\Models\User;
+use Symfony\Component\Console\Helper\ProgressBar;
 
 class RebuildSearchIndex extends Command
 {
@@ -65,32 +66,56 @@ class RebuildSearchIndex extends Command
         $totalPlaylists = Playlist::withTrashed()->count();
         $totalUsers = User::count();
 
-        Track::withTrashed()->chunk(200, function(Collection $tracks) {
+        $trackProgress = $this->output->createProgressBar($totalTracks);
+        $this->info("Processing tracks...");
+        Track::withTrashed()->chunk(200, function(Collection $tracks) use ($trackProgress) {
             foreach($tracks as $track) {
-                $this->info("Processing track #{$track->id}...");
+                /** @var Track $track */
-                $track->ensureElasticsearchEntryIsUpToDate();
+                $trackProgress->advance();
+                $track->updateElasticsearchEntry();
             }
         });
+        $trackProgress->finish();
+        $this->line('');
 
-        Album::withTrashed()->chunk(200, function(Collection $albums) {
+
+        $albumProgress = $this->output->createProgressBar($totalAlbums);
+        $this->info("Processing albums...");
+        Album::withTrashed()->chunk(200, function(Collection $albums) use ($albumProgress) {
             foreach($albums as $album) {
-                $this->info("Processing album #{$album->id}...");
+                /** @var Album $album */
-                $album->ensureElasticsearchEntryIsUpToDate();
+                $albumProgress->advance();
+                $album->updateElasticsearchEntry();
             }
         });
+        $albumProgress->finish();
+        $this->line('');
 
-        Playlist::withTrashed()->chunk(200, function(Collection $playlists) {
+
+        $playlistProgress = $this->output->createProgressBar($totalPlaylists);
+        $this->info("Processing playlists...");
+        Playlist::withTrashed()->chunk(200, function(Collection $playlists) use ($playlistProgress) {
             foreach($playlists as $playlist) {
-                $this->info("Processing playlist #{$playlist->id}...");
+                /** @var Playlist $playlist */
-                $playlist->ensureElasticsearchEntryIsUpToDate();
+                $playlistProgress->advance();
+                $playlist->updateElasticsearchEntry();
             }
         });
+        $playlistProgress->finish();
+        $this->line('');
 
-        User::chunk(200, function(Collection $users) {
+
+        $userProgress = $this->output->createProgressBar($totalUsers);
+        $this->info("Processing users...");
+        User::chunk(200, function(Collection $users) use ($userProgress) {
             foreach($users as $user) {
-                $this->info("Processing user #{$user->id}...");
+                /** @var User $user */
-                $user->ensureElasticsearchEntryIsUpToDate();
+                $userProgress->advance();
+                $user->updateElasticsearchEntry();
             }
         });
+        $userProgress->finish();
+        $this->line('');
+        $this->info('Everything has been queued for re-indexing!');
     }
 }

app/Contracts/Searchable.php

@@ -0,0 +1,39 @@
+<?php
+
+/**
+ * Pony.fm - A community for pony fan music.
+ * Copyright (C) 2016 Peter Deltchev
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+namespace Poniverse\Ponyfm\Contracts;
+
+interface Searchable {
+    /**
+     * Returns this model in Elasticsearch-friendly form. The array returned by
+     * this method should match the current mapping for this model's ES type.
+     *
+     * @return array
+     */
+    public function toElasticsearch():array;
+
+    /**
+     * @return bool whether this particular object should be indexed or not
+     */
+    public function shouldBeIndexed():bool;
+
+    public function updateElasticsearchEntry();
+    public function updateElasticsearchEntrySynchronously();
+}

app/Http/routes.php

@@ -82,7 +82,7 @@ Route::group(['prefix' => 'api/v1', 'middleware' => 'json-exceptions'], function
 
 Route::group(['prefix' => 'api/web'], function() {
     Route::get('/taxonomies/all', 'Api\Web\TaxonomiesController@getAll');
-    Route::get('/search', ['middleware' => ['auth', 'can:access-search'], 'Api\Web\SearchController@getSearch']);
+    Route::get('/search', ['middleware' => ['auth', 'can:access-search'], 'uses' => 'Api\Web\SearchController@getSearch']);
 
     Route::get('/tracks', 'Api\Web\TracksController@getIndex');
     Route::get('/tracks/{id}', 'Api\Web\TracksController@getShow')->where('id', '\d+');

app/Jobs/UpdateSearchIndexForEntity.php

@@ -0,0 +1,56 @@
+<?php
+
+/**
+ * Pony.fm - A community for pony fan music.
+ * Copyright (C) 2016 Peter Deltchev
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+namespace Poniverse\Ponyfm\Jobs;
+
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Database\Eloquent\Model;
+use Illuminate\Queue\InteractsWithQueue;
+use Poniverse\Ponyfm\Contracts\Searchable;
+use Poniverse\Ponyfm\Jobs\Job;
+use Illuminate\Contracts\Bus\SelfHandling;
+use SerializesModels;
+
+class UpdateSearchIndexForEntity extends Job implements SelfHandling, ShouldQueue
+{
+    use InteractsWithQueue, SerializesModels;
+
+    protected $entity;
+
+    /**
+     * Create a new job instance.
+     *
+     * @param Model $entity
+     */
+    public function __construct(Searchable $entity)
+    {
+        $this->entity = $entity;
+    }
+
+    /**
+     * Execute the job.
+     *
+     * @return void
+     */
+    public function handle()
+    {
+        $this->entity->updateElasticsearchEntrySynchronously();
+    }
+}

app/Models/Album.php

@@ -27,6 +27,7 @@ use Illuminate\Database\Eloquent\SoftDeletes;
 use Illuminate\Foundation\Bus\DispatchesJobs;
 use Auth;
 use Cache;
+use Poniverse\Ponyfm\Contracts\Searchable;
 use Poniverse\Ponyfm\Exceptions\TrackFileNotFoundException;
 use Poniverse\Ponyfm\Traits\IndexedInElasticsearchTrait;
 use Poniverse\Ponyfm\Traits\TrackCollection;
@@ -60,9 +61,9 @@ use Venturecraft\Revisionable\RevisionableTrait;
  * @property-read \Illuminate\Database\Eloquent\Collection|\Venturecraft\Revisionable\Revision[] $revisionHistory
  * @method static \Illuminate\Database\Query\Builder|\Poniverse\Ponyfm\Models\Album userDetails()
  */
-class Album extends Model
+class Album extends Model implements Searchable
 {
-    use SoftDeletes, SlugTrait, DispatchesJobs, TrackCollection, RevisionableTrait, IndexedInElasticsearchTrait;
+    use SoftDeletes, SlugTrait, TrackCollection, RevisionableTrait, IndexedInElasticsearchTrait;
 
     protected $elasticsearchType = 'album';
 
@@ -413,7 +414,7 @@ class Album extends Model
      *
      * @return array
      */
-    public function toElasticsearch() {
+    public function toElasticsearch():array {
         return [
             'title' => $this->title,
             'artist' => $this->user->display_name,

app/Models/Playlist.php

@@ -26,6 +26,7 @@ use Illuminate\Database\Eloquent\SoftDeletes;
 use Illuminate\Foundation\Bus\DispatchesJobs;
 use Auth;
 use Cache;
+use Poniverse\Ponyfm\Contracts\Searchable;
 use Poniverse\Ponyfm\Exceptions\TrackFileNotFoundException;
 use Poniverse\Ponyfm\Traits\IndexedInElasticsearchTrait;
 use Poniverse\Ponyfm\Traits\TrackCollection;
@@ -59,9 +60,9 @@ use Venturecraft\Revisionable\RevisionableTrait;
  * @property-read \Illuminate\Database\Eloquent\Collection|\Venturecraft\Revisionable\Revision[] $revisionHistory
  * @method static \Illuminate\Database\Query\Builder|\Poniverse\Ponyfm\Models\Playlist userDetails()
  */
-class Playlist extends Model
+class Playlist extends Model implements Searchable
 {
-    use SoftDeletes, SlugTrait, DispatchesJobs, TrackCollection, RevisionableTrait, IndexedInElasticsearchTrait;
+    use SoftDeletes, SlugTrait, TrackCollection, RevisionableTrait, IndexedInElasticsearchTrait;
 
     protected $elasticsearchType = 'playlist';
 
@@ -307,7 +308,7 @@ class Playlist extends Model
      *
      * @return array
      */
-    public function toElasticsearch() {
+    public function toElasticsearch():array {
         return [
             'title'     => $this->title,
             'curator'   => $this->user->display_name,

app/Models/Track.php

@@ -25,6 +25,7 @@ use Cache;
 use Config;
 use DB;
 use Elasticsearch;
+use Poniverse\Ponyfm\Contracts\Searchable;
 use Poniverse\Ponyfm\Exceptions\TrackFileNotFoundException;
 use Poniverse\Ponyfm\Traits\IndexedInElasticsearchTrait;
 use Poniverse\Ponyfm\Traits\SlugTrait;
@@ -95,7 +96,7 @@ use Venturecraft\Revisionable\RevisionableTrait;
  * @method static \Illuminate\Database\Query\Builder|\Poniverse\Ponyfm\Models\Track withComments()
  * @method static \Illuminate\Database\Query\Builder|\Poniverse\Ponyfm\Models\Track mlpma()
  */
-class Track extends Model
+class Track extends Model implements Searchable
 {
     use SoftDeletes, IndexedInElasticsearchTrait;
 

app/Models/User.php

@@ -29,6 +29,7 @@ use Illuminate\Database\Eloquent\Model;
 use Illuminate\Foundation\Auth\Access\Authorizable;
 use Auth;
 use Illuminate\Support\Str;
+use Poniverse\Ponyfm\Contracts\Searchable;
 use Poniverse\Ponyfm\Traits\IndexedInElasticsearchTrait;
 use Venturecraft\Revisionable\RevisionableTrait;
 
@@ -63,7 +64,7 @@ use Venturecraft\Revisionable\RevisionableTrait;
  * @property-read \Illuminate\Database\Eloquent\Collection|\Venturecraft\Revisionable\Revision[] $revisionHistory
  * @method static \Illuminate\Database\Query\Builder|\Poniverse\Ponyfm\Models\User userDetails()
  */
-class User extends Model implements AuthenticatableContract, CanResetPasswordContract, \Illuminate\Contracts\Auth\Access\Authorizable
+class User extends Model implements AuthenticatableContract, CanResetPasswordContract, \Illuminate\Contracts\Auth\Access\Authorizable, Searchable
 {
     use Authenticatable, CanResetPassword, Authorizable, RevisionableTrait, IndexedInElasticsearchTrait;
 
@@ -272,7 +273,7 @@ class User extends Model implements AuthenticatableContract, CanResetPasswordCon
      *
      * @return array
      */
-    public function toElasticsearch() {
+    public function toElasticsearch():array {
         return [
             'username'      => $this->username,
             'display_name'  => $this->display_name,

app/Traits/IndexedInElasticsearchTrait.php

@@ -20,40 +20,39 @@
 
 namespace Poniverse\Ponyfm\Traits;
 
+use Config;
 use Elasticsearch;
 use Elasticsearch\Common\Exceptions\Missing404Exception;
+use Illuminate\Foundation\Bus\DispatchesJobs;
+use Poniverse\Ponyfm\Contracts\Searchable;
+use Poniverse\Ponyfm\Jobs\UpdateSearchIndexForEntity;
 
 /**
  * Class IndexedInElasticsearch
  *
- * Classes using this trait must declare the `$elasticsearchType` property
+ * Classes using this trait must declare the `$elasticsearchType` property and
- * and use the `SoftDeletes` trait.
+ * implement the `Searchable` interface.
  *
  * @package Poniverse\Ponyfm\Traits
  */
 trait IndexedInElasticsearchTrait
 {
-    /**
+    use DispatchesJobs;
-     * Returns this model in Elasticsearch-friendly form. The array returned by
-     * this method should match the current mapping for this model's ES type.
-     *
-     * @return array
-     */
-    abstract public function toElasticsearch();
 
-    /**
+    // These two functions are from the Searchable interface. They're included
-     * @return bool whether this particular object should be indexed or not
+    // here, without being implemented, to assist IDE's when editing this trait.
-     */
+    public abstract function toElasticsearch():array;
-    abstract public function shouldBeIndexed():bool;
+    public abstract function shouldBeIndexed():bool;
 
 
+    // Laravel automatically runs this method based on the trait's name. #magic
     public static function bootIndexedInElasticsearchTrait() {
-        static::saved(function ($model) {
+        static::saved(function (Searchable $entity) {
-            $model->ensureElasticsearchEntryIsUpToDate();
+            $entity->updateElasticsearchEntry();
         });
 
-        static::deleted(function ($model) {
+        static::deleted(function (Searchable $entity) {
-            $model->ensureElasticsearchEntryIsUpToDate();
+            $entity->updateElasticsearchEntry();
         });
     }
 
@@ -63,7 +62,7 @@ trait IndexedInElasticsearchTrait
      */
     private function getElasticsearchParameters(bool $includeBody = true) {
         $parameters = [
-            'index' => 'ponyfm',
+            'index' => Config::get('ponyfm.elasticsearch_index'),
             'type'  => $this->elasticsearchType,
             'id'    => $this->id,
         ];
@@ -89,7 +88,20 @@ trait IndexedInElasticsearchTrait
         }
     }
 
-    public function ensureElasticsearchEntryIsUpToDate() {
+    /**
+     * Asynchronously updates the Elasticsearch entry.
+     * When in doubt, this is the method to use.
+     */
+    public function updateElasticsearchEntry() {
+        $job = (new UpdateSearchIndexForEntity($this))->onQueue(Config::get('ponyfm.indexing_queue'));
+        $this->dispatch($job);
+    }
+
+    /**
+     * Synchronously updates the Elasticsearch entry. This should only be
+     * called from the UpdateSearchIndexForEntity job.
+     */
+    public function updateElasticsearchEntrySynchronously() {
         if ($this->shouldBeIndexed()) {
             $this->createOrUpdateElasticsearchEntry();
         } else {

config/ponyfm.php

@@ -76,4 +76,17 @@ return [
 
     'elasticsearch_index' => 'ponyfm',
 
+    /*
+    |--------------------------------------------------------------------------
+    | Indexing queue name
+    |--------------------------------------------------------------------------
+    |
+    | The name of the queue to process re-indexing jobs on. This is separated
+    | from the default queue to avoid having a site-wide re-index clog uploads
+    | and downloads.
+    |
+    */
+
+    'indexing_queue' => 'indexing',
+
 ];

config/queue.php

@@ -35,13 +35,6 @@ return [
             'driver' => 'sync',
         ],
 
-        'database' => [
-            'driver' => 'database',
-            'table' => 'jobs',
-            'queue' => 'default',
-            'expire' => 60,
-        ],
-
         'beanstalkd' => [
             'driver' => 'beanstalkd',
             'host'   => 'localhost',
@@ -49,30 +42,6 @@ return [
             'ttr'    => 60,
         ],
 
-        'sqs' => [
-            'driver' => 'sqs',
-            'key'    => 'your-public-key',
-            'secret' => 'your-secret-key',
-            'queue'  => 'your-queue-url',
-            'region' => 'us-east-1',
-        ],
-
-        'iron' => [
-            'driver'  => 'iron',
-            'host'    => 'mq-aws-us-east-1.iron.io',
-            'token'   => 'your-token',
-            'project' => 'your-project-id',
-            'queue'   => 'your-queue-name',
-            'encrypt' => true,
-        ],
-
-        'redis' => [
-            'driver' => 'redis',
-            'connection' => 'default',
-            'queue'  => 'default',
-            'expire' => 60,
-        ],
-
     ],
 
     /*