From 845449c8cc0152f8a34284cc8b9ec648445d35c0 Mon Sep 17 00:00:00 2001 From: Peter Deltchev Date: Sun, 17 Jan 2016 07:16:16 -0800 Subject: [PATCH] #1: Reindexing now runs on its own queue + lots of code cleanup. --- app/Console/Commands/RebuildSearchIndex.php | 49 +++++++++++++----- app/Contracts/Searchable.php | 39 ++++++++++++++ app/Http/routes.php | 2 +- app/Jobs/UpdateSearchIndexForEntity.php | 56 +++++++++++++++++++++ app/Models/Album.php | 7 +-- app/Models/Playlist.php | 7 +-- app/Models/Track.php | 3 +- app/Models/User.php | 5 +- app/Traits/IndexedInElasticsearchTrait.php | 50 +++++++++++------- config/ponyfm.php | 13 +++++ config/queue.php | 31 ------------ 11 files changed, 190 insertions(+), 72 deletions(-) create mode 100644 app/Contracts/Searchable.php create mode 100644 app/Jobs/UpdateSearchIndexForEntity.php diff --git a/app/Console/Commands/RebuildSearchIndex.php b/app/Console/Commands/RebuildSearchIndex.php index 72e66066..41431e51 100644 --- a/app/Console/Commands/RebuildSearchIndex.php +++ b/app/Console/Commands/RebuildSearchIndex.php @@ -26,6 +26,7 @@ use Poniverse\Ponyfm\Models\Album; use Poniverse\Ponyfm\Models\Playlist; use Poniverse\Ponyfm\Models\Track; use Poniverse\Ponyfm\Models\User; +use Symfony\Component\Console\Helper\ProgressBar; class RebuildSearchIndex extends Command { @@ -65,32 +66,56 @@ class RebuildSearchIndex extends Command $totalPlaylists = Playlist::withTrashed()->count(); $totalUsers = User::count(); - Track::withTrashed()->chunk(200, function(Collection $tracks) { + $trackProgress = $this->output->createProgressBar($totalTracks); + $this->info("Processing tracks..."); + Track::withTrashed()->chunk(200, function(Collection $tracks) use ($trackProgress) { foreach($tracks as $track) { - $this->info("Processing track #{$track->id}..."); - $track->ensureElasticsearchEntryIsUpToDate(); + /** @var Track $track */ + $trackProgress->advance(); + $track->updateElasticsearchEntry(); } }); + $trackProgress->finish(); + $this->line(''); - Album::withTrashed()->chunk(200, function(Collection $albums) { + + $albumProgress = $this->output->createProgressBar($totalAlbums); + $this->info("Processing albums..."); + Album::withTrashed()->chunk(200, function(Collection $albums) use ($albumProgress) { foreach($albums as $album) { - $this->info("Processing album #{$album->id}..."); - $album->ensureElasticsearchEntryIsUpToDate(); + /** @var Album $album */ + $albumProgress->advance(); + $album->updateElasticsearchEntry(); } }); + $albumProgress->finish(); + $this->line(''); - Playlist::withTrashed()->chunk(200, function(Collection $playlists) { + + $playlistProgress = $this->output->createProgressBar($totalPlaylists); + $this->info("Processing playlists..."); + Playlist::withTrashed()->chunk(200, function(Collection $playlists) use ($playlistProgress) { foreach($playlists as $playlist) { - $this->info("Processing playlist #{$playlist->id}..."); - $playlist->ensureElasticsearchEntryIsUpToDate(); + /** @var Playlist $playlist */ + $playlistProgress->advance(); + $playlist->updateElasticsearchEntry(); } }); + $playlistProgress->finish(); + $this->line(''); - User::chunk(200, function(Collection $users) { + + $userProgress = $this->output->createProgressBar($totalUsers); + $this->info("Processing users..."); + User::chunk(200, function(Collection $users) use ($userProgress) { foreach($users as $user) { - $this->info("Processing user #{$user->id}..."); - $user->ensureElasticsearchEntryIsUpToDate(); + /** @var User $user */ + $userProgress->advance(); + $user->updateElasticsearchEntry(); } }); + $userProgress->finish(); + $this->line(''); + $this->info('Everything has been queued for re-indexing!'); } } diff --git a/app/Contracts/Searchable.php b/app/Contracts/Searchable.php new file mode 100644 index 00000000..8b81b440 --- /dev/null +++ b/app/Contracts/Searchable.php @@ -0,0 +1,39 @@ +. + */ + +namespace Poniverse\Ponyfm\Contracts; + +interface Searchable { + /** + * Returns this model in Elasticsearch-friendly form. The array returned by + * this method should match the current mapping for this model's ES type. + * + * @return array + */ + public function toElasticsearch():array; + + /** + * @return bool whether this particular object should be indexed or not + */ + public function shouldBeIndexed():bool; + + public function updateElasticsearchEntry(); + public function updateElasticsearchEntrySynchronously(); +} diff --git a/app/Http/routes.php b/app/Http/routes.php index 401fd813..a3889ec7 100644 --- a/app/Http/routes.php +++ b/app/Http/routes.php @@ -82,7 +82,7 @@ Route::group(['prefix' => 'api/v1', 'middleware' => 'json-exceptions'], function Route::group(['prefix' => 'api/web'], function() { Route::get('/taxonomies/all', 'Api\Web\TaxonomiesController@getAll'); - Route::get('/search', ['middleware' => ['auth', 'can:access-search'], 'Api\Web\SearchController@getSearch']); + Route::get('/search', ['middleware' => ['auth', 'can:access-search'], 'uses' => 'Api\Web\SearchController@getSearch']); Route::get('/tracks', 'Api\Web\TracksController@getIndex'); Route::get('/tracks/{id}', 'Api\Web\TracksController@getShow')->where('id', '\d+'); diff --git a/app/Jobs/UpdateSearchIndexForEntity.php b/app/Jobs/UpdateSearchIndexForEntity.php new file mode 100644 index 00000000..21b32131 --- /dev/null +++ b/app/Jobs/UpdateSearchIndexForEntity.php @@ -0,0 +1,56 @@ +. + */ + +namespace Poniverse\Ponyfm\Jobs; + +use Illuminate\Contracts\Queue\ShouldQueue; +use Illuminate\Database\Eloquent\Model; +use Illuminate\Queue\InteractsWithQueue; +use Poniverse\Ponyfm\Contracts\Searchable; +use Poniverse\Ponyfm\Jobs\Job; +use Illuminate\Contracts\Bus\SelfHandling; +use SerializesModels; + +class UpdateSearchIndexForEntity extends Job implements SelfHandling, ShouldQueue +{ + use InteractsWithQueue, SerializesModels; + + protected $entity; + + /** + * Create a new job instance. + * + * @param Model $entity + */ + public function __construct(Searchable $entity) + { + $this->entity = $entity; + } + + /** + * Execute the job. + * + * @return void + */ + public function handle() + { + $this->entity->updateElasticsearchEntrySynchronously(); + } +} diff --git a/app/Models/Album.php b/app/Models/Album.php index 38ff9588..64d709f6 100644 --- a/app/Models/Album.php +++ b/app/Models/Album.php @@ -27,6 +27,7 @@ use Illuminate\Database\Eloquent\SoftDeletes; use Illuminate\Foundation\Bus\DispatchesJobs; use Auth; use Cache; +use Poniverse\Ponyfm\Contracts\Searchable; use Poniverse\Ponyfm\Exceptions\TrackFileNotFoundException; use Poniverse\Ponyfm\Traits\IndexedInElasticsearchTrait; use Poniverse\Ponyfm\Traits\TrackCollection; @@ -60,9 +61,9 @@ use Venturecraft\Revisionable\RevisionableTrait; * @property-read \Illuminate\Database\Eloquent\Collection|\Venturecraft\Revisionable\Revision[] $revisionHistory * @method static \Illuminate\Database\Query\Builder|\Poniverse\Ponyfm\Models\Album userDetails() */ -class Album extends Model +class Album extends Model implements Searchable { - use SoftDeletes, SlugTrait, DispatchesJobs, TrackCollection, RevisionableTrait, IndexedInElasticsearchTrait; + use SoftDeletes, SlugTrait, TrackCollection, RevisionableTrait, IndexedInElasticsearchTrait; protected $elasticsearchType = 'album'; @@ -413,7 +414,7 @@ class Album extends Model * * @return array */ - public function toElasticsearch() { + public function toElasticsearch():array { return [ 'title' => $this->title, 'artist' => $this->user->display_name, diff --git a/app/Models/Playlist.php b/app/Models/Playlist.php index 8a12d3fd..c9f4d7ad 100644 --- a/app/Models/Playlist.php +++ b/app/Models/Playlist.php @@ -26,6 +26,7 @@ use Illuminate\Database\Eloquent\SoftDeletes; use Illuminate\Foundation\Bus\DispatchesJobs; use Auth; use Cache; +use Poniverse\Ponyfm\Contracts\Searchable; use Poniverse\Ponyfm\Exceptions\TrackFileNotFoundException; use Poniverse\Ponyfm\Traits\IndexedInElasticsearchTrait; use Poniverse\Ponyfm\Traits\TrackCollection; @@ -59,9 +60,9 @@ use Venturecraft\Revisionable\RevisionableTrait; * @property-read \Illuminate\Database\Eloquent\Collection|\Venturecraft\Revisionable\Revision[] $revisionHistory * @method static \Illuminate\Database\Query\Builder|\Poniverse\Ponyfm\Models\Playlist userDetails() */ -class Playlist extends Model +class Playlist extends Model implements Searchable { - use SoftDeletes, SlugTrait, DispatchesJobs, TrackCollection, RevisionableTrait, IndexedInElasticsearchTrait; + use SoftDeletes, SlugTrait, TrackCollection, RevisionableTrait, IndexedInElasticsearchTrait; protected $elasticsearchType = 'playlist'; @@ -307,7 +308,7 @@ class Playlist extends Model * * @return array */ - public function toElasticsearch() { + public function toElasticsearch():array { return [ 'title' => $this->title, 'curator' => $this->user->display_name, diff --git a/app/Models/Track.php b/app/Models/Track.php index f3cbaf66..7b29e6b6 100644 --- a/app/Models/Track.php +++ b/app/Models/Track.php @@ -25,6 +25,7 @@ use Cache; use Config; use DB; use Elasticsearch; +use Poniverse\Ponyfm\Contracts\Searchable; use Poniverse\Ponyfm\Exceptions\TrackFileNotFoundException; use Poniverse\Ponyfm\Traits\IndexedInElasticsearchTrait; use Poniverse\Ponyfm\Traits\SlugTrait; @@ -95,7 +96,7 @@ use Venturecraft\Revisionable\RevisionableTrait; * @method static \Illuminate\Database\Query\Builder|\Poniverse\Ponyfm\Models\Track withComments() * @method static \Illuminate\Database\Query\Builder|\Poniverse\Ponyfm\Models\Track mlpma() */ -class Track extends Model +class Track extends Model implements Searchable { use SoftDeletes, IndexedInElasticsearchTrait; diff --git a/app/Models/User.php b/app/Models/User.php index 63371a66..5e83f5d8 100644 --- a/app/Models/User.php +++ b/app/Models/User.php @@ -29,6 +29,7 @@ use Illuminate\Database\Eloquent\Model; use Illuminate\Foundation\Auth\Access\Authorizable; use Auth; use Illuminate\Support\Str; +use Poniverse\Ponyfm\Contracts\Searchable; use Poniverse\Ponyfm\Traits\IndexedInElasticsearchTrait; use Venturecraft\Revisionable\RevisionableTrait; @@ -63,7 +64,7 @@ use Venturecraft\Revisionable\RevisionableTrait; * @property-read \Illuminate\Database\Eloquent\Collection|\Venturecraft\Revisionable\Revision[] $revisionHistory * @method static \Illuminate\Database\Query\Builder|\Poniverse\Ponyfm\Models\User userDetails() */ -class User extends Model implements AuthenticatableContract, CanResetPasswordContract, \Illuminate\Contracts\Auth\Access\Authorizable +class User extends Model implements AuthenticatableContract, CanResetPasswordContract, \Illuminate\Contracts\Auth\Access\Authorizable, Searchable { use Authenticatable, CanResetPassword, Authorizable, RevisionableTrait, IndexedInElasticsearchTrait; @@ -272,7 +273,7 @@ class User extends Model implements AuthenticatableContract, CanResetPasswordCon * * @return array */ - public function toElasticsearch() { + public function toElasticsearch():array { return [ 'username' => $this->username, 'display_name' => $this->display_name, diff --git a/app/Traits/IndexedInElasticsearchTrait.php b/app/Traits/IndexedInElasticsearchTrait.php index 92297a9a..8254521a 100644 --- a/app/Traits/IndexedInElasticsearchTrait.php +++ b/app/Traits/IndexedInElasticsearchTrait.php @@ -20,40 +20,39 @@ namespace Poniverse\Ponyfm\Traits; +use Config; use Elasticsearch; use Elasticsearch\Common\Exceptions\Missing404Exception; +use Illuminate\Foundation\Bus\DispatchesJobs; +use Poniverse\Ponyfm\Contracts\Searchable; +use Poniverse\Ponyfm\Jobs\UpdateSearchIndexForEntity; /** * Class IndexedInElasticsearch * - * Classes using this trait must declare the `$elasticsearchType` property - * and use the `SoftDeletes` trait. + * Classes using this trait must declare the `$elasticsearchType` property and + * implement the `Searchable` interface. * * @package Poniverse\Ponyfm\Traits */ trait IndexedInElasticsearchTrait { - /** - * Returns this model in Elasticsearch-friendly form. The array returned by - * this method should match the current mapping for this model's ES type. - * - * @return array - */ - abstract public function toElasticsearch(); + use DispatchesJobs; - /** - * @return bool whether this particular object should be indexed or not - */ - abstract public function shouldBeIndexed():bool; + // These two functions are from the Searchable interface. They're included + // here, without being implemented, to assist IDE's when editing this trait. + public abstract function toElasticsearch():array; + public abstract function shouldBeIndexed():bool; + // Laravel automatically runs this method based on the trait's name. #magic public static function bootIndexedInElasticsearchTrait() { - static::saved(function ($model) { - $model->ensureElasticsearchEntryIsUpToDate(); + static::saved(function (Searchable $entity) { + $entity->updateElasticsearchEntry(); }); - static::deleted(function ($model) { - $model->ensureElasticsearchEntryIsUpToDate(); + static::deleted(function (Searchable $entity) { + $entity->updateElasticsearchEntry(); }); } @@ -63,7 +62,7 @@ trait IndexedInElasticsearchTrait */ private function getElasticsearchParameters(bool $includeBody = true) { $parameters = [ - 'index' => 'ponyfm', + 'index' => Config::get('ponyfm.elasticsearch_index'), 'type' => $this->elasticsearchType, 'id' => $this->id, ]; @@ -89,7 +88,20 @@ trait IndexedInElasticsearchTrait } } - public function ensureElasticsearchEntryIsUpToDate() { + /** + * Asynchronously updates the Elasticsearch entry. + * When in doubt, this is the method to use. + */ + public function updateElasticsearchEntry() { + $job = (new UpdateSearchIndexForEntity($this))->onQueue(Config::get('ponyfm.indexing_queue')); + $this->dispatch($job); + } + + /** + * Synchronously updates the Elasticsearch entry. This should only be + * called from the UpdateSearchIndexForEntity job. + */ + public function updateElasticsearchEntrySynchronously() { if ($this->shouldBeIndexed()) { $this->createOrUpdateElasticsearchEntry(); } else { diff --git a/config/ponyfm.php b/config/ponyfm.php index af45bf27..c94ed5c1 100644 --- a/config/ponyfm.php +++ b/config/ponyfm.php @@ -76,4 +76,17 @@ return [ 'elasticsearch_index' => 'ponyfm', + /* + |-------------------------------------------------------------------------- + | Indexing queue name + |-------------------------------------------------------------------------- + | + | The name of the queue to process re-indexing jobs on. This is separated + | from the default queue to avoid having a site-wide re-index clog uploads + | and downloads. + | + */ + + 'indexing_queue' => 'indexing', + ]; diff --git a/config/queue.php b/config/queue.php index cf9b09da..2092ae44 100644 --- a/config/queue.php +++ b/config/queue.php @@ -35,13 +35,6 @@ return [ 'driver' => 'sync', ], - 'database' => [ - 'driver' => 'database', - 'table' => 'jobs', - 'queue' => 'default', - 'expire' => 60, - ], - 'beanstalkd' => [ 'driver' => 'beanstalkd', 'host' => 'localhost', @@ -49,30 +42,6 @@ return [ 'ttr' => 60, ], - 'sqs' => [ - 'driver' => 'sqs', - 'key' => 'your-public-key', - 'secret' => 'your-secret-key', - 'queue' => 'your-queue-url', - 'region' => 'us-east-1', - ], - - 'iron' => [ - 'driver' => 'iron', - 'host' => 'mq-aws-us-east-1.iron.io', - 'token' => 'your-token', - 'project' => 'your-project-id', - 'queue' => 'your-queue-name', - 'encrypt' => true, - ], - - 'redis' => [ - 'driver' => 'redis', - 'connection' => 'default', - 'queue' => 'default', - 'expire' => 60, - ], - ], /*