From e0faefee2390a3394be2d6ae077ed55e0c940173 Mon Sep 17 00:00:00 2001 From: Peter Deltchev Date: Thu, 7 Jan 2016 10:16:37 -0800 Subject: [PATCH 01/15] #1: Progress commit. --- app/Console/Commands/RebuildSearchIndex.php | 93 +++++++ app/Console/Kernel.php | 1 + .../Controllers/Api/Web/SearchController.php | 59 ++++ app/Http/routes.php | 4 +- app/Models/Album.php | 19 +- app/Models/Playlist.php | 16 +- app/Models/Track.php | 17 +- app/Models/User.php | 15 +- app/Traits/IndexedInElasticsearch.php | 92 +++++++ composer.json | 3 +- composer.lock | 252 +++++++++++++++++- config/app.php | 2 + config/elasticsearch.php | 167 ++++++++++++ .../2016_01_14_021607_setup_elasticsearch.php | 88 ++++++ vagrant/install.sh | 25 +- 15 files changed, 838 insertions(+), 15 deletions(-) create mode 100644 app/Console/Commands/RebuildSearchIndex.php create mode 100644 app/Http/Controllers/Api/Web/SearchController.php create mode 100644 app/Traits/IndexedInElasticsearch.php create mode 100644 config/elasticsearch.php create mode 100644 database/migrations/2016_01_14_021607_setup_elasticsearch.php diff --git a/app/Console/Commands/RebuildSearchIndex.php b/app/Console/Commands/RebuildSearchIndex.php new file mode 100644 index 00000000..ec583e4c --- /dev/null +++ b/app/Console/Commands/RebuildSearchIndex.php @@ -0,0 +1,93 @@ +. + */ + +namespace Poniverse\Ponyfm\Console\Commands; + +use Illuminate\Console\Command; +use Illuminate\Database\Eloquent\Collection; +use Poniverse\Ponyfm\Models\Album; +use Poniverse\Ponyfm\Models\Playlist; +use Poniverse\Ponyfm\Models\Track; +use Poniverse\Ponyfm\Models\User; + +class RebuildSearchIndex extends Command +{ + /** + * The name and signature of the console command. + * + * @var string + */ + protected $signature = 'rebuild:search'; + + /** + * The console command description. + * + * @var string + */ + protected $description = 'Rebuilds the Elasticsearch index.'; + + /** + * Create a new command instance. + * + * @return void + */ + public function __construct() + { + parent::__construct(); + } + + /** + * Execute the console command. + * + * @return mixed + */ + public function handle() + { + $totalTracks = Track::withTrashed()->count(); + $totalAlbums = Album::withTrashed()->count(); + $totalPlaylists = Playlist::withTrashed()->count(); + $totalUsers = User::count(); + + Track::withTrashed()->chunk(200, function(Collection $tracks) { + foreach($tracks as $track) { + $this->info("Processing track #{$track->id}..."); + $track->ensureElasticsearchEntryIsUpToDate(); + } + }); + + Album::withTrashed()->chunk(200, function(Collection $albums) { + foreach($albums as $album) { + $this->info("Processing album #{$album->id}..."); + $album->ensureElasticsearchEntryIsUpToDate(); + } + }); + +// Playlist::withTrashed()->chunk(200, function(Collection $playlists) { +// foreach($playlists as $playlist) { +// $this->info("Processing playlist #{$playlist->id}..."); +// $playlist->ensureElasticsearchEntryIsUpToDate(); +// } +// }); +// +// User::withTrashed()->chunk(200, function(User $user) { +// $user->ensureElasticsearchEntryIsUpToDate(); +// }); + } +} diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 4b1d2233..93446c6d 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -42,6 +42,7 @@ class Kernel extends ConsoleKernel \Poniverse\Ponyfm\Console\Commands\RebuildTrackCache::class, \Poniverse\Ponyfm\Console\Commands\RebuildTrack::class, \Poniverse\Ponyfm\Console\Commands\RebuildFilesizes::class, + \Poniverse\Ponyfm\Console\Commands\RebuildSearchIndex::class, \Poniverse\Ponyfm\Console\Commands\MergeDuplicateAccounts::class, ]; diff --git a/app/Http/Controllers/Api/Web/SearchController.php b/app/Http/Controllers/Api/Web/SearchController.php new file mode 100644 index 00000000..5025259c --- /dev/null +++ b/app/Http/Controllers/Api/Web/SearchController.php @@ -0,0 +1,59 @@ +. + */ + +namespace Poniverse\Ponyfm\Http\Controllers\Api\Web; + +use Elasticsearch; +use Poniverse\Ponyfm\Http\Controllers\ApiControllerBase; +use Input; +use Response; + +class SearchController extends ApiControllerBase +{ + public function getSearch() + { + $input = Input::all(); + + $elasticsearch = Elasticsearch::connection(); + + $results = $elasticsearch->search([ + 'index' => 'ponyfm', + 'type' => 'track,album', + 'body' => [ + 'query' => [ + 'multi_match' => [ + 'query' => $input['query'], + 'fields' => [ + 'track.title', + 'album.title', + 'track.artist', + 'album.artist', + 'track.genre', + ] + ] + ] + ] + ]); + + return Response::json([ + 'results' => $results, + ], 200); + } +} diff --git a/app/Http/routes.php b/app/Http/routes.php index ec54763b..57f3edb5 100644 --- a/app/Http/routes.php +++ b/app/Http/routes.php @@ -82,8 +82,7 @@ Route::group(['prefix' => 'api/v1', 'middleware' => 'json-exceptions'], function Route::group(['prefix' => 'api/web'], function() { Route::get('/taxonomies/all', 'Api\Web\TaxonomiesController@getAll'); - - Route::get('/playlists/show/{id}', 'Api\Web\PlaylistsController@getShow'); + Route::get('/search', 'Api\Web\SearchController@getSearch'); Route::get('/tracks', 'Api\Web\TracksController@getIndex'); Route::get('/tracks/{id}', 'Api\Web\TracksController@getShow')->where('id', '\d+'); @@ -94,6 +93,7 @@ Route::group(['prefix' => 'api/web'], function() { Route::get('/albums/cached/{id}/{format}', 'Api\Web\AlbumsController@getCachedAlbum')->where(['id' => '\d+', 'format' => '.+']); Route::get('/playlists', 'Api\Web\PlaylistsController@getIndex'); + Route::get('/playlists/show/{id}', 'Api\Web\PlaylistsController@getShow'); Route::get('/playlists/{id}', 'Api\Web\PlaylistsController@getShow')->where('id', '\d+'); Route::get('/playlists/cached/{id}/{format}', 'Api\Web\PlaylistsController@getCachedPlaylist')->where(['id' => '\d+', 'format' => '.+']); diff --git a/app/Models/Album.php b/app/Models/Album.php index 6666a4eb..023ce002 100644 --- a/app/Models/Album.php +++ b/app/Models/Album.php @@ -28,6 +28,7 @@ use Illuminate\Foundation\Bus\DispatchesJobs; use Auth; use Cache; use Poniverse\Ponyfm\Exceptions\TrackFileNotFoundException; +use Poniverse\Ponyfm\Traits\IndexedInElasticsearch; use Poniverse\Ponyfm\Traits\TrackCollection; use Poniverse\Ponyfm\Traits\SlugTrait; use Venturecraft\Revisionable\RevisionableTrait; @@ -61,7 +62,9 @@ use Venturecraft\Revisionable\RevisionableTrait; */ class Album extends Model { - use SoftDeletes, SlugTrait, DispatchesJobs, TrackCollection, RevisionableTrait; + use SoftDeletes, SlugTrait, DispatchesJobs, TrackCollection, RevisionableTrait, IndexedInElasticsearch; + + protected $elasticsearchType = 'album'; protected $dates = ['deleted_at']; protected $fillable = ['user_id', 'title', 'slug']; @@ -403,4 +406,18 @@ class Album extends Model protected function recountTracks() { $this->track_count = $this->tracks->count(); } + + /** + * Returns this model in Elasticsearch-friendly form. The array returned by + * this method should match the current mapping for this model's ES type. + * + * @return array + */ + public function toElasticsearch() { + return [ + 'title' => $this->title, + 'artist' => $this->user->display_name, + 'tracks' => $this->tracks->pluck('title'), + ]; + } } diff --git a/app/Models/Playlist.php b/app/Models/Playlist.php index cc3c3be2..d84080d8 100644 --- a/app/Models/Playlist.php +++ b/app/Models/Playlist.php @@ -27,6 +27,7 @@ use Illuminate\Foundation\Bus\DispatchesJobs; use Auth; use Cache; use Poniverse\Ponyfm\Exceptions\TrackFileNotFoundException; +use Poniverse\Ponyfm\Traits\IndexedInElasticsearch; use Poniverse\Ponyfm\Traits\TrackCollection; use Poniverse\Ponyfm\Traits\SlugTrait; use Venturecraft\Revisionable\RevisionableTrait; @@ -60,10 +61,11 @@ use Venturecraft\Revisionable\RevisionableTrait; */ class Playlist extends Model { - use SoftDeletes, SlugTrait, DispatchesJobs, TrackCollection, RevisionableTrait; + use SoftDeletes, SlugTrait, DispatchesJobs, TrackCollection, RevisionableTrait, IndexedInElasticsearch; + + protected $elasticsearchType = 'playlist'; protected $table = 'playlists'; - protected $dates = ['deleted_at']; public static function summary() @@ -285,4 +287,14 @@ class Playlist extends Model { return 'playlist-' . $this->id . '-' . $key; } + + /** + * Returns this model in Elasticsearch-friendly form. The array returned by + * this method should match the current mapping for this model's ES type. + * + * @return array + */ + public function toElasticsearch() { + return $this->toArray(); + } } diff --git a/app/Models/Track.php b/app/Models/Track.php index c32c5aba..b977be4a 100644 --- a/app/Models/Track.php +++ b/app/Models/Track.php @@ -24,7 +24,9 @@ use Auth; use Cache; use Config; use DB; +use Elasticsearch; use Poniverse\Ponyfm\Exceptions\TrackFileNotFoundException; +use Poniverse\Ponyfm\Traits\IndexedInElasticsearch; use Poniverse\Ponyfm\Traits\SlugTrait; use Exception; use External; @@ -95,7 +97,9 @@ use Venturecraft\Revisionable\RevisionableTrait; */ class Track extends Model { - use SoftDeletes; + use SoftDeletes, IndexedInElasticsearch; + + protected $elasticsearchType = 'track'; protected $dates = ['deleted_at', 'published_at', 'released_at']; protected $hidden = ['original_tags', 'metadata']; @@ -826,4 +830,15 @@ class Track extends Model { return 'track-' . $this->id . '-' . $key; } + + public function toElasticsearch() { + return [ + 'title' => $this->title, + 'artist' => $this->user->display_name, + 'published_at' => $this->published_at ? $this->published_at->toIso8601String() : null, + 'genre' => $this->genre->name, + 'track_type' => $this->trackType->title, + 'show_songs' => $this->showSongs->pluck('title') + ]; + } } diff --git a/app/Models/User.php b/app/Models/User.php index 9a956028..97b86b05 100644 --- a/app/Models/User.php +++ b/app/Models/User.php @@ -29,6 +29,7 @@ use Illuminate\Database\Eloquent\Model; use Illuminate\Foundation\Auth\Access\Authorizable; use Auth; use Illuminate\Support\Str; +use Poniverse\Ponyfm\Traits\IndexedInElasticsearch; use Venturecraft\Revisionable\RevisionableTrait; /** @@ -64,7 +65,9 @@ use Venturecraft\Revisionable\RevisionableTrait; */ class User extends Model implements AuthenticatableContract, CanResetPasswordContract, \Illuminate\Contracts\Auth\Access\Authorizable { - use Authenticatable, CanResetPassword, Authorizable, RevisionableTrait; + use Authenticatable, CanResetPassword, Authorizable, RevisionableTrait, IndexedInElasticsearch; + + protected $elasticsearchType = 'user'; protected $table = 'users'; protected $casts = [ @@ -247,4 +250,14 @@ class User extends Model implements AuthenticatableContract, CanResetPasswordCon return false; } + + /** + * Returns this model in Elasticsearch-friendly form. The array returned by + * this method should match the current mapping for this model's ES type. + * + * @return array + */ + public function toElasticsearch() { + return $this->toArray(); + } } diff --git a/app/Traits/IndexedInElasticsearch.php b/app/Traits/IndexedInElasticsearch.php new file mode 100644 index 00000000..7e94b2d2 --- /dev/null +++ b/app/Traits/IndexedInElasticsearch.php @@ -0,0 +1,92 @@ +. + */ + +namespace Poniverse\Ponyfm\Traits; + +use Elasticsearch; + +/** + * Class IndexedInElasticsearch + * + * Classes using this trait must declare the `$elasticsearchType` property + * and use the `SoftDeletes` trait. + * + * @package Poniverse\Ponyfm\Traits + */ +trait IndexedInElasticsearch +{ + /** + * Returns this model in Elasticsearch-friendly form. The array returned by + * this method should match the current mapping for this model's ES type. + * + * @return array + */ + abstract public function toElasticsearch(); + + public static function bootIndexedInElasticsearch() { + static::saved(function ($model) { + $model->createOrUpdateElasticsearchEntry(); + }); + + static::deleted(function ($model) { + $model->deleteElasticsearchEntry(); + }); + } + + /** + * @param bool $includeBody set to false when deleting documents + * @return array + */ + private function getElasticsearchParameters(bool $includeBody = true) { + $parameters = [ + 'index' => 'ponyfm', + 'type' => $this->elasticsearchType, + 'id' => $this->id, + ]; + + if ($includeBody) { + $parameters['body'] = $this->toElasticsearch(); + } + + return $parameters; + } + + private function createOrUpdateElasticsearchEntry() { + Elasticsearch::connection()->index($this->getElasticsearchParameters()); + } + + private function deleteElasticsearchEntry() { + try { + Elasticsearch::connection()->delete($this->getElasticsearchParameters(false)); + + } catch (\Elasticsearch\Common\Exceptions\Missing404Exception $e) { + // If the track we're trying to delete isn't indexed in Elasticsearch, + // that's fine. + } + } + + public function ensureElasticsearchEntryIsUpToDate() { + if ($this->trashed()) { + $this->deleteElasticsearchEntry(); + } else { + $this->createOrUpdateElasticsearchEntry(); + } + } +} diff --git a/composer.json b/composer.json index 3c072416..bae0215b 100644 --- a/composer.json +++ b/composer.json @@ -13,7 +13,8 @@ "guzzlehttp/guzzle": "~6.0", "doctrine/dbal": "^2.5", "venturecraft/revisionable": "^1.23", - "pda/pheanstalk": "~3.0" + "pda/pheanstalk": "~3.0", + "cviebrock/laravel-elasticsearch": "^1.0" }, "require-dev": { "fzaninotto/faker": "~1.4", diff --git a/composer.lock b/composer.lock index 56be2782..6066c1cd 100644 --- a/composer.lock +++ b/composer.lock @@ -4,8 +4,8 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", "This file is @generated automatically" ], - "hash": "d99fa4165b8a1c9e929248bbb16183ad", - "content-hash": "31347c82515003f78f1f9adef010f1f5", + "hash": "0deb7713636ee82aadee47da3a9217cc", + "content-hash": "9dea148233d815e53eb636413f2bcaed", "packages": [ { "name": "barryvdh/laravel-ide-helper", @@ -168,6 +168,55 @@ ], "time": "2013-05-05 09:10:04" }, + { + "name": "cviebrock/laravel-elasticsearch", + "version": "1.0.0", + "source": { + "type": "git", + "url": "https://github.com/cviebrock/laravel-elasticsearch.git", + "reference": "52aa1f8228006cb0bb60954e26c068af523bf47b" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/cviebrock/laravel-elasticsearch/zipball/52aa1f8228006cb0bb60954e26c068af523bf47b", + "reference": "52aa1f8228006cb0bb60954e26c068af523bf47b", + "shasum": "" + }, + "require": { + "elasticsearch/elasticsearch": "^2.0", + "illuminate/support": "~4|~5", + "monolog/monolog": "~1", + "php": ">=5.4.0" + }, + "type": "library", + "autoload": { + "psr-4": { + "Cviebrock\\LaravelElasticsearch\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Colin Viebrock", + "email": "colin@viebrock.ca" + }, + { + "name": "Brandon Martel", + "email": "brandonmartel@gmail.com" + } + ], + "description": "An easy way to use the official PHP ElasticSearch client in your Laravel applications", + "keywords": [ + "client", + "elasticsearch", + "laravel", + "search" + ], + "time": "2016-01-06 15:58:07" + }, { "name": "danielstjules/stringy", "version": "1.10.0", @@ -726,6 +775,60 @@ ], "time": "2014-09-09 13:34:57" }, + { + "name": "elasticsearch/elasticsearch", + "version": "v2.1.3", + "source": { + "type": "git", + "url": "https://github.com/elastic/elasticsearch-php.git", + "reference": "7086a86cab241a77f19cdd653ae3d2e023b41699" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/elastic/elasticsearch-php/zipball/7086a86cab241a77f19cdd653ae3d2e023b41699", + "reference": "7086a86cab241a77f19cdd653ae3d2e023b41699", + "shasum": "" + }, + "require": { + "guzzlehttp/ringphp": "~1.0", + "php": ">=5.4", + "psr/log": "~1.0" + }, + "require-dev": { + "athletic/athletic": "~0.1", + "cpliakas/git-wrapper": "~1.0", + "mockery/mockery": "0.9.4", + "phpunit/phpunit": "~4.7", + "symfony/yaml": "2.4.3 as 2.4.2", + "twig/twig": "1.*" + }, + "suggest": { + "ext-curl": "*", + "monolog/monolog": "Allows for client-level logging and tracing" + }, + "type": "library", + "autoload": { + "psr-4": { + "Elasticsearch\\": "src/Elasticsearch/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache 2" + ], + "authors": [ + { + "name": "Zachary Tong" + } + ], + "description": "PHP Client for Elasticsearch", + "keywords": [ + "client", + "elasticsearch", + "search" + ], + "time": "2015-12-15 18:42:26" + }, { "name": "guzzlehttp/guzzle", "version": "6.1.1", @@ -897,6 +1000,107 @@ ], "time": "2015-11-03 01:34:55" }, + { + "name": "guzzlehttp/ringphp", + "version": "1.1.0", + "source": { + "type": "git", + "url": "https://github.com/guzzle/RingPHP.git", + "reference": "dbbb91d7f6c191e5e405e900e3102ac7f261bc0b" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/guzzle/RingPHP/zipball/dbbb91d7f6c191e5e405e900e3102ac7f261bc0b", + "reference": "dbbb91d7f6c191e5e405e900e3102ac7f261bc0b", + "shasum": "" + }, + "require": { + "guzzlehttp/streams": "~3.0", + "php": ">=5.4.0", + "react/promise": "~2.0" + }, + "require-dev": { + "ext-curl": "*", + "phpunit/phpunit": "~4.0" + }, + "suggest": { + "ext-curl": "Guzzle will use specific adapters if cURL is present" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.1-dev" + } + }, + "autoload": { + "psr-4": { + "GuzzleHttp\\Ring\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Michael Dowling", + "email": "mtdowling@gmail.com", + "homepage": "https://github.com/mtdowling" + } + ], + "description": "Provides a simple API and specification that abstracts away the details of HTTP into a single PHP function.", + "time": "2015-05-20 03:37:09" + }, + { + "name": "guzzlehttp/streams", + "version": "3.0.0", + "source": { + "type": "git", + "url": "https://github.com/guzzle/streams.git", + "reference": "47aaa48e27dae43d39fc1cea0ccf0d84ac1a2ba5" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/guzzle/streams/zipball/47aaa48e27dae43d39fc1cea0ccf0d84ac1a2ba5", + "reference": "47aaa48e27dae43d39fc1cea0ccf0d84ac1a2ba5", + "shasum": "" + }, + "require": { + "php": ">=5.4.0" + }, + "require-dev": { + "phpunit/phpunit": "~4.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "3.0-dev" + } + }, + "autoload": { + "psr-4": { + "GuzzleHttp\\Stream\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Michael Dowling", + "email": "mtdowling@gmail.com", + "homepage": "https://github.com/mtdowling" + } + ], + "description": "Provides a simple abstraction over streams of data", + "homepage": "http://guzzlephp.org/", + "keywords": [ + "Guzzle", + "stream" + ], + "time": "2014-10-12 19:18:40" + }, { "name": "intouch/laravel-newrelic", "version": "2.0.0", @@ -1864,6 +2068,50 @@ ], "time": "2015-11-12 16:18:56" }, + { + "name": "react/promise", + "version": "v2.2.1", + "source": { + "type": "git", + "url": "https://github.com/reactphp/promise.git", + "reference": "3b6fca09c7d56321057fa8867c8dbe1abf648627" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/reactphp/promise/zipball/3b6fca09c7d56321057fa8867c8dbe1abf648627", + "reference": "3b6fca09c7d56321057fa8867c8dbe1abf648627", + "shasum": "" + }, + "require": { + "php": ">=5.4.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.0-dev" + } + }, + "autoload": { + "psr-4": { + "React\\Promise\\": "src/" + }, + "files": [ + "src/functions_include.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Jan Sorgalla", + "email": "jsorgalla@gmail.com" + } + ], + "description": "A lightweight implementation of CommonJS Promises/A for PHP", + "time": "2015-07-03 13:48:55" + }, { "name": "swiftmailer/swiftmailer", "version": "v5.4.1", diff --git a/config/app.php b/config/app.php index 8fcbfdd7..119b1bf4 100644 --- a/config/app.php +++ b/config/app.php @@ -147,6 +147,7 @@ return [ Intouch\LaravelNewrelic\NewrelicServiceProvider::class, Barryvdh\LaravelIdeHelper\IdeHelperServiceProvider::class, + Cviebrock\LaravelElasticsearch\ServiceProvider::class, ], @@ -197,6 +198,7 @@ return [ 'Validator' => Illuminate\Support\Facades\Validator::class, 'View' => Illuminate\Support\Facades\View::class, + 'Elasticsearch' => Cviebrock\LaravelElasticsearch\Facade::class, 'Newrelic' => Intouch\LaravelNewrelic\Facades\Newrelic::class, ], diff --git a/config/elasticsearch.php b/config/elasticsearch.php new file mode 100644 index 00000000..faa27549 --- /dev/null +++ b/config/elasticsearch.php @@ -0,0 +1,167 @@ + 'default', + + /** + * These are the connection parameters used when building a client. + */ + + 'connections' => [ + + 'default' => [ + + /** + * Hosts + * + * This is an array of hosts that the client will connect to. It can be a + * single host name, or an array if you are running a cluster of Elasticsearch + * instances. + * + * This is the only configuration value that is mandatory. + * + * @see https://www.elastic.co/guide/en/elasticsearch/client/php-api/2.0/_configuration.html#_host_configuration + */ + + 'hosts' => [ + 'localhost:9200' + ], + + /** + * SSL + * + * If your Elasticsearch instance uses an out-dated or self-signed SSL + * certificate, you will need to pass in the certificate bundle. This can + * either be the path to the certificate file (for self-signed certs), or a + * package like https://github.com/Kdyby/CurlCaBundle. See the documentation + * below for all the details. + * + * If you are using SSL instances, and the certificates are up-to-date and + * signed by a public certificate authority, then you can leave this null and + * just use "https" in the host path(s) above and you should be fine. + * + * @see https://www.elastic.co/guide/en/elasticsearch/client/php-api/2.0/_security.html#_ssl_encryption_2 + */ + + 'sslVerification' => null, + + /** + * Logging + * + * Logging is handled by passing in an instance of Monolog\Logger (which + * coincidentally is what Laravel's default logger is). + * + * If logging is enabled, you either need to set the path and log level + * (some defaults are given for you below), or you can use a custom logger by + * setting 'logObject' to an instance of Psr\Log\LoggerInterface. In fact, + * if you just want to use the default Laravel logger, then set 'logObject' + * to \Log::getMonolog(). + * + * Note: 'logObject' takes precedent over 'logPath'/'logLevel', so set + * 'logObject' null if you just want file-based logging to a custom path. + * + * @see https://www.elastic.co/guide/en/elasticsearch/client/php-api/2.0/_configuration.html#enabling_logger + */ + + 'logging' => false, + + // If you have an existing instance of Monolog you can use it here. + //'logObject' => \Log::getMonolog(), + + 'logPath' => storage_path('logs/elasticsearch.log'), + + 'logLevel' => Monolog\Logger::INFO, + + /** + * Retries + * + * By default, the client will retry n times, where n = number of nodes in + * your cluster. If you would like to disable retries, or change the number, + * you can do so here. + * + * @see https://www.elastic.co/guide/en/elasticsearch/client/php-api/2.0/_configuration.html#_set_retries + */ + + 'retries' => null, + + /** + * The remainder of the configuration options can almost always be left + * as-is unless you have specific reasons to change them. Refer to the + * appropriate sections in the Elasticsearch documentation for what each option + * does and what values it expects. + */ + + /** + * Sniff On Start + * + * @see https://www.elastic.co/guide/en/elasticsearch/client/php-api/2.0/_configuration.html + */ + + 'sniffOnStart' => false, + + /** + * HTTP Handler + * + * @see https://www.elastic.co/guide/en/elasticsearch/client/php-api/2.0/_configuration.html#_configure_the_http_handler + * @see http://ringphp.readthedocs.org/en/latest/client_handlers.html + */ + + 'httpHandler' => null, + + /** + * Connection Pool + * + * @see https://www.elastic.co/guide/en/elasticsearch/client/php-api/2.0/_configuration.html#_setting_the_connection_pool + * @see https://www.elastic.co/guide/en/elasticsearch/client/php-api/2.0/_connection_pool.html + */ + + 'connectionPool' => null, + + /** + * Connection Selector + * + * @see https://www.elastic.co/guide/en/elasticsearch/client/php-api/2.0/_configuration.html#_setting_the_connection_selector + * @see https://www.elastic.co/guide/en/elasticsearch/client/php-api/2.0/_selectors.html + */ + + 'connectionSelector' => null, + + /** + * Serializer + * + * @see https://www.elastic.co/guide/en/elasticsearch/client/php-api/2.0/_configuration.html#_setting_the_serializer + * @see https://www.elastic.co/guide/en/elasticsearch/client/php-api/2.0/_serializers.html + */ + + 'serializer' => null, + + /** + * Connection Factory + * + * @see https://www.elastic.co/guide/en/elasticsearch/client/php-api/2.0/_configuration.html#_setting_a_custom_connectionfactory + */ + + 'connectionFactory' => null, + + /** + * Endpoint + * + * @see https://www.elastic.co/guide/en/elasticsearch/client/php-api/2.0/_configuration.html#_set_the_endpoint_closure + */ + + 'endpoint' => null, + + ] + ] + +]; diff --git a/database/migrations/2016_01_14_021607_setup_elasticsearch.php b/database/migrations/2016_01_14_021607_setup_elasticsearch.php new file mode 100644 index 00000000..f021afff --- /dev/null +++ b/database/migrations/2016_01_14_021607_setup_elasticsearch.php @@ -0,0 +1,88 @@ +. + */ + +use Illuminate\Database\Schema\Blueprint; +use Illuminate\Database\Migrations\Migration; +use Poniverse\Ponyfm\Console\Commands\RebuildSearchIndex; + +class SetupElasticsearch extends Migration +{ + /** + * Run the migrations. + * + * @return void + */ + public function up() + { + $elasticsearch = Elasticsearch::connection(); + + $elasticsearch->indices()->create([ + 'index' => 'ponyfm', + 'body' => [ + 'mappings' => [ + 'track' => [ + '_source' => ['enabled' => true], + 'dynamic' => 'strict', + 'properties' => [ + 'title' => ['type' => 'string'], + 'artist' => ['type' => 'string'], + 'published_at' => ['type' => 'date'], + 'genre' => ['type' => 'string'], + 'track_type' => ['type' => 'string'], + + // This field is intended to be used as an array. + // Note that all Elasticsearch fields can technically be used as arrays. + // See: https://www.elastic.co/guide/en/elasticsearch/reference/current/array.html + 'show_songs' => ['type' => 'string'], + ] + ], + + 'album' => [ + '_source' => ['enabled' => true], + 'dynamic' => 'strict', + 'properties' => [ + 'title' => ['type' => 'string'], + 'artist' => ['type' => 'string'], + + // This field is intended to be used as an array. + // Note that all Elasticsearch fields can technically be used as arrays. + // See: https://www.elastic.co/guide/en/elasticsearch/reference/current/array.html + 'tracks' => ['type' => 'string'] + ] + ] + ] + ] + ]); + + Artisan::call('rebuild:search'); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + $elasticsearch = Elasticsearch::connection(); + + $elasticsearch->indices()->delete(['index' => 'ponyfm']); + } +} diff --git a/vagrant/install.sh b/vagrant/install.sh index 4b913554..654b251e 100755 --- a/vagrant/install.sh +++ b/vagrant/install.sh @@ -1,13 +1,28 @@ #!/usr/bin/env bash + +if type java &>/dev/null; then + echo "Java is installed!" +else + sudo add-apt-repository -y ppa:webupd8team/java + echo /usr/bin/debconf shared/accepted-oracle-license-v1-1 select true | sudo debconf-set-selections + echo /usr/bin/debconf shared/accepted-oracle-license-v1-1 seen true | sudo debconf-set-selections +fi + + +if type /usr/share/elasticsearch/bin/elasticsearch &>/dev/null; then + echo "ElasticSearch is installed!" +else + wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - + echo "deb http://packages.elastic.co/elasticsearch/2.x/debian stable main" | sudo tee -a /etc/apt/sources.list.d/elasticsearch-2.x.list +fi + + echo "Running apt-get update..." sudo apt-get -qq update -echo "Installing tagging tools..." -sudo apt-get -qq install -y AtomicParsley flac vorbis-tools imagemagick - -echo "Installing ffmpeg dependencies.." -sudo apt-get -qq install -y pkg-config yasm libfaac-dev libmp3lame-dev libvorbis-dev libtheora-dev +echo "Installing tagging tools & other dependencies..." +sudo apt-get -qq install -y AtomicParsley flac vorbis-tools imagemagick oracle-java8-installer pkg-config yasm libfaac-dev libmp3lame-dev libvorbis-dev libtheora-dev if type ffmpeg &>/dev/null; then From 990916171cac4147fdf82f1992dc49bf0a5d70ec Mon Sep 17 00:00:00 2001 From: Peter Deltchev Date: Thu, 14 Jan 2016 22:20:28 -0800 Subject: [PATCH 02/15] #1: Progress commit - some semblance of this is working now. --- .../Controllers/Api/Web/SearchController.php | 24 +- app/Library/Search.php | 160 +++++++++++ app/Models/Track.php | 2 + app/Providers/AppServiceProvider.php | 9 +- composer.lock | 254 +++++++++--------- config/ponyfm.php | 13 +- .../2016_01_14_021607_setup_elasticsearch.php | 7 +- 7 files changed, 315 insertions(+), 154 deletions(-) create mode 100644 app/Library/Search.php diff --git a/app/Http/Controllers/Api/Web/SearchController.php b/app/Http/Controllers/Api/Web/SearchController.php index 5025259c..367e08a4 100644 --- a/app/Http/Controllers/Api/Web/SearchController.php +++ b/app/Http/Controllers/Api/Web/SearchController.php @@ -23,34 +23,16 @@ namespace Poniverse\Ponyfm\Http\Controllers\Api\Web; use Elasticsearch; use Poniverse\Ponyfm\Http\Controllers\ApiControllerBase; use Input; +use Poniverse\Ponyfm\Library\Search; use Response; class SearchController extends ApiControllerBase { - public function getSearch() + public function getSearch(Search $search) { $input = Input::all(); - $elasticsearch = Elasticsearch::connection(); - - $results = $elasticsearch->search([ - 'index' => 'ponyfm', - 'type' => 'track,album', - 'body' => [ - 'query' => [ - 'multi_match' => [ - 'query' => $input['query'], - 'fields' => [ - 'track.title', - 'album.title', - 'track.artist', - 'album.artist', - 'track.genre', - ] - ] - ] - ] - ]); + $results = $search->searchAllContent($input['query']); return Response::json([ 'results' => $results, diff --git a/app/Library/Search.php b/app/Library/Search.php new file mode 100644 index 00000000..3580fdf2 --- /dev/null +++ b/app/Library/Search.php @@ -0,0 +1,160 @@ +. + */ + +namespace Poniverse\Ponyfm\Library; + +use DB; +use Elasticsearch\Client; +use Illuminate\Database\Eloquent\Collection; +use Poniverse\Ponyfm\Models\Album; +use Poniverse\Ponyfm\Models\Playlist; +use Poniverse\Ponyfm\Models\Track; +use Poniverse\Ponyfm\Models\User; + +class Search { + protected $elasticsearch; + protected $index; + + public function __construct(Client $connection, string $indexName) { + $this->elasticsearch = $connection; + $this->index = $indexName; + } + + /** + * @param string $query + * @param int $resultsPerContentType + * @return array + */ + public function searchAllContent(string $query, int $resultsPerContentType = 10) { + $results = $this->elasticsearch->msearch([ + 'index' => $this->index, + 'body' => [ + //===== Tracks=====// + ['type' => 'track'], + [ + 'query' => [ + 'multi_match' => [ + 'query' => $query, + 'fields' => [ + 'title', + 'artist', + 'genre', + 'track_type', + 'show_songs', + ], + ], + ], + 'size' => $resultsPerContentType + ], + + //===== Albums =====// + ['type' => 'album'], + [ + 'query' => [ + 'multi_match' => [ + 'query' => $query, + 'fields' => [ + 'title', + 'artist', + 'tracks', + ], + ], + ], + 'size' => $resultsPerContentType + ], + + //===== Playlists =====// + ['type' => 'playlist'], + [ + 'query' => [ + 'multi_match' => [ + 'query' => $query, + 'fields' => [ + 'title', + 'user', + ], + ], + ], + 'size' => $resultsPerContentType + ], + + //===== Users =====// + ['type' => 'user'], + [ + 'query' => [ + 'multi_match' => [ + 'query' => $query, + 'fields' => [ + 'display_name', + ], + ], + ], + 'size' => $resultsPerContentType + ], + ] + ]); + + $tracks = $this->transformToEloquent(Track::class, $results['responses'][0]['hits']['hits']); + $albums = $this->transformToEloquent(Album::class, $results['responses'][1]['hits']['hits']); + $playlists = $this->transformToEloquent(Playlist::class, $results['responses'][2]['hits']['hits']); + $users = $this->transformToEloquent(User::class, $results['responses'][3]['hits']['hits']); + + return [ + 'tracks' => $tracks, + 'albums' => $albums, + 'playlists' => $playlists, + 'users' => $users + ]; + } + + /** + * Transforms the given Elasticsearch results into a collection of corresponding + * Eloquent models. + * + * This method assumes that the given class uses soft deletes. + * + * @param string $modelClass The Eloquent model class to instantiate these results as + * @param array $searchHits + * @return \Illuminate\Database\Eloquent\Collection + */ + protected function transformToEloquent(string $modelClass, array $searchHits) { + if (empty($searchHits)) { + return new Collection(); + } + + $ids = []; + $caseStatement = 'CASE id '; + + $i = 0; + foreach ($searchHits as $result) { + $ids[$result['_id']] = $result['_score']; + $caseStatement .= "WHEN ${result['_id']} THEN $i "; + $i++; + } + $caseStatement .= 'END'; + + $modelInstances = $modelClass::withTrashed() + ->whereIn('id', array_keys($ids)) + ->orderBy(DB::raw($caseStatement)) + ->get(); + + return $modelInstances; + } +} diff --git a/app/Models/Track.php b/app/Models/Track.php index b977be4a..862f4ce9 100644 --- a/app/Models/Track.php +++ b/app/Models/Track.php @@ -831,6 +831,8 @@ class Track extends Model return 'track-' . $this->id . '-' . $key; } + //============= Elasticsearch stuff ==================// + public function toElasticsearch() { return [ 'title' => $this->title, diff --git a/app/Providers/AppServiceProvider.php b/app/Providers/AppServiceProvider.php index df875e1a..58cc94cf 100644 --- a/app/Providers/AppServiceProvider.php +++ b/app/Providers/AppServiceProvider.php @@ -20,8 +20,6 @@ namespace Poniverse\Ponyfm\Providers; -use DB; -use Illuminate\Database\SQLiteConnection; use Illuminate\Foundation\Application; use Illuminate\Support\ServiceProvider; use PfmValidator; @@ -53,5 +51,12 @@ class AppServiceProvider extends ServiceProvider $this->app->bind(Poniverse::class, function(Application $app) { return new Poniverse($app['config']->get('poniverse.client_id'), $app['config']->get('poniverse.secret')); }); + + $this->app->bind(Poniverse\Ponyfm\Library\Search::class, function(Application $app) { + return new Poniverse\Ponyfm\Library\Search( + \Elasticsearch::connection(), + $app['config']->get('ponyfm.elasticsearch_index') + ); + }); } } diff --git a/composer.lock b/composer.lock index 6066c1cd..f7ef8573 100644 --- a/composer.lock +++ b/composer.lock @@ -376,33 +376,33 @@ }, { "name": "doctrine/cache", - "version": "v1.5.4", + "version": "v1.6.0", "source": { "type": "git", "url": "https://github.com/doctrine/cache.git", - "reference": "47cdc76ceb95cc591d9c79a36dc3794975b5d136" + "reference": "f8af318d14bdb0eff0336795b428b547bd39ccb6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/doctrine/cache/zipball/47cdc76ceb95cc591d9c79a36dc3794975b5d136", - "reference": "47cdc76ceb95cc591d9c79a36dc3794975b5d136", + "url": "https://api.github.com/repos/doctrine/cache/zipball/f8af318d14bdb0eff0336795b428b547bd39ccb6", + "reference": "f8af318d14bdb0eff0336795b428b547bd39ccb6", "shasum": "" }, "require": { - "php": ">=5.3.2" + "php": "~5.5|~7.0" }, "conflict": { "doctrine/common": ">2.2,<2.4" }, "require-dev": { - "phpunit/phpunit": ">=3.7", + "phpunit/phpunit": "~4.8|~5.0", "predis/predis": "~1.0", "satooshi/php-coveralls": "~0.6" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "1.5.x-dev" + "dev-master": "1.6.x-dev" } }, "autoload": { @@ -442,7 +442,7 @@ "cache", "caching" ], - "time": "2015-12-19 05:03:47" + "time": "2015-12-31 16:37:02" }, { "name": "doctrine/collections", @@ -512,16 +512,16 @@ }, { "name": "doctrine/common", - "version": "v2.5.2", + "version": "v2.6.1", "source": { "type": "git", "url": "https://github.com/doctrine/common.git", - "reference": "311001fd9865a4d0d59efff4eac6d7dcb3f5270c" + "reference": "a579557bc689580c19fee4e27487a67fe60defc0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/doctrine/common/zipball/311001fd9865a4d0d59efff4eac6d7dcb3f5270c", - "reference": "311001fd9865a4d0d59efff4eac6d7dcb3f5270c", + "url": "https://api.github.com/repos/doctrine/common/zipball/a579557bc689580c19fee4e27487a67fe60defc0", + "reference": "a579557bc689580c19fee4e27487a67fe60defc0", "shasum": "" }, "require": { @@ -530,20 +530,20 @@ "doctrine/collections": "1.*", "doctrine/inflector": "1.*", "doctrine/lexer": "1.*", - "php": ">=5.3.2" + "php": "~5.5|~7.0" }, "require-dev": { - "phpunit/phpunit": "~3.7" + "phpunit/phpunit": "~4.8|~5.0" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "2.5.x-dev" + "dev-master": "2.7.x-dev" } }, "autoload": { - "psr-0": { - "Doctrine\\Common\\": "lib/" + "psr-4": { + "Doctrine\\Common\\": "lib/Doctrine/Common" } }, "notification-url": "https://packagist.org/downloads/", @@ -581,24 +581,24 @@ "persistence", "spl" ], - "time": "2015-12-04 12:49:42" + "time": "2015-12-25 13:18:31" }, { "name": "doctrine/dbal", - "version": "v2.5.2", + "version": "v2.5.4", "source": { "type": "git", "url": "https://github.com/doctrine/dbal.git", - "reference": "01dbcbc5cd0a913d751418e635434a18a2f2a75c" + "reference": "abbdfd1cff43a7b99d027af3be709bc8fc7d4769" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/doctrine/dbal/zipball/01dbcbc5cd0a913d751418e635434a18a2f2a75c", - "reference": "01dbcbc5cd0a913d751418e635434a18a2f2a75c", + "url": "https://api.github.com/repos/doctrine/dbal/zipball/abbdfd1cff43a7b99d027af3be709bc8fc7d4769", + "reference": "abbdfd1cff43a7b99d027af3be709bc8fc7d4769", "shasum": "" }, "require": { - "doctrine/common": ">=2.4,<2.6-dev", + "doctrine/common": ">=2.4,<2.7-dev", "php": ">=5.3.2" }, "require-dev": { @@ -652,7 +652,7 @@ "persistence", "queryobject" ], - "time": "2015-09-16 16:29:33" + "time": "2016-01-05 22:11:12" }, { "name": "doctrine/inflector", @@ -777,16 +777,16 @@ }, { "name": "elasticsearch/elasticsearch", - "version": "v2.1.3", + "version": "v2.0.3", "source": { "type": "git", "url": "https://github.com/elastic/elasticsearch-php.git", - "reference": "7086a86cab241a77f19cdd653ae3d2e023b41699" + "reference": "9ce5bd7606f6c185d434de4f80863f998f74e179" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/elastic/elasticsearch-php/zipball/7086a86cab241a77f19cdd653ae3d2e023b41699", - "reference": "7086a86cab241a77f19cdd653ae3d2e023b41699", + "url": "https://api.github.com/repos/elastic/elasticsearch-php/zipball/9ce5bd7606f6c185d434de4f80863f998f74e179", + "reference": "9ce5bd7606f6c185d434de4f80863f998f74e179", "shasum": "" }, "require": { @@ -797,8 +797,8 @@ "require-dev": { "athletic/athletic": "~0.1", "cpliakas/git-wrapper": "~1.0", - "mockery/mockery": "0.9.4", - "phpunit/phpunit": "~4.7", + "mockery/mockery": "dev-master@dev", + "phpunit/phpunit": "3.7.*", "symfony/yaml": "2.4.3 as 2.4.2", "twig/twig": "1.*" }, @@ -827,7 +827,7 @@ "elasticsearch", "search" ], - "time": "2015-12-15 18:42:26" + "time": "2015-11-05 15:29:21" }, { "name": "guzzlehttp/guzzle", @@ -1333,16 +1333,16 @@ }, { "name": "laravel/framework", - "version": "v5.1.27", + "version": "v5.1.28", "source": { "type": "git", "url": "https://github.com/laravel/framework.git", - "reference": "b16f80878fd3603022d3c84593397cedd9af0bcf" + "reference": "3f0fd27939dfdafb1e50058423cd24e640894ba2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/framework/zipball/b16f80878fd3603022d3c84593397cedd9af0bcf", - "reference": "b16f80878fd3603022d3c84593397cedd9af0bcf", + "url": "https://api.github.com/repos/laravel/framework/zipball/3f0fd27939dfdafb1e50058423cd24e640894ba2", + "reference": "3f0fd27939dfdafb1e50058423cd24e640894ba2", "shasum": "" }, "require": { @@ -1457,7 +1457,7 @@ "framework", "laravel" ], - "time": "2015-12-17 20:35:38" + "time": "2015-12-31 17:41:30" }, { "name": "league/flysystem", @@ -1764,16 +1764,16 @@ }, { "name": "paragonie/random_compat", - "version": "1.1.4", + "version": "1.1.5", "source": { "type": "git", "url": "https://github.com/paragonie/random_compat.git", - "reference": "d762ee5b099a29044603cd4649851e81aa66cb47" + "reference": "dd8998b7c846f6909f4e7a5f67fabebfc412a4f7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/paragonie/random_compat/zipball/d762ee5b099a29044603cd4649851e81aa66cb47", - "reference": "d762ee5b099a29044603cd4649851e81aa66cb47", + "url": "https://api.github.com/repos/paragonie/random_compat/zipball/dd8998b7c846f6909f4e7a5f67fabebfc412a4f7", + "reference": "dd8998b7c846f6909f4e7a5f67fabebfc412a4f7", "shasum": "" }, "require": { @@ -1808,7 +1808,7 @@ "pseudorandom", "random" ], - "time": "2015-12-10 14:48:13" + "time": "2016-01-06 13:31:20" }, { "name": "pda/pheanstalk", @@ -2167,16 +2167,16 @@ }, { "name": "symfony/class-loader", - "version": "v2.8.0", + "version": "v2.8.2", "source": { "type": "git", "url": "https://github.com/symfony/class-loader.git", - "reference": "51f83451bf0ddfc696e47e4642d6cd10fcfce160" + "reference": "98e9089a428ed0e39423b67352c57ef5910a3269" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/class-loader/zipball/51f83451bf0ddfc696e47e4642d6cd10fcfce160", - "reference": "51f83451bf0ddfc696e47e4642d6cd10fcfce160", + "url": "https://api.github.com/repos/symfony/class-loader/zipball/98e9089a428ed0e39423b67352c57ef5910a3269", + "reference": "98e9089a428ed0e39423b67352c57ef5910a3269", "shasum": "" }, "require": { @@ -2215,20 +2215,20 @@ ], "description": "Symfony ClassLoader Component", "homepage": "https://symfony.com", - "time": "2015-11-26 07:00:59" + "time": "2016-01-03 15:33:41" }, { "name": "symfony/console", - "version": "v2.7.7", + "version": "v2.7.9", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "16bb1cb86df43c90931df65f529e7ebd79636750" + "reference": "d3fc138b6ed8f8074591821d3416d8f9c04d6ca6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/16bb1cb86df43c90931df65f529e7ebd79636750", - "reference": "16bb1cb86df43c90931df65f529e7ebd79636750", + "url": "https://api.github.com/repos/symfony/console/zipball/d3fc138b6ed8f8074591821d3416d8f9c04d6ca6", + "reference": "d3fc138b6ed8f8074591821d3416d8f9c04d6ca6", "shasum": "" }, "require": { @@ -2274,20 +2274,20 @@ ], "description": "Symfony Console Component", "homepage": "https://symfony.com", - "time": "2015-11-18 09:54:26" + "time": "2016-01-14 08:26:43" }, { "name": "symfony/css-selector", - "version": "v2.7.7", + "version": "v2.7.9", "source": { "type": "git", "url": "https://github.com/symfony/css-selector.git", - "reference": "abb47717fb88aebd9437da2fc8bb01a50a36679f" + "reference": "1a869e59cc3b2802961fc2124139659e12b72fe5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/css-selector/zipball/abb47717fb88aebd9437da2fc8bb01a50a36679f", - "reference": "abb47717fb88aebd9437da2fc8bb01a50a36679f", + "url": "https://api.github.com/repos/symfony/css-selector/zipball/1a869e59cc3b2802961fc2124139659e12b72fe5", + "reference": "1a869e59cc3b2802961fc2124139659e12b72fe5", "shasum": "" }, "require": { @@ -2327,20 +2327,20 @@ ], "description": "Symfony CssSelector Component", "homepage": "https://symfony.com", - "time": "2015-10-30 20:10:21" + "time": "2016-01-03 15:32:00" }, { "name": "symfony/debug", - "version": "v2.7.7", + "version": "v2.7.9", "source": { "type": "git", "url": "https://github.com/symfony/debug.git", - "reference": "0dbc119596f4afc82d9b2eb2a7e6a4af1ee763fa" + "reference": "5aca4aa9600b943287b4a1799a4d1d78b5388175" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/debug/zipball/0dbc119596f4afc82d9b2eb2a7e6a4af1ee763fa", - "reference": "0dbc119596f4afc82d9b2eb2a7e6a4af1ee763fa", + "url": "https://api.github.com/repos/symfony/debug/zipball/5aca4aa9600b943287b4a1799a4d1d78b5388175", + "reference": "5aca4aa9600b943287b4a1799a4d1d78b5388175", "shasum": "" }, "require": { @@ -2384,20 +2384,20 @@ ], "description": "Symfony Debug Component", "homepage": "https://symfony.com", - "time": "2015-10-30 20:10:21" + "time": "2016-01-13 07:57:33" }, { "name": "symfony/dom-crawler", - "version": "v2.7.7", + "version": "v2.7.9", "source": { "type": "git", "url": "https://github.com/symfony/dom-crawler.git", - "reference": "b33593cbfe1d81b50d48353f338aca76a08658d8" + "reference": "55cc79a177193eb3bd74ac54b353691fbb211d3a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/dom-crawler/zipball/b33593cbfe1d81b50d48353f338aca76a08658d8", - "reference": "b33593cbfe1d81b50d48353f338aca76a08658d8", + "url": "https://api.github.com/repos/symfony/dom-crawler/zipball/55cc79a177193eb3bd74ac54b353691fbb211d3a", + "reference": "55cc79a177193eb3bd74ac54b353691fbb211d3a", "shasum": "" }, "require": { @@ -2439,20 +2439,20 @@ ], "description": "Symfony DomCrawler Component", "homepage": "https://symfony.com", - "time": "2015-11-02 20:20:53" + "time": "2016-01-03 15:32:00" }, { "name": "symfony/event-dispatcher", - "version": "v2.8.0", + "version": "v2.8.2", "source": { "type": "git", "url": "https://github.com/symfony/event-dispatcher.git", - "reference": "a5eb815363c0388e83247e7e9853e5dbc14999cc" + "reference": "ee278f7c851533e58ca307f66305ccb9188aceda" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/event-dispatcher/zipball/a5eb815363c0388e83247e7e9853e5dbc14999cc", - "reference": "a5eb815363c0388e83247e7e9853e5dbc14999cc", + "url": "https://api.github.com/repos/symfony/event-dispatcher/zipball/ee278f7c851533e58ca307f66305ccb9188aceda", + "reference": "ee278f7c851533e58ca307f66305ccb9188aceda", "shasum": "" }, "require": { @@ -2499,20 +2499,20 @@ ], "description": "Symfony EventDispatcher Component", "homepage": "https://symfony.com", - "time": "2015-10-30 20:15:42" + "time": "2016-01-13 10:28:07" }, { "name": "symfony/finder", - "version": "v2.7.7", + "version": "v2.7.9", "source": { "type": "git", "url": "https://github.com/symfony/finder.git", - "reference": "a06a0c0ff7db3736a50d530c908cca547bf13da9" + "reference": "d20ac81c81a67ab898b0c0afa435f3e9a7d460cf" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/finder/zipball/a06a0c0ff7db3736a50d530c908cca547bf13da9", - "reference": "a06a0c0ff7db3736a50d530c908cca547bf13da9", + "url": "https://api.github.com/repos/symfony/finder/zipball/d20ac81c81a67ab898b0c0afa435f3e9a7d460cf", + "reference": "d20ac81c81a67ab898b0c0afa435f3e9a7d460cf", "shasum": "" }, "require": { @@ -2548,20 +2548,20 @@ ], "description": "Symfony Finder Component", "homepage": "https://symfony.com", - "time": "2015-10-30 20:10:21" + "time": "2016-01-14 08:26:43" }, { "name": "symfony/http-foundation", - "version": "v2.7.7", + "version": "v2.7.9", "source": { "type": "git", "url": "https://github.com/symfony/http-foundation.git", - "reference": "e83a3d105ddaf5a113e803c904fdec552d1f1c35" + "reference": "2f9d240056f026af5f7ba7f7052b0c6709bf288c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-foundation/zipball/e83a3d105ddaf5a113e803c904fdec552d1f1c35", - "reference": "e83a3d105ddaf5a113e803c904fdec552d1f1c35", + "url": "https://api.github.com/repos/symfony/http-foundation/zipball/2f9d240056f026af5f7ba7f7052b0c6709bf288c", + "reference": "2f9d240056f026af5f7ba7f7052b0c6709bf288c", "shasum": "" }, "require": { @@ -2603,20 +2603,20 @@ ], "description": "Symfony HttpFoundation Component", "homepage": "https://symfony.com", - "time": "2015-11-20 17:41:18" + "time": "2016-01-13 10:26:43" }, { "name": "symfony/http-kernel", - "version": "v2.7.7", + "version": "v2.7.9", "source": { "type": "git", "url": "https://github.com/symfony/http-kernel.git", - "reference": "5570de31e8fbc03777a8c61eb24f9b626e5e5941" + "reference": "aa2f1e544d6cb862452504b5479a5095b7bfc53f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-kernel/zipball/5570de31e8fbc03777a8c61eb24f9b626e5e5941", - "reference": "5570de31e8fbc03777a8c61eb24f9b626e5e5941", + "url": "https://api.github.com/repos/symfony/http-kernel/zipball/aa2f1e544d6cb862452504b5479a5095b7bfc53f", + "reference": "aa2f1e544d6cb862452504b5479a5095b7bfc53f", "shasum": "" }, "require": { @@ -2685,20 +2685,20 @@ ], "description": "Symfony HttpKernel Component", "homepage": "https://symfony.com", - "time": "2015-11-23 11:57:49" + "time": "2016-01-14 10:41:45" }, { "name": "symfony/polyfill-php56", - "version": "v1.0.0", + "version": "v1.0.1", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php56.git", - "reference": "a6bd4770a6967517e6610529e14afaa3111094a3" + "reference": "e2e77609a9e2328eb370fbb0e0d8b2000ebb488f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php56/zipball/a6bd4770a6967517e6610529e14afaa3111094a3", - "reference": "a6bd4770a6967517e6610529e14afaa3111094a3", + "url": "https://api.github.com/repos/symfony/polyfill-php56/zipball/e2e77609a9e2328eb370fbb0e0d8b2000ebb488f", + "reference": "e2e77609a9e2328eb370fbb0e0d8b2000ebb488f", "shasum": "" }, "require": { @@ -2741,11 +2741,11 @@ "portable", "shim" ], - "time": "2015-11-04 20:28:58" + "time": "2015-12-18 15:10:25" }, { "name": "symfony/polyfill-util", - "version": "v1.0.0", + "version": "v1.0.1", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-util.git", @@ -2797,16 +2797,16 @@ }, { "name": "symfony/process", - "version": "v2.7.7", + "version": "v2.7.9", "source": { "type": "git", "url": "https://github.com/symfony/process.git", - "reference": "f6290983c8725d0afa29bdc3e5295879de3e58f5" + "reference": "0570b9ca51135ee7da0f19239eaf7b07ffb87034" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/process/zipball/f6290983c8725d0afa29bdc3e5295879de3e58f5", - "reference": "f6290983c8725d0afa29bdc3e5295879de3e58f5", + "url": "https://api.github.com/repos/symfony/process/zipball/0570b9ca51135ee7da0f19239eaf7b07ffb87034", + "reference": "0570b9ca51135ee7da0f19239eaf7b07ffb87034", "shasum": "" }, "require": { @@ -2842,20 +2842,20 @@ ], "description": "Symfony Process Component", "homepage": "https://symfony.com", - "time": "2015-11-19 16:11:24" + "time": "2016-01-06 09:57:37" }, { "name": "symfony/routing", - "version": "v2.7.7", + "version": "v2.7.9", "source": { "type": "git", "url": "https://github.com/symfony/routing.git", - "reference": "7450f6196711b124fb8b04a12286d01a0401ddfe" + "reference": "6fec77993acfe19aecf60544b9c7d32f3d5b2506" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/routing/zipball/7450f6196711b124fb8b04a12286d01a0401ddfe", - "reference": "7450f6196711b124fb8b04a12286d01a0401ddfe", + "url": "https://api.github.com/repos/symfony/routing/zipball/6fec77993acfe19aecf60544b9c7d32f3d5b2506", + "reference": "6fec77993acfe19aecf60544b9c7d32f3d5b2506", "shasum": "" }, "require": { @@ -2915,20 +2915,20 @@ "uri", "url" ], - "time": "2015-11-18 13:41:01" + "time": "2016-01-03 15:32:00" }, { "name": "symfony/translation", - "version": "v2.7.7", + "version": "v2.7.9", "source": { "type": "git", "url": "https://github.com/symfony/translation.git", - "reference": "e4ecb9c3ba1304eaf24de15c2d7a428101c1982f" + "reference": "8cbab8445ad4269427077ba02fff8718cb397e22" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/translation/zipball/e4ecb9c3ba1304eaf24de15c2d7a428101c1982f", - "reference": "e4ecb9c3ba1304eaf24de15c2d7a428101c1982f", + "url": "https://api.github.com/repos/symfony/translation/zipball/8cbab8445ad4269427077ba02fff8718cb397e22", + "reference": "8cbab8445ad4269427077ba02fff8718cb397e22", "shasum": "" }, "require": { @@ -2978,20 +2978,20 @@ ], "description": "Symfony Translation Component", "homepage": "https://symfony.com", - "time": "2015-11-18 13:41:01" + "time": "2016-01-03 15:32:00" }, { "name": "symfony/var-dumper", - "version": "v2.7.7", + "version": "v2.7.9", "source": { "type": "git", "url": "https://github.com/symfony/var-dumper.git", - "reference": "72bcb27411780eaee9469729aace73c0d46fb2b8" + "reference": "ad39199e91f2f845a0181b14d459fda13a622138" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/var-dumper/zipball/72bcb27411780eaee9469729aace73c0d46fb2b8", - "reference": "72bcb27411780eaee9469729aace73c0d46fb2b8", + "url": "https://api.github.com/repos/symfony/var-dumper/zipball/ad39199e91f2f845a0181b14d459fda13a622138", + "reference": "ad39199e91f2f845a0181b14d459fda13a622138", "shasum": "" }, "require": { @@ -3037,20 +3037,20 @@ "debug", "dump" ], - "time": "2015-11-18 13:41:01" + "time": "2016-01-07 11:12:32" }, { "name": "venturecraft/revisionable", - "version": "1.24.0", + "version": "1.26.0", "source": { "type": "git", "url": "https://github.com/VentureCraft/revisionable.git", - "reference": "99c27d94f80ae9240cec89c4276f61e748e989a5" + "reference": "7a3d5304de6c10d43cfb0d9ebe0bbdbb6e5b82ee" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/VentureCraft/revisionable/zipball/99c27d94f80ae9240cec89c4276f61e748e989a5", - "reference": "99c27d94f80ae9240cec89c4276f61e748e989a5", + "url": "https://api.github.com/repos/VentureCraft/revisionable/zipball/7a3d5304de6c10d43cfb0d9ebe0bbdbb6e5b82ee", + "reference": "7a3d5304de6c10d43cfb0d9ebe0bbdbb6e5b82ee", "shasum": "" }, "require": { @@ -3085,7 +3085,7 @@ "model", "revision" ], - "time": "2015-12-09 21:48:10" + "time": "2016-01-13 12:14:05" }, { "name": "vlucas/phpdotenv", @@ -3387,16 +3387,16 @@ }, { "name": "phpspec/phpspec", - "version": "2.4.0", + "version": "2.4.1", "source": { "type": "git", "url": "https://github.com/phpspec/phpspec.git", - "reference": "1d3938e6d9ffb1bd4805ea8ddac62ea48767f358" + "reference": "5528ce1e93a1efa090c9404aba3395c329b4e6ed" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpspec/phpspec/zipball/1d3938e6d9ffb1bd4805ea8ddac62ea48767f358", - "reference": "1d3938e6d9ffb1bd4805ea8ddac62ea48767f358", + "url": "https://api.github.com/repos/phpspec/phpspec/zipball/5528ce1e93a1efa090c9404aba3395c329b4e6ed", + "reference": "5528ce1e93a1efa090c9404aba3395c329b4e6ed", "shasum": "" }, "require": { @@ -3461,7 +3461,7 @@ "testing", "tests" ], - "time": "2015-11-29 02:03:49" + "time": "2016-01-01 10:17:54" }, { "name": "phpspec/prophecy", @@ -4264,16 +4264,16 @@ }, { "name": "symfony/yaml", - "version": "v3.0.0", + "version": "v3.0.1", "source": { "type": "git", "url": "https://github.com/symfony/yaml.git", - "reference": "177a015cb0e19ff4a49e0e2e2c5fc1c1bee07002" + "reference": "3df409958a646dad2bc5046c3fb671ee24a1a691" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/yaml/zipball/177a015cb0e19ff4a49e0e2e2c5fc1c1bee07002", - "reference": "177a015cb0e19ff4a49e0e2e2c5fc1c1bee07002", + "url": "https://api.github.com/repos/symfony/yaml/zipball/3df409958a646dad2bc5046c3fb671ee24a1a691", + "reference": "3df409958a646dad2bc5046c3fb671ee24a1a691", "shasum": "" }, "require": { @@ -4309,7 +4309,7 @@ ], "description": "Symfony Yaml Component", "homepage": "https://symfony.com", - "time": "2015-11-30 12:36:17" + "time": "2015-12-26 13:39:53" } ], "aliases": [], diff --git a/config/ponyfm.php b/config/ponyfm.php index 1470dcd4..af45bf27 100644 --- a/config/ponyfm.php +++ b/config/ponyfm.php @@ -56,7 +56,7 @@ return [ /* |-------------------------------------------------------------------------- - | Cache Duration + | Cache duration |-------------------------------------------------------------------------- | | Duration in minutes for track files to be stored in cache. @@ -65,4 +65,15 @@ return [ 'track_file_cache_duration' => 1440, + /* + |-------------------------------------------------------------------------- + | Elasticsearch index name + |-------------------------------------------------------------------------- + | + | The name of the Elasticsearch index to store Pony.fm's search data in. + | + */ + + 'elasticsearch_index' => 'ponyfm', + ]; diff --git a/database/migrations/2016_01_14_021607_setup_elasticsearch.php b/database/migrations/2016_01_14_021607_setup_elasticsearch.php index f021afff..364da970 100644 --- a/database/migrations/2016_01_14_021607_setup_elasticsearch.php +++ b/database/migrations/2016_01_14_021607_setup_elasticsearch.php @@ -21,6 +21,7 @@ use Illuminate\Database\Schema\Blueprint; use Illuminate\Database\Migrations\Migration; use Poniverse\Ponyfm\Console\Commands\RebuildSearchIndex; +use Poniverse\Ponyfm\Models\Track; class SetupElasticsearch extends Migration { @@ -44,13 +45,13 @@ class SetupElasticsearch extends Migration 'title' => ['type' => 'string'], 'artist' => ['type' => 'string'], 'published_at' => ['type' => 'date'], - 'genre' => ['type' => 'string'], - 'track_type' => ['type' => 'string'], + 'genre' => ['type' => 'string', 'index' => 'not_analyzed'], + 'track_type' => ['type' => 'string', 'index' => 'not_analyzed'], // This field is intended to be used as an array. // Note that all Elasticsearch fields can technically be used as arrays. // See: https://www.elastic.co/guide/en/elasticsearch/reference/current/array.html - 'show_songs' => ['type' => 'string'], + 'show_songs' => ['type' => 'string', 'index' => 'not_analyzed'], ] ], From a5bb117ae7274de41f28b877f6c4a166b4218699 Mon Sep 17 00:00:00 2001 From: Peter Deltchev Date: Fri, 15 Jan 2016 14:36:53 -0800 Subject: [PATCH 03/15] #1: Don't analyze the artist field in Elasticsearch. --- .../migrations/2016_01_14_021607_setup_elasticsearch.php | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/database/migrations/2016_01_14_021607_setup_elasticsearch.php b/database/migrations/2016_01_14_021607_setup_elasticsearch.php index 364da970..c6869aec 100644 --- a/database/migrations/2016_01_14_021607_setup_elasticsearch.php +++ b/database/migrations/2016_01_14_021607_setup_elasticsearch.php @@ -18,10 +18,8 @@ * along with this program. If not, see . */ -use Illuminate\Database\Schema\Blueprint; use Illuminate\Database\Migrations\Migration; use Poniverse\Ponyfm\Console\Commands\RebuildSearchIndex; -use Poniverse\Ponyfm\Models\Track; class SetupElasticsearch extends Migration { @@ -43,7 +41,7 @@ class SetupElasticsearch extends Migration 'dynamic' => 'strict', 'properties' => [ 'title' => ['type' => 'string'], - 'artist' => ['type' => 'string'], + 'artist' => ['type' => 'string', 'index' => 'not_analyzed'], 'published_at' => ['type' => 'date'], 'genre' => ['type' => 'string', 'index' => 'not_analyzed'], 'track_type' => ['type' => 'string', 'index' => 'not_analyzed'], @@ -60,7 +58,7 @@ class SetupElasticsearch extends Migration 'dynamic' => 'strict', 'properties' => [ 'title' => ['type' => 'string'], - 'artist' => ['type' => 'string'], + 'artist' => ['type' => 'string', 'index' => 'not_analyzed'], // This field is intended to be used as an array. // Note that all Elasticsearch fields can technically be used as arrays. From dbbaa03542480ab3e9ccdfba43effa450ecac0a6 Mon Sep 17 00:00:00 2001 From: Peter Deltchev Date: Fri, 15 Jan 2016 20:10:20 -0800 Subject: [PATCH 04/15] #1: Implemented Elasticsearch mappings for playlists and users. --- app/Commands/AddTrackToPlaylistCommand.php | 1 + app/Console/Commands/RebuildSearchIndex.php | 23 ++++++++------ app/Library/Search.php | 26 +++++++++++----- app/Models/Album.php | 4 +-- app/Models/Playlist.php | 10 ++++-- app/Models/Track.php | 4 +-- app/Models/User.php | 10 ++++-- ...ch.php => IndexedInElasticsearchTrait.php} | 4 +-- .../2016_01_14_021607_setup_elasticsearch.php | 31 ++++++++++++++++++- 9 files changed, 82 insertions(+), 31 deletions(-) rename app/Traits/{IndexedInElasticsearch.php => IndexedInElasticsearchTrait.php} (96%) diff --git a/app/Commands/AddTrackToPlaylistCommand.php b/app/Commands/AddTrackToPlaylistCommand.php index 314e6ce2..499fcf74 100644 --- a/app/Commands/AddTrackToPlaylistCommand.php +++ b/app/Commands/AddTrackToPlaylistCommand.php @@ -54,6 +54,7 @@ class AddTrackToPlaylistCommand extends CommandBase { $songIndex = $this->_playlist->tracks()->count() + 1; $this->_playlist->tracks()->attach($this->_track, ['position' => $songIndex]); + $this->_playlist->touch(); Playlist::whereId($this->_playlist->id)->update([ 'track_count' => DB::raw('(SELECT COUNT(id) FROM playlist_track WHERE playlist_id = ' . $this->_playlist->id . ')') diff --git a/app/Console/Commands/RebuildSearchIndex.php b/app/Console/Commands/RebuildSearchIndex.php index ec583e4c..72e66066 100644 --- a/app/Console/Commands/RebuildSearchIndex.php +++ b/app/Console/Commands/RebuildSearchIndex.php @@ -79,15 +79,18 @@ class RebuildSearchIndex extends Command } }); -// Playlist::withTrashed()->chunk(200, function(Collection $playlists) { -// foreach($playlists as $playlist) { -// $this->info("Processing playlist #{$playlist->id}..."); -// $playlist->ensureElasticsearchEntryIsUpToDate(); -// } -// }); -// -// User::withTrashed()->chunk(200, function(User $user) { -// $user->ensureElasticsearchEntryIsUpToDate(); -// }); + Playlist::withTrashed()->chunk(200, function(Collection $playlists) { + foreach($playlists as $playlist) { + $this->info("Processing playlist #{$playlist->id}..."); + $playlist->ensureElasticsearchEntryIsUpToDate(); + } + }); + + User::chunk(200, function(Collection $users) { + foreach($users as $user) { + $this->info("Processing user #{$user->id}..."); + $user->ensureElasticsearchEntryIsUpToDate(); + } + }); } } diff --git a/app/Library/Search.php b/app/Library/Search.php index 3580fdf2..d7259807 100644 --- a/app/Library/Search.php +++ b/app/Library/Search.php @@ -22,6 +22,7 @@ namespace Poniverse\Ponyfm\Library; use DB; use Elasticsearch\Client; +use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Collection; use Poniverse\Ponyfm\Models\Album; use Poniverse\Ponyfm\Models\Playlist; @@ -53,11 +54,11 @@ class Search { 'multi_match' => [ 'query' => $query, 'fields' => [ - 'title', - 'artist', + 'title^3', + 'artist^2', 'genre', 'track_type', - 'show_songs', + 'show_songs^2', ], ], ], @@ -71,7 +72,7 @@ class Search { 'multi_match' => [ 'query' => $query, 'fields' => [ - 'title', + 'title^2', 'artist', 'tracks', ], @@ -87,8 +88,9 @@ class Search { 'multi_match' => [ 'query' => $query, 'fields' => [ - 'title', - 'user', + 'title^3', + 'curator', + 'tracks^2', ], ], ], @@ -102,7 +104,8 @@ class Search { 'multi_match' => [ 'query' => $query, 'fields' => [ - 'display_name', + 'display_name^2', + 'tracks', ], ], ], @@ -150,7 +153,14 @@ class Search { } $caseStatement .= 'END'; - $modelInstances = $modelClass::withTrashed() + /** @var Builder $modelInstances */ + $modelInstances = $modelClass::query(); + + if (method_exists($modelClass, 'withTrashed')) { + $modelInstances = $modelInstances->withTrashed(); + } + + $modelInstances = $modelInstances ->whereIn('id', array_keys($ids)) ->orderBy(DB::raw($caseStatement)) ->get(); diff --git a/app/Models/Album.php b/app/Models/Album.php index 023ce002..f32c69a8 100644 --- a/app/Models/Album.php +++ b/app/Models/Album.php @@ -28,7 +28,7 @@ use Illuminate\Foundation\Bus\DispatchesJobs; use Auth; use Cache; use Poniverse\Ponyfm\Exceptions\TrackFileNotFoundException; -use Poniverse\Ponyfm\Traits\IndexedInElasticsearch; +use Poniverse\Ponyfm\Traits\IndexedInElasticsearchTrait; use Poniverse\Ponyfm\Traits\TrackCollection; use Poniverse\Ponyfm\Traits\SlugTrait; use Venturecraft\Revisionable\RevisionableTrait; @@ -62,7 +62,7 @@ use Venturecraft\Revisionable\RevisionableTrait; */ class Album extends Model { - use SoftDeletes, SlugTrait, DispatchesJobs, TrackCollection, RevisionableTrait, IndexedInElasticsearch; + use SoftDeletes, SlugTrait, DispatchesJobs, TrackCollection, RevisionableTrait, IndexedInElasticsearchTrait; protected $elasticsearchType = 'album'; diff --git a/app/Models/Playlist.php b/app/Models/Playlist.php index d84080d8..327dd67e 100644 --- a/app/Models/Playlist.php +++ b/app/Models/Playlist.php @@ -27,7 +27,7 @@ use Illuminate\Foundation\Bus\DispatchesJobs; use Auth; use Cache; use Poniverse\Ponyfm\Exceptions\TrackFileNotFoundException; -use Poniverse\Ponyfm\Traits\IndexedInElasticsearch; +use Poniverse\Ponyfm\Traits\IndexedInElasticsearchTrait; use Poniverse\Ponyfm\Traits\TrackCollection; use Poniverse\Ponyfm\Traits\SlugTrait; use Venturecraft\Revisionable\RevisionableTrait; @@ -61,7 +61,7 @@ use Venturecraft\Revisionable\RevisionableTrait; */ class Playlist extends Model { - use SoftDeletes, SlugTrait, DispatchesJobs, TrackCollection, RevisionableTrait, IndexedInElasticsearch; + use SoftDeletes, SlugTrait, DispatchesJobs, TrackCollection, RevisionableTrait, IndexedInElasticsearchTrait; protected $elasticsearchType = 'playlist'; @@ -295,6 +295,10 @@ class Playlist extends Model * @return array */ public function toElasticsearch() { - return $this->toArray(); + return [ + 'title' => $this->title, + 'curator' => $this->user->display_name, + 'tracks' => $this->tracks->pluck('title'), + ]; } } diff --git a/app/Models/Track.php b/app/Models/Track.php index 862f4ce9..6dbd1af2 100644 --- a/app/Models/Track.php +++ b/app/Models/Track.php @@ -26,7 +26,7 @@ use Config; use DB; use Elasticsearch; use Poniverse\Ponyfm\Exceptions\TrackFileNotFoundException; -use Poniverse\Ponyfm\Traits\IndexedInElasticsearch; +use Poniverse\Ponyfm\Traits\IndexedInElasticsearchTrait; use Poniverse\Ponyfm\Traits\SlugTrait; use Exception; use External; @@ -97,7 +97,7 @@ use Venturecraft\Revisionable\RevisionableTrait; */ class Track extends Model { - use SoftDeletes, IndexedInElasticsearch; + use SoftDeletes, IndexedInElasticsearchTrait; protected $elasticsearchType = 'track'; diff --git a/app/Models/User.php b/app/Models/User.php index 97b86b05..d2f77bfd 100644 --- a/app/Models/User.php +++ b/app/Models/User.php @@ -29,7 +29,7 @@ use Illuminate\Database\Eloquent\Model; use Illuminate\Foundation\Auth\Access\Authorizable; use Auth; use Illuminate\Support\Str; -use Poniverse\Ponyfm\Traits\IndexedInElasticsearch; +use Poniverse\Ponyfm\Traits\IndexedInElasticsearchTrait; use Venturecraft\Revisionable\RevisionableTrait; /** @@ -65,7 +65,7 @@ use Venturecraft\Revisionable\RevisionableTrait; */ class User extends Model implements AuthenticatableContract, CanResetPasswordContract, \Illuminate\Contracts\Auth\Access\Authorizable { - use Authenticatable, CanResetPassword, Authorizable, RevisionableTrait, IndexedInElasticsearch; + use Authenticatable, CanResetPassword, Authorizable, RevisionableTrait, IndexedInElasticsearchTrait; protected $elasticsearchType = 'user'; @@ -258,6 +258,10 @@ class User extends Model implements AuthenticatableContract, CanResetPasswordCon * @return array */ public function toElasticsearch() { - return $this->toArray(); + return [ + 'username' => $this->username, + 'display_name' => $this->display_name, + 'tracks' => $this->tracks->pluck('title'), + ]; } } diff --git a/app/Traits/IndexedInElasticsearch.php b/app/Traits/IndexedInElasticsearchTrait.php similarity index 96% rename from app/Traits/IndexedInElasticsearch.php rename to app/Traits/IndexedInElasticsearchTrait.php index 7e94b2d2..de0c0ac6 100644 --- a/app/Traits/IndexedInElasticsearch.php +++ b/app/Traits/IndexedInElasticsearchTrait.php @@ -30,7 +30,7 @@ use Elasticsearch; * * @package Poniverse\Ponyfm\Traits */ -trait IndexedInElasticsearch +trait IndexedInElasticsearchTrait { /** * Returns this model in Elasticsearch-friendly form. The array returned by @@ -83,7 +83,7 @@ trait IndexedInElasticsearch } public function ensureElasticsearchEntryIsUpToDate() { - if ($this->trashed()) { + if (method_exists($this, 'trashed') && $this->trashed()) { $this->deleteElasticsearchEntry(); } else { $this->createOrUpdateElasticsearchEntry(); diff --git a/database/migrations/2016_01_14_021607_setup_elasticsearch.php b/database/migrations/2016_01_14_021607_setup_elasticsearch.php index c6869aec..2c697187 100644 --- a/database/migrations/2016_01_14_021607_setup_elasticsearch.php +++ b/database/migrations/2016_01_14_021607_setup_elasticsearch.php @@ -42,6 +42,7 @@ class SetupElasticsearch extends Migration 'properties' => [ 'title' => ['type' => 'string'], 'artist' => ['type' => 'string', 'index' => 'not_analyzed'], + 'published_at' => ['type' => 'date'], 'genre' => ['type' => 'string', 'index' => 'not_analyzed'], 'track_type' => ['type' => 'string', 'index' => 'not_analyzed'], @@ -65,7 +66,35 @@ class SetupElasticsearch extends Migration // See: https://www.elastic.co/guide/en/elasticsearch/reference/current/array.html 'tracks' => ['type' => 'string'] ] - ] + ], + + 'playlist' => [ + '_source' => ['enabled' => true], + 'dynamic' => 'strict', + 'properties' => [ + 'title' => ['type' => 'string'], + 'curator' => ['type' => 'string', 'index' => 'not_analyzed'], + + // This field is intended to be used as an array. + // Note that all Elasticsearch fields can technically be used as arrays. + // See: https://www.elastic.co/guide/en/elasticsearch/reference/current/array.html + 'tracks' => ['type' => 'string'] + ] + ], + + 'user' => [ + '_source' => ['enabled' => true], + 'dynamic' => 'strict', + 'properties' => [ + 'username' => ['type' => 'string', 'index' => 'not_analyzed'], + 'display_name' => ['type' => 'string', 'index' => 'not_analyzed'], + + // This field is intended to be used as an array. + // Note that all Elasticsearch fields can technically be used as arrays. + // See: https://www.elastic.co/guide/en/elasticsearch/reference/current/array.html + 'tracks' => ['type' => 'string'] + ] + ], ] ] ]); From 6b5d3f67a3ee2aaecfa16049c2358e615df63ca3 Mon Sep 17 00:00:00 2001 From: Peter Deltchev Date: Sat, 16 Jan 2016 01:10:07 -0800 Subject: [PATCH 05/15] #1: Beginning of the search front-end. --- app/Library/Search.php | 30 +++++++++- public/templates/directives/search.html | 19 ++++++ .../scripts/app/directives/search.coffee | 59 +++++++++++++++++++ .../assets/scripts/app/services/search.coffee | 32 ++++++++++ resources/assets/styles/app.less | 1 + resources/assets/styles/search.less | 58 ++++++++++++++++++ resources/views/shared/_app_layout.blade.php | 1 + 7 files changed, 197 insertions(+), 3 deletions(-) create mode 100644 public/templates/directives/search.html create mode 100644 resources/assets/scripts/app/directives/search.coffee create mode 100644 resources/assets/scripts/app/services/search.coffee create mode 100644 resources/assets/styles/search.less diff --git a/app/Library/Search.php b/app/Library/Search.php index d7259807..f28e5f9a 100644 --- a/app/Library/Search.php +++ b/app/Library/Search.php @@ -114,9 +114,9 @@ class Search { ] ]); - $tracks = $this->transformToEloquent(Track::class, $results['responses'][0]['hits']['hits']); - $albums = $this->transformToEloquent(Album::class, $results['responses'][1]['hits']['hits']); - $playlists = $this->transformToEloquent(Playlist::class, $results['responses'][2]['hits']['hits']); + $tracks = $this->transformTracks($results['responses'][0]['hits']['hits']); + $albums = $this->transformAlbums($results['responses'][1]['hits']['hits']); + $playlists = $this->transformPlaylists($results['responses'][2]['hits']['hits']); $users = $this->transformToEloquent(User::class, $results['responses'][3]['hits']['hits']); return [ @@ -127,6 +127,30 @@ class Search { ]; } + protected function transformTracks(array $searchHits) { + $tracks = $this->transformToEloquent(Track::class, $searchHits); + $tracks = $tracks->map(function (Track $track) { + return Track::mapPublicTrackSummary($track); + }); + return $tracks; + } + + protected function transformAlbums(array $searchHits) { + $albums = $this->transformToEloquent(Album::class, $searchHits); + $albums = $albums->map(function (Album $track) { + return Album::mapPublicAlbumSummary($track); + }); + return $albums; + } + + protected function transformPlaylists(array $searchHits) { + $playlists = $this->transformToEloquent(Playlist::class, $searchHits); + $playlists = $playlists->map(function (Playlist $track) { + return Playlist::mapPublicPlaylistSummary($track); + }); + return $playlists; + } + /** * Transforms the given Elasticsearch results into a collection of corresponding * Eloquent models. diff --git a/public/templates/directives/search.html b/public/templates/directives/search.html new file mode 100644 index 00000000..f86ce766 --- /dev/null +++ b/public/templates/directives/search.html @@ -0,0 +1,19 @@ + diff --git a/resources/assets/scripts/app/directives/search.coffee b/resources/assets/scripts/app/directives/search.coffee new file mode 100644 index 00000000..62ca04df --- /dev/null +++ b/resources/assets/scripts/app/directives/search.coffee @@ -0,0 +1,59 @@ +# Pony.fm - A community for pony fan music. +# Copyright (C) 2016 Peter Deltchev +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +angular.module('ponyfm').directive 'pfmSearch', () -> + restrict: 'E' + templateUrl: '/templates/directives/search.html' + scope: + resource: '=resource', + type: '@type' + + controller: [ + '$scope', 'search' + ($scope, search) -> + $scope.searchQuery = null + $scope.searchInProgress = false + + $scope.tracks = [] + $scope.albums = [] + $scope.playlists = [] + $scope.users = [] + + clearResults = ()-> + $scope.tracks = [] + $scope.albums = [] + $scope.playlists = [] + $scope.users = [] + + $scope.quickSearch = ()-> + clearResults() + $scope.searchInProgress = true + + search.searchAllContent($scope.searchQuery) + .done (results)-> + for track in results.tracks + $scope.tracks.push(track) + + for album in results.albums + $scope.albums.push(album) + + for playlist in results.playlists + $scope.playlists.push(playlist) + + for user in results.users + $scope.users.push(user) + + ] diff --git a/resources/assets/scripts/app/services/search.coffee b/resources/assets/scripts/app/services/search.coffee new file mode 100644 index 00000000..acc3fd37 --- /dev/null +++ b/resources/assets/scripts/app/services/search.coffee @@ -0,0 +1,32 @@ +# Pony.fm - A community for pony fan music. +# Copyright (C) 2016 Peter Deltchev +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +angular.module('ponyfm').factory('search', [ + '$http' + ($http) -> + + self = + searchAllContent: (query) -> + searchDef = new $.Deferred() + + $http.get('/api/web/search?query=' + encodeURIComponent(query)) + .success (results)-> + searchDef.resolve(results.results) + + searchDef.promise() + + self +]) diff --git a/resources/assets/styles/app.less b/resources/assets/styles/app.less index 73a7deae..33c7095c 100644 --- a/resources/assets/styles/app.less +++ b/resources/assets/styles/app.less @@ -32,3 +32,4 @@ @import 'content'; @import 'dashboard'; @import 'uploader'; +@import 'search'; diff --git a/resources/assets/styles/search.less b/resources/assets/styles/search.less new file mode 100644 index 00000000..612873ac --- /dev/null +++ b/resources/assets/styles/search.less @@ -0,0 +1,58 @@ +/** + * Pony.fm - A community for pony fan music. + * Copyright (C) 2016 Peter Deltchev + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +@import 'base/bootstrap/bootstrap'; +@import 'mixins'; +@import 'variables'; + + +.search-input { + +} + +.search-results { + position: absolute; + width: 500px; + padding: 10px; + + background: #fff; + + ol li { + list-style: disc; + } + + li a { + padding: 0; + padding-right: 0; + overflow: hidden; + color: @pfm-light-grey; + } + + .-section-header { + background: transparent; + color: @pfm-purple; + + padding-left: 0; + } + + .albums-listing, .playlists-listing { + li { + width: 100%; + } + } +} diff --git a/resources/views/shared/_app_layout.blade.php b/resources/views/shared/_app_layout.blade.php index 9970e97a..0ad04930 100644 --- a/resources/views/shared/_app_layout.blade.php +++ b/resources/views/shared/_app_layout.blade.php @@ -66,6 +66,7 @@