From 447815c5ee129f1bd4660eda9759071325cf1a98 Mon Sep 17 00:00:00 2001 From: Peter Deltchev Date: Fri, 4 Sep 2015 09:59:18 -0700 Subject: [PATCH 1/8] T357: Initial commit of the MLPMA import script. --- app/commands/ImportMLPMA.php | 278 ++++++++++++++++++ .../2015_09_04_160648_make_email_nullable.php | 27 ++ app/start/artisan.php | 3 +- vagrant/config/app.php | 3 +- 4 files changed, 309 insertions(+), 2 deletions(-) create mode 100644 app/commands/ImportMLPMA.php create mode 100644 app/database/migrations/2015_09_04_160648_make_email_nullable.php diff --git a/app/commands/ImportMLPMA.php b/app/commands/ImportMLPMA.php new file mode 100644 index 00000000..1e6a076e --- /dev/null +++ b/app/commands/ImportMLPMA.php @@ -0,0 +1,278 @@ +comment('Enumerating MLP Music Archive source files...'); + $files = File::allFiles($mlpmaPath); + $this->info(sizeof($files).' files found!'); + + $this->comment('Enumerating artists...'); + $artists = File::directories($mlpmaPath); + $this->info(sizeof($artists).' artists found!'); + + $this->comment('Importing tracks...'.PHP_EOL); + + foreach($files as $file) { + $this->comment('Importing track ['. $file->getFilename() .']...'); + + if (in_array($file->getExtension(), $this->ignoredExtensions)) { + $this->comment('This is not an audio file! Skipping...'.PHP_EOL); + continue; + } + + + //========================================================================================================== + // Extract the original tags. + //========================================================================================================== + $getId3 = new getID3; + $tags = $getId3->analyze($file->getPathname()); + + $parsedTags = []; + if ($file->getExtension() === 'mp3') { + $parsedTags = $this->getId3Tags($tags); + + } else if ($file->getExtension() === 'm4a') { + $parsedTags = $this->getAtomTags($tags); + } + + + //========================================================================================================== + // Determine the release date. + //========================================================================================================== + $modifiedDate = Carbon::createFromTimeStampUTC(File::lastModified($file->getPathname())); + $taggedYear = $parsedTags['year']; + + $this->info('Modification year: '.$modifiedDate->year); + $this->info('Tagged year: '.$taggedYear); + + if ($taggedYear !== null && $modifiedDate->year === $taggedYear) { + $released_at = $modifiedDate; + + } else if ($taggedYear !== null && $modifiedDate->year !== $taggedYear) { + $this->error('Release years do not match! Using the tagged year...'); + $released_at = Carbon::create($taggedYear); + + } else { + // $taggedYear is null + $this->error('This track isn\'t tagged with its release year! Using the track\'s last modified date...'); + $released_at = $modifiedDate; + } + + //========================================================================================================== + // Does this track have vocals? + //========================================================================================================== + $is_vocal = $parsedTags['lyrics'] !== null; + + + //========================================================================================================== + // Determine which artist account this file belongs to using the containing directory. + //========================================================================================================== + $this->info('Path to file: '.$file->getRelativePath()); + $path_components = explode(DIRECTORY_SEPARATOR, $file->getRelativePath()); + $artist_name = $path_components[0]; + $album_name = array_key_exists(1, $path_components) ? $path_components[1] : null; + + $this->info('Artist: '.$artist_name); + $this->info('Album: '.$album_name); + + $artist = User::where('display_name', '=', $artist_name)->first(); + + if (!$artist) { + $artist = new User; + $artist->display_name = $artist_name; + $artist->email = null; + $artist->is_archived = true; + + $artist->slug = Str::slug($artist_name); + + $slugExists = User::where('slug', '=', $artist->slug)->first(); + if ($slugExists) { + $this->error('Horsefeathers! The slug '.$artist->slug.' is already taken!'); + $artist->slug = $artist->slug.'-'.Str::random(4); + } + + $artist = $artist->save(); + } + + //========================================================================================================== + // Extract the cover art, if any exists. + //========================================================================================================== + $cover_id = null; + if (array_key_exists('comments', $tags) && array_key_exists('picture', $tags['comments'])) { + $image = $tags['comments']['picture'][0]; + + if ($image['image_mime'] === 'image/png') { + $extension = 'png'; + + } else if ($image['image_mime'] === 'image/jpeg') { + $extension = 'jpg'; + + } else if ($image['image_mime'] === 'image/gif') { + $extension = 'gif'; + + } else { + $this->error('Unknown cover art format!'); + } + + // write temporary image file + $imageFilename = $file->getFilename() . ".cover.$extension"; + $imageFilePath = "$tmpPath/".$imageFilename; + File::put($imageFilePath, $image['data']); + + + $imageFile = new UploadedFile($imageFilePath, $imageFilename, $image['image_mime']); + + $cover_id = Image::upload($imageFile, $artist); + + } else { + $this->error('No cover art found!'); + } + + + //========================================================================================================== + // Is this part of an album? + //========================================================================================================== + + // TODO: find/create the album + + + //========================================================================================================== + // Original, show song remix, fan song remix, show audio remix, or ponified song? + //========================================================================================================== + + // TODO: implement this + + //========================================================================================================== + // Save this track. + //========================================================================================================== + + // TODO: use these variables + $cover_id; + $released_at; + $is_vocal; + + echo PHP_EOL; + } + } + + /** + * Get the console command arguments. + * + * @return array + */ + protected function getArguments() + { + return array(); + } + + /** + * Get the console command options. + * + * @return array + */ + protected function getOptions() + { + return array(); + } + + + /** + * @param array $rawTags + * @return array + */ + protected function getId3Tags($rawTags) { + $tags = $rawTags['tags']['id3v2']; + + return [ + 'title' => $tags['title'][0], + 'artist' => $tags['artist'][0], + 'band' => isset($tags['band']) ? $tags['band'][0] : null, + 'genre' => isset($tags['genre']) ? $tags['genre'][0] : null, + 'track_number' => isset($tags['track_number']) ? $tags['track_number'][0] : null, + 'album' => isset($tags['album']) ? $tags['album'][0] : null, + 'year' => isset($tags['year']) ? (int) $tags['year'][0] : null, + 'comments' => isset($tags['comments']) ? $tags['comments'][0] : null, + 'lyrics' => isset($tags['unsynchronised_lyric']) ? $tags['unsynchronised_lyric'][0] : null, + ]; + } + + /** + * @param array $rawTags + * @return array + */ + protected function getAtomTags($rawTags) { + // TODO: finish this + print_r($rawTags['tags']['quicktime']); + print_r($rawTags['quicktime']['comments']); + + return [ + 'title' => null, + 'artist' => null, + 'band' => null, + 'genre' => null, + 'track_number' => null, + 'album' => null, + 'year' => null, + 'comments' => null, + 'lyrics' => null, + ]; + } + +} diff --git a/app/database/migrations/2015_09_04_160648_make_email_nullable.php b/app/database/migrations/2015_09_04_160648_make_email_nullable.php new file mode 100644 index 00000000..39d32d40 --- /dev/null +++ b/app/database/migrations/2015_09_04_160648_make_email_nullable.php @@ -0,0 +1,27 @@ + true, 'url' => 'pony.fm.local', 'files_directory' => '/vagrant-files/', + 'mlpma_directory' => '/vagrant-files/mlpma/', 'node' => null, 'node_paths' => ['/usr/lib/node_modules/'], 'secure' => false, -); \ No newline at end of file +); From b7084de22841e261c6fa64f548cba491a88fc75f Mon Sep 17 00:00:00 2001 From: Peter Deltchev Date: Sat, 5 Sep 2015 06:39:48 -0700 Subject: [PATCH 2/8] T357: Added track type constants and a progress commit for the importer script. --- app/commands/ImportMLPMA.php | 114 ++++++++++++++++++++++++++++-- app/models/Entities/TrackType.php | 8 ++- 2 files changed, 117 insertions(+), 5 deletions(-) diff --git a/app/commands/ImportMLPMA.php b/app/commands/ImportMLPMA.php index 1e6a076e..ff3f4edd 100644 --- a/app/commands/ImportMLPMA.php +++ b/app/commands/ImportMLPMA.php @@ -4,8 +4,11 @@ use Illuminate\Console\Command; use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Input\InputArgument; use Illuminate\Support\Facades\File; +use Entities\Album; use Entities\Image; use Entities\User; +use Entities\ShowSong; +use Entities\TrackType; use Symfony\Component\HttpFoundation\File\UploadedFile; use Carbon\Carbon; @@ -69,8 +72,12 @@ class ImportMLPMA extends Command { $this->comment('Importing tracks...'.PHP_EOL); + $totalFiles = sizeof($files); + $currentFile = 0; + foreach($files as $file) { - $this->comment('Importing track ['. $file->getFilename() .']...'); + $currentFile++; + $this->comment('['.$currentFile.'/'.$totalFiles.'] Importing track ['. $file->getFilename() .']...'); if (in_array($file->getExtension(), $this->ignoredExtensions)) { $this->comment('This is not an audio file! Skipping...'.PHP_EOL); @@ -106,7 +113,7 @@ class ImportMLPMA extends Command { $released_at = $modifiedDate; } else if ($taggedYear !== null && $modifiedDate->year !== $taggedYear) { - $this->error('Release years do not match! Using the tagged year...'); + $this->error('Release years don\'t match! Using the tagged year...'); $released_at = Carbon::create($taggedYear); } else { @@ -179,7 +186,7 @@ class ImportMLPMA extends Command { $imageFile = new UploadedFile($imageFilePath, $imageFilename, $image['image_mime']); - $cover_id = Image::upload($imageFile, $artist); + $cover = Image::upload($imageFile, $artist); } else { $this->error('No cover art found!'); @@ -191,27 +198,126 @@ class ImportMLPMA extends Command { //========================================================================================================== // TODO: find/create the album + $album_name = $parsedTags['album']; + if ($album_name !== null) { + $album = Album::where('user_id', '=', $artist->id) + ->where('title', '=', $album_name) + ->first(); + + if (!$album) { + $album = new Album; + + $album->title = $album_name; + $album->user_id = $artist->id; + $album->cover_id = $cover->id; + + $album->save(); + } + } //========================================================================================================== // Original, show song remix, fan song remix, show audio remix, or ponified song? //========================================================================================================== + $track_type = TrackType::ORIGINAL_TRACK; + + $sanitized_track_title = $parsedTags['title']; + $sanitized_track_title = str_replace(' - ', ' ', $sanitized_track_title); + $sanitized_track_title = str_replace('ft. ', '', $sanitized_track_title); + $sanitized_track_title = str_replace('*', '', $sanitized_track_title); + + $queriedTitle = DB::connection()->getPdo()->quote($sanitized_track_title); + $officialSongs = ShowSong::select(['id', 'title']) + ->whereRaw(" + MATCH (title) + AGAINST ($queriedTitle IN BOOLEAN MODE) + ") + ->get(); + + + // It it has "Ingram" in the name, it's definitely an official song remix. + if (Str::contains(Str::lower($file->getFilename()), 'ingram')) { + $track_type = TrackType::OFFICIAL_TRACK_REMIX; + $this->comment('This is an official song remix!'); + + foreach($officialSongs as $song) { + $this->comment('=> Matched official song: ['.$song->id.'] '.$song->title); + } + + if (sizeof($officialSongs) === 1) { + $linkedSongIds = [$officialSongs[0]->id]; + } else { + list($track_type, $linkedSongIds) = $this->classifyTrack($officialSongs); + + } + + + // If it has "remix" in the name, it's definitely a remix. + } else if (Str::contains(Str::lower($sanitized_track_title), 'remix')) { + $this->comment('This is some kind of remix!'); + + foreach($officialSongs as $song) { + $this->comment('=> Matched official song: [' . $song->id . '] ' . $song->title); + } + } - // TODO: implement this //========================================================================================================== // Save this track. //========================================================================================================== // TODO: use these variables + $cover; + $album; $cover_id; $released_at; $is_vocal; + $track_type; + + // TODO: mark imported tracks as needing QA echo PHP_EOL; } } + protected function classifyTrack() + { + $trackTypeId = null; + $linkedSongIds = []; + + $this->question('Multiple official songs matched! Please enter the ID of the correct one.'); + $this->question('If this is a medley, multiple song ID\'s can be separated by commas.'); + $this->question(' Other options:'); + $this->question(' a = show audio remix'); + $this->question(' f = fan track remix'); + $this->question(' p = ponified track'); + $this->question(' o = original track'); + $input = $this->ask('[#/a/f/p/o]: '); + + switch($input) { + case 'a': + $trackTypeId = TrackType::OFFICIAL_AUDIO_REMIX; + break; + + case 'f': + $trackTypeId = TrackType::FAN_TRACK_REMIX; + break; + + case 'p': + $trackTypeId = TrackType::PONIFIED_TRACK; + break; + + case 'o': + $trackTypeId = TrackType::ORIGINAL_TRACK; + break; + + default: + $trackTypeId = TrackType::OFFICIAL_TRACK_REMIX; + $linkedSongIds = explode(',', $input); + $linkedSongIds = array_map(function($item){ return (int) $item; }, $linkedSongIds); + } + } + /** * Get the console command arguments. * diff --git a/app/models/Entities/TrackType.php b/app/models/Entities/TrackType.php index 336c0e5a..43ea73ea 100644 --- a/app/models/Entities/TrackType.php +++ b/app/models/Entities/TrackType.php @@ -4,4 +4,10 @@ class TrackType extends \Eloquent { protected $table = 'track_types'; - } \ No newline at end of file + + const ORIGINAL_TRACK = 1; + const OFFICIAL_TRACK_REMIX = 2; + const FAN_TRACK_REMIX = 3; + const PONIFIED_TRACK = 4; + const OFFICIAL_AUDIO_REMIX = 5; + } From dbab3a9ecc1bb10d2688c25ac87ee4877c1d2a3a Mon Sep 17 00:00:00 2001 From: Peter Deltchev Date: Sun, 6 Sep 2015 08:33:12 -0700 Subject: [PATCH 3/8] T357: MLPMA import script is mostly implemented now. --- app/commands/ImportMLPMA.php | 224 +++++++++++------- .../2015_09_05_113647_add_new_indices.php | 45 ++++ 2 files changed, 184 insertions(+), 85 deletions(-) create mode 100644 app/database/migrations/2015_09_05_113647_add_new_indices.php diff --git a/app/commands/ImportMLPMA.php b/app/commands/ImportMLPMA.php index ff3f4edd..8047bd33 100644 --- a/app/commands/ImportMLPMA.php +++ b/app/commands/ImportMLPMA.php @@ -8,6 +8,7 @@ use Entities\Album; use Entities\Image; use Entities\User; use Entities\ShowSong; +use Entities\Track; use Entities\TrackType; use Symfony\Component\HttpFoundation\File\UploadedFile; use Carbon\Carbon; @@ -110,22 +111,22 @@ class ImportMLPMA extends Command { $this->info('Tagged year: '.$taggedYear); if ($taggedYear !== null && $modifiedDate->year === $taggedYear) { - $released_at = $modifiedDate; + $releasedAt = $modifiedDate; } else if ($taggedYear !== null && $modifiedDate->year !== $taggedYear) { $this->error('Release years don\'t match! Using the tagged year...'); - $released_at = Carbon::create($taggedYear); + $releasedAt = Carbon::create($taggedYear); } else { // $taggedYear is null $this->error('This track isn\'t tagged with its release year! Using the track\'s last modified date...'); - $released_at = $modifiedDate; + $releasedAt = $modifiedDate; } //========================================================================================================== // Does this track have vocals? //========================================================================================================== - $is_vocal = $parsedTags['lyrics'] !== null; + $isVocal = $parsedTags['lyrics'] !== null; //========================================================================================================== @@ -161,7 +162,7 @@ class ImportMLPMA extends Command { //========================================================================================================== // Extract the cover art, if any exists. //========================================================================================================== - $cover_id = null; + $coverId = null; if (array_key_exists('comments', $tags) && array_key_exists('picture', $tags['comments'])) { $image = $tags['comments']['picture'][0]; @@ -187,6 +188,7 @@ class ImportMLPMA extends Command { $imageFile = new UploadedFile($imageFilePath, $imageFilename, $image['image_mime']); $cover = Image::upload($imageFile, $artist); + $coverId = $cover->id; } else { $this->error('No cover art found!'); @@ -196,37 +198,40 @@ class ImportMLPMA extends Command { //========================================================================================================== // Is this part of an album? //========================================================================================================== + // Find the album if it exists and create it if it doesn't. + $albumId = null; + $albumName = $parsedTags['album']; - // TODO: find/create the album - $album_name = $parsedTags['album']; - - if ($album_name !== null) { + if ($albumName !== null) { $album = Album::where('user_id', '=', $artist->id) - ->where('title', '=', $album_name) + ->where('title', '=', $albumName) ->first(); if (!$album) { $album = new Album; - $album->title = $album_name; + $album->title = $albumName; $album->user_id = $artist->id; - $album->cover_id = $cover->id; + $album->cover_id = $coverId; $album->save(); } + + $albumId = $album->id; } //========================================================================================================== // Original, show song remix, fan song remix, show audio remix, or ponified song? //========================================================================================================== - $track_type = TrackType::ORIGINAL_TRACK; + $trackType = TrackType::ORIGINAL_TRACK; + $linkedSongIds = []; - $sanitized_track_title = $parsedTags['title']; - $sanitized_track_title = str_replace(' - ', ' ', $sanitized_track_title); - $sanitized_track_title = str_replace('ft. ', '', $sanitized_track_title); - $sanitized_track_title = str_replace('*', '', $sanitized_track_title); + $sanitizedTrackTitle = $parsedTags['title']; + $sanitizedTrackTitle = str_replace(' - ', ' ', $sanitizedTrackTitle); + $sanitizedTrackTitle = str_replace('ft. ', '', $sanitizedTrackTitle); + $sanitizedTrackTitle = str_replace('*', '', $sanitizedTrackTitle); - $queriedTitle = DB::connection()->getPdo()->quote($sanitized_track_title); + $queriedTitle = DB::connection()->getPdo()->quote($sanitizedTrackTitle); $officialSongs = ShowSong::select(['id', 'title']) ->whereRaw(" MATCH (title) @@ -235,87 +240,131 @@ class ImportMLPMA extends Command { ->get(); - // It it has "Ingram" in the name, it's definitely an official song remix. + // If it has "Ingram" in the name, it's definitely an official song remix. if (Str::contains(Str::lower($file->getFilename()), 'ingram')) { - $track_type = TrackType::OFFICIAL_TRACK_REMIX; $this->comment('This is an official song remix!'); - foreach($officialSongs as $song) { - $this->comment('=> Matched official song: ['.$song->id.'] '.$song->title); - } - - if (sizeof($officialSongs) === 1) { - $linkedSongIds = [$officialSongs[0]->id]; - } else { - list($track_type, $linkedSongIds) = $this->classifyTrack($officialSongs); - - } + list($trackType, $linkedSongIds) = $this->classifyTrack($file, $officialSongs, true); // If it has "remix" in the name, it's definitely a remix. - } else if (Str::contains(Str::lower($sanitized_track_title), 'remix')) { + } else if (Str::contains(Str::lower($sanitizedTrackTitle), 'remix')) { $this->comment('This is some kind of remix!'); - foreach($officialSongs as $song) { - $this->comment('=> Matched official song: [' . $song->id . '] ' . $song->title); - } + list($trackType, $linkedSongIds) = $this->classifyTrack($file, $officialSongs); } //========================================================================================================== // Save this track. //========================================================================================================== + $title = $parsedTags['title']; - // TODO: use these variables - $cover; - $album; - $cover_id; - $released_at; - $is_vocal; - $track_type; + // Has this track already been imported? + $track = Track::where('user_id', '=', $artist->id) + ->where('title', '=', $title) + ->first(); - // TODO: mark imported tracks as needing QA + if (!$track) { + $track = new Track; - echo PHP_EOL; + $track->user_id = $artist->id; + $track->title = $parsedTags['title']; + $track->cover_id = $coverId; + $track->album_id = $albumId; + $track->released_at = $releasedAt; + $track->is_vocal = $isVocal; + $track->track_type_id = $trackType; + + $track->save(); + + if (sizeof($linkedSongIds) > 0) { + $track->showSongs()->attach($linkedSongIds); + } + + // TODO: mark imported tracks as needing QA + } else { + $this->comment('This track has already been imported!'); + } + + + + echo PHP_EOL.PHP_EOL; } } - protected function classifyTrack() + protected function classifyTrack($file, $officialSongs, $isRemixOfOfficialTrack = false) { $trackTypeId = null; $linkedSongIds = []; - $this->question('Multiple official songs matched! Please enter the ID of the correct one.'); - $this->question('If this is a medley, multiple song ID\'s can be separated by commas.'); - $this->question(' Other options:'); - $this->question(' a = show audio remix'); - $this->question(' f = fan track remix'); - $this->question(' p = ponified track'); - $this->question(' o = original track'); - $input = $this->ask('[#/a/f/p/o]: '); - switch($input) { - case 'a': - $trackTypeId = TrackType::OFFICIAL_AUDIO_REMIX; - break; - - case 'f': - $trackTypeId = TrackType::FAN_TRACK_REMIX; - break; - - case 'p': - $trackTypeId = TrackType::PONIFIED_TRACK; - break; - - case 'o': - $trackTypeId = TrackType::ORIGINAL_TRACK; - break; - - default: - $trackTypeId = TrackType::OFFICIAL_TRACK_REMIX; - $linkedSongIds = explode(',', $input); - $linkedSongIds = array_map(function($item){ return (int) $item; }, $linkedSongIds); + foreach ($officialSongs as $song) { + $this->comment('=> Matched official song: [' . $song->id . '] ' . $song->title); } + + if ($isRemixOfOfficialTrack && sizeof($officialSongs) === 1) { + $linkedSongIds = [$officialSongs[0]->id]; + + } else { + if ($isRemixOfOfficialTrack) { + $this->question('Multiple official songs matched! Please enter the ID of the correct one.'); + + } else if (sizeof($officialSongs) > 0) { + $this->question('This looks like a remix of an official song!'); + $this->question('Press "r" if the match above is right!'); + + } else { + $this->question('Exactly what kind of track is this?'); + + } + $this->question('If this is a medley, multiple song ID\'s can be separated by commas. '); + $this->question(' '); + $this->question(' '.$file->getFilename().' '); + $this->question(' '); + $this->question(' r = official song remix (accept all "guessed" matches) '); + $this->question(' # = official song remix (enter the ID(s) of the show song(s)) '); + $this->question(' a = show audio remix '); + $this->question(' f = fan track remix '); + $this->question(' p = ponified track '); + $this->question(' o = original track '); + $this->question(' '); + $input = $this->ask('[r/#/a/f/p/o]: '); + + switch ($input) { + case 'r': + $trackTypeId = TrackType::OFFICIAL_TRACK_REMIX; + foreach ($officialSongs as $officialSong) { + $linkedSongIds[] = (int) $officialSong->id; + } + break; + + case 'a': + $trackTypeId = TrackType::OFFICIAL_AUDIO_REMIX; + break; + + case 'f': + $trackTypeId = TrackType::FAN_TRACK_REMIX; + break; + + case 'p': + $trackTypeId = TrackType::PONIFIED_TRACK; + break; + + case 'o': + $trackTypeId = TrackType::ORIGINAL_TRACK; + break; + + default: + $trackTypeId = TrackType::OFFICIAL_TRACK_REMIX; + $linkedSongIds = explode(',', $input); + $linkedSongIds = array_map(function ($item) { + return (int) $item; + }, $linkedSongIds); + } + } + + return [$trackTypeId, $linkedSongIds]; } /** @@ -364,20 +413,25 @@ class ImportMLPMA extends Command { * @return array */ protected function getAtomTags($rawTags) { - // TODO: finish this - print_r($rawTags['tags']['quicktime']); - print_r($rawTags['quicktime']['comments']); + $tags = $rawTags['tags']['quicktime']; + + $trackNumber = null; + if (isset($tags['track_number'])) { + $trackNumberComponents = explode('/', $tags['track_number'][0]); + $trackNumber = $trackNumberComponents[0]; + } return [ - 'title' => null, - 'artist' => null, - 'band' => null, - 'genre' => null, - 'track_number' => null, - 'album' => null, - 'year' => null, - 'comments' => null, - 'lyrics' => null, + 'title' => $tags['title'][0], + 'artist' => $tags['artist'][0], + 'band' => isset($tags['band']) ? $tags['band'][0] : null, + 'album_artist' => isset($tags['album_artist']) ? $tags['album_artist'][0] : null, + 'genre' => isset($tags['genre']) ? $tags['genre'][0] : null, + 'track_number' => $trackNumber, + 'album' => isset($tags['album']) ? $tags['album'][0] : null, + 'year' => isset($tags['year']) ? (int) $tags['year'][0] : null, + 'comments' => isset($tags['comments']) ? $tags['comments'][0] : null, + 'lyrics' => isset($tags['lyrics']) ? $tags['lyrics'][0] : null, ]; } diff --git a/app/database/migrations/2015_09_05_113647_add_new_indices.php b/app/database/migrations/2015_09_05_113647_add_new_indices.php new file mode 100644 index 00000000..d3eec00d --- /dev/null +++ b/app/database/migrations/2015_09_05_113647_add_new_indices.php @@ -0,0 +1,45 @@ +index('hash'); + }); + + Schema::table('track_files', function ($table) { + $table->index('is_master'); + $table->index('format'); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + DB::statement('ALTER TABLE `show_songs` DROP INDEX show_songs_title_fulltext'); + + Schema::table('images', function ($table) { + $table->dropIndex('images_hash_index'); + }); + + Schema::table('track_files', function ($table) { + $table->dropIndex('track_files_is_master_index'); + $table->dropIndex('track_files_format_index'); + }); + } + +} From f17e82458673a83a87d3fd269cb7f89c2963e2a1 Mon Sep 17 00:00:00 2001 From: Peter Deltchev Date: Mon, 7 Sep 2015 04:50:35 -0700 Subject: [PATCH 4/8] T357: Separated track publishing and classification into its own script, fixed an issue with reading comments from ID3 tags, and added lossy support to UploadTrackCommand. --- app/commands/ClassifyMLPMA.php | 210 +++++++++++++++ app/commands/ImportMLPMA.php | 251 +++++++----------- .../2015_09_05_143300_create_mlpma_table.php | 38 +++ app/library/PfmValidator.php | 3 +- app/models/Commands/UploadTrackCommand.php | 54 +++- app/models/Entities/Track.php | 16 +- app/start/artisan.php | 1 + 7 files changed, 412 insertions(+), 161 deletions(-) create mode 100644 app/commands/ClassifyMLPMA.php create mode 100644 app/database/migrations/2015_09_05_143300_create_mlpma_table.php diff --git a/app/commands/ClassifyMLPMA.php b/app/commands/ClassifyMLPMA.php new file mode 100644 index 00000000..3f2ab684 --- /dev/null +++ b/app/commands/ClassifyMLPMA.php @@ -0,0 +1,210 @@ +orderBy('id') + ->get(); + + foreach ($tracks as $track) { + $parsedTags = json_decode($track->parsed_tags); + + + //========================================================================================================== + // Original, show song remix, fan song remix, show audio remix, or ponified song? + //========================================================================================================== + $trackType = TrackType::ORIGINAL_TRACK; + $linkedSongIds = []; + + $sanitizedTrackTitle = $parsedTags['title']; + $sanitizedTrackTitle = str_replace([' - ', 'ft.', '*'], ' ', $sanitizedTrackTitle); + + $queriedTitle = DB::connection()->getPdo()->quote($sanitizedTrackTitle); + $officialSongs = ShowSong::select(['id', 'title']) + ->whereRaw(" + MATCH (title) + AGAINST ($queriedTitle IN BOOLEAN MODE) + ") + ->get(); + + + // If it has "Ingram" in the name, it's definitely an official song remix. + if (Str::contains(Str::lower($track->filename), 'ingram')) { + $this->comment('This is an official song remix!'); + + list($trackType, $linkedSongIds) = $this->classifyTrack($track->filename, $officialSongs, true); + + + // If it has "remix" in the name, it's definitely a remix. + } else if (Str::contains(Str::lower($sanitizedTrackTitle), 'remix')) { + $this->comment('This is some kind of remix!'); + + list($trackType, $linkedSongIds) = $this->classifyTrack($track->filename, $officialSongs); + } + + + //========================================================================================================== + // Attach the data and publish the track! + //========================================================================================================== + + $track = Track::find($track->track_id); + + $track->track_type_id = $trackType; + $track->published_at = $parsedTags['released_at']; + $track->save(); + + if (sizeof($linkedSongIds) > 0) { + $track->showSongs()->attach($linkedSongIds); + } + + } + + } + + /** + * Get the console command arguments. + * + * @return array + */ + protected function getArguments() + { + return array( + array('example', InputArgument::REQUIRED, 'An example argument.'), + ); + } + + /** + * Get the console command options. + * + * @return array + */ + protected function getOptions() + { + return array( + array('example', null, InputOption::VALUE_OPTIONAL, 'An example option.', null), + ); + } + + + /** + * Determines what type of track the given file is. If unable to guess, the user + * is asked to identify it interactively. + * + * @param string $filename + * @param \Entities\ShowSong[] $officialSongs + * @param bool|false $isRemixOfOfficialTrack + * @return array + */ + protected function classifyTrack($filename, $officialSongs, $isRemixOfOfficialTrack = false) { + $trackTypeId = null; + $linkedSongIds = []; + + + foreach ($officialSongs as $song) { + $this->comment('=> Matched official song: [' . $song->id . '] ' . $song->title); + } + + if ($isRemixOfOfficialTrack && sizeof($officialSongs) === 1) { + $linkedSongIds = [$officialSongs[0]->id]; + + } else { + if ($isRemixOfOfficialTrack) { + $this->question('Multiple official songs matched! Please enter the ID of the correct one.'); + + } else if (sizeof($officialSongs) > 0) { + $this->question('This looks like a remix of an official song!'); + $this->question('Press "r" if the match above is right!'); + + } else { + $this->question('Exactly what kind of track is this?'); + + } + $this->question('If this is a medley, multiple song ID\'s can be separated by commas. '); + $this->question(' '); + $this->question(' ' . $filename . ' '); + $this->question(' '); + $this->question(' r = official song remix (accept all "guessed" matches) '); + $this->question(' # = official song remix (enter the ID(s) of the show song(s)) '); + $this->question(' a = show audio remix '); + $this->question(' f = fan track remix '); + $this->question(' p = ponified track '); + $this->question(' o = original track '); + $this->question(' '); + $input = $this->ask('[r/#/a/f/p/o]: '); + + switch ($input) { + case 'r': + $trackTypeId = TrackType::OFFICIAL_TRACK_REMIX; + foreach ($officialSongs as $officialSong) { + $linkedSongIds[] = (int) $officialSong->id; + } + break; + + case 'a': + $trackTypeId = TrackType::OFFICIAL_AUDIO_REMIX; + break; + + case 'f': + $trackTypeId = TrackType::FAN_TRACK_REMIX; + break; + + case 'p': + $trackTypeId = TrackType::PONIFIED_TRACK; + break; + + case 'o': + $trackTypeId = TrackType::ORIGINAL_TRACK; + break; + + default: + $trackTypeId = TrackType::OFFICIAL_TRACK_REMIX; + $linkedSongIds = explode(',', $input); + $linkedSongIds = array_map(function ($item) { + return (int) $item; + }, $linkedSongIds); + } + } + + return [$trackTypeId, $linkedSongIds]; + } + +} diff --git a/app/commands/ImportMLPMA.php b/app/commands/ImportMLPMA.php index 8047bd33..bf31c5ee 100644 --- a/app/commands/ImportMLPMA.php +++ b/app/commands/ImportMLPMA.php @@ -10,6 +10,7 @@ use Entities\User; use Entities\ShowSong; use Entities\Track; use Entities\TrackType; +use Commands\UploadTrackCommand; use Symfony\Component\HttpFoundation\File\UploadedFile; use Carbon\Carbon; @@ -23,7 +24,7 @@ class ImportMLPMA extends Command { * * @var string */ - protected $name = 'import-mlpma'; + protected $name = 'mlpma:import'; /** * The console command description. @@ -86,18 +87,36 @@ class ImportMLPMA extends Command { } + // Has this track already been imported? + $importedTrack = DB::table('mlpma_tracks') + ->where('filename', '=', $file->getFilename()) + ->first(); + + if ($importedTrack) { + $this->comment('This track has already been imported! Skipping...' . PHP_EOL); + continue; + } + + //========================================================================================================== // Extract the original tags. //========================================================================================================== $getId3 = new getID3; - $tags = $getId3->analyze($file->getPathname()); + // all tags read by getID3, including the cover art + $allTags = $getId3->analyze($file->getPathname()); + + // tags specific to a file format (ID3 or Atom), pre-normalization but with cover art removed + $rawTags = []; + + // normalized tags used by Pony.fm $parsedTags = []; + if ($file->getExtension() === 'mp3') { - $parsedTags = $this->getId3Tags($tags); + list($parsedTags, $rawTags) = $this->getId3Tags($allTags); } else if ($file->getExtension() === 'm4a') { - $parsedTags = $this->getAtomTags($tags); + list($parsedTags, $rawTags) = $this->getAtomTags($allTags); } @@ -123,6 +142,9 @@ class ImportMLPMA extends Command { $releasedAt = $modifiedDate; } + // This is later used by the classification/publishing script to determine the publication date. + $parsedTags['released_at'] = $releasedAt; + //========================================================================================================== // Does this track have vocals? //========================================================================================================== @@ -163,8 +185,8 @@ class ImportMLPMA extends Command { // Extract the cover art, if any exists. //========================================================================================================== $coverId = null; - if (array_key_exists('comments', $tags) && array_key_exists('picture', $tags['comments'])) { - $image = $tags['comments']['picture'][0]; + if (array_key_exists('comments', $allTags) && array_key_exists('picture', $allTags['comments'])) { + $image = $allTags['comments']['picture'][0]; if ($image['image_mime'] === 'image/png') { $extension = 'png'; @@ -220,153 +242,58 @@ class ImportMLPMA extends Command { $albumId = $album->id; } - //========================================================================================================== - // Original, show song remix, fan song remix, show audio remix, or ponified song? - //========================================================================================================== - $trackType = TrackType::ORIGINAL_TRACK; - $linkedSongIds = []; - - $sanitizedTrackTitle = $parsedTags['title']; - $sanitizedTrackTitle = str_replace(' - ', ' ', $sanitizedTrackTitle); - $sanitizedTrackTitle = str_replace('ft. ', '', $sanitizedTrackTitle); - $sanitizedTrackTitle = str_replace('*', '', $sanitizedTrackTitle); - - $queriedTitle = DB::connection()->getPdo()->quote($sanitizedTrackTitle); - $officialSongs = ShowSong::select(['id', 'title']) - ->whereRaw(" - MATCH (title) - AGAINST ($queriedTitle IN BOOLEAN MODE) - ") - ->get(); - - - // If it has "Ingram" in the name, it's definitely an official song remix. - if (Str::contains(Str::lower($file->getFilename()), 'ingram')) { - $this->comment('This is an official song remix!'); - - list($trackType, $linkedSongIds) = $this->classifyTrack($file, $officialSongs, true); - - - // If it has "remix" in the name, it's definitely a remix. - } else if (Str::contains(Str::lower($sanitizedTrackTitle), 'remix')) { - $this->comment('This is some kind of remix!'); - - list($trackType, $linkedSongIds) = $this->classifyTrack($file, $officialSongs); - } - //========================================================================================================== // Save this track. //========================================================================================================== $title = $parsedTags['title']; +// +// $track = Track::where('user_id', '=', $artist->id) +// ->where('title', '=', $title) +// ->first(); - // Has this track already been imported? - $track = Track::where('user_id', '=', $artist->id) - ->where('title', '=', $title) - ->first(); + // "upload" the track to Pony.fm + Auth::loginUsingId($artist->id); - if (!$track) { - $track = new Track; + $trackFile = new UploadedFile($file->getPathname(), $file->getFilename(), $allTags['mime_type']); + Input::instance()->files->add(['track' => $trackFile]); + + $upload = new UploadTrackCommand(true); + $result = $upload->execute(); + +// var_dump(null); + + if ($result->didFail()) { + $this->error(json_encode($result->getValidator()->messages()->getMessages(), JSON_PRETTY_PRINT)); + } else { + DB::table('mlpma_tracks') + ->insert([ + 'track_id' => $result['id'], + 'path' => $file->getRelativePath(), + 'filename' => $file->getFilename(), + 'extension' => $file->getExtension(), + 'imported_at' => Carbon::now(), + 'parsed_tags' => json_encode($parsedTags), + 'raw_tags' => json_encode($rawTags), + ]); + + $track = Track::find($result['id']); + var_dump($track); - $track->user_id = $artist->id; $track->title = $parsedTags['title']; $track->cover_id = $coverId; $track->album_id = $albumId; + $track->track_number = $parsedTags['track_number']; $track->released_at = $releasedAt; $track->is_vocal = $isVocal; - $track->track_type_id = $trackType; - $track->save(); - - if (sizeof($linkedSongIds) > 0) { - $track->showSongs()->attach($linkedSongIds); - } - - // TODO: mark imported tracks as needing QA - } else { - $this->comment('This track has already been imported!'); } - echo PHP_EOL.PHP_EOL; } } - protected function classifyTrack($file, $officialSongs, $isRemixOfOfficialTrack = false) - { - $trackTypeId = null; - $linkedSongIds = []; - - - foreach ($officialSongs as $song) { - $this->comment('=> Matched official song: [' . $song->id . '] ' . $song->title); - } - - if ($isRemixOfOfficialTrack && sizeof($officialSongs) === 1) { - $linkedSongIds = [$officialSongs[0]->id]; - - } else { - if ($isRemixOfOfficialTrack) { - $this->question('Multiple official songs matched! Please enter the ID of the correct one.'); - - } else if (sizeof($officialSongs) > 0) { - $this->question('This looks like a remix of an official song!'); - $this->question('Press "r" if the match above is right!'); - - } else { - $this->question('Exactly what kind of track is this?'); - - } - $this->question('If this is a medley, multiple song ID\'s can be separated by commas. '); - $this->question(' '); - $this->question(' '.$file->getFilename().' '); - $this->question(' '); - $this->question(' r = official song remix (accept all "guessed" matches) '); - $this->question(' # = official song remix (enter the ID(s) of the show song(s)) '); - $this->question(' a = show audio remix '); - $this->question(' f = fan track remix '); - $this->question(' p = ponified track '); - $this->question(' o = original track '); - $this->question(' '); - $input = $this->ask('[r/#/a/f/p/o]: '); - - switch ($input) { - case 'r': - $trackTypeId = TrackType::OFFICIAL_TRACK_REMIX; - foreach ($officialSongs as $officialSong) { - $linkedSongIds[] = (int) $officialSong->id; - } - break; - - case 'a': - $trackTypeId = TrackType::OFFICIAL_AUDIO_REMIX; - break; - - case 'f': - $trackTypeId = TrackType::FAN_TRACK_REMIX; - break; - - case 'p': - $trackTypeId = TrackType::PONIFIED_TRACK; - break; - - case 'o': - $trackTypeId = TrackType::ORIGINAL_TRACK; - break; - - default: - $trackTypeId = TrackType::OFFICIAL_TRACK_REMIX; - $linkedSongIds = explode(',', $input); - $linkedSongIds = array_map(function ($item) { - return (int) $item; - }, $linkedSongIds); - } - } - - return [$trackTypeId, $linkedSongIds]; - } - /** * Get the console command arguments. * @@ -394,18 +321,34 @@ class ImportMLPMA extends Command { */ protected function getId3Tags($rawTags) { $tags = $rawTags['tags']['id3v2']; + $comment = null; + + if (isset($tags['comment'])) { + // The "comment" tag comes in with a badly encoded string index + // so its array key has to be used implicitly. + $key = array_keys($tags['comment'])[0]; + + // The comment may have a null byte at the end. trim() removes it. + $comment = trim($tags['comment'][$key]); + + // Replace the malformed comment with the "fixed" one. + unset($tags['comment'][$key]); + $tags['comment'][0] = $comment; + } return [ - 'title' => $tags['title'][0], - 'artist' => $tags['artist'][0], - 'band' => isset($tags['band']) ? $tags['band'][0] : null, - 'genre' => isset($tags['genre']) ? $tags['genre'][0] : null, - 'track_number' => isset($tags['track_number']) ? $tags['track_number'][0] : null, - 'album' => isset($tags['album']) ? $tags['album'][0] : null, - 'year' => isset($tags['year']) ? (int) $tags['year'][0] : null, - 'comments' => isset($tags['comments']) ? $tags['comments'][0] : null, - 'lyrics' => isset($tags['unsynchronised_lyric']) ? $tags['unsynchronised_lyric'][0] : null, - ]; + [ + 'title' => $tags['title'][0], + 'artist' => $tags['artist'][0], + 'band' => isset($tags['band']) ? $tags['band'][0] : null, + 'genre' => isset($tags['genre']) ? $tags['genre'][0] : null, + 'track_number' => isset($tags['track_number']) ? $tags['track_number'][0] : null, + 'album' => isset($tags['album']) ? $tags['album'][0] : null, + 'year' => isset($tags['year']) ? (int) $tags['year'][0] : null, + 'comments' => $comment, + 'lyrics' => isset($tags['unsynchronised_lyric']) ? $tags['unsynchronised_lyric'][0] : null, + ], + $tags]; } /** @@ -422,17 +365,19 @@ class ImportMLPMA extends Command { } return [ - 'title' => $tags['title'][0], - 'artist' => $tags['artist'][0], - 'band' => isset($tags['band']) ? $tags['band'][0] : null, - 'album_artist' => isset($tags['album_artist']) ? $tags['album_artist'][0] : null, - 'genre' => isset($tags['genre']) ? $tags['genre'][0] : null, - 'track_number' => $trackNumber, - 'album' => isset($tags['album']) ? $tags['album'][0] : null, - 'year' => isset($tags['year']) ? (int) $tags['year'][0] : null, - 'comments' => isset($tags['comments']) ? $tags['comments'][0] : null, - 'lyrics' => isset($tags['lyrics']) ? $tags['lyrics'][0] : null, - ]; + [ + 'title' => $tags['title'][0], + 'artist' => $tags['artist'][0], + 'band' => isset($tags['band']) ? $tags['band'][0] : null, + 'album_artist' => isset($tags['album_artist']) ? $tags['album_artist'][0] : null, + 'genre' => isset($tags['genre']) ? $tags['genre'][0] : null, + 'track_number' => $trackNumber, + 'album' => isset($tags['album']) ? $tags['album'][0] : null, + 'year' => isset($tags['year']) ? (int) $tags['year'][0] : null, + 'comments' => isset($tags['comments']) ? $tags['comments'][0] : null, + 'lyrics' => isset($tags['lyrics']) ? $tags['lyrics'][0] : null, + ], + $tags]; } } diff --git a/app/database/migrations/2015_09_05_143300_create_mlpma_table.php b/app/database/migrations/2015_09_05_143300_create_mlpma_table.php new file mode 100644 index 00000000..be114960 --- /dev/null +++ b/app/database/migrations/2015_09_05_143300_create_mlpma_table.php @@ -0,0 +1,38 @@ +increments('id'); + $table->integer('track_id')->unsigned()->index(); + $table->string('path')->index(); + $table->string('filename')->index(); + $table->string('extension')->index(); + $table->dateTime('imported_at'); + $table->longText('parsed_tags'); + $table->longText('raw_tags'); + + $table->foreign('track_id')->references('id')->on('tracks'); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::drop('mlpma_tracks'); + } + +} diff --git a/app/library/PfmValidator.php b/app/library/PfmValidator.php index aa1df4ad..0d98c6cf 100644 --- a/app/library/PfmValidator.php +++ b/app/library/PfmValidator.php @@ -25,6 +25,7 @@ // value is the file array itself // parameters is a list of formats the file can be, verified via ffmpeg $file = AudioCache::get($value->getPathname()); + var_dump($file->getAudioCodec()); return in_array($file->getAudioCodec(), $parameters); } @@ -144,4 +145,4 @@ public function validateTextareaLength($attribute, $value, $parameters) { return strlen(str_replace("\r\n", "\n", $value)) <= $parameters[0]; } - } \ No newline at end of file + } diff --git a/app/models/Commands/UploadTrackCommand.php b/app/models/Commands/UploadTrackCommand.php index a6954b1c..c241b980 100644 --- a/app/models/Commands/UploadTrackCommand.php +++ b/app/models/Commands/UploadTrackCommand.php @@ -4,9 +4,28 @@ use Entities\Track; use Entities\TrackFile; + use Illuminate\Support\Facades\File; use Illuminate\Support\Facades\Log; + use AudioCache; + use Illuminate\Support\Str; class UploadTrackCommand extends CommandBase { + private $_allowLossy; + private $_losslessFormats = [ + 'flac', + 'pcm_s16le ([1][0][0][0] / 0x0001)', + 'pcm_s16be', + 'adpcm_ms ([2][0][0][0] / 0x0002)', + 'pcm_s24le ([1][0][0][0] / 0x0001)', + 'pcm_s24be', + 'pcm_f32le ([3][0][0][0] / 0x0003)', + 'pcm_f32be (fl32 / 0x32336C66)' + ]; + + public function __construct($allowLossy = false) { + $this->_allowLossy = $allowLossy; + } + /** * @return bool */ @@ -26,7 +45,7 @@ $validator = \Validator::make(['track' => $trackFile], [ 'track' => 'required|' - . 'audio_format:flac,pcm_s16le ([1][0][0][0] / 0x0001),pcm_s16be,adpcm_ms ([2][0][0][0] / 0x0002),pcm_s24le ([1][0][0][0] / 0x0001),pcm_s24be,pcm_f32le ([3][0][0][0] / 0x0003),pcm_f32be (fl32 / 0x32336C66)|' + . $this->_allowLossy ? '' : 'audio_format:'.implode(',', $this->_losslessFormats).'|' . 'audio_channels:1,2|' . 'sample_rate:44100,48000,88200,96000,176400,192000|' . 'min_duration:30' @@ -53,7 +72,40 @@ $processes = []; + // Lossy uploads need to be identified and set as the master file + // without being re-encoded. + $audioObject = AudioCache::get($source); + $isLossyUpload = !in_array($audioObject->getAudioCodec(), $this->_losslessFormats); + + if ($isLossyUpload) { + + if ($audioObject->getAudioCodec() === 'mp3') { + $masterFormat = 'MP3'; + + } else if (Str::startsWith($audioObject->getAudioCodec(), 'aac')) { + $masterFormat = 'AAC'; + + } else { + $validator->messages()->add('track', 'The track does not contain audio in a known lossy format.'); + return CommandResponse::fail($validator); + } + + $trackFile = new TrackFile(); + $trackFile->is_master = true; + $trackFile->format = $masterFormat; + + // Lossy masters are copied into the datastore - no re-encoding involved. + File::copy($source, $trackFile->getFilename()); + $track->trackFiles()->save($trackFile); + } + foreach (Track::$Formats as $name => $format) { + // Don't bother with lossless transcodes of lossy uploads, and + // don't re-encode the lossy master. + if ($isLossyUpload && ($format['is_lossless'] || $name === $masterFormat)) { + continue; + } + $trackFile = new TrackFile(); $trackFile->is_master = $name === 'FLAC' ? true : false; $trackFile->format = $name; diff --git a/app/models/Entities/Track.php b/app/models/Entities/Track.php index 4c8569f8..337b6b8f 100644 --- a/app/models/Entities/Track.php +++ b/app/models/Entities/Track.php @@ -24,11 +24,11 @@ } public static $Formats = [ - 'FLAC' => ['index' => 0, 'extension' => 'flac', 'tag_format' => 'metaflac', 'tag_method' => 'updateTagsWithGetId3', 'mime_type' => 'audio/flac', 'command' => 'ffmpeg 2>&1 -y -i {$source} -acodec flac -aq 8 -f flac {$target}'], - 'MP3' => ['index' => 1, 'extension' => 'mp3', 'tag_format' => 'id3v2.3', 'tag_method' => 'updateTagsWithGetId3', 'mime_type' => 'audio/mpeg', 'command' => 'ffmpeg 2>&1 -y -i {$source} -acodec libmp3lame -ab 320k -f mp3 {$target}'], - 'OGG Vorbis' => ['index' => 2, 'extension' => 'ogg', 'tag_format' => 'vorbiscomment', 'tag_method' => 'updateTagsWithGetId3', 'mime_type' => 'audio/ogg', 'command' => 'ffmpeg 2>&1 -y -i {$source} -acodec libvorbis -aq 7 -f ogg {$target}'], - 'AAC' => ['index' => 3, 'extension' => 'm4a', 'tag_format' => 'AtomicParsley', 'tag_method' => 'updateTagsWithAtomicParsley', 'mime_type' => 'audio/mp4', 'command' => 'ffmpeg 2>&1 -y -i {$source} -acodec libfaac -ab 256k -f mp4 {$target}'], - 'ALAC' => ['index' => 4, 'extension' => 'alac.m4a', 'tag_format' => 'AtomicParsley', 'tag_method' => 'updateTagsWithAtomicParsley', 'mime_type' => 'audio/mp4', 'command' => 'ffmpeg 2>&1 -y -i {$source} -acodec alac {$target}'], + 'FLAC' => ['index' => 0, 'is_lossless' => true, 'extension' => 'flac', 'tag_format' => 'metaflac', 'tag_method' => 'updateTagsWithGetId3', 'mime_type' => 'audio/flac', 'command' => 'ffmpeg 2>&1 -y -i {$source} -acodec flac -aq 8 -f flac {$target}'], + 'MP3' => ['index' => 1, 'is_lossless' => false, 'extension' => 'mp3', 'tag_format' => 'id3v2.3', 'tag_method' => 'updateTagsWithGetId3', 'mime_type' => 'audio/mpeg', 'command' => 'ffmpeg 2>&1 -y -i {$source} -acodec libmp3lame -ab 320k -f mp3 {$target}'], + 'OGG Vorbis' => ['index' => 2, 'is_lossless' => false, 'extension' => 'ogg', 'tag_format' => 'vorbiscomment', 'tag_method' => 'updateTagsWithGetId3', 'mime_type' => 'audio/ogg', 'command' => 'ffmpeg 2>&1 -y -i {$source} -acodec libvorbis -aq 7 -f ogg {$target}'], + 'AAC' => ['index' => 3, 'is_lossless' => false, 'extension' => 'm4a', 'tag_format' => 'AtomicParsley', 'tag_method' => 'updateTagsWithAtomicParsley', 'mime_type' => 'audio/mp4', 'command' => 'ffmpeg 2>&1 -y -i {$source} -acodec libfaac -ab 256k -f mp4 {$target}'], + 'ALAC' => ['index' => 4, 'is_lossless' => true, 'extension' => 'alac.m4a', 'tag_format' => 'AtomicParsley', 'tag_method' => 'updateTagsWithAtomicParsley', 'mime_type' => 'audio/mp4', 'command' => 'ffmpeg 2>&1 -y -i {$source} -acodec alac {$target}'], ]; public static function summary() { @@ -430,7 +430,11 @@ public function updateTags() { $this->trackFiles()->touch(); - foreach (self::$Formats as $format => $data) { + + foreach ($this->trackFiles as $trackFile) { + $format = $trackFile->format; + $data = self::$Formats[$format]; + $this->{$data['tag_method']}($format); } } diff --git a/app/start/artisan.php b/app/start/artisan.php index d7082915..9a804152 100644 --- a/app/start/artisan.php +++ b/app/start/artisan.php @@ -14,3 +14,4 @@ Artisan::add(new MigrateOldData); Artisan::add(new RefreshCache); Artisan::add(new ImportMLPMA); + Artisan::add(new ClassifyMLPMA); From 15760a1040e002f3b3026c4d250a67c41945faef Mon Sep 17 00:00:00 2001 From: Peter Deltchev Date: Mon, 7 Sep 2015 07:02:25 -0700 Subject: [PATCH 5/8] T357, T125: Added SIGINT handling and a skip feature to the import script, implemented importing of lossy files, made the import script actually import tracks now, and other improvements. --- app/commands/ClassifyMLPMA.php | 42 +++++++--- app/commands/ImportMLPMA.php | 96 +++++++++++++++------- app/models/Commands/UploadTrackCommand.php | 6 +- app/models/Entities/Track.php | 3 +- 4 files changed, 101 insertions(+), 46 deletions(-) diff --git a/app/commands/ClassifyMLPMA.php b/app/commands/ClassifyMLPMA.php index 3f2ab684..782ea61e 100644 --- a/app/commands/ClassifyMLPMA.php +++ b/app/commands/ClassifyMLPMA.php @@ -21,7 +21,14 @@ class ClassifyMLPMA extends Command { * * @var string */ - protected $description = 'Add Pony.fm-specific metadata to imported MLPMA tracks.'; + protected $description = 'Adds Pony.fm-specific metadata to imported MLPMA tracks.'; + + /** + * A counter for the number of processed tracks. + * + * @var int + */ + protected $currentTrack = 0; /** * Create a new command instance. @@ -42,11 +49,26 @@ class ClassifyMLPMA extends Command { { // Get the list of tracks that need classification $tracks = DB::table('mlpma_tracks') - ->orderBy('id') + ->orderBy('mlpma_tracks.id') + ->join('tracks', 'tracks.id', '=', 'mlpma_tracks.track_id') + ->whereNull('tracks.published_at') ->get(); + $this->comment('Importing tracks...'); + + $totalTracks = sizeof($tracks); + + $fileToStartAt = (int) $this->option('startAt') - 1; + $this->comment("Skipping $fileToStartAt files..." . PHP_EOL); + + $tracks = array_slice($tracks, $fileToStartAt); + $this->currentTrack = $fileToStartAt; + foreach ($tracks as $track) { - $parsedTags = json_decode($track->parsed_tags); + $this->currentTrack++; + $this->comment('[' . $this->currentTrack . '/' . $totalTracks . '] Classifying track [' . $track->filename . ']...'); + + $parsedTags = json_decode($track->parsed_tags, true); //========================================================================================================== @@ -69,14 +91,14 @@ class ClassifyMLPMA extends Command { // If it has "Ingram" in the name, it's definitely an official song remix. if (Str::contains(Str::lower($track->filename), 'ingram')) { - $this->comment('This is an official song remix!'); + $this->info('This is an official song remix!'); list($trackType, $linkedSongIds) = $this->classifyTrack($track->filename, $officialSongs, true); // If it has "remix" in the name, it's definitely a remix. } else if (Str::contains(Str::lower($sanitizedTrackTitle), 'remix')) { - $this->comment('This is some kind of remix!'); + $this->info('This is some kind of remix!'); list($trackType, $linkedSongIds) = $this->classifyTrack($track->filename, $officialSongs); } @@ -96,6 +118,7 @@ class ClassifyMLPMA extends Command { $track->showSongs()->attach($linkedSongIds); } + echo PHP_EOL; } } @@ -107,9 +130,7 @@ class ClassifyMLPMA extends Command { */ protected function getArguments() { - return array( - array('example', InputArgument::REQUIRED, 'An example argument.'), - ); + return array(); } /** @@ -120,7 +141,7 @@ class ClassifyMLPMA extends Command { protected function getOptions() { return array( - array('example', null, InputOption::VALUE_OPTIONAL, 'An example option.', null), + array('startAt', null, InputOption::VALUE_OPTIONAL, 'Track to start importing from. Useful for resuming an interrupted import.', 1), ); } @@ -143,11 +164,12 @@ class ClassifyMLPMA extends Command { $this->comment('=> Matched official song: [' . $song->id . '] ' . $song->title); } + if ($isRemixOfOfficialTrack && sizeof($officialSongs) === 1) { $linkedSongIds = [$officialSongs[0]->id]; } else { - if ($isRemixOfOfficialTrack) { + if ($isRemixOfOfficialTrack && sizeof($officialSongs) > 1) { $this->question('Multiple official songs matched! Please enter the ID of the correct one.'); } else if (sizeof($officialSongs) > 0) { diff --git a/app/commands/ImportMLPMA.php b/app/commands/ImportMLPMA.php index bf31c5ee..8e0ab371 100644 --- a/app/commands/ImportMLPMA.php +++ b/app/commands/ImportMLPMA.php @@ -7,9 +7,7 @@ use Illuminate\Support\Facades\File; use Entities\Album; use Entities\Image; use Entities\User; -use Entities\ShowSong; use Entities\Track; -use Entities\TrackType; use Commands\UploadTrackCommand; use Symfony\Component\HttpFoundation\File\UploadedFile; use Carbon\Carbon; @@ -40,6 +38,20 @@ class ImportMLPMA extends Command { */ protected $ignoredExtensions = ['db', 'jpg', 'png']; + /** + * Used to stop the import process when a SIGINT is received. + * + * @var bool + */ + protected $isInterrupted = false; + + /** + * A counter for the number of processed tracks. + * + * @var int + */ + protected $currentFile; + /** * Create a new command instance. * @@ -50,6 +62,12 @@ class ImportMLPMA extends Command { parent::__construct(); } + public function handleInterrupt($signo) { + $this->error('Import aborted!'); + $this->error('Resume it from here using: --startAt='.$this->currentFile); + $this->isInterrupted = true; + } + /** * Execute the console command. * @@ -57,6 +75,8 @@ class ImportMLPMA extends Command { */ public function fire() { + pcntl_signal(SIGINT, [$this, 'handleInterrupt']); + $mlpmaPath = Config::get('app.files_directory').'mlpma'; $tmpPath = Config::get('app.files_directory').'tmp'; @@ -72,14 +92,25 @@ class ImportMLPMA extends Command { $artists = File::directories($mlpmaPath); $this->info(sizeof($artists).' artists found!'); - $this->comment('Importing tracks...'.PHP_EOL); + $this->comment('Importing tracks...'); $totalFiles = sizeof($files); - $currentFile = 0; + + $fileToStartAt = (int) $this->option('startAt') - 1; + $this->comment("Skipping $fileToStartAt files...".PHP_EOL); + + $files = array_slice($files, $fileToStartAt); + $this->currentFile = $fileToStartAt; foreach($files as $file) { - $currentFile++; - $this->comment('['.$currentFile.'/'.$totalFiles.'] Importing track ['. $file->getFilename() .']...'); + $this->currentFile++; + + pcntl_signal_dispatch(); + if ($this->isInterrupted) { + break; + } + + $this->comment('['.$this->currentFile.'/'.$totalFiles.'] Importing track ['. $file->getFilename() .']...'); if (in_array($file->getExtension(), $this->ignoredExtensions)) { $this->comment('This is not an audio file! Skipping...'.PHP_EOL); @@ -143,7 +174,7 @@ class ImportMLPMA extends Command { } // This is later used by the classification/publishing script to determine the publication date. - $parsedTags['released_at'] = $releasedAt; + $parsedTags['released_at'] = $releasedAt->toDateTimeString(); //========================================================================================================== // Does this track have vocals? @@ -178,12 +209,14 @@ class ImportMLPMA extends Command { $artist->slug = $artist->slug.'-'.Str::random(4); } - $artist = $artist->save(); + $artist->save(); } //========================================================================================================== // Extract the cover art, if any exists. //========================================================================================================== + + $this->comment('Extracting cover art!'); $coverId = null; if (array_key_exists('comments', $allTags) && array_key_exists('picture', $allTags['comments'])) { $image = $allTags['comments']['picture'][0]; @@ -213,7 +246,7 @@ class ImportMLPMA extends Command { $coverId = $cover->id; } else { - $this->error('No cover art found!'); + $this->comment('No cover art found!'); } @@ -246,13 +279,9 @@ class ImportMLPMA extends Command { //========================================================================================================== // Save this track. //========================================================================================================== - $title = $parsedTags['title']; -// -// $track = Track::where('user_id', '=', $artist->id) -// ->where('title', '=', $title) -// ->first(); - // "upload" the track to Pony.fm + // "Upload" the track to Pony.fm + $this->comment('Transcoding the track!'); Auth::loginUsingId($artist->id); $trackFile = new UploadedFile($file->getPathname(), $file->getFilename(), $allTags['mime_type']); @@ -261,14 +290,29 @@ class ImportMLPMA extends Command { $upload = new UploadTrackCommand(true); $result = $upload->execute(); -// var_dump(null); - if ($result->didFail()) { $this->error(json_encode($result->getValidator()->messages()->getMessages(), JSON_PRETTY_PRINT)); + } else { + // Save metadata. + $track = Track::find($result->getResponse()['id']); + + $track->title = $parsedTags['title']; + $track->cover_id = $coverId; + $track->album_id = $albumId; + $track->track_number = $parsedTags['track_number']; + $track->released_at = $releasedAt; + $track->description = $parsedTags['comments']; + $track->is_downloadable = true; + $track->lyrics = $parsedTags['lyrics']; + $track->is_vocal = $isVocal; + $track->license_id = 2; + $track->save(); + + // If we made it to here, the track is intact! Log the import. DB::table('mlpma_tracks') ->insert([ - 'track_id' => $result['id'], + 'track_id' => $result->getResponse()['id'], 'path' => $file->getRelativePath(), 'filename' => $file->getFilename(), 'extension' => $file->getExtension(), @@ -276,20 +320,8 @@ class ImportMLPMA extends Command { 'parsed_tags' => json_encode($parsedTags), 'raw_tags' => json_encode($rawTags), ]); - - $track = Track::find($result['id']); - var_dump($track); - - $track->title = $parsedTags['title']; - $track->cover_id = $coverId; - $track->album_id = $albumId; - $track->track_number = $parsedTags['track_number']; - $track->released_at = $releasedAt; - $track->is_vocal = $isVocal; - $track->save(); } - echo PHP_EOL.PHP_EOL; } } @@ -311,7 +343,9 @@ class ImportMLPMA extends Command { */ protected function getOptions() { - return array(); + return array( + array('startAt', null, InputOption::VALUE_OPTIONAL, 'Track to start importing from. Useful for resuming an interrupted import.', 1), + ); } diff --git a/app/models/Commands/UploadTrackCommand.php b/app/models/Commands/UploadTrackCommand.php index c241b980..17467db7 100644 --- a/app/models/Commands/UploadTrackCommand.php +++ b/app/models/Commands/UploadTrackCommand.php @@ -78,7 +78,6 @@ $isLossyUpload = !in_array($audioObject->getAudioCodec(), $this->_losslessFormats); if ($isLossyUpload) { - if ($audioObject->getAudioCodec() === 'mp3') { $masterFormat = 'MP3'; @@ -93,10 +92,11 @@ $trackFile = new TrackFile(); $trackFile->is_master = true; $trackFile->format = $masterFormat; + $trackFile->track_id = $track->id; + $trackFile->save(); // Lossy masters are copied into the datastore - no re-encoding involved. - File::copy($source, $trackFile->getFilename()); - $track->trackFiles()->save($trackFile); + File::copy($source, $trackFile->getFile()); } foreach (Track::$Formats as $name => $format) { diff --git a/app/models/Entities/Track.php b/app/models/Entities/Track.php index 337b6b8f..8edf51bb 100644 --- a/app/models/Entities/Track.php +++ b/app/models/Entities/Track.php @@ -448,8 +448,7 @@ $command .= '--genre ' . escapeshellarg($this->genre != null ? $this->genre->name : '') . ' '; $command .= '--copyright ' . escapeshellarg('© '.$this->year.' '.$this->user->display_name).' '; $command .= '--comment "' . 'Downloaded from: https://pony.fm/' . '" '; - $command .= '--encodingTool "' . 'Pony.fm' . '" '; - $command .= '--encodedBy "' . 'Pony.fm - https://pony.fm/' . '" '; + $command .= '--encodingTool "' . 'Pony.fm - https://pony.fm/' . '" '; if ($this->album_id !== NULL) { $command .= '--album ' . escapeshellarg($this->album->title) . ' '; From 9d087a94155dddc6b12992758879a1aba56d137f Mon Sep 17 00:00:00 2001 From: Peter Deltchev Date: Mon, 7 Sep 2015 07:11:28 -0700 Subject: [PATCH 6/8] T357: Actually classify tracks in the classifier and allow re-classification of published MLPMA tracks. --- app/commands/ClassifyMLPMA.php | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/app/commands/ClassifyMLPMA.php b/app/commands/ClassifyMLPMA.php index 782ea61e..b44cbf20 100644 --- a/app/commands/ClassifyMLPMA.php +++ b/app/commands/ClassifyMLPMA.php @@ -49,9 +49,7 @@ class ClassifyMLPMA extends Command { { // Get the list of tracks that need classification $tracks = DB::table('mlpma_tracks') - ->orderBy('mlpma_tracks.id') - ->join('tracks', 'tracks.id', '=', 'mlpma_tracks.track_id') - ->whereNull('tracks.published_at') + ->orderBy('id') ->get(); $this->comment('Importing tracks...'); @@ -74,9 +72,6 @@ class ClassifyMLPMA extends Command { //========================================================================================================== // Original, show song remix, fan song remix, show audio remix, or ponified song? //========================================================================================================== - $trackType = TrackType::ORIGINAL_TRACK; - $linkedSongIds = []; - $sanitizedTrackTitle = $parsedTags['title']; $sanitizedTrackTitle = str_replace([' - ', 'ft.', '*'], ' ', $sanitizedTrackTitle); @@ -96,11 +91,15 @@ class ClassifyMLPMA extends Command { list($trackType, $linkedSongIds) = $this->classifyTrack($track->filename, $officialSongs, true); - // If it has "remix" in the name, it's definitely a remix. + // If it has "remix" in the name, it's definitely a remix. } else if (Str::contains(Str::lower($sanitizedTrackTitle), 'remix')) { $this->info('This is some kind of remix!'); list($trackType, $linkedSongIds) = $this->classifyTrack($track->filename, $officialSongs); + + // No idea what this is. Have the pony at the terminal figure it out! + } else { + list($trackType, $linkedSongIds) = $this->classifyTrack($track->filename, $officialSongs); } From 05ecc34e1284b31195bd55f65db98197143471ac Mon Sep 17 00:00:00 2001 From: Peter Deltchev Date: Mon, 7 Sep 2015 07:19:41 -0700 Subject: [PATCH 7/8] Made history immutable for Arcanist. --- .arcconfig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.arcconfig b/.arcconfig index fe223088..adbd0155 100644 --- a/.arcconfig +++ b/.arcconfig @@ -1,3 +1,4 @@ { - "phabricator.uri" : "https://phabricator.poniverse.net/" + "phabricator.uri" : "https://phabricator.poniverse.net/", + "history.immutable" : true } From 82dc200f17836b1d655eb3640fa480677caa29b9 Mon Sep 17 00:00:00 2001 From: Peter Deltchev Date: Mon, 7 Sep 2015 07:35:25 -0700 Subject: [PATCH 8/8] T357: Continue restricting lossy user-facing uploads. --- app/library/PfmValidator.php | 1 - app/models/Commands/UploadTrackCommand.php | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/app/library/PfmValidator.php b/app/library/PfmValidator.php index 0d98c6cf..ffb24c9e 100644 --- a/app/library/PfmValidator.php +++ b/app/library/PfmValidator.php @@ -25,7 +25,6 @@ // value is the file array itself // parameters is a list of formats the file can be, verified via ffmpeg $file = AudioCache::get($value->getPathname()); - var_dump($file->getAudioCodec()); return in_array($file->getAudioCodec(), $parameters); } diff --git a/app/models/Commands/UploadTrackCommand.php b/app/models/Commands/UploadTrackCommand.php index 17467db7..2a9f8cdb 100644 --- a/app/models/Commands/UploadTrackCommand.php +++ b/app/models/Commands/UploadTrackCommand.php @@ -45,7 +45,7 @@ $validator = \Validator::make(['track' => $trackFile], [ 'track' => 'required|' - . $this->_allowLossy ? '' : 'audio_format:'.implode(',', $this->_losslessFormats).'|' + . ($this->_allowLossy ? '' : 'audio_format:'. implode(',', $this->_losslessFormats).'|') . 'audio_channels:1,2|' . 'sample_rate:44100,48000,88200,96000,176400,192000|' . 'min_duration:30'