T357: Separated track publishing and classification into its own script, fixed an issue with reading comments from ID3 tags, and added lossy support to UploadTrackCommand.

This commit is contained in:
Peter Deltchev 2015-09-07 04:50:35 -07:00
parent dbab3a9ecc
commit f17e824586
7 changed files with 412 additions and 161 deletions

View file

@ -0,0 +1,210 @@
<?php
use Illuminate\Console\Command;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Input\InputArgument;
use Entities\ShowSong;
use Entities\Track;
use Entities\TrackType;
class ClassifyMLPMA extends Command {
/**
* The console command name.
*
* @var string
*/
protected $name = 'mlpma:classify';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Add Pony.fm-specific metadata to imported MLPMA tracks.';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return void
*/
public function fire()
{
// Get the list of tracks that need classification
$tracks = DB::table('mlpma_tracks')
->orderBy('id')
->get();
foreach ($tracks as $track) {
$parsedTags = json_decode($track->parsed_tags);
//==========================================================================================================
// Original, show song remix, fan song remix, show audio remix, or ponified song?
//==========================================================================================================
$trackType = TrackType::ORIGINAL_TRACK;
$linkedSongIds = [];
$sanitizedTrackTitle = $parsedTags['title'];
$sanitizedTrackTitle = str_replace([' - ', 'ft.', '*'], ' ', $sanitizedTrackTitle);
$queriedTitle = DB::connection()->getPdo()->quote($sanitizedTrackTitle);
$officialSongs = ShowSong::select(['id', 'title'])
->whereRaw("
MATCH (title)
AGAINST ($queriedTitle IN BOOLEAN MODE)
")
->get();
// If it has "Ingram" in the name, it's definitely an official song remix.
if (Str::contains(Str::lower($track->filename), 'ingram')) {
$this->comment('This is an official song remix!');
list($trackType, $linkedSongIds) = $this->classifyTrack($track->filename, $officialSongs, true);
// If it has "remix" in the name, it's definitely a remix.
} else if (Str::contains(Str::lower($sanitizedTrackTitle), 'remix')) {
$this->comment('This is some kind of remix!');
list($trackType, $linkedSongIds) = $this->classifyTrack($track->filename, $officialSongs);
}
//==========================================================================================================
// Attach the data and publish the track!
//==========================================================================================================
$track = Track::find($track->track_id);
$track->track_type_id = $trackType;
$track->published_at = $parsedTags['released_at'];
$track->save();
if (sizeof($linkedSongIds) > 0) {
$track->showSongs()->attach($linkedSongIds);
}
}
}
/**
* Get the console command arguments.
*
* @return array
*/
protected function getArguments()
{
return array(
array('example', InputArgument::REQUIRED, 'An example argument.'),
);
}
/**
* Get the console command options.
*
* @return array
*/
protected function getOptions()
{
return array(
array('example', null, InputOption::VALUE_OPTIONAL, 'An example option.', null),
);
}
/**
* Determines what type of track the given file is. If unable to guess, the user
* is asked to identify it interactively.
*
* @param string $filename
* @param \Entities\ShowSong[] $officialSongs
* @param bool|false $isRemixOfOfficialTrack
* @return array
*/
protected function classifyTrack($filename, $officialSongs, $isRemixOfOfficialTrack = false) {
$trackTypeId = null;
$linkedSongIds = [];
foreach ($officialSongs as $song) {
$this->comment('=> Matched official song: [' . $song->id . '] ' . $song->title);
}
if ($isRemixOfOfficialTrack && sizeof($officialSongs) === 1) {
$linkedSongIds = [$officialSongs[0]->id];
} else {
if ($isRemixOfOfficialTrack) {
$this->question('Multiple official songs matched! Please enter the ID of the correct one.');
} else if (sizeof($officialSongs) > 0) {
$this->question('This looks like a remix of an official song!');
$this->question('Press "r" if the match above is right!');
} else {
$this->question('Exactly what kind of track is this?');
}
$this->question('If this is a medley, multiple song ID\'s can be separated by commas. ');
$this->question(' ');
$this->question(' ' . $filename . ' ');
$this->question(' ');
$this->question(' r = official song remix (accept all "guessed" matches) ');
$this->question(' # = official song remix (enter the ID(s) of the show song(s)) ');
$this->question(' a = show audio remix ');
$this->question(' f = fan track remix ');
$this->question(' p = ponified track ');
$this->question(' o = original track ');
$this->question(' ');
$input = $this->ask('[r/#/a/f/p/o]: ');
switch ($input) {
case 'r':
$trackTypeId = TrackType::OFFICIAL_TRACK_REMIX;
foreach ($officialSongs as $officialSong) {
$linkedSongIds[] = (int) $officialSong->id;
}
break;
case 'a':
$trackTypeId = TrackType::OFFICIAL_AUDIO_REMIX;
break;
case 'f':
$trackTypeId = TrackType::FAN_TRACK_REMIX;
break;
case 'p':
$trackTypeId = TrackType::PONIFIED_TRACK;
break;
case 'o':
$trackTypeId = TrackType::ORIGINAL_TRACK;
break;
default:
$trackTypeId = TrackType::OFFICIAL_TRACK_REMIX;
$linkedSongIds = explode(',', $input);
$linkedSongIds = array_map(function ($item) {
return (int) $item;
}, $linkedSongIds);
}
}
return [$trackTypeId, $linkedSongIds];
}
}

View file

@ -10,6 +10,7 @@ use Entities\User;
use Entities\ShowSong;
use Entities\Track;
use Entities\TrackType;
use Commands\UploadTrackCommand;
use Symfony\Component\HttpFoundation\File\UploadedFile;
use Carbon\Carbon;
@ -23,7 +24,7 @@ class ImportMLPMA extends Command {
*
* @var string
*/
protected $name = 'import-mlpma';
protected $name = 'mlpma:import';
/**
* The console command description.
@ -86,18 +87,36 @@ class ImportMLPMA extends Command {
}
// Has this track already been imported?
$importedTrack = DB::table('mlpma_tracks')
->where('filename', '=', $file->getFilename())
->first();
if ($importedTrack) {
$this->comment('This track has already been imported! Skipping...' . PHP_EOL);
continue;
}
//==========================================================================================================
// Extract the original tags.
//==========================================================================================================
$getId3 = new getID3;
$tags = $getId3->analyze($file->getPathname());
// all tags read by getID3, including the cover art
$allTags = $getId3->analyze($file->getPathname());
// tags specific to a file format (ID3 or Atom), pre-normalization but with cover art removed
$rawTags = [];
// normalized tags used by Pony.fm
$parsedTags = [];
if ($file->getExtension() === 'mp3') {
$parsedTags = $this->getId3Tags($tags);
list($parsedTags, $rawTags) = $this->getId3Tags($allTags);
} else if ($file->getExtension() === 'm4a') {
$parsedTags = $this->getAtomTags($tags);
list($parsedTags, $rawTags) = $this->getAtomTags($allTags);
}
@ -123,6 +142,9 @@ class ImportMLPMA extends Command {
$releasedAt = $modifiedDate;
}
// This is later used by the classification/publishing script to determine the publication date.
$parsedTags['released_at'] = $releasedAt;
//==========================================================================================================
// Does this track have vocals?
//==========================================================================================================
@ -163,8 +185,8 @@ class ImportMLPMA extends Command {
// Extract the cover art, if any exists.
//==========================================================================================================
$coverId = null;
if (array_key_exists('comments', $tags) && array_key_exists('picture', $tags['comments'])) {
$image = $tags['comments']['picture'][0];
if (array_key_exists('comments', $allTags) && array_key_exists('picture', $allTags['comments'])) {
$image = $allTags['comments']['picture'][0];
if ($image['image_mime'] === 'image/png') {
$extension = 'png';
@ -220,153 +242,58 @@ class ImportMLPMA extends Command {
$albumId = $album->id;
}
//==========================================================================================================
// Original, show song remix, fan song remix, show audio remix, or ponified song?
//==========================================================================================================
$trackType = TrackType::ORIGINAL_TRACK;
$linkedSongIds = [];
$sanitizedTrackTitle = $parsedTags['title'];
$sanitizedTrackTitle = str_replace(' - ', ' ', $sanitizedTrackTitle);
$sanitizedTrackTitle = str_replace('ft. ', '', $sanitizedTrackTitle);
$sanitizedTrackTitle = str_replace('*', '', $sanitizedTrackTitle);
$queriedTitle = DB::connection()->getPdo()->quote($sanitizedTrackTitle);
$officialSongs = ShowSong::select(['id', 'title'])
->whereRaw("
MATCH (title)
AGAINST ($queriedTitle IN BOOLEAN MODE)
")
->get();
// If it has "Ingram" in the name, it's definitely an official song remix.
if (Str::contains(Str::lower($file->getFilename()), 'ingram')) {
$this->comment('This is an official song remix!');
list($trackType, $linkedSongIds) = $this->classifyTrack($file, $officialSongs, true);
// If it has "remix" in the name, it's definitely a remix.
} else if (Str::contains(Str::lower($sanitizedTrackTitle), 'remix')) {
$this->comment('This is some kind of remix!');
list($trackType, $linkedSongIds) = $this->classifyTrack($file, $officialSongs);
}
//==========================================================================================================
// Save this track.
//==========================================================================================================
$title = $parsedTags['title'];
//
// $track = Track::where('user_id', '=', $artist->id)
// ->where('title', '=', $title)
// ->first();
// Has this track already been imported?
$track = Track::where('user_id', '=', $artist->id)
->where('title', '=', $title)
->first();
// "upload" the track to Pony.fm
Auth::loginUsingId($artist->id);
if (!$track) {
$track = new Track;
$trackFile = new UploadedFile($file->getPathname(), $file->getFilename(), $allTags['mime_type']);
Input::instance()->files->add(['track' => $trackFile]);
$upload = new UploadTrackCommand(true);
$result = $upload->execute();
// var_dump(null);
if ($result->didFail()) {
$this->error(json_encode($result->getValidator()->messages()->getMessages(), JSON_PRETTY_PRINT));
} else {
DB::table('mlpma_tracks')
->insert([
'track_id' => $result['id'],
'path' => $file->getRelativePath(),
'filename' => $file->getFilename(),
'extension' => $file->getExtension(),
'imported_at' => Carbon::now(),
'parsed_tags' => json_encode($parsedTags),
'raw_tags' => json_encode($rawTags),
]);
$track = Track::find($result['id']);
var_dump($track);
$track->user_id = $artist->id;
$track->title = $parsedTags['title'];
$track->cover_id = $coverId;
$track->album_id = $albumId;
$track->track_number = $parsedTags['track_number'];
$track->released_at = $releasedAt;
$track->is_vocal = $isVocal;
$track->track_type_id = $trackType;
$track->save();
if (sizeof($linkedSongIds) > 0) {
$track->showSongs()->attach($linkedSongIds);
}
// TODO: mark imported tracks as needing QA
} else {
$this->comment('This track has already been imported!');
}
echo PHP_EOL.PHP_EOL;
}
}
protected function classifyTrack($file, $officialSongs, $isRemixOfOfficialTrack = false)
{
$trackTypeId = null;
$linkedSongIds = [];
foreach ($officialSongs as $song) {
$this->comment('=> Matched official song: [' . $song->id . '] ' . $song->title);
}
if ($isRemixOfOfficialTrack && sizeof($officialSongs) === 1) {
$linkedSongIds = [$officialSongs[0]->id];
} else {
if ($isRemixOfOfficialTrack) {
$this->question('Multiple official songs matched! Please enter the ID of the correct one.');
} else if (sizeof($officialSongs) > 0) {
$this->question('This looks like a remix of an official song!');
$this->question('Press "r" if the match above is right!');
} else {
$this->question('Exactly what kind of track is this?');
}
$this->question('If this is a medley, multiple song ID\'s can be separated by commas. ');
$this->question(' ');
$this->question(' '.$file->getFilename().' ');
$this->question(' ');
$this->question(' r = official song remix (accept all "guessed" matches) ');
$this->question(' # = official song remix (enter the ID(s) of the show song(s)) ');
$this->question(' a = show audio remix ');
$this->question(' f = fan track remix ');
$this->question(' p = ponified track ');
$this->question(' o = original track ');
$this->question(' ');
$input = $this->ask('[r/#/a/f/p/o]: ');
switch ($input) {
case 'r':
$trackTypeId = TrackType::OFFICIAL_TRACK_REMIX;
foreach ($officialSongs as $officialSong) {
$linkedSongIds[] = (int) $officialSong->id;
}
break;
case 'a':
$trackTypeId = TrackType::OFFICIAL_AUDIO_REMIX;
break;
case 'f':
$trackTypeId = TrackType::FAN_TRACK_REMIX;
break;
case 'p':
$trackTypeId = TrackType::PONIFIED_TRACK;
break;
case 'o':
$trackTypeId = TrackType::ORIGINAL_TRACK;
break;
default:
$trackTypeId = TrackType::OFFICIAL_TRACK_REMIX;
$linkedSongIds = explode(',', $input);
$linkedSongIds = array_map(function ($item) {
return (int) $item;
}, $linkedSongIds);
}
}
return [$trackTypeId, $linkedSongIds];
}
/**
* Get the console command arguments.
*
@ -394,18 +321,34 @@ class ImportMLPMA extends Command {
*/
protected function getId3Tags($rawTags) {
$tags = $rawTags['tags']['id3v2'];
$comment = null;
if (isset($tags['comment'])) {
// The "comment" tag comes in with a badly encoded string index
// so its array key has to be used implicitly.
$key = array_keys($tags['comment'])[0];
// The comment may have a null byte at the end. trim() removes it.
$comment = trim($tags['comment'][$key]);
// Replace the malformed comment with the "fixed" one.
unset($tags['comment'][$key]);
$tags['comment'][0] = $comment;
}
return [
'title' => $tags['title'][0],
'artist' => $tags['artist'][0],
'band' => isset($tags['band']) ? $tags['band'][0] : null,
'genre' => isset($tags['genre']) ? $tags['genre'][0] : null,
'track_number' => isset($tags['track_number']) ? $tags['track_number'][0] : null,
'album' => isset($tags['album']) ? $tags['album'][0] : null,
'year' => isset($tags['year']) ? (int) $tags['year'][0] : null,
'comments' => isset($tags['comments']) ? $tags['comments'][0] : null,
'lyrics' => isset($tags['unsynchronised_lyric']) ? $tags['unsynchronised_lyric'][0] : null,
];
[
'title' => $tags['title'][0],
'artist' => $tags['artist'][0],
'band' => isset($tags['band']) ? $tags['band'][0] : null,
'genre' => isset($tags['genre']) ? $tags['genre'][0] : null,
'track_number' => isset($tags['track_number']) ? $tags['track_number'][0] : null,
'album' => isset($tags['album']) ? $tags['album'][0] : null,
'year' => isset($tags['year']) ? (int) $tags['year'][0] : null,
'comments' => $comment,
'lyrics' => isset($tags['unsynchronised_lyric']) ? $tags['unsynchronised_lyric'][0] : null,
],
$tags];
}
/**
@ -422,17 +365,19 @@ class ImportMLPMA extends Command {
}
return [
'title' => $tags['title'][0],
'artist' => $tags['artist'][0],
'band' => isset($tags['band']) ? $tags['band'][0] : null,
'album_artist' => isset($tags['album_artist']) ? $tags['album_artist'][0] : null,
'genre' => isset($tags['genre']) ? $tags['genre'][0] : null,
'track_number' => $trackNumber,
'album' => isset($tags['album']) ? $tags['album'][0] : null,
'year' => isset($tags['year']) ? (int) $tags['year'][0] : null,
'comments' => isset($tags['comments']) ? $tags['comments'][0] : null,
'lyrics' => isset($tags['lyrics']) ? $tags['lyrics'][0] : null,
];
[
'title' => $tags['title'][0],
'artist' => $tags['artist'][0],
'band' => isset($tags['band']) ? $tags['band'][0] : null,
'album_artist' => isset($tags['album_artist']) ? $tags['album_artist'][0] : null,
'genre' => isset($tags['genre']) ? $tags['genre'][0] : null,
'track_number' => $trackNumber,
'album' => isset($tags['album']) ? $tags['album'][0] : null,
'year' => isset($tags['year']) ? (int) $tags['year'][0] : null,
'comments' => isset($tags['comments']) ? $tags['comments'][0] : null,
'lyrics' => isset($tags['lyrics']) ? $tags['lyrics'][0] : null,
],
$tags];
}
}

View file

@ -0,0 +1,38 @@
<?php
use Illuminate\Database\Migrations\Migration;
class CreateMlpmaTable extends Migration {
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::create('mlpma_tracks', function(\Illuminate\Database\Schema\Blueprint $table) {
$table->increments('id');
$table->integer('track_id')->unsigned()->index();
$table->string('path')->index();
$table->string('filename')->index();
$table->string('extension')->index();
$table->dateTime('imported_at');
$table->longText('parsed_tags');
$table->longText('raw_tags');
$table->foreign('track_id')->references('id')->on('tracks');
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::drop('mlpma_tracks');
}
}

View file

@ -25,6 +25,7 @@
// value is the file array itself
// parameters is a list of formats the file can be, verified via ffmpeg
$file = AudioCache::get($value->getPathname());
var_dump($file->getAudioCodec());
return in_array($file->getAudioCodec(), $parameters);
}
@ -144,4 +145,4 @@
public function validateTextareaLength($attribute, $value, $parameters) {
return strlen(str_replace("\r\n", "\n", $value)) <= $parameters[0];
}
}
}

View file

@ -4,9 +4,28 @@
use Entities\Track;
use Entities\TrackFile;
use Illuminate\Support\Facades\File;
use Illuminate\Support\Facades\Log;
use AudioCache;
use Illuminate\Support\Str;
class UploadTrackCommand extends CommandBase {
private $_allowLossy;
private $_losslessFormats = [
'flac',
'pcm_s16le ([1][0][0][0] / 0x0001)',
'pcm_s16be',
'adpcm_ms ([2][0][0][0] / 0x0002)',
'pcm_s24le ([1][0][0][0] / 0x0001)',
'pcm_s24be',
'pcm_f32le ([3][0][0][0] / 0x0003)',
'pcm_f32be (fl32 / 0x32336C66)'
];
public function __construct($allowLossy = false) {
$this->_allowLossy = $allowLossy;
}
/**
* @return bool
*/
@ -26,7 +45,7 @@
$validator = \Validator::make(['track' => $trackFile], [
'track' =>
'required|'
. 'audio_format:flac,pcm_s16le ([1][0][0][0] / 0x0001),pcm_s16be,adpcm_ms ([2][0][0][0] / 0x0002),pcm_s24le ([1][0][0][0] / 0x0001),pcm_s24be,pcm_f32le ([3][0][0][0] / 0x0003),pcm_f32be (fl32 / 0x32336C66)|'
. $this->_allowLossy ? '' : 'audio_format:'.implode(',', $this->_losslessFormats).'|'
. 'audio_channels:1,2|'
. 'sample_rate:44100,48000,88200,96000,176400,192000|'
. 'min_duration:30'
@ -53,7 +72,40 @@
$processes = [];
// Lossy uploads need to be identified and set as the master file
// without being re-encoded.
$audioObject = AudioCache::get($source);
$isLossyUpload = !in_array($audioObject->getAudioCodec(), $this->_losslessFormats);
if ($isLossyUpload) {
if ($audioObject->getAudioCodec() === 'mp3') {
$masterFormat = 'MP3';
} else if (Str::startsWith($audioObject->getAudioCodec(), 'aac')) {
$masterFormat = 'AAC';
} else {
$validator->messages()->add('track', 'The track does not contain audio in a known lossy format.');
return CommandResponse::fail($validator);
}
$trackFile = new TrackFile();
$trackFile->is_master = true;
$trackFile->format = $masterFormat;
// Lossy masters are copied into the datastore - no re-encoding involved.
File::copy($source, $trackFile->getFilename());
$track->trackFiles()->save($trackFile);
}
foreach (Track::$Formats as $name => $format) {
// Don't bother with lossless transcodes of lossy uploads, and
// don't re-encode the lossy master.
if ($isLossyUpload && ($format['is_lossless'] || $name === $masterFormat)) {
continue;
}
$trackFile = new TrackFile();
$trackFile->is_master = $name === 'FLAC' ? true : false;
$trackFile->format = $name;

View file

@ -24,11 +24,11 @@
}
public static $Formats = [
'FLAC' => ['index' => 0, 'extension' => 'flac', 'tag_format' => 'metaflac', 'tag_method' => 'updateTagsWithGetId3', 'mime_type' => 'audio/flac', 'command' => 'ffmpeg 2>&1 -y -i {$source} -acodec flac -aq 8 -f flac {$target}'],
'MP3' => ['index' => 1, 'extension' => 'mp3', 'tag_format' => 'id3v2.3', 'tag_method' => 'updateTagsWithGetId3', 'mime_type' => 'audio/mpeg', 'command' => 'ffmpeg 2>&1 -y -i {$source} -acodec libmp3lame -ab 320k -f mp3 {$target}'],
'OGG Vorbis' => ['index' => 2, 'extension' => 'ogg', 'tag_format' => 'vorbiscomment', 'tag_method' => 'updateTagsWithGetId3', 'mime_type' => 'audio/ogg', 'command' => 'ffmpeg 2>&1 -y -i {$source} -acodec libvorbis -aq 7 -f ogg {$target}'],
'AAC' => ['index' => 3, 'extension' => 'm4a', 'tag_format' => 'AtomicParsley', 'tag_method' => 'updateTagsWithAtomicParsley', 'mime_type' => 'audio/mp4', 'command' => 'ffmpeg 2>&1 -y -i {$source} -acodec libfaac -ab 256k -f mp4 {$target}'],
'ALAC' => ['index' => 4, 'extension' => 'alac.m4a', 'tag_format' => 'AtomicParsley', 'tag_method' => 'updateTagsWithAtomicParsley', 'mime_type' => 'audio/mp4', 'command' => 'ffmpeg 2>&1 -y -i {$source} -acodec alac {$target}'],
'FLAC' => ['index' => 0, 'is_lossless' => true, 'extension' => 'flac', 'tag_format' => 'metaflac', 'tag_method' => 'updateTagsWithGetId3', 'mime_type' => 'audio/flac', 'command' => 'ffmpeg 2>&1 -y -i {$source} -acodec flac -aq 8 -f flac {$target}'],
'MP3' => ['index' => 1, 'is_lossless' => false, 'extension' => 'mp3', 'tag_format' => 'id3v2.3', 'tag_method' => 'updateTagsWithGetId3', 'mime_type' => 'audio/mpeg', 'command' => 'ffmpeg 2>&1 -y -i {$source} -acodec libmp3lame -ab 320k -f mp3 {$target}'],
'OGG Vorbis' => ['index' => 2, 'is_lossless' => false, 'extension' => 'ogg', 'tag_format' => 'vorbiscomment', 'tag_method' => 'updateTagsWithGetId3', 'mime_type' => 'audio/ogg', 'command' => 'ffmpeg 2>&1 -y -i {$source} -acodec libvorbis -aq 7 -f ogg {$target}'],
'AAC' => ['index' => 3, 'is_lossless' => false, 'extension' => 'm4a', 'tag_format' => 'AtomicParsley', 'tag_method' => 'updateTagsWithAtomicParsley', 'mime_type' => 'audio/mp4', 'command' => 'ffmpeg 2>&1 -y -i {$source} -acodec libfaac -ab 256k -f mp4 {$target}'],
'ALAC' => ['index' => 4, 'is_lossless' => true, 'extension' => 'alac.m4a', 'tag_format' => 'AtomicParsley', 'tag_method' => 'updateTagsWithAtomicParsley', 'mime_type' => 'audio/mp4', 'command' => 'ffmpeg 2>&1 -y -i {$source} -acodec alac {$target}'],
];
public static function summary() {
@ -430,7 +430,11 @@
public function updateTags() {
$this->trackFiles()->touch();
foreach (self::$Formats as $format => $data) {
foreach ($this->trackFiles as $trackFile) {
$format = $trackFile->format;
$data = self::$Formats[$format];
$this->{$data['tag_method']}($format);
}
}

View file

@ -14,3 +14,4 @@
Artisan::add(new MigrateOldData);
Artisan::add(new RefreshCache);
Artisan::add(new ImportMLPMA);
Artisan::add(new ClassifyMLPMA);