From d7a59b4131a8d5ac0bdb588153cf566ca795674a Mon Sep 17 00:00:00 2001 From: Josef Citrine Date: Thu, 2 Feb 2017 00:08:29 +0000 Subject: [PATCH] #100: Start of Ponify import --- app/Commands/UploadTrackCommand.php | 20 +- app/Console/Commands/ImportPonify.php | 316 ++++++++++++++++++++++++++ app/Console/Kernel.php | 3 +- 3 files changed, 334 insertions(+), 5 deletions(-) create mode 100644 app/Console/Commands/ImportPonify.php diff --git a/app/Commands/UploadTrackCommand.php b/app/Commands/UploadTrackCommand.php index 2bbd221b..0648aa77 100644 --- a/app/Commands/UploadTrackCommand.php +++ b/app/Commands/UploadTrackCommand.php @@ -43,6 +43,8 @@ class UploadTrackCommand extends CommandBase private $_version; private $_isReplacingTrack; + public $_file; + /** * @return bool */ @@ -94,7 +96,16 @@ class UploadTrackCommand extends CommandBase */ public function execute() { - $trackFile = Request::file('track', null); + $trackFile = null; + $source = 'direct_upload'; + + if ($this->_file !== null) { + $trackFile = $this->_file; + $source = 'ponify'; + } else { + $trackFile = Request::file('track', null); + } + if (!$this->_isReplacingTrack) { $coverFile = Request::file('cover', null); } @@ -114,7 +125,7 @@ class UploadTrackCommand extends CommandBase $this->_track->user_id = $this->_artist->id; // The title set here is a placeholder; it'll be replaced by ParseTrackTagsCommand // if the file contains a title tag. - $this->_track->title = Request::get('title', pathinfo($trackFile->getClientOriginalName(), PATHINFO_FILENAME)); + $this->_track->title = mb_strimwidth(Request::get('title', pathinfo($trackFile->getClientOriginalName(), PATHINFO_FILENAME)), 0, 100, '...'); // The duration/version of the track cannot be changed until the encoding is successful $this->_track->duration = $audio->getDuration(); $this->_track->current_version = $this->_version; @@ -129,7 +140,8 @@ class UploadTrackCommand extends CommandBase if (!is_dir(Config::get('ponyfm.files_directory').'/queued-tracks')) { mkdir(Config::get('ponyfm.files_directory').'/queued-tracks', 0755, true); } - $trackFile = $trackFile->move(Config::get('ponyfm.files_directory').'/queued-tracks', $this->_track->id . 'v' . $this->_version); + + $trackFile = $trackFile->move(Config::get('ponyfm.files_directory') . '/queued-tracks', $this->_track->id . 'v' . $this->_version); $input = Request::all(); $input['track'] = $trackFile; @@ -183,7 +195,7 @@ class UploadTrackCommand extends CommandBase $this->_track->metadata = json_decode(Request::get('metadata', null)); } $autoPublish = (bool)($input['auto_publish'] ?? $this->_autoPublishByDefault); - $this->_track->source = $this->_customTrackSource ?? 'direct_upload'; + $this->_track->source = $this->_customTrackSource ?? $source; $this->_track->save(); if (!$this->_isReplacingTrack) { diff --git a/app/Console/Commands/ImportPonify.php b/app/Console/Commands/ImportPonify.php new file mode 100644 index 00000000..656671ec --- /dev/null +++ b/app/Console/Commands/ImportPonify.php @@ -0,0 +1,316 @@ +error('Import aborted!'); + $this->error('Resume it from here using: --startAt=' . $this->currentFile); + $this->isInterrupted = true; + } + + /** + * Execute the console command. + * + * @return mixed + */ + public function handle() + { + // Most of this is the same as the old ImportMLPMA.php command with a few tweaks + // to use the new upload system and the newer version of Laravel + + pcntl_signal(SIGINT, [$this, 'handleInterrupt']); + + $ponifyPath = Config::get('ponyfm.files_directory').'/ponify'; + $tmpPath = Config::get('ponyfm.files_directory').'/tmp'; + + if (!File::exists($tmpPath)) { + File::makeDirectory($tmpPath); + } + + //========================================================================================================== + // Get the list of files and artists + //========================================================================================================== + $this->comment('Enumerating Ponify files...'); + $files = File::allFiles($ponifyPath); + $this->info(sizeof($files) . ' files found!'); + + $this->comment('Enumerating artists...'); + $artists = File::directories($ponifyPath); + $this->info(sizeof($artists) . ' artists found!'); + + $this->comment('Importing tracks...'); + + $totalFiles = sizeof($files); + + $fileToStartAt = (int)$this->option('startAt') - 1; + $this->comment("Skipping $fileToStartAt files..." . PHP_EOL); + + $files = array_slice($files, $fileToStartAt); + $this->currentFile = $fileToStartAt; + + foreach ($files as $file) { + $this->currentFile++; + + pcntl_signal_dispatch(); + + if ($this->isInterrupted) { + break; + } + + $this->comment('[' . $this->currentFile . '/' . $totalFiles . '] Importing track [' . $file->getFilename() . ']...'); + + if (in_array($file->getExtension(), $this->ignoredExtensions)) { + $this->comment('This is not an audio file! Skipping...' . PHP_EOL); + continue; + } + + $this->info('Path to file: ' . $file->getRelativePath()); + $path_components = explode(DIRECTORY_SEPARATOR, $file->getRelativePath()); + $artist_name = $path_components[0]; + $album_name = array_key_exists(1, $path_components) ? $path_components[1] : null; + + $this->info('Artist: ' . $artist_name); + $this->info('Album: ' . $album_name); + + //========================================================================================================== + // Analyse the track so we can find the MIME type and album art + //========================================================================================================== + + $getId3 = new getID3; + + // all tags read by getID3, including the cover art + $allTags = $getId3->analyze($file->getPathname()); + + // tags specific to a file format (ID3 or Atom), pre-normalization but with cover art removed + $rawTags = []; + + // normalized tags used by Pony.fm + $parsedTags = []; + + if ($file->getExtension() === 'mp3') { + list($parsedTags, $rawTags) = $this->getId3Tags($allTags); + } else { + if ($file->getExtension() === 'm4a') { + list($parsedTags, $rawTags) = $this->getAtomTags($allTags); + } + } + + //========================================================================================================== + // Create new user for the artist if one doesn't exist + //========================================================================================================== + + $artist = User::where('display_name', '=', $artist_name)->first(); + + if (!$artist) { + $artist = new User; + $artist->display_name = $artist_name; + $artist->email = null; + $artist->is_archived = true; + + $artist->slug = Str::slug($artist_name); + + $slugExists = User::where('slug', '=', $artist->slug)->first(); + if ($slugExists) { + $this->error('Horsefeathers! The slug ' . $artist->slug . ' is already taken!'); + $artist->slug = $artist->slug . '-' . Str::random(4); + } + + $artist->save(); + } + + //========================================================================================================== + // Grab the image and save it so we can pass that along when the track gets uploaded + //========================================================================================================== + + $this->comment('Extracting cover art!'); + $coverId = null; + if (array_key_exists('comments', $allTags) && array_key_exists('picture', $allTags['comments'])) { + $image = $allTags['comments']['picture'][0]; + + if ($image['image_mime'] === 'image/png') { + $extension = 'png'; + } else { + if ($image['image_mime'] === 'image/jpeg') { + $extension = 'jpg'; + } else { + if ($image['image_mime'] === 'image/gif') { + $extension = 'gif'; + } else { + $this->error('Unknown cover art format!'); + } + } + } + // write temporary image file + $imageFilename = $file->getFilename() . ".cover.$extension"; + $imageFilePath = "$tmpPath/" . $imageFilename; + File::put($imageFilePath, $image['data']); + + $imageFile = new UploadedFile($imageFilePath, $imageFilename, $image['image_mime'], null, null, true); + $cover = Image::upload($imageFile, $artist); + $coverId = $cover->id; + } else { + $this->comment('No cover art found!'); + } + + //========================================================================================================== + // Send the track into the upload system like a user just uploaded a track + //========================================================================================================== + + $this->comment('Transcoding the track!'); + Auth::loginUsingId($artist->id); + + $getID3 = new getID3; + $getID3->analyze($file->getPathname()); + + $mime = null; + + if (isset($getID3->info['mime_type'])) $mime = $getID3->info['mime_type']; + + $trackFile = new UploadedFile($file->getPathname(), $file->getFilename(), $mime, null, null, true); + + $upload = new UploadTrackCommand(true); + $upload->_file = $trackFile; + $result = $upload->execute(); + + if ($result->didFail()) { + $this->error(json_encode($result->getValidator()->messages()->getMessages(), JSON_PRETTY_PRINT)); + } else { + $track = Track::find($result->getResponse()['id']); + $track->license_id = 2; + $track->save(); + } + + echo PHP_EOL . PHP_EOL; + } + } + + /** + * @param array $rawTags + * @return array + */ + protected function getId3Tags($rawTags) + { + $tags = $rawTags['tags']['id3v2']; + $comment = null; + + if (isset($tags['comment'])) { + // The "comment" tag comes in with a badly encoded string index + // so its array key has to be used implicitly. + $key = array_keys($tags['comment'])[0]; + + // The comment may have a null byte at the end. trim() removes it. + $comment = trim($tags['comment'][$key]); + + // Replace the malformed comment with the "fixed" one. + unset($tags['comment'][$key]); + $tags['comment'][0] = $comment; + } + + return [ + [ + 'title' => $tags['title'][0], + 'artist' => $tags['artist'][0], + 'band' => isset($tags['band']) ? $tags['band'][0] : null, + 'genre' => isset($tags['genre']) ? $tags['genre'][0] : null, + 'track_number' => isset($tags['track_number']) ? $tags['track_number'][0] : null, + 'album' => isset($tags['album']) ? $tags['album'][0] : null, + 'year' => isset($tags['year']) ? (int)$tags['year'][0] : null, + 'comments' => $comment, + 'lyrics' => isset($tags['unsynchronised_lyric']) ? $tags['unsynchronised_lyric'][0] : null, + ], + $tags + ]; + } + + /** + * @param array $rawTags + * @return array + */ + protected function getAtomTags($rawTags) + { + $tags = $rawTags['tags']['quicktime']; + + $trackNumber = null; + if (isset($tags['track_number'])) { + $trackNumberComponents = explode('/', $tags['track_number'][0]); + $trackNumber = $trackNumberComponents[0]; + } + + return [ + [ + 'title' => $tags['title'][0], + 'artist' => $tags['artist'][0], + 'band' => isset($tags['band']) ? $tags['band'][0] : null, + 'album_artist' => isset($tags['album_artist']) ? $tags['album_artist'][0] : null, + 'genre' => isset($tags['genre']) ? $tags['genre'][0] : null, + 'track_number' => $trackNumber, + 'album' => isset($tags['album']) ? $tags['album'][0] : null, + 'year' => isset($tags['year']) ? (int)$tags['year'][0] : null, + 'comments' => isset($tags['comments']) ? $tags['comments'][0] : null, + 'lyrics' => isset($tags['lyrics']) ? $tags['lyrics'][0] : null, + ], + $tags + ]; + } +} diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 1c84903f..d2ef8ff7 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -46,7 +46,8 @@ class Kernel extends ConsoleKernel \Poniverse\Ponyfm\Console\Commands\MergeAccounts::class, \Poniverse\Ponyfm\Console\Commands\SyncPoniverseAccounts::class, \Poniverse\Ponyfm\Console\Commands\FixMLPMAImages::class, - \Poniverse\Ponyfm\Console\Commands\VersionFiles::class + \Poniverse\Ponyfm\Console\Commands\VersionFiles::class, + \Poniverse\Ponyfm\Console\Commands\ImportPonify::class, ]; /**