error('Import aborted!'); $this->error('Resume it from here using: --startAt=' . $this->currentFile); $this->isInterrupted = true; } /** * Execute the console command. * * @return mixed */ public function handle() { // Most of this is the same as the old ImportMLPMA.php command with a few tweaks // to use the new upload system and the newer version of Laravel pcntl_signal(SIGINT, [$this, 'handleInterrupt']); $ponifyPath = Config::get('ponyfm.ponify_directory'); $tmpPath = Config::get('ponyfm.files_directory').'/tmp'; if (!File::exists($tmpPath)) { File::makeDirectory($tmpPath); } $UNKNOWN_GENRE = Genre::firstOrCreate([ 'name' => 'Unknown', 'slug' => 'unknown' ]); //========================================================================================================== // Get the list of files and artists //========================================================================================================== $this->comment('Enumerating Ponify files...'); $files = File::allFiles($ponifyPath); $this->info(sizeof($files) . ' files found!'); $this->comment('Enumerating artists...'); $artists = File::directories($ponifyPath); $this->info(sizeof($artists) . ' artists found!'); $this->comment('Importing tracks...'); $totalFiles = sizeof($files); $fileToStartAt = (int)$this->option('startAt') - 1; $this->comment("Skipping $fileToStartAt files..." . PHP_EOL); $files = array_slice($files, $fileToStartAt); $this->currentFile = $fileToStartAt; foreach ($files as $file) { $this->currentFile++; pcntl_signal_dispatch(); if ($this->isInterrupted) { break; } $this->comment('[' . $this->currentFile . '/' . $totalFiles . '] Importing track [' . $file->getFilename() . ']...'); if (!in_array($file->getExtension(), $this->allowedExtensions)) { $this->comment('This is not an audio file! Skipping...' . PHP_EOL); continue; } $this->info('Path to file: ' . $file->getRelativePath()); $path_components = explode(DIRECTORY_SEPARATOR, $file->getRelativePath()); $artist_name = $path_components[0]; $album_name = array_key_exists(1, $path_components) ? $path_components[1] : null; $this->info('Artist: ' . $artist_name); $this->info('Album: ' . $album_name); //========================================================================================================== // Analyse the track so we can find the MIME type and album art //========================================================================================================== $getId3 = new getID3; // all tags read by getID3, including the cover art $allTags = $getId3->analyze($file->getPathname()); // tags specific to a file format (ID3 or Atom), pre-normalization but with cover art removed $rawTags = []; // normalized tags used by Pony.fm $parsedTags = []; list($parsedTags, $rawTags) = $this->parseTags($file, $allTags); //$imageFilename = $file->getFilename() . ".tags.txt"; //$imageFilePath = "$tmpPath/" . $imageFilename; //File::put($imageFilePath, print_r($allTags, true)); // Parse out the hyphen in the file name $hyphen = ' - '; if (is_null($parsedTags['title'])) { $fileName = pathinfo($file->getFilename(), PATHINFO_FILENAME); $parsedTags['title'] = substr($fileName, strpos($fileName, $hyphen) + strlen($hyphen)); } $this->info('Title: ' . $parsedTags['title']); //========================================================================================================== // Determine the release date. //========================================================================================================== $modifiedDate = Carbon::createFromTimeStampUTC(File::lastModified($file->getPathname())); $taggedYear = $parsedTags['year']; $this->info('Modification year: '.$modifiedDate->year); $this->info('Tagged year: '.$taggedYear); if ($taggedYear !== null && $modifiedDate->year === $taggedYear) { $releasedAt = $modifiedDate; } else if ($taggedYear !== null && $modifiedDate->year !== $taggedYear) { $this->error('Release years don\'t match! Using the tagged year...'); $releasedAt = Carbon::create($taggedYear); } else { // $taggedYear is null $this->error('This track isn\'t tagged with its release year! Using the track\'s last modified date...'); $releasedAt = $modifiedDate; } // This is later used by the classification/publishing script to determine the publication date. $parsedTags['released_at'] = $releasedAt->toDateTimeString(); //========================================================================================================== // Does this track have vocals? //========================================================================================================== $isVocal = $parsedTags['lyrics'] !== null; //========================================================================================================== // Determine the genre //========================================================================================================== $genreName = $parsedTags['genre']; $this->info('Genre: '.$genreName); if ($genreName) { $genre = Genre::where('name', '=', $genreName)->first(); if ($genre) { $genreId = $genre->id; } else { $genre = new Genre(); $genre->name = $genreName; $genre->slug = Str::slug($genreName); $genre->save(); $genreId = $genre->id; $this->comment('Created a new genre!'); } } else { $genreId = $UNKNOWN_GENRE->id; // "Unknown" genre ID } //========================================================================================================== // Check to see if we have this track already, if so, compare hashes of the two files //========================================================================================================== $artist = User::where('display_name', '=', $artist_name)->first(); $artistId = null; $this->comment("Checking for duplicates"); if ($artist) { $artistId = $artist->id; } $existingTrack = Track::where('title', '=', $parsedTags['title']) ->where('user_id', '=', $artistId) ->first(); if ($existingTrack) { // We got one!! // Ok, let's not get too excited // First let's see if we have a matching file type $importFormat = $this->getFormat($file->getExtension()); if ($importFormat == null) { // No idea what this is, skip file $this->comment(sprintf("Not an audio file (%s), skipping...", $importFormat)); continue; } $existingFile = null; foreach ($existingTrack->trackFiles as $trackFile) { if ($trackFile->format == $importFormat) { $existingFile = $trackFile; } } if ($existingFile === null) { // Can't find a matching format // See if we have a higher quality source file if (Track::$Formats[$importFormat]['is_lossless']) { // Source is lossless, is the existing track lossy? if ($existingFile->isMasterLossy()) { // Cool! Let's replace it $this->comment('Replacing (' . $existingTrack->id . ') ' . $existingTrack->title); $this->replaceTrack($file, $existingTrack, $artist, $allTags['mime_type']); continue; } } continue; } else { $this->comment("Found existing file"); // Found a matching format, are they the same? // Before we check it, see if it came from MLPMA // We're only replacing tracks with the same format if they're archived $mlpmaTrack = DB::table('mlpma_tracks')->where('track_id', '=', $existingTrack->id)->first(); if (!is_null($mlpmaTrack)) { $getId3_source = new getID3; $getId3_source->option_md5_data = true; $getId3_source->option_md5_data_source = true; $sourceWithMd5 = $getId3_source->analyze($file->getPathname()); $getId3_existing = new getID3; $getId3_existing->option_md5_data = true; $getId3_existing->option_md5_data_source = true; $existingFileTags = $getId3_existing->analyze($existingFile->getFile()); $importHash = array_key_exists('md5_data_source', $sourceWithMd5) ? $sourceWithMd5['md5_data_source'] : $sourceWithMd5['md5_data']; $targetHash = array_key_exists('md5_data_source', $existingFileTags) ? $existingFileTags['md5_data_source'] : $existingFileTags['md5_data']; $this->info("Archive hash: " . $importHash); $this->info("Pony.fm hash: " . $targetHash); if ($importHash == $targetHash) { // Audio is identical, no need to reupload // We can update the metadata though $this->comment("Versions are the same. Updating metadata...\n"); $changedMetadata = false; if (strlen($existingTrack->description) < strlen($parsedTags['comments'])) { $existingTrack->description = $parsedTags['comments']; $changedMetadata = true; $this->comment("Updated description"); } if (strlen($existingTrack->lyrics) < strlen($parsedTags['lyrics'])) { $existingTrack->lyrics = $parsedTags['lyrics']; $changedMetadata = true; $this->comment("Updated lyrics"); } if ($changedMetadata) $existingTrack->save(); continue; } else { // Audio is different, let's replace it $this->comment('Replacing (' . $existingTrack->id . ') ' . $existingTrack->title); $this->replaceTrack($file, $existingTrack, $artist, $allTags['mime_type']); continue; } } else { $this->comment("Not replacing, user uploaded"); // We can update the metadata though $changedMetadata = false; if (strlen($existingTrack->description) < strlen($parsedTags['comments'])) { $existingTrack->description = $parsedTags['comments']; $changedMetadata = true; $this->comment("Updated description"); } if (strlen($existingTrack->lyrics) < strlen($parsedTags['lyrics'])) { $existingTrack->lyrics = $parsedTags['lyrics']; $changedMetadata = true; $this->comment("Updated lyrics"); } if ($changedMetadata) $existingTrack->save(); continue; } } } else { $this->comment("No duplicates"); } //========================================================================================================== // Create new user for the artist if one doesn't exist //========================================================================================================== if (!$artist) { $artist = new User; $artist->display_name = $artist_name; $artist->email = null; $artist->is_archived = true; $artist->slug = Str::slug($artist_name); $slugExists = User::where('slug', '=', $artist->slug)->first(); if ($slugExists) { $this->error('Horsefeathers! The slug ' . $artist->slug . ' is already taken!'); $artist->slug = $artist->slug . '-' . Str::random(4); } $artist->save(); } //========================================================================================================== // Grab the image and save it so we can pass that along when the track gets uploaded //========================================================================================================== $this->comment('Extracting cover art!'); $coverId = null; $image = null; if (array_key_exists('comments', $allTags) && array_key_exists('picture', $allTags['comments'])) { $image = $allTags['comments']['picture'][0]; } else if (array_key_exists('id3v2', $allTags) && array_key_exists('APIC', $allTags['id3v2'])) { $image = $allTags['id3v2']['APIC'][0]; } if ($image !== null) { if ($image['image_mime'] === 'image/png') { $extension = 'png'; } else { if ($image['image_mime'] === 'image/jpeg') { $extension = 'jpg'; } else { if ($image['image_mime'] === 'image/gif') { $extension = 'gif'; } else { $this->error('Unknown cover art format!'); } } } // write temporary image file $imageFilename = $file->getFilename() . ".cover.$extension"; $imageFilePath = "$tmpPath/" . $imageFilename; File::put($imageFilePath, $image['data']); $imageFile = new UploadedFile($imageFilePath, $imageFilename, $image['image_mime'], null, null, true); $cover = Image::upload($imageFile, $artist); $coverId = $cover->id; } else { $this->comment('No cover art found!'); } //========================================================================================================== // Is this part of an album? //========================================================================================================== $albumId = null; $albumName = $parsedTags['album']; if ($albumName !== null) { $album = Album::where('user_id', '=', $artist->id) ->where('title', '=', $albumName) ->first(); if (!$album) { $album = new Album; $album->title = $albumName; $album->user_id = $artist->id; $album->cover_id = $coverId; $album->save(); } $albumId = $album->id; } //========================================================================================================== // Send the track into the upload system like a user just uploaded a track //========================================================================================================== $this->comment('Transcoding the track!'); Auth::loginUsingId($artist->id); $mime = $allTags['mime_type']; File::copy($file->getPathname(), "$tmpPath/" . $file->getFilename()); $trackFile = new UploadedFile("$tmpPath/" . $file->getFilename(), $file->getFilename(), $mime, null, null, true); $upload = new UploadTrackCommand(true); $upload->_file = $trackFile; $result = $upload->execute(); if ($result->didFail()) { $this->error(json_encode($result->getValidator()->messages()->getMessages(), JSON_PRETTY_PRINT)); } else { $track = Track::find($result->getResponse()['id']); $track->cover_id = $coverId; $track->album_id = $albumId; $track->genre_id = $genreId; $track->track_number = $parsedTags['track_number']; $track->released_at = $releasedAt; $track->is_downloadable = true; $track->is_vocal = $isVocal; $track->license_id = 2; if (!is_null($parsedTags['comments'])) { $track->title = $parsedTags['comments']; } if (!is_null($parsedTags['lyrics'])) { $track->title = $parsedTags['lyrics']; } if (!is_null($parsedTags['title'])) { $track->title = $parsedTags['title']; } $track->save(); // If we made it to here, the track is intact! Log the import. DB::table('ponify_tracks') ->insert([ 'track_id' => $result->getResponse()['id'], 'path' => $file->getRelativePath(), 'filename' => $file->getFilename(), 'extension' => $file->getExtension(), 'imported_at' => Carbon::now(), 'parsed_tags' => json_encode($parsedTags), 'raw_tags' => json_encode($rawTags), ]); } echo PHP_EOL . PHP_EOL; } } protected function hashAudio($filepath) { $hash = hash_file('crc32b', $filepath); $array = unpack('N', pack('H*', $hash)); return $array[1]; } protected function getFormat($extension) { foreach(Track::$Formats as $name => $format) { if ($format['extension'] == $extension) { return $name; } } return null; } public function parseTags($file, $allTags) { $audioCodec = $file->getExtension(); //========================================================================================================== // Extract the original tags. //========================================================================================================== // $rawTags => tags specific to a file format (ID3 or Atom), pre-normalization but with cover art removed // $parsedTags => normalized tags used by Pony.fm if ($audioCodec === 'mp3') { list($parsedTags, $rawTags) = $this->getId3Tags($allTags); } elseif (Str::startsWith($audioCodec, ['aac', 'alac'])) { list($parsedTags, $rawTags) = $this->getAtomTags($allTags); } elseif (in_array($audioCodec, ['vorbis', 'flac'])) { list($parsedTags, $rawTags) = $this->getVorbisTags($allTags); } elseif (Str::startsWith($audioCodec, ['pcm', 'adpcm'])) { list($parsedTags, $rawTags) = $this->getAtomTags($allTags); } else { // Assume the file is untagged if it's in an unknown format. $parsedTags = [ 'title' => null, 'artist' => null, 'band' => null, 'genre' => null, 'track_number' => null, 'album' => null, 'year' => null, 'release_date' => null, 'comments' => null, 'lyrics' => null, ]; $rawTags = []; } return [$parsedTags, $rawTags]; } /** * @param array $rawTags * @return array */ protected function getId3Tags($rawTags) { $tags = []; if (array_key_exists('tags', $rawTags)) { if (array_key_exists('id3v2', $rawTags['tags'])) { $tags = $rawTags['tags']['id3v2']; } elseif (array_key_exists('id3v1', $rawTags['tags'])) { $tags = $rawTags['tags']['id3v1']; } else { $tags = []; } } else { $tags = []; } $comment = null; if (isset($tags['comment'])) { // The "comment" tag comes in with a badly encoded string index // so its array key has to be used implicitly. $key = array_keys($tags['comment'])[0]; // The comment may have a null byte at the end. trim() removes it. $comment = trim($tags['comment'][$key]); // Replace the malformed comment with the "fixed" one. unset($tags['comment'][$key]); $tags['comment'][0] = $comment; } $trackNumber = 1; if (isset($tags['track_number'])) { $trackNumberComponents = explode('/', $tags['track_number'][0]); $trackNumber = $trackNumberComponents[0]; } return [ [ 'title' => isset($tags['title']) ? $tags['title'][0] : null, 'artist' => isset($tags['artist']) ? $tags['artist'][0] : null, 'band' => isset($tags['band']) ? $tags['band'][0] : null, 'genre' => isset($tags['genre']) ? $tags['genre'][0] : null, 'track_number' => $trackNumber, 'album' => isset($tags['album']) ? $tags['album'][0] : null, 'year' => isset($tags['year']) ? (int) $tags['year'][0] : null, 'release_date' => isset($tags['release_date']) ? $this->parseDateString($tags['release_date'][0]) : null, 'comments' => $comment, 'lyrics' => isset($tags['unsynchronised_lyric']) ? $tags['unsynchronised_lyric'][0] : null, ], $tags ]; } /** * @param array $rawTags * @return array */ protected function getAtomTags($rawTags) { if (array_key_exists('tags', $rawTags) && array_key_exists('quicktime', $rawTags['tags'])) { $tags = $rawTags['tags']['quicktime']; } else { $tags = []; } $trackNumber = 1; if (isset($tags['track_number'])) { $trackNumberComponents = explode('/', $tags['track_number'][0]); $trackNumber = $trackNumberComponents[0]; } if (isset($tags['release_date'])) { $releaseDate = $this->parseDateString($tags['release_date'][0]); } elseif (isset($tags['creation_date'])) { $releaseDate = $this->parseDateString($tags['creation_date'][0]); } else { $releaseDate = null; } return [ [ 'title' => isset($tags['title']) ? $tags['title'][0] : null, 'artist' => isset($tags['artist']) ? $tags['artist'][0] : null, 'band' => isset($tags['band']) ? $tags['band'][0] : null, 'album_artist' => isset($tags['album_artist']) ? $tags['album_artist'][0] : null, 'genre' => isset($tags['genre']) ? $tags['genre'][0] : null, 'track_number' => $trackNumber, 'album' => isset($tags['album']) ? $tags['album'][0] : null, 'year' => isset($tags['year']) ? (int) $tags['year'][0] : null, 'release_date' => $releaseDate, 'comments' => isset($tags['comments']) ? $tags['comments'][0] : null, 'lyrics' => isset($tags['lyrics']) ? $tags['lyrics'][0] : null, ], $tags ]; } /** * @param array $rawTags * @return array */ protected function getVorbisTags($rawTags) { if (array_key_exists('tags', $rawTags) && array_key_exists('vorbiscomment', $rawTags['tags'])) { $tags = $rawTags['tags']['vorbiscomment']; } else { $tags = []; } $trackNumber = 1; if (isset($tags['track_number'])) { $trackNumberComponents = explode('/', $tags['track_number'][0]); $trackNumber = $trackNumberComponents[0]; } return [ [ 'title' => isset($tags['title']) ? $tags['title'][0] : null, 'artist' => isset($tags['artist']) ? $tags['artist'][0] : null, 'band' => isset($tags['band']) ? $tags['band'][0] : null, 'album_artist' => isset($tags['album_artist']) ? $tags['album_artist'][0] : null, 'genre' => isset($tags['genre']) ? $tags['genre'][0] : null, 'track_number' => $trackNumber, 'album' => isset($tags['album']) ? $tags['album'][0] : null, 'year' => isset($tags['year']) ? (int) $tags['year'][0] : null, 'release_date' => isset($tags['date']) ? $this->parseDateString($tags['date'][0]) : null, 'comments' => isset($tags['comments']) ? $tags['comments'][0] : null, 'lyrics' => isset($tags['lyrics']) ? $tags['lyrics'][0] : null, ], $tags ]; } /** * Parses a potentially-partial date string into a proper date object. * * The tagging formats we deal with base their date format on ISO 8601, but * the timestamp may be incomplete. * * @link https://code.google.com/p/mp4v2/wiki/iTunesMetadata * @link https://wiki.xiph.org/VorbisComment#Date_and_time * @link http://id3.org/id3v2.4.0-frames * * @param string $dateString * @return null|Carbon */ protected function parseDateString(string $dateString) { switch (Str::length($dateString)) { // YYYY case 4: try { return Carbon::createFromFormat('Y', $dateString) ->month(1) ->day(1); } catch (\InvalidArgumentException $e) { return null; } // YYYY-MM case 7: try { return Carbon::createFromFormat('Y m', str_replace("-", " ", $dateString)) ->day(1); } catch (\InvalidArgumentException $e) { return null; } // YYYY-MM-DD case 10: try { return Carbon::createFromFormat('Y m d', str_replace("-", " ", $dateString)); } catch (\InvalidArgumentException $e) { return null; } break; default: // We might have an ISO-8601 string in our hooves. // If not, give up. try { return Carbon::createFromFormat(Carbon::ISO8601, $dateString); } catch (\InvalidArgumentException $e) { return null; } } } protected function replaceTrack($toBeUploaded, $targetTrack, $artist, $mime) { Auth::loginUsingId($artist->id); $trackFile = new UploadedFile($toBeUploaded->getPathname(), $toBeUploaded->getFilename(), $mime, null, null, true); $upload = new UploadTrackCommand(true, false, null, false, $targetTrack->getNextVersion(), $targetTrack); $upload->_file = $trackFile; $result = $upload->execute(); if ($result->didFail()) { $this->error(json_encode($result->getValidator()->messages()->getMessages(), JSON_PRETTY_PRINT)); } else { $track = Track::find($result->getResponse()['id']); $track->license_id = 2; $track->save(); } } }