From f50f5bd65c6bd8e43eb58b3e168a0f36937ff324 Mon Sep 17 00:00:00 2001 From: Joakim Soderlund Date: Sun, 21 Jul 2019 16:32:51 +0000 Subject: [PATCH] Add fetcher module for archive reading --- Cargo.lock | 70 +++++++++++++++++++++++++++++++++ Cargo.toml | 5 +++ src/fetcher.rs | 102 +++++++++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 1 + 4 files changed, 178 insertions(+) create mode 100644 src/fetcher.rs diff --git a/Cargo.lock b/Cargo.lock index 0245136..a06ce06 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,10 +1,25 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +[[package]] +name = "adler32" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "autocfg" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "byteorder" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cfg-if" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "chrono" version = "0.4.7" @@ -17,6 +32,14 @@ dependencies = [ "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "crc32fast" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "fimfareader" version = "0.1.0" @@ -25,6 +48,7 @@ dependencies = [ "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.97 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)", + "zip 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -42,6 +66,18 @@ name = "libc" version = "0.2.60" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "libflate" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "adler32 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rle-decode-fast 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "take_mut 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "num-integer" version = "0.1.41" @@ -59,6 +95,11 @@ dependencies = [ "autocfg 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "podio" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "proc-macro2" version = "0.4.30" @@ -80,6 +121,11 @@ name = "redox_syscall" version = "0.1.56" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "rle-decode-fast" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "ryu" version = "1.0.0" @@ -123,6 +169,11 @@ dependencies = [ "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "take_mut" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "time" version = "0.1.42" @@ -157,24 +208,43 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "zip" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libflate 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", + "podio 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [metadata] +"checksum adler32 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7e522997b529f05601e05166c07ed17789691f562762c7f3b987263d2dedee5c" "checksum autocfg 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "22130e92352b948e7e82a49cdb0aa94f2211761117f29e052dd397c1ac33542b" +"checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5" +"checksum cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "b486ce3ccf7ffd79fdeb678eac06a9e6c09fc88d33836340becb8fffe87c5e33" "checksum chrono 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "77d81f58b7301084de3b958691458a53c3f7e0b1d702f77e550b6a88e3a88abe" +"checksum crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1" "checksum hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "805026a5d0141ffc30abb3be3173848ad46a1b1664fe632428479619a3644d77" "checksum itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "501266b7edd0174f8530248f87f99c88fbe60ca4ef3dd486835b8d8d53136f7f" "checksum libc 0.2.60 (registry+https://github.com/rust-lang/crates.io-index)" = "d44e80633f007889c7eff624b709ab43c92d708caad982295768a7b13ca3b5eb" +"checksum libflate 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)" = "90c6f86f4b0caa347206f916f8b687b51d77c6ef8ff18d52dd007491fd580529" "checksum num-integer 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)" = "b85e541ef8255f6cf42bbfe4ef361305c6c135d10919ecc26126c4e5ae94bc09" "checksum num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "6ba9a427cfca2be13aa6f6403b0b7e7368fe982bfa16fccc450ce74c46cd9b32" +"checksum podio 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "780fb4b6698bbf9cf2444ea5d22411cef2953f0824b98f33cf454ec5615645bd" "checksum proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)" = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" "checksum quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" "checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" +"checksum rle-decode-fast 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cabe4fa914dec5870285fa7f71f602645da47c486e68486d2b4ceb4a343e90ac" "checksum ryu 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c92464b447c0ee8c4fb3824ecc8383b81717b9f1e74ba2e72540aef7b9f82997" "checksum serde 1.0.97 (registry+https://github.com/rust-lang/crates.io-index)" = "d46b3dfedb19360a74316866cef04687cd4d6a70df8e6a506c63512790769b72" "checksum serde_derive 1.0.97 (registry+https://github.com/rust-lang/crates.io-index)" = "c22a0820adfe2f257b098714323563dd06426502abbbce4f51b72ef544c5027f" "checksum serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)" = "051c49229f282f7c6f3813f8286cc1e3323e8051823fce42c7ea80fe13521704" "checksum syn 0.15.40 (registry+https://github.com/rust-lang/crates.io-index)" = "bc945221ccf4a7e8c31222b9d1fc77aefdd6638eb901a6ce457a3dc29d4c31e8" +"checksum take_mut 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" "checksum time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" "checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" "checksum winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "f10e386af2b13e47c89e7236a7a14a086791a2b88ebad6df9bf42040195cf770" "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +"checksum zip 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c18fc320faf909036e46ac785ea827f72e485304877faf1a3a39538d3714dbc3" diff --git a/Cargo.toml b/Cargo.toml index da1eb73..4814f9d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,3 +20,8 @@ features = ["derive"] [dependencies.serde_json] version = "*" + +[dependencies.zip] +version = "*" +features = ["deflate"] +default-features = false diff --git a/src/fetcher.rs b/src/fetcher.rs new file mode 100644 index 0000000..ce73b3b --- /dev/null +++ b/src/fetcher.rs @@ -0,0 +1,102 @@ +//! Archive fetcher. + +use std::fs::File; +use std::io::{BufReader, Read, Seek}; +use std::path::Path; +use std::sync::Mutex; + +use zip::read::ZipArchive; +use zip::result::ZipError; + +use crate::error::{Error, Result}; +use crate::parser::parse; +use crate::story::Story; + +pub struct Fetcher +where + T: Read + Seek, +{ + archive: Mutex>, + index: Vec, +} + +impl Fetcher> { + pub fn from(path: impl AsRef) -> Result { + use Error::*; + + let file = File::open(path).map_err(|e| match e { + _ => SourceError("Could not open archive file."), + })?; + + Self::from_reader(BufReader::with_capacity(8_000_000, file)) + } +} + +impl Fetcher +where + T: Read + Seek, +{ + pub fn from_reader(reader: T) -> Result { + let mut handle = Self::open(reader)?; + let index = Self::load(&mut handle)?; + let archive = Mutex::new(handle); + + Ok(Self { archive, index }) + } + + fn open(archive: T) -> Result> { + use Error::*; + use ZipError::*; + + ZipArchive::new(archive).map_err(|e| match e { + InvalidArchive(e) => SourceError(e), + UnsupportedArchive(e) => SourceError(e), + _ => SourceError("Could not read archive."), + }) + } + + fn load(archive: &mut ZipArchive) -> Result> { + use Error::*; + use ZipError::*; + + let file = archive.by_name("index.json").map_err(|e| match e { + FileNotFound => SourceError("Missing archive index."), + _ => SourceError("Could not open archive index."), + })?; + + parse(BufReader::with_capacity(8_000_000, file)) + } + + pub fn fetch<'a>(&'a self, key: i64) -> Option<&'a Story> { + match self.index.binary_search_by_key(&key, |story| story.id) { + Ok(i) => self.index.get(i), + Err(_) => None, + } + } + + pub fn read<'a>(&self, path: &str) -> Result> { + use Error::*; + use ZipError::*; + + let mut archive = self.archive.lock().map_err(|e| match e { + _ => SourceError("Could not acquire archive lock."), + })?; + + let mut file = archive.by_name(path).map_err(|e| match e { + FileNotFound => SourceError("File not found."), + _ => SourceError("Could not open file."), + })?; + + let size = file.size() as usize; + let mut buf = Vec::with_capacity(size); + + file.read_to_end(&mut buf) + .map_err(|_| SourceError("Could not read file."))?; + + Ok(buf) + } + + pub fn iter<'a>(&'a self) -> impl Iterator { + self.index.iter() + } +} diff --git a/src/main.rs b/src/main.rs index d675c0b..43f06fd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,7 @@ //! Main module. pub mod error; +pub mod fetcher; pub mod parser; pub mod story;