From c08fd29a45c9addcc2f41d278e5fa3bcd7ecb0ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orhun=20Parmaks=C4=B1z?= Date: Tue, 12 Oct 2021 19:35:06 +0300 Subject: [PATCH] feat(paste): make duplicate uploads optional (#7) --- Cargo.lock | 67 ++++++++++++++++++++++++++++++++++++++------- Cargo.toml | 1 + README.md | 1 + config.toml | 1 + src/config.rs | 2 ++ src/file.rs | 75 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 3 +++ src/server.rs | 19 +++++++++++++ src/util.rs | 38 ++++++++++++++++++++++++++ 9 files changed, 197 insertions(+), 10 deletions(-) create mode 100644 src/file.rs diff --git a/Cargo.lock b/Cargo.lock index 47a2b57..c18bd89 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -980,6 +980,15 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" +[[package]] +name = "js-sys" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cc9ffccd38c451a86bf13657df244e9c3f37493cce8e5e21e940963777acc84" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "kernel32-sys" version = "0.2.2" @@ -1493,6 +1502,21 @@ dependencies = [ "quick-error", ] +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi 0.3.9", +] + [[package]] name = "rustc_version" version = "0.2.3" @@ -1532,6 +1556,7 @@ dependencies = [ "petname", "rand 0.8.4", "regex", + "ring", "serde", "serde_regex", "url", @@ -1686,6 +1711,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "standback" version = "0.2.17" @@ -2060,6 +2091,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "url" version = "2.2.2" @@ -2136,9 +2173,9 @@ checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" [[package]] name = "wasm-bindgen" -version = "0.2.76" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce9b1b516211d33767048e5d47fa2a381ed8b76fc48d2ce4aa39877f9f183e0" +checksum = "632f73e236b219150ea279196e54e610f5dbafa5d61786303d4da54f84e47fce" dependencies = [ "cfg-if 1.0.0", "wasm-bindgen-macro", @@ -2146,9 +2183,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.76" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe8dc78e2326ba5f845f4b5bf548401604fa20b1dd1d365fb73b6c1d6364041" +checksum = "a317bf8f9fba2476b4b2c85ef4c4af8ff39c3c7f0cdfeed4f82c34a880aa837b" dependencies = [ "bumpalo", "lazy_static", @@ -2161,9 +2198,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.76" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44468aa53335841d9d6b6c023eaab07c0cd4bddbcfdee3e2bb1e8d2cb8069fef" +checksum = "d56146e7c495528bf6587663bea13a8eb588d39b36b679d83972e1a2dbbdacf9" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2171,9 +2208,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.76" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0195807922713af1e67dc66132c7328206ed9766af3858164fb583eedc25fbad" +checksum = "7803e0eea25835f8abdc585cd3021b3deb11543c6fe226dcd30b228857c5c5ab" dependencies = [ "proc-macro2", "quote", @@ -2184,9 +2221,19 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.76" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acdb075a845574a1fa5f09fd77e43f7747599301ea3417a9fbffdeedfc1f4a29" +checksum = "0237232789cf037d5480773fe568aac745bfe2afbc11a863e97901780a6b47cc" + +[[package]] +name = "web-sys" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38eb105f1c59d9eaa6b5cdc92b859d85b926e82cb2e0945cd0c9259faa6fe9fb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] [[package]] name = "widestring" diff --git a/Cargo.toml b/Cargo.toml index 3d3d56b..ad181f4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ regex = "1.5.4" serde_regex = "1.1.0" humantime = "2.1.0" glob = "0.3.0" +ring = "0.16.20" [dependencies.config] version = "0.11.0" diff --git a/README.md b/README.md index a4ba5d1..bcf9354 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ some text - supports one shot links (can only be viewed once) - guesses MIME types - supports overriding and blacklisting + - no duplicate uploads (optional) - Single binary - [binary releases](https://github.com/orhun/rustypaste/releases) - Easy to deploy diff --git a/config.toml b/config.toml index 5f57b77..5f83dca 100644 --- a/config.toml +++ b/config.toml @@ -22,3 +22,4 @@ mime_blacklist = [ "application/java-archive", "application/java-vm" ] +duplicate_files = false diff --git a/src/config.rs b/src/config.rs index 8dbb168..9035af4 100644 --- a/src/config.rs +++ b/src/config.rs @@ -37,6 +37,8 @@ pub struct PasteConfig { pub mime_override: Vec, /// Media type blacklist. pub mime_blacklist: Vec, + /// Allow duplicate uploads + pub duplicate_files: Option, } impl Config { diff --git a/src/file.rs b/src/file.rs new file mode 100644 index 0000000..e33d359 --- /dev/null +++ b/src/file.rs @@ -0,0 +1,75 @@ +use crate::util; +use actix_web::{error, Error as ActixError}; +use glob::glob; +use std::convert::TryFrom; +use std::fs::File as OsFile; +use std::path::{Path, PathBuf}; + +/// [`PathBuf`] wrapper for storing checksums. +#[derive(Debug)] +pub struct File { + /// Path of the file. + pub path: PathBuf, + /// SHA256 checksum. + pub sha256sum: String, +} + +/// Directory that contains [`File`]s. +pub struct Directory { + /// Files in the directory. + pub files: Vec, +} + +impl<'a> TryFrom<&'a Path> for Directory { + type Error = ActixError; + fn try_from(directory: &'a Path) -> Result { + let files = glob(directory.join("**").join("*").to_str().ok_or_else(|| { + error::ErrorInternalServerError("directory contains invalid characters") + })?) + .map_err(error::ErrorInternalServerError)? + .filter_map(Result::ok) + .filter(|path| !path.is_dir()) + .filter_map(|path| match OsFile::open(&path) { + Ok(file) => Some((path, file)), + _ => None, + }) + .filter_map(|(path, file)| match util::sha256_digest(file) { + Ok(sha256sum) => Some(File { path, sha256sum }), + _ => None, + }) + .collect(); + Ok(Self { files }) + } +} + +impl Directory { + /// Returns the file that matches the given checksum. + pub fn get_file>(self, sha256sum: S) -> Option { + self.files + .into_iter() + .find(|file| file.sha256sum == sha256sum.as_ref()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_file_checksum() -> Result<(), ActixError> { + assert_eq!( + "rustypaste_logo.png", + Directory::try_from( + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("img") + .as_path() + )? + .get_file("2073f6f567dcba3b468c568d29cf8ed2e9d3f0f7305b9ab1b5a22861f5922e61") + .unwrap() + .path + .file_name() + .unwrap() + ); + Ok(()) + } +} diff --git a/src/lib.rs b/src/lib.rs index 06a7ddc..9793cc6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,6 +19,9 @@ pub mod auth; /// Storage handler. pub mod paste; +/// File metadata handler. +pub mod file; + /// Media type handler. pub mod mime; diff --git a/src/server.rs b/src/server.rs index 5b51637..c331ee9 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1,5 +1,6 @@ use crate::auth; use crate::config::Config; +use crate::file::Directory; use crate::header::{self, ContentDisposition}; use crate::mime; use crate::paste::{Paste, PasteType}; @@ -101,6 +102,24 @@ async fn upload( log::warn!("{} sent zero bytes", host); return Err(error::ErrorBadRequest("invalid file size")); } + if paste_type != PasteType::Oneshot && !config.paste.duplicate_files.unwrap_or(true) { + let bytes_checksum = util::sha256_digest(&*bytes)?; + if let Some(file) = Directory::try_from(config.server.upload_path.as_path())? + .get_file(bytes_checksum) + { + urls.push(format!( + "{}://{}/{}\n", + connection.scheme(), + connection.host(), + file.path + .file_name() + .map(|v| v.to_string_lossy()) + .unwrap_or_default() + .to_string() + )); + continue; + } + } let bytes_unit = Byte::from_bytes(bytes.len() as u128).get_appropriate_unit(false); let paste = Paste { data: bytes.to_vec(), diff --git a/src/util.rs b/src/util.rs index d00da24..9c265f6 100644 --- a/src/util.rs +++ b/src/util.rs @@ -1,5 +1,7 @@ use actix_web::{error, Error as ActixError}; use glob::glob; +use ring::digest::{Context, SHA256}; +use std::io::{BufReader, Read}; use std::path::PathBuf; use std::time::Duration; use std::time::{SystemTime, UNIX_EPOCH}; @@ -41,6 +43,29 @@ pub fn glob_match_file(mut path: PathBuf) -> Result { Ok(path) } +/// Returns the SHA256 digest of the given input. +pub fn sha256_digest(input: R) -> Result { + let mut reader = BufReader::new(input); + let mut context = Context::new(&SHA256); + let mut buffer = [0; 1024]; + loop { + let bytes_read = reader.read(&mut buffer)?; + if bytes_read != 0 { + context.update(&buffer[..bytes_read]); + } else { + break; + } + } + Ok(context + .finish() + .as_ref() + .iter() + .collect::>() + .iter() + .map(|byte| format!("{:02x}", byte)) + .collect::()) +} + #[cfg(test)] mod tests { use super::*; @@ -72,4 +97,17 @@ mod tests { Ok(()) } + + #[test] + fn test_sha256sum() -> Result<(), ActixError> { + assert_eq!( + "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", + sha256_digest(String::from("test").as_bytes())? + ); + assert_eq!( + "2fc36f72540bb9145e95e67c41dccdc440c95173257032e32e111ebd7b6df960", + sha256_digest(env!("CARGO_PKG_NAME").as_bytes())? + ); + Ok(()) + } }