From 7a6842e1819e360defe2c2788f54bdc456428dc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orhun=20Parmaks=C4=B1z?= Date: Sat, 6 Nov 2021 23:55:55 +0300 Subject: [PATCH] feat(paste): support pasting files from remote URLs --- Cargo.lock | 174 +++++++++++++++++++++++++------------------------- Cargo.toml | 4 +- README.md | 13 ++-- src/main.rs | 7 ++ src/paste.rs | 78 +++++++++++++++++++++- src/server.rs | 28 ++++++-- 6 files changed, 202 insertions(+), 102 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8390e17..9bac1f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,7 +14,7 @@ dependencies = [ "futures-sink", "log", "pin-project 0.4.28", - "tokio 0.2.25", + "tokio", "tokio-util", ] @@ -25,7 +25,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "177837a10863f15ba8d3ae3ec12fac1099099529ed20083a27fdfe247381d0dc" dependencies = [ "actix-codec", - "actix-rt 1.1.1", + "actix-rt", "actix-service", "actix-utils", "derive_more", @@ -33,8 +33,11 @@ dependencies = [ "futures-util", "http", "log", + "rustls", + "tokio-rustls", "trust-dns-proto", "trust-dns-resolver", + "webpki", ] [[package]] @@ -65,11 +68,12 @@ checksum = "5cb8958da437716f3f31b0e76f8daf36554128517d7df37ceba7df00f09622ee" dependencies = [ "actix-codec", "actix-connect", - "actix-rt 1.1.1", + "actix-rt", "actix-service", "actix-threadpool", + "actix-tls", "actix-utils", - "base64", + "base64 0.13.0", "bitflags", "brotli2", "bytes 0.5.6", @@ -114,16 +118,6 @@ dependencies = [ "syn", ] -[[package]] -name = "actix-macros" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2f86cd6857c135e6e9fe57b1619a88d1f94a7df34c00e11fe13e64fd3438837" -dependencies = [ - "quote", - "syn", -] - [[package]] name = "actix-multipart" version = "0.3.0" @@ -161,24 +155,13 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "143fcc2912e0d1de2bcf4e2f720d2a60c28652ab4179685a1ee159e0fb3db227" dependencies = [ - "actix-macros 0.1.3", + "actix-macros", "actix-threadpool", "copyless", "futures-channel", "futures-util", "smallvec", - "tokio 0.2.25", -] - -[[package]] -name = "actix-rt" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea360596a50aa9af459850737f99293e5cb9114ae831118cb6026b3bbc7583ad" -dependencies = [ - "actix-macros 0.2.1", - "futures-core", - "tokio 1.10.1", + "tokio", ] [[package]] @@ -188,13 +171,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45407e6e672ca24784baa667c5d32ef109ccdd8d5e0b5ebb9ef8a67f4dfb708e" dependencies = [ "actix-codec", - "actix-rt 1.1.1", + "actix-rt", "actix-service", "actix-utils", "futures-channel", "futures-util", "log", - "mio 0.6.23", + "mio", "mio-uds", "num_cpus", "slab", @@ -217,8 +200,8 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47239ca38799ab74ee6a8a94d1ce857014b2ac36f242f70f3f75a66f691e791c" dependencies = [ - "actix-macros 0.1.3", - "actix-rt 1.1.1", + "actix-macros", + "actix-rt", "actix-server", "actix-service", "log", @@ -250,6 +233,10 @@ dependencies = [ "actix-service", "actix-utils", "futures-util", + "rustls", + "tokio-rustls", + "webpki", + "webpki-roots", ] [[package]] @@ -259,7 +246,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e9022dec56632d1d7979e59af14f0597a28a830a9c1c7fec8b2327eb9f16b5a" dependencies = [ "actix-codec", - "actix-rt 1.1.1", + "actix-rt", "actix-service", "bitflags", "bytes 0.5.6", @@ -280,9 +267,9 @@ checksum = "e641d4a172e7faa0862241a20ff4f1f5ab0ab7c279f00c2d4587b77483477b86" dependencies = [ "actix-codec", "actix-http", - "actix-macros 0.1.3", + "actix-macros", "actix-router", - "actix-rt 1.1.1", + "actix-rt", "actix-server", "actix-service", "actix-testing", @@ -302,6 +289,7 @@ dependencies = [ "mime", "pin-project 1.0.8", "regex", + "rustls", "serde", "serde_json", "serde_urlencoded", @@ -379,9 +367,9 @@ checksum = "b381e490e7b0cfc37ebc54079b0413d8093ef43d14a4e4747083f7fa47a9e691" dependencies = [ "actix-codec", "actix-http", - "actix-rt 1.1.1", + "actix-rt", "actix-service", - "base64", + "base64 0.13.0", "bytes 0.5.6", "cfg-if 1.0.0", "derive_more", @@ -390,6 +378,7 @@ dependencies = [ "mime", "percent-encoding", "rand 0.7.3", + "rustls", "serde", "serde_json", "serde_urlencoded", @@ -401,6 +390,12 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4521f3e3d031370679b3b140beb36dfe4801b09ac77e30c61941f97df3ef28b" +[[package]] +name = "base64" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" + [[package]] name = "base64" version = "0.13.0" @@ -844,7 +839,7 @@ dependencies = [ "http", "indexmap", "slab", - "tokio 0.2.25", + "tokio", "tokio-util", "tracing", "tracing-futures", @@ -1120,25 +1115,12 @@ dependencies = [ "kernel32-sys", "libc", "log", - "miow 0.2.2", + "miow", "net2", "slab", "winapi 0.2.8", ] -[[package]] -name = "mio" -version = "0.7.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c2bdb6314ec10835cd3293dd268473a835c02b7b352e788be788b3c6ca6bb16" -dependencies = [ - "libc", - "log", - "miow 0.3.7", - "ntapi", - "winapi 0.3.9", -] - [[package]] name = "mio-uds" version = "0.6.8" @@ -1147,7 +1129,7 @@ checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" dependencies = [ "iovec", "libc", - "mio 0.6.23", + "mio", ] [[package]] @@ -1162,15 +1144,6 @@ dependencies = [ "ws2_32-sys", ] -[[package]] -name = "miow" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" -dependencies = [ - "winapi 0.3.9", -] - [[package]] name = "net2" version = "0.2.37" @@ -1203,15 +1176,6 @@ dependencies = [ "version_check 0.9.3", ] -[[package]] -name = "ntapi" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" -dependencies = [ - "winapi 0.3.9", -] - [[package]] name = "num_cpus" version = "1.13.0" @@ -1535,13 +1499,26 @@ dependencies = [ "semver 0.11.0", ] +[[package]] +name = "rustls" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d1126dcf58e93cee7d098dbda643b5f92ed724f1f6a63007c1116eed6700c81" +dependencies = [ + "base64 0.12.3", + "log", + "ring", + "sct", + "webpki", +] + [[package]] name = "rustypaste" version = "0.5.0" dependencies = [ "actix-files", "actix-multipart", - "actix-rt 2.3.0", + "actix-rt", "actix-web", "byte-unit", "config", @@ -1574,6 +1551,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sct" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "semver" version = "0.9.0" @@ -1904,7 +1891,7 @@ dependencies = [ "lazy_static", "libc", "memchr", - "mio 0.6.23", + "mio", "mio-uds", "pin-project-lite 0.1.12", "signal-hook-registry", @@ -1913,19 +1900,15 @@ dependencies = [ ] [[package]] -name = "tokio" -version = "1.10.1" +name = "tokio-rustls" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92036be488bb6594459f2e03b60e42df6f937fe6ca5c5ffdcb539c6b84dc40f5" +checksum = "e12831b255bcfa39dc0436b01e19fea231a37db570686c06ee72c423479f889a" dependencies = [ - "autocfg", - "libc", - "mio 0.7.13", - "once_cell", - "parking_lot", - "pin-project-lite 0.2.7", - "signal-hook-registry", - "winapi 0.3.9", + "futures-core", + "rustls", + "tokio", + "webpki", ] [[package]] @@ -1939,7 +1922,7 @@ dependencies = [ "futures-sink", "log", "pin-project-lite 0.1.12", - "tokio 0.2.25", + "tokio", ] [[package]] @@ -1998,7 +1981,7 @@ dependencies = [ "rand 0.7.3", "smallvec", "thiserror", - "tokio 0.2.25", + "tokio", "url", ] @@ -2017,7 +2000,7 @@ dependencies = [ "resolv-conf", "smallvec", "thiserror", - "tokio 0.2.25", + "tokio", "trust-dns-proto", ] @@ -2235,6 +2218,25 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f20dea7535251981a9670857150d571846545088359b28e4951d350bdaf179f" +dependencies = [ + "webpki", +] + [[package]] name = "widestring" version = "0.4.3" diff --git a/Cargo.toml b/Cargo.toml index 24225dd..83e750d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ categories = ["web-programming::http-server"] include = ["src/**/*", "Cargo.*", "LICENSE", "README.md", "CHANGELOG.md"] [dependencies] -actix-web = "3.3.2" +actix-web = { version = "3.3.2", features = ["rustls"] } actix-multipart = "0.3.0" actix-files = "0.5.0" env_logger = "0.9.0" @@ -45,7 +45,7 @@ version = "0.5.0" default-features = false [dev-dependencies] -actix-rt = "2.3.0" +actix-rt = "1.1.1" [profile.dev] opt-level = 0 diff --git a/README.md b/README.md index c4659bd..96efc59 100644 --- a/README.md +++ b/README.md @@ -85,24 +85,30 @@ $ curl -F "file=@x.txt" -H "expire:10min" "" (supported units: `ns`, `us`, `ms`, `sec`, `min`, `hours`, `days`, `weeks`, `months`, `years`) -#### One Shot +#### One shot ```sh $ curl -F "oneshot=@x.txt" "" ``` -#### Cleaning Up Expired Files +#### Cleaning up expired files ```sh $ find upload/ -maxdepth 2 -type f -iname "*.[0-9]*" -exec rm -v {} \; ``` -#### URL Shortening +#### URL shortening ```sh $ curl -F "url=https://example.com/some/long/url" "" ``` +### Paste file from remote URL + +```sh +$ curl -F "remote=https://example.com/file.png" "" +``` + ### Server To start the server: @@ -178,7 +184,6 @@ http { ### Roadmap -- Support uploading files from given URL - Hot reload the configuration file ### Contributing diff --git a/src/main.rs b/src/main.rs index d870213..43e0fc5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,4 @@ +use actix_web::client::ClientBuilder; use actix_web::middleware::Logger; use actix_web::{App, HttpServer}; use rustypaste::config::Config; @@ -6,6 +7,7 @@ use rustypaste::server; use std::env; use std::fs; use std::io::Result as IoResult; +use std::time::Duration; #[actix_web::main] async fn main() -> IoResult<()> { @@ -19,8 +21,13 @@ async fn main() -> IoResult<()> { fs::create_dir_all(paste_type.get_path(&server_config.upload_path))?; } let mut http_server = HttpServer::new(move || { + let http_client = ClientBuilder::default() + .timeout(Duration::from_secs(30)) + .disable_redirects() + .finish(); App::new() .data(config.clone()) + .data(http_client) .wrap(Logger::default()) .configure(server::configure_routes) }) diff --git a/src/paste.rs b/src/paste.rs index 5eb7a0f..f9f6507 100644 --- a/src/paste.rs +++ b/src/paste.rs @@ -1,5 +1,9 @@ use crate::config::Config; +use crate::file::Directory; use crate::header::ContentDisposition; +use crate::util; +use actix_web::client::Client; +use actix_web::{error, Error}; use std::convert::TryFrom; use std::fs::{self, File}; use std::io::{Error as IoError, ErrorKind as IoErrorKind, Result as IoResult, Write}; @@ -12,6 +16,8 @@ use url::Url; pub enum PasteType { /// Any type of file. File, + /// A file that is on a remote URL. + RemoteFile, /// A file that allowed to be accessed once. Oneshot, /// A file that only contains an URL. @@ -23,6 +29,8 @@ impl<'a> TryFrom<&'a ContentDisposition> for PasteType { fn try_from(content_disposition: &'a ContentDisposition) -> Result { if content_disposition.has_form_field("file") { Ok(Self::File) + } else if content_disposition.has_form_field("remote") { + Ok(Self::RemoteFile) } else if content_disposition.has_form_field("oneshot") { Ok(Self::Oneshot) } else if content_disposition.has_form_field("url") { @@ -37,7 +45,7 @@ impl PasteType { /// Returns the corresponding directory of the paste type. pub fn get_dir(&self) -> String { match self { - Self::File => String::new(), + Self::File | Self::RemoteFile => String::new(), Self::Oneshot => String::from("oneshot"), Self::Url => String::from("url"), } @@ -138,6 +146,50 @@ impl Paste { Ok(file_name) } + /// Downloads a file from URL and stores it with [`store_file`]. + /// + /// - File name is inferred from URL if the last URL segment is a file. + /// - Same content length configuration is applied for download limit. + /// - Checks SHA256 digest of the downloaded file for preventing duplication. + /// - Assumes `self.data` contains a valid URL, otherwise returns an error. + pub async fn store_remote_file( + &mut self, + expiry_date: Option, + client: &Client, + config: &Config, + ) -> Result { + let data = str::from_utf8(&self.data).map_err(error::ErrorBadRequest)?; + let url = Url::parse(data).map_err(error::ErrorBadRequest)?; + let file_name = url + .path_segments() + .and_then(|segments| segments.last()) + .and_then(|name| if name.is_empty() { None } else { Some(name) }) + .unwrap_or("file"); + let mut response = client.get(url.as_str()).send().await?; + let payload_limit = config + .server + .max_content_length + .get_bytes() + .try_into() + .map_err(error::ErrorInternalServerError)?; + let bytes = response.body().limit(payload_limit).await?.to_vec(); + let bytes_checksum = util::sha256_digest(&*bytes)?; + self.data = bytes; + if !config.paste.duplicate_files.unwrap_or(true) { + if let Some(file) = + Directory::try_from(config.server.upload_path.as_path())?.get_file(bytes_checksum) + { + return Ok(file + .path + .file_name() + .map(|v| v.to_string_lossy()) + .unwrap_or_default() + .to_string()); + } + } + Ok(self.store_file(file_name, expiry_date, config)?) + } + /// Writes an URL to a file in upload directory. /// /// - Checks if the data is a valid URL. @@ -169,10 +221,13 @@ mod tests { use super::*; use crate::random::{RandomURLConfig, RandomURLType}; use crate::util; + use actix_web::client::Client; + use actix_web::web::Data; + use byte_unit::Byte; use std::env; - #[test] - fn test_paste_data() -> IoResult<()> { + #[actix_rt::test] + async fn test_paste_data() -> Result<(), Error> { let mut config = Config::default(); config.server.upload_path = env::current_dir()?; config.paste.random_url = RandomURLConfig { @@ -257,6 +312,23 @@ mod tests { }; assert!(paste.store_url(None, &config).is_err()); + config.server.max_content_length = Byte::from_str("30k").unwrap(); + let url = String::from("https://upload.wikimedia.org/wikipedia/en/a/a9/Example.jpg"); + let mut paste = Paste { + data: url.as_bytes().to_vec(), + type_: PasteType::RemoteFile, + }; + let client_data = Data::new(Client::default()); + let file_name = paste.store_remote_file(None, &client_data, &config).await?; + let file_path = PasteType::RemoteFile + .get_path(&config.server.upload_path) + .join(&file_name); + assert_eq!( + "8c712905b799905357b8202d0cb7a244cefeeccf7aa5eb79896645ac50158ffa", + util::sha256_digest(&*paste.data)? + ); + fs::remove_file(file_path)?; + for paste_type in &[PasteType::Url, PasteType::Oneshot] { fs::remove_dir(paste_type.get_path(&config.server.upload_path))?; } diff --git a/src/server.rs b/src/server.rs index c331ee9..a734f74 100644 --- a/src/server.rs +++ b/src/server.rs @@ -7,6 +7,7 @@ use crate::paste::{Paste, PasteType}; use crate::util; use actix_files::NamedFile; use actix_multipart::Multipart; +use actix_web::client::Client; use actix_web::{error, get, post, web, Error, HttpRequest, HttpResponse, Responder}; use byte_unit::Byte; use futures_util::stream::StreamExt; @@ -37,7 +38,7 @@ async fn serve( let alt_path = type_.get_path(&config.server.upload_path).join(&*file); let alt_path = util::glob_match_file(alt_path)?; if alt_path.exists() - || path.file_name().map(|v| v.to_str()).flatten() == Some(&type_.get_dir()) + || path.file_name().and_then(|v| v.to_str()) == Some(&type_.get_dir()) { path = alt_path; paste_type = *type_; @@ -46,7 +47,7 @@ async fn serve( } } match paste_type { - PasteType::File | PasteType::Oneshot => { + PasteType::File | PasteType::RemoteFile | PasteType::Oneshot => { let response = NamedFile::open(&path) .map_err(|_| error::ErrorNotFound("file is not found or expired :("))? .disable_content_disposition() @@ -79,6 +80,7 @@ async fn serve( async fn upload( request: HttpRequest, mut payload: Multipart, + client: web::Data, config: web::Data, ) -> Result { let connection = request.connection_info(); @@ -102,7 +104,10 @@ async fn upload( log::warn!("{} sent zero bytes", host); return Err(error::ErrorBadRequest("invalid file size")); } - if paste_type != PasteType::Oneshot && !config.paste.duplicate_files.unwrap_or(true) { + if paste_type != PasteType::Oneshot + && paste_type != PasteType::RemoteFile + && !config.paste.duplicate_files.unwrap_or(true) + { let bytes_checksum = util::sha256_digest(&*bytes)?; if let Some(file) = Directory::try_from(config.server.upload_path.as_path())? .get_file(bytes_checksum) @@ -120,18 +125,27 @@ async fn upload( continue; } } - let bytes_unit = Byte::from_bytes(bytes.len() as u128).get_appropriate_unit(false); - let paste = Paste { + let mut paste = Paste { data: bytes.to_vec(), type_: paste_type, }; - let file_name = match paste_type { + let file_name = match paste.type_ { PasteType::File | PasteType::Oneshot => { paste.store_file(content.get_file_name()?, expiry_date, &config)? } + PasteType::RemoteFile => { + paste + .store_remote_file(expiry_date, &client, &config) + .await? + } PasteType::Url => paste.store_url(expiry_date, &config)?, }; - log::info!("{} ({}) is uploaded from {}", file_name, bytes_unit, host); + log::info!( + "{} ({}) is uploaded from {}", + file_name, + Byte::from_bytes(paste.data.len() as u128).get_appropriate_unit(false), + host + ); urls.push(format!( "{}://{}/{}\n", connection.scheme(),