feat(paste): support pasting files from remote URLs

Cargo.lock

@@ -14,7 +14,7 @@ dependencies = [
  "futures-sink",
  "log",
  "pin-project 0.4.28",
- "tokio 0.2.25",
+ "tokio",
  "tokio-util",
 ]
 
@@ -25,7 +25,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "177837a10863f15ba8d3ae3ec12fac1099099529ed20083a27fdfe247381d0dc"
 dependencies = [
  "actix-codec",
- "actix-rt 1.1.1",
+ "actix-rt",
  "actix-service",
  "actix-utils",
  "derive_more",
@@ -33,8 +33,11 @@ dependencies = [
  "futures-util",
  "http",
  "log",
+ "rustls",
+ "tokio-rustls",
  "trust-dns-proto",
  "trust-dns-resolver",
+ "webpki",
 ]
 
 [[package]]
@@ -65,11 +68,12 @@ checksum = "5cb8958da437716f3f31b0e76f8daf36554128517d7df37ceba7df00f09622ee"
 dependencies = [
  "actix-codec",
  "actix-connect",
- "actix-rt 1.1.1",
+ "actix-rt",
  "actix-service",
  "actix-threadpool",
+ "actix-tls",
  "actix-utils",
- "base64",
+ "base64 0.13.0",
  "bitflags",
  "brotli2",
  "bytes 0.5.6",
@@ -114,16 +118,6 @@ dependencies = [
  "syn",
 ]
 
-[[package]]
-name = "actix-macros"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c2f86cd6857c135e6e9fe57b1619a88d1f94a7df34c00e11fe13e64fd3438837"
-dependencies = [
- "quote",
- "syn",
-]
-
 [[package]]
 name = "actix-multipart"
 version = "0.3.0"
@@ -161,24 +155,13 @@ version = "1.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "143fcc2912e0d1de2bcf4e2f720d2a60c28652ab4179685a1ee159e0fb3db227"
 dependencies = [
- "actix-macros 0.1.3",
+ "actix-macros",
  "actix-threadpool",
  "copyless",
  "futures-channel",
  "futures-util",
  "smallvec",
- "tokio 0.2.25",
+ "tokio",
-]
-
-[[package]]
-name = "actix-rt"
-version = "2.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea360596a50aa9af459850737f99293e5cb9114ae831118cb6026b3bbc7583ad"
-dependencies = [
- "actix-macros 0.2.1",
- "futures-core",
- "tokio 1.10.1",
 ]
 
 [[package]]
@@ -188,13 +171,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "45407e6e672ca24784baa667c5d32ef109ccdd8d5e0b5ebb9ef8a67f4dfb708e"
 dependencies = [
  "actix-codec",
- "actix-rt 1.1.1",
+ "actix-rt",
  "actix-service",
  "actix-utils",
  "futures-channel",
  "futures-util",
  "log",
- "mio 0.6.23",
+ "mio",
  "mio-uds",
  "num_cpus",
  "slab",
@@ -217,8 +200,8 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "47239ca38799ab74ee6a8a94d1ce857014b2ac36f242f70f3f75a66f691e791c"
 dependencies = [
- "actix-macros 0.1.3",
+ "actix-macros",
- "actix-rt 1.1.1",
+ "actix-rt",
  "actix-server",
  "actix-service",
  "log",
@@ -250,6 +233,10 @@ dependencies = [
  "actix-service",
  "actix-utils",
  "futures-util",
+ "rustls",
+ "tokio-rustls",
+ "webpki",
+ "webpki-roots",
 ]
 
 [[package]]
@@ -259,7 +246,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2e9022dec56632d1d7979e59af14f0597a28a830a9c1c7fec8b2327eb9f16b5a"
 dependencies = [
  "actix-codec",
- "actix-rt 1.1.1",
+ "actix-rt",
  "actix-service",
  "bitflags",
  "bytes 0.5.6",
@@ -280,9 +267,9 @@ checksum = "e641d4a172e7faa0862241a20ff4f1f5ab0ab7c279f00c2d4587b77483477b86"
 dependencies = [
  "actix-codec",
  "actix-http",
- "actix-macros 0.1.3",
+ "actix-macros",
  "actix-router",
- "actix-rt 1.1.1",
+ "actix-rt",
  "actix-server",
  "actix-service",
  "actix-testing",
@@ -302,6 +289,7 @@ dependencies = [
  "mime",
  "pin-project 1.0.8",
  "regex",
+ "rustls",
  "serde",
  "serde_json",
  "serde_urlencoded",
@@ -379,9 +367,9 @@ checksum = "b381e490e7b0cfc37ebc54079b0413d8093ef43d14a4e4747083f7fa47a9e691"
 dependencies = [
  "actix-codec",
  "actix-http",
- "actix-rt 1.1.1",
+ "actix-rt",
  "actix-service",
- "base64",
+ "base64 0.13.0",
  "bytes 0.5.6",
  "cfg-if 1.0.0",
  "derive_more",
@@ -390,6 +378,7 @@ dependencies = [
  "mime",
  "percent-encoding",
  "rand 0.7.3",
+ "rustls",
  "serde",
  "serde_json",
  "serde_urlencoded",
@@ -401,6 +390,12 @@ version = "0.2.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a4521f3e3d031370679b3b140beb36dfe4801b09ac77e30c61941f97df3ef28b"
 
+[[package]]
+name = "base64"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff"
+
 [[package]]
 name = "base64"
 version = "0.13.0"
@@ -844,7 +839,7 @@ dependencies = [
  "http",
  "indexmap",
  "slab",
- "tokio 0.2.25",
+ "tokio",
  "tokio-util",
  "tracing",
  "tracing-futures",
@@ -1120,25 +1115,12 @@ dependencies = [
  "kernel32-sys",
  "libc",
  "log",
- "miow 0.2.2",
+ "miow",
  "net2",
  "slab",
  "winapi 0.2.8",
 ]
 
-[[package]]
-name = "mio"
-version = "0.7.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c2bdb6314ec10835cd3293dd268473a835c02b7b352e788be788b3c6ca6bb16"
-dependencies = [
- "libc",
- "log",
- "miow 0.3.7",
- "ntapi",
- "winapi 0.3.9",
-]
-
 [[package]]
 name = "mio-uds"
 version = "0.6.8"
@@ -1147,7 +1129,7 @@ checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0"
 dependencies = [
  "iovec",
  "libc",
- "mio 0.6.23",
+ "mio",
 ]
 
 [[package]]
@@ -1162,15 +1144,6 @@ dependencies = [
  "ws2_32-sys",
 ]
 
-[[package]]
-name = "miow"
-version = "0.3.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21"
-dependencies = [
- "winapi 0.3.9",
-]
-
 [[package]]
 name = "net2"
 version = "0.2.37"
@@ -1203,15 +1176,6 @@ dependencies = [
  "version_check 0.9.3",
 ]
 
-[[package]]
-name = "ntapi"
-version = "0.3.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44"
-dependencies = [
- "winapi 0.3.9",
-]
-
 [[package]]
 name = "num_cpus"
 version = "1.13.0"
@@ -1535,13 +1499,26 @@ dependencies = [
  "semver 0.11.0",
 ]
 
+[[package]]
+name = "rustls"
+version = "0.18.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d1126dcf58e93cee7d098dbda643b5f92ed724f1f6a63007c1116eed6700c81"
+dependencies = [
+ "base64 0.12.3",
+ "log",
+ "ring",
+ "sct",
+ "webpki",
+]
+
 [[package]]
 name = "rustypaste"
 version = "0.5.0"
 dependencies = [
  "actix-files",
  "actix-multipart",
- "actix-rt 2.3.0",
+ "actix-rt",
  "actix-web",
  "byte-unit",
  "config",
@@ -1574,6 +1551,16 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
+[[package]]
+name = "sct"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce"
+dependencies = [
+ "ring",
+ "untrusted",
+]
+
 [[package]]
 name = "semver"
 version = "0.9.0"
@@ -1904,7 +1891,7 @@ dependencies = [
  "lazy_static",
  "libc",
  "memchr",
- "mio 0.6.23",
+ "mio",
  "mio-uds",
  "pin-project-lite 0.1.12",
  "signal-hook-registry",
@@ -1913,19 +1900,15 @@ dependencies = [
 ]
 
 [[package]]
-name = "tokio"
+name = "tokio-rustls"
-version = "1.10.1"
+version = "0.14.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92036be488bb6594459f2e03b60e42df6f937fe6ca5c5ffdcb539c6b84dc40f5"
+checksum = "e12831b255bcfa39dc0436b01e19fea231a37db570686c06ee72c423479f889a"
 dependencies = [
- "autocfg",
+ "futures-core",
- "libc",
+ "rustls",
- "mio 0.7.13",
+ "tokio",
- "once_cell",
+ "webpki",
- "parking_lot",
- "pin-project-lite 0.2.7",
- "signal-hook-registry",
- "winapi 0.3.9",
 ]
 
 [[package]]
@@ -1939,7 +1922,7 @@ dependencies = [
  "futures-sink",
  "log",
  "pin-project-lite 0.1.12",
- "tokio 0.2.25",
+ "tokio",
 ]
 
 [[package]]
@@ -1998,7 +1981,7 @@ dependencies = [
  "rand 0.7.3",
  "smallvec",
  "thiserror",
- "tokio 0.2.25",
+ "tokio",
  "url",
 ]
 
@@ -2017,7 +2000,7 @@ dependencies = [
  "resolv-conf",
  "smallvec",
  "thiserror",
- "tokio 0.2.25",
+ "tokio",
  "trust-dns-proto",
 ]
 
@@ -2235,6 +2218,25 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "webpki"
+version = "0.21.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea"
+dependencies = [
+ "ring",
+ "untrusted",
+]
+
+[[package]]
+name = "webpki-roots"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f20dea7535251981a9670857150d571846545088359b28e4951d350bdaf179f"
+dependencies = [
+ "webpki",
+]
+
 [[package]]
 name = "widestring"
 version = "0.4.3"

Cargo.toml

@@ -13,7 +13,7 @@ categories = ["web-programming::http-server"]
 include = ["src/**/*", "Cargo.*", "LICENSE", "README.md", "CHANGELOG.md"]
 
 [dependencies]
-actix-web = "3.3.2"
+actix-web = { version = "3.3.2", features = ["rustls"] }
 actix-multipart = "0.3.0"
 actix-files = "0.5.0"
 env_logger = "0.9.0"
@@ -45,7 +45,7 @@ version = "0.5.0"
 default-features = false
 
 [dev-dependencies]
-actix-rt = "2.3.0"
+actix-rt = "1.1.1"
 
 [profile.dev]
 opt-level = 0

README.md

@@ -85,24 +85,30 @@ $ curl -F "file=@x.txt" -H "expire:10min" "<server_address>"
 
 (supported units: `ns`, `us`, `ms`, `sec`, `min`, `hours`, `days`, `weeks`, `months`, `years`)
 
-#### One Shot
+#### One shot
 
 ```sh
 $ curl -F "oneshot=@x.txt" "<server_address>"
 ```
 
-#### Cleaning Up Expired Files
+#### Cleaning up expired files
 
 ```sh
 $ find upload/ -maxdepth 2 -type f -iname "*.[0-9]*" -exec rm -v {} \;
 ```
 
-#### URL Shortening
+#### URL shortening
 
 ```sh
 $ curl -F "url=https://example.com/some/long/url" "<server_address>"
 ```
 
+### Paste file from remote URL
+
+```sh
+$ curl -F "remote=https://example.com/file.png" "<server_address>"
+```
+
 ### Server
 
 To start the server:
@@ -178,7 +184,6 @@ http {
 
 ### Roadmap
 
-- Support uploading files from given URL
 - Hot reload the configuration file
 
 ### Contributing

src/main.rs

@@ -1,3 +1,4 @@
+use actix_web::client::ClientBuilder;
 use actix_web::middleware::Logger;
 use actix_web::{App, HttpServer};
 use rustypaste::config::Config;
@@ -6,6 +7,7 @@ use rustypaste::server;
 use std::env;
 use std::fs;
 use std::io::Result as IoResult;
+use std::time::Duration;
 
 #[actix_web::main]
 async fn main() -> IoResult<()> {
@@ -19,8 +21,13 @@ async fn main() -> IoResult<()> {
         fs::create_dir_all(paste_type.get_path(&server_config.upload_path))?;
     }
     let mut http_server = HttpServer::new(move || {
+        let http_client = ClientBuilder::default()
+            .timeout(Duration::from_secs(30))
+            .disable_redirects()
+            .finish();
         App::new()
             .data(config.clone())
+            .data(http_client)
             .wrap(Logger::default())
             .configure(server::configure_routes)
     })

src/paste.rs

@@ -1,5 +1,9 @@
 use crate::config::Config;
+use crate::file::Directory;
 use crate::header::ContentDisposition;
+use crate::util;
+use actix_web::client::Client;
+use actix_web::{error, Error};
 use std::convert::TryFrom;
 use std::fs::{self, File};
 use std::io::{Error as IoError, ErrorKind as IoErrorKind, Result as IoResult, Write};
@@ -12,6 +16,8 @@ use url::Url;
 pub enum PasteType {
     /// Any type of file.
     File,
+    /// A file that is on a remote URL.
+    RemoteFile,
     /// A file that allowed to be accessed once.
     Oneshot,
     /// A file that only contains an URL.
@@ -23,6 +29,8 @@ impl<'a> TryFrom<&'a ContentDisposition> for PasteType {
     fn try_from(content_disposition: &'a ContentDisposition) -> Result<Self, Self::Error> {
         if content_disposition.has_form_field("file") {
             Ok(Self::File)
+        } else if content_disposition.has_form_field("remote") {
+            Ok(Self::RemoteFile)
         } else if content_disposition.has_form_field("oneshot") {
             Ok(Self::Oneshot)
         } else if content_disposition.has_form_field("url") {
@@ -37,7 +45,7 @@ impl PasteType {
     /// Returns the corresponding directory of the paste type.
     pub fn get_dir(&self) -> String {
         match self {
-            Self::File => String::new(),
+            Self::File | Self::RemoteFile => String::new(),
             Self::Oneshot => String::from("oneshot"),
             Self::Url => String::from("url"),
         }
@@ -138,6 +146,50 @@ impl Paste {
         Ok(file_name)
     }
 
+    /// Downloads a file from URL and stores it with [`store_file`].
+    ///
+    /// - File name is inferred from URL if the last URL segment is a file.
+    /// - Same content length configuration is applied for download limit.
+    /// - Checks SHA256 digest of the downloaded file for preventing duplication.
+    /// - Assumes `self.data` contains a valid URL, otherwise returns an error.
+    pub async fn store_remote_file(
+        &mut self,
+        expiry_date: Option<u128>,
+        client: &Client,
+        config: &Config,
+    ) -> Result<String, Error> {
+        let data = str::from_utf8(&self.data).map_err(error::ErrorBadRequest)?;
+        let url = Url::parse(data).map_err(error::ErrorBadRequest)?;
+        let file_name = url
+            .path_segments()
+            .and_then(|segments| segments.last())
+            .and_then(|name| if name.is_empty() { None } else { Some(name) })
+            .unwrap_or("file");
+        let mut response = client.get(url.as_str()).send().await?;
+        let payload_limit = config
+            .server
+            .max_content_length
+            .get_bytes()
+            .try_into()
+            .map_err(error::ErrorInternalServerError)?;
+        let bytes = response.body().limit(payload_limit).await?.to_vec();
+        let bytes_checksum = util::sha256_digest(&*bytes)?;
+        self.data = bytes;
+        if !config.paste.duplicate_files.unwrap_or(true) {
+            if let Some(file) =
+                Directory::try_from(config.server.upload_path.as_path())?.get_file(bytes_checksum)
+            {
+                return Ok(file
+                    .path
+                    .file_name()
+                    .map(|v| v.to_string_lossy())
+                    .unwrap_or_default()
+                    .to_string());
+            }
+        }
+        Ok(self.store_file(file_name, expiry_date, config)?)
+    }
+
     /// Writes an URL to a file in upload directory.
     ///
     /// - Checks if the data is a valid URL.
@@ -169,10 +221,13 @@ mod tests {
     use super::*;
     use crate::random::{RandomURLConfig, RandomURLType};
     use crate::util;
+    use actix_web::client::Client;
+    use actix_web::web::Data;
+    use byte_unit::Byte;
     use std::env;
 
-    #[test]
+    #[actix_rt::test]
-    fn test_paste_data() -> IoResult<()> {
+    async fn test_paste_data() -> Result<(), Error> {
         let mut config = Config::default();
         config.server.upload_path = env::current_dir()?;
         config.paste.random_url = RandomURLConfig {
@@ -257,6 +312,23 @@ mod tests {
         };
         assert!(paste.store_url(None, &config).is_err());
 
+        config.server.max_content_length = Byte::from_str("30k").unwrap();
+        let url = String::from("https://upload.wikimedia.org/wikipedia/en/a/a9/Example.jpg");
+        let mut paste = Paste {
+            data: url.as_bytes().to_vec(),
+            type_: PasteType::RemoteFile,
+        };
+        let client_data = Data::new(Client::default());
+        let file_name = paste.store_remote_file(None, &client_data, &config).await?;
+        let file_path = PasteType::RemoteFile
+            .get_path(&config.server.upload_path)
+            .join(&file_name);
+        assert_eq!(
+            "8c712905b799905357b8202d0cb7a244cefeeccf7aa5eb79896645ac50158ffa",
+            util::sha256_digest(&*paste.data)?
+        );
+        fs::remove_file(file_path)?;
+
         for paste_type in &[PasteType::Url, PasteType::Oneshot] {
             fs::remove_dir(paste_type.get_path(&config.server.upload_path))?;
         }

src/server.rs

@@ -7,6 +7,7 @@ use crate::paste::{Paste, PasteType};
 use crate::util;
 use actix_files::NamedFile;
 use actix_multipart::Multipart;
+use actix_web::client::Client;
 use actix_web::{error, get, post, web, Error, HttpRequest, HttpResponse, Responder};
 use byte_unit::Byte;
 use futures_util::stream::StreamExt;
@@ -37,7 +38,7 @@ async fn serve(
             let alt_path = type_.get_path(&config.server.upload_path).join(&*file);
             let alt_path = util::glob_match_file(alt_path)?;
             if alt_path.exists()
-                || path.file_name().map(|v| v.to_str()).flatten() == Some(&type_.get_dir())
+                || path.file_name().and_then(|v| v.to_str()) == Some(&type_.get_dir())
             {
                 path = alt_path;
                 paste_type = *type_;
@@ -46,7 +47,7 @@ async fn serve(
         }
     }
     match paste_type {
-        PasteType::File | PasteType::Oneshot => {
+        PasteType::File | PasteType::RemoteFile | PasteType::Oneshot => {
             let response = NamedFile::open(&path)
                 .map_err(|_| error::ErrorNotFound("file is not found or expired :("))?
                 .disable_content_disposition()
@@ -79,6 +80,7 @@ async fn serve(
 async fn upload(
     request: HttpRequest,
     mut payload: Multipart,
+    client: web::Data<Client>,
     config: web::Data<Config>,
 ) -> Result<HttpResponse, Error> {
     let connection = request.connection_info();
@@ -102,7 +104,10 @@ async fn upload(
                 log::warn!("{} sent zero bytes", host);
                 return Err(error::ErrorBadRequest("invalid file size"));
             }
-            if paste_type != PasteType::Oneshot && !config.paste.duplicate_files.unwrap_or(true) {
+            if paste_type != PasteType::Oneshot
+                && paste_type != PasteType::RemoteFile
+                && !config.paste.duplicate_files.unwrap_or(true)
+            {
                 let bytes_checksum = util::sha256_digest(&*bytes)?;
                 if let Some(file) = Directory::try_from(config.server.upload_path.as_path())?
                     .get_file(bytes_checksum)
@@ -120,18 +125,27 @@ async fn upload(
                     continue;
                 }
             }
-            let bytes_unit = Byte::from_bytes(bytes.len() as u128).get_appropriate_unit(false);
+            let mut paste = Paste {
-            let paste = Paste {
                 data: bytes.to_vec(),
                 type_: paste_type,
             };
-            let file_name = match paste_type {
+            let file_name = match paste.type_ {
                 PasteType::File | PasteType::Oneshot => {
                     paste.store_file(content.get_file_name()?, expiry_date, &config)?
                 }
+                PasteType::RemoteFile => {
+                    paste
+                        .store_remote_file(expiry_date, &client, &config)
+                        .await?
+                }
                 PasteType::Url => paste.store_url(expiry_date, &config)?,
             };
-            log::info!("{} ({}) is uploaded from {}", file_name, bytes_unit, host);
+            log::info!(
+                "{} ({}) is uploaded from {}",
+                file_name,
+                Byte::from_bytes(paste.data.len() as u128).get_appropriate_unit(false),
+                host
+            );
             urls.push(format!(
                 "{}://{}/{}\n",
                 connection.scheme(),