diff --git a/CHANGELOG.md b/CHANGELOG.md index b306c621..2d131453 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,28 @@ # CHANGELOG.md -## 0.16.1 +## unreleased + +### Uploads + +This release is all about a long awaited feature: file uploads. +Your SQLPage website can now accept file uploads from users, store them either in a directory or directly in a database table. + +#### New functions + +##### Handle uploaded files + + - [`sqlpage.uploaded_file_path`](https://sql.ophir.dev/functions.sql?function=uploaded_file_path#function) to get the temprary local path of a file uploaded by the user. This path will be valid until the end of the current request, and will be located in a temporary directory (customizable with `TMPDIR`). You can use [`sqlpage.exec`](https://sql.ophir.dev/functions.sql?function=exec#function) to operate on the file, for instance to move it to a permanent location. + - [`sqlpage.uploaded_file_mime_type`](https://sql.ophir.dev/functions.sql?function=uploaded_file_name#function) to get the type of file uploaded by the user. This is the MIME type of the file, such as `image/png` or `text/csv`. You can use this to easily check that the file is of the expected type before storing it. + +##### Read files + +These new functions are useful to read the content of a file uploaded by the user, +but can also be used to read any file on the server. + + - [`sqlpage.read_file_as_text`](https://sql.ophir.dev/functions.sql?function=read_file#function) reads the contents of a file on the server and returns a text string. + - [`sqlpage.read_file_as_data_url`](https://sql.ophir.dev/functions.sql?function=read_file#function) reads the contents of a file on the server and returns a [data URL](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs). This is useful to embed images directly in web pages, or make link + +## 0.16.1 (2023-11-22) - fix a bug where setting a variable to a non-string value would always set it to null - clearer debug logs (https://github.com/wooorm/markdown-rs/pull/92) diff --git a/Cargo.lock b/Cargo.lock index 24e54069..851b3b1c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -69,6 +69,44 @@ dependencies = [ "syn 2.0.39", ] +[[package]] +name = "actix-multipart" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b960e2aea75f49c8f069108063d12a48d329fc8b60b786dfc7552a9d5918d2d" +dependencies = [ + "actix-multipart-derive", + "actix-utils", + "actix-web", + "bytes", + "derive_more", + "futures-core", + "futures-util", + "httparse", + "local-waker", + "log", + "memchr", + "mime", + "serde", + "serde_json", + "serde_plain", + "tempfile", + "tokio", +] + +[[package]] +name = "actix-multipart-derive" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a0a77f836d869f700e5b47ac7c3c8b9c8bc82e4aec861954c6198abee3ebd4d" +dependencies = [ + "darling", + "parse-size", + "proc-macro2", + "quote", + "syn 2.0.39", +] + [[package]] name = "actix-router" version = "0.5.1" @@ -694,6 +732,41 @@ dependencies = [ "typenum", ] +[[package]] +name = "darling" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.39", +] + +[[package]] +name = "darling_macro" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.39", +] + [[package]] name = "dary_heap" version = "0.3.6" @@ -846,6 +919,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + [[package]] name = "finl_unicode" version = "1.2.0" @@ -1210,6 +1289,12 @@ dependencies = [ "cc", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "0.4.0" @@ -1685,6 +1770,12 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "parse-size" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "944553dd59c802559559161f9816429058b869003836120e262e8caec061b7ae" + [[package]] name = "password-hash" version = "0.5.0" @@ -2143,6 +2234,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_plain" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce1fc6db65a611022b23a0dec6975d63fb80a302cb3388835ff02c097258d50" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -2271,6 +2371,7 @@ dependencies = [ name = "sqlpage" version = "0.16.1" dependencies = [ + "actix-multipart", "actix-rt", "actix-web", "actix-web-httpauth", @@ -2280,6 +2381,7 @@ dependencies = [ "async-stream", "async-trait", "awc", + "base64 0.21.5", "chrono", "config", "dashmap", @@ -2439,6 +2541,12 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "subtle" version = "2.5.0" @@ -2467,6 +2575,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tempfile" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" +dependencies = [ + "cfg-if", + "fastrand", + "redox_syscall", + "rustix", + "windows-sys", +] + [[package]] name = "termcolor" version = "1.4.0" diff --git a/Cargo.toml b/Cargo.toml index 10819825..3f806844 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,6 +45,8 @@ password-hash = "0.5.0" argon2 = "0.5.0" actix-web-httpauth = "0.8.0" rand = "0.8.5" +actix-multipart = "0.6.1" +base64 = "0.21.5" [build-dependencies] awc = { version = "3", features = ["rustls"] } diff --git a/configuration.md b/configuration.md index 30f28534..e8e5669e 100644 --- a/configuration.md +++ b/configuration.md @@ -16,6 +16,7 @@ on a [JSON](https://en.wikipedia.org/wiki/JSON) file placed in `sqlpage/sqlpage. | `sqlite_extensions` | | An array of SQLite extensions to load, such as `mod_spatialite` | | `web_root` | `.` | The root directory of the web server, where the `index.sql` file is located. | | `allow_exec` | false | Allow usage of the `sqlpage.exec` function. Do this only if all users with write access to sqlpage query files and to the optional `sqlpage_files` table on the database are trusted. | +| `max_uploaded_file_size` | 10485760 | Maximum size of uploaded files in bytes. Defaults to 10 MiB. | You can find an example configuration file in [`sqlpage/sqlpage.json`](./sqlpage/sqlpage.json). @@ -28,6 +29,9 @@ but in uppercase. The environment variable name can optionally be prefixed with `SQLPAGE_`. +Additionnally, when troubleshooting, you can set the [`RUST_LOG`](https://docs.rs/env_logger/latest/env_logger/#enabling-logging) +environment variable to `sqlpage=debug` to get more detailed logs and see exactly what SQLPage is doing. + ### Example ```bash diff --git a/examples/official-site/examples/handle_picture_upload.sql b/examples/official-site/examples/handle_picture_upload.sql new file mode 100644 index 00000000..8f8d297d --- /dev/null +++ b/examples/official-site/examples/handle_picture_upload.sql @@ -0,0 +1,5 @@ +select 'card' as component; +select 'Your picture' as title; + +select 'debug' as component; +select :my_file as file; \ No newline at end of file diff --git a/examples/official-site/sqlpage/migrations/01_documentation.sql b/examples/official-site/sqlpage/migrations/01_documentation.sql index 68b943fb..b5fc73e4 100644 --- a/examples/official-site/sqlpage/migrations/01_documentation.sql +++ b/examples/official-site/sqlpage/migrations/01_documentation.sql @@ -378,6 +378,22 @@ if your website has authenticated users that can perform sensitive actions throu {"name": "password", "label": "Password", "type": "password", "width": 6}, {"name": "password_confirmation", "label": "Password confirmation", "type": "password", "width": 6}, {"name": "terms", "label": "I accept the terms and conditions", "type": "checkbox", "required": true} + ]')), + ('form', ' +## File upload + +You can use the `file` type to allow the user to upload a file. +The file will be uploaded to the server, and you will be able to access it using the +[`sqlpage.uploaded_file_path`](functions.sql?function=uploaded_file_path#function) function. + +Here is how you could save the uploaded file to a table in the database: + +```sql +INSERT INTO uploaded_file(name, data) VALUES(:filename, sqlpage.uploaded_file_data_url(:filename)) +``` +', + json('[{"component":"form", "title": "Upload a picture", "validate": "Upload", "action": "examples/handle_picture_upload.sql"}, + {"name": "my_file", "type": "file", "accept": "image/png, image/jpeg", "label": "Picture", "description": "Upload a nice picture", "required": true} ]')) ; diff --git a/examples/official-site/sqlpage/migrations/23_uploaded_file_functions.sql b/examples/official-site/sqlpage/migrations/23_uploaded_file_functions.sql new file mode 100644 index 00000000..da624abd --- /dev/null +++ b/examples/official-site/sqlpage/migrations/23_uploaded_file_functions.sql @@ -0,0 +1,99 @@ +-- Insert the 'variables' function into sqlpage_functions table +INSERT INTO sqlpage_functions ( + "name", + "introduced_in_version", + "icon", + "description_md" +) +VALUES ( + 'uploaded_file_path', + '0.17.0', + 'upload', + 'Returns the path to a temporary file containing the contents of an uploaded file. + +## Example: handling a picture upload + +### Making a form + +```sql +select ''form'' as component, ''handle_picture_upload.sql'' as action; +select ''myfile'' as name, ''file'' as type, ''Picture'' as label; +select ''title'' as name, ''text'' as type, ''Title'' as label; +``` + +### Handling the form response + +In `handle_picture_upload.sql`, one can process the form results like this: + +```sql +insert into pictures (title, path) values (:title, sqlpage.read_file_as_data_url(sqlpage.uploaded_file_path(''myfile''))); +``` +' +), +( + 'uploaded_file_mime_type', + '0.17.0', + 'file-settings', + 'Returns the MIME type of an uploaded file. + +## Example: handling a picture upload + +When letting the user upload a picture, you may want to check that the uploaded file is indeed an image. + +```sql +select ''redirect'' as component, + ''invalid_file.sql'' as link +where sqlpage.uploaded_file_mime_type(''myfile'') not like ''image/%''; +``` + +In `invalid_file.sql`, you can display an error message to the user: + +```sql +select ''alert'' as component, ''Error'' as title, + ''Invalid file type'' as description, + ''alert-circle'' as icon, ''red'' as color; +``` + +## Example: white-listing file types + +You could have a database table containing the allowed MIME types, and check that the uploaded file is of one of those types: + +```sql +select ''redirect'' as component, + ''invalid_file.sql'' as link +where sqlpage.uploaded_file_mime_type(''myfile'') not in (select mime_type from allowed_mime_types); +``` +' +); + +INSERT INTO sqlpage_function_parameters ( + "function", + "index", + "name", + "description_md", + "type" +) +VALUES ( + 'uploaded_file_path', + 1, + 'name', + 'Name of the file input field in the form.', + 'TEXT' +), +( + 'uploaded_file_path', + 2, + 'allowed_mime_type', + 'Makes the function return NULL if the uploaded file is not of the specified MIME type. + If omitted, any MIME type is allowed. + This makes it possible to restrict the function to only accept certain file types.', + 'TEXT' +), +( + 'uploaded_file_mime_type', + 1, + 'name', + 'Name of the file input field in the form.', + 'TEXT' +) +; diff --git a/examples/official-site/sqlpage/migrations/24_read_file_as_data_url.sql b/examples/official-site/sqlpage/migrations/24_read_file_as_data_url.sql new file mode 100644 index 00000000..e083381f --- /dev/null +++ b/examples/official-site/sqlpage/migrations/24_read_file_as_data_url.sql @@ -0,0 +1,41 @@ +-- Insert the 'variables' function into sqlpage_functions table +INSERT INTO sqlpage_functions ( + "name", + "introduced_in_version", + "icon", + "description_md" +) +VALUES ( + 'read_file_as_data_url', + '0.17.0', + 'file-dollar', + 'Returns a [data URL](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs) +containing the contents of the given file. + +## Example: inlining a picture + +```sql +select ''card'' as component; +select ''Picture'' as title, sqlpage.read_file_as_data_url(''/path/to/picture.jpg'') as top_image; +``` + +> **Note:** Data URLs are larger than the original file they represent, so they should only be used for small files (a few kilobytes). +> Otherwise, the page will take a long time to load. +'); + +-- Insert the parameters for the 'variables' function into sqlpage_function_parameters table +-- Parameter 1: 'method' parameter +INSERT INTO sqlpage_function_parameters ( + "function", + "index", + "name", + "description_md", + "type" +) +VALUES ( + 'read_file_as_data_url', + 1, + 'name', + 'Path to the file to read.', + 'TEXT' +); diff --git a/examples/official-site/sqlpage/migrations/25_read_file_as_text.sql b/examples/official-site/sqlpage/migrations/25_read_file_as_text.sql new file mode 100644 index 00000000..5e7572a4 --- /dev/null +++ b/examples/official-site/sqlpage/migrations/25_read_file_as_text.sql @@ -0,0 +1,40 @@ +-- Insert the 'variables' function into sqlpage_functions table +INSERT INTO sqlpage_functions ( + "name", + "introduced_in_version", + "icon", + "description_md" +) +VALUES ( + 'read_file_as_text', + '0.17.0', + 'file-invoice', + 'Returns a string containing the contents of the given file. + +The file must be a raw text file using UTF-8 encoding. + +## Example + +### Rendering a markdown file + +```sql +select ''text'' as component, sqlpage.read_file_as_text(''/path/to/file.md'') as text; +``` +'); + +-- Insert the parameters for the 'variables' function into sqlpage_function_parameters table +-- Parameter 1: 'method' parameter +INSERT INTO sqlpage_function_parameters ( + "function", + "index", + "name", + "description_md", + "type" +) +VALUES ( + 'read_file_as_text', + 1, + 'name', + 'Path to the file to read.', + 'TEXT' +); diff --git a/sqlpage/templates/form.handlebars b/sqlpage/templates/form.handlebars index 8fad78bb..b635a83c 100644 --- a/sqlpage/templates/form.handlebars +++ b/sqlpage/templates/form.handlebars @@ -85,6 +85,7 @@ {{~#if formtarget}}formtarget="{{formtarget}}" {{/if~}} {{~#if list}}list="{{list}}" {{/if~}} {{~#if multiple}}multiple="{{multiple}}" {{/if~}} + {{~#if accept}}accept="{{accept}}" {{/if~}} {{~#if autofocus}}autofocus {{/if~}} > {{/if}} @@ -95,6 +96,10 @@ {{/if}} {{/if}} + + {{#if (eq type "file")}} + {{#delay}}formenctype="multipart/form-data"{{/delay}} + {{/if}} {{/each_row}} {{#if (ne validate '')}} @@ -103,6 +108,7 @@ {{#if validate_shape}} btn-{{validate_shape}} {{/if}} {{#if validate_outline}} btn-outline-{{validate_outline}} {{/if}} {{#if validate_size}} btn-{{validate_size}} {{/if}}" + {{flush_delayed}} type="submit" {{#if validate}}value="{{validate}}"{{/if}}> {{/if}} diff --git a/src/app_config.rs b/src/app_config.rs index 2ee7b71c..cab074b1 100644 --- a/src/app_config.rs +++ b/src/app_config.rs @@ -44,6 +44,10 @@ pub struct AppConfig { /// them the ability to execute arbitrary shell commands on the server. #[serde(default)] pub allow_exec: bool, + + /// Maximum size of uploaded files in bytes. The default is 10MiB (10 * 1024 * 1024 bytes) + #[serde(default = "default_max_file_size")] + pub max_uploaded_file_size: usize, } pub fn load() -> anyhow::Result { @@ -130,6 +134,10 @@ fn default_web_root() -> PathBuf { }) } +fn default_max_file_size() -> usize { + 10 * 1024 * 1024 +} + #[cfg(test)] pub mod tests { use super::AppConfig; diff --git a/src/filesystem.rs b/src/filesystem.rs index dcca3226..15f3266b 100644 --- a/src/filesystem.rs +++ b/src/filesystem.rs @@ -207,14 +207,14 @@ async fn test_sql_file_read_utf8() -> anyhow::Result<()> { .db .connection .execute( - r#" + r" CREATE TABLE sqlpage_files( path VARCHAR(255) NOT NULL PRIMARY KEY, contents BLOB, last_modified TIMESTAMP DEFAULT CURRENT_TIMESTAMP ); INSERT INTO sqlpage_files(path, contents) VALUES ('unit test file.txt', 'Héllö world! 😀'); - "#, + ", ) .await?; let fs = FileSystem::init("/", &state.db).await; diff --git a/src/main.rs b/src/main.rs index c78a0398..eddd6d0d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,6 +15,11 @@ async fn main() { async fn start() -> anyhow::Result<()> { let app_config = app_config::load()?; log::debug!("Starting with the following configuration: {app_config:?}"); + std::env::set_current_dir(&app_config.web_root)?; + log::info!( + "Set the working directory to {}", + app_config.web_root.display() + ); let state = AppState::init(&app_config).await?; webserver::database::migrations::apply(&state.db).await?; let listen_on = app_config.listen_on; @@ -46,5 +51,8 @@ async fn log_welcome_message(config: &AppConfig) { } fn init_logging() { - env_logger::init_from_env(env_logger::Env::new().default_filter_or("info")); + let env = env_logger::Env::new().default_filter_or("info"); + let mut logging = env_logger::Builder::from_env(env); + logging.format_timestamp_millis(); + logging.init(); } diff --git a/src/webserver/database/execute_queries.rs b/src/webserver/database/execute_queries.rs index 366c11a7..1495de01 100644 --- a/src/webserver/database/execute_queries.rs +++ b/src/webserver/database/execute_queries.rs @@ -7,7 +7,8 @@ use std::collections::HashMap; use super::sql::{ParsedSqlFile, ParsedStatement, StmtWithParams}; use crate::webserver::database::sql_pseudofunctions::extract_req_param; use crate::webserver::database::sql_to_json::row_to_string; -use crate::webserver::http::{RequestInfo, SingleOrVec}; +use crate::webserver::http::SingleOrVec; +use crate::webserver::http_request_info::RequestInfo; use sqlx::any::{AnyArguments, AnyQueryResult, AnyRow, AnyStatement, AnyTypeInfo}; use sqlx::pool::PoolConnection; diff --git a/src/webserver/database/sql_pseudofunctions.rs b/src/webserver/database/sql_pseudofunctions.rs index b9d950c7..174dd2d8 100644 --- a/src/webserver/database/sql_pseudofunctions.rs +++ b/src/webserver/database/sql_pseudofunctions.rs @@ -2,12 +2,11 @@ use std::{borrow::Cow, collections::HashMap}; use actix_web::http::StatusCode; use actix_web_httpauth::headers::authorization::Basic; +use base64::Engine; +use mime_guess::{mime::APPLICATION_OCTET_STREAM, Mime}; use sqlparser::ast::FunctionArg; -use crate::webserver::{ - http::{RequestInfo, SingleOrVec}, - ErrorWithStatus, -}; +use crate::webserver::{http::SingleOrVec, http_request_info::RequestInfo, ErrorWithStatus}; use super::sql::{ extract_integer, extract_single_quoted_string, extract_single_quoted_string_optional, @@ -35,6 +34,9 @@ pub(super) enum StmtParam { EnvironmentVariable(String), SqlPageVersion, Literal(String), + UploadedFilePath(String), + ReadFileAsText(Box), + ReadFileAsDataUrl(Box), Path, } @@ -89,6 +91,15 @@ pub(super) fn func_call_to_param(func_name: &str, arguments: &mut [FunctionArg]) "version" => StmtParam::SqlPageVersion, "variables" => parse_get_or_post(extract_single_quoted_string_optional(arguments)), "path" => StmtParam::Path, + "uploaded_file_path" => extract_single_quoted_string("uploaded_file_path", arguments) + .map_or_else(StmtParam::Error, StmtParam::UploadedFilePath), + "read_file_as_text" => StmtParam::ReadFileAsText(Box::new(extract_variable_argument( + "read_file_as_text", + arguments, + ))), + "read_file_as_data_url" => StmtParam::ReadFileAsDataUrl(Box::new( + extract_variable_argument("read_file_as_data_url", arguments), + )), unknown_name => StmtParam::Error(format!( "Unknown function {unknown_name}({})", FormatArguments(arguments) @@ -106,6 +117,8 @@ pub(super) async fn extract_req_param<'a>( StmtParam::HashPassword(inner) => has_password_param(inner, request).await?, StmtParam::Exec(args_params) => exec_external_command(args_params, request).await?, StmtParam::UrlEncode(inner) => url_encode(inner, request)?, + StmtParam::ReadFileAsText(inner) => read_file_as_text(inner, request).await?, + StmtParam::ReadFileAsDataUrl(inner) => read_file_as_data_url(inner, request).await?, _ => extract_req_param_non_nested(param, request)?, }) } @@ -176,6 +189,55 @@ async fn exec_external_command<'a>( ))) } +async fn read_file_as_text<'a>( + param0: &StmtParam, + request: &'a RequestInfo, +) -> Result>, anyhow::Error> { + let Some(evaluated_param) = extract_req_param_non_nested(param0, request)? else { + log::debug!("read_file: first argument is NULL, returning NULL"); + return Ok(None); + }; + let bytes = tokio::fs::read(evaluated_param.as_ref()) + .await + .with_context(|| format!("Unable to read file {evaluated_param}"))?; + let as_str = String::from_utf8(bytes) + .with_context(|| format!("read_file_as_text: {param0:?} does not contain raw UTF8 text"))?; + Ok(Some(Cow::Owned(as_str))) +} + +async fn read_file_as_data_url<'a>( + param0: &StmtParam, + request: &'a RequestInfo, +) -> Result>, anyhow::Error> { + let Some(evaluated_param) = extract_req_param_non_nested(param0, request)? else { + log::debug!("read_file: first argument is NULL, returning NULL"); + return Ok(None); + }; + let bytes = tokio::fs::read(evaluated_param.as_ref()) + .await + .with_context(|| format!("Unable to read file {evaluated_param}"))?; + let mime = mime_from_upload(param0, request).map_or_else( + || Cow::Owned(mime_guess_from_filename(&evaluated_param)), + Cow::Borrowed, + ); + let mut data_url = format!("data:{}/{};base64,", mime.type_(), mime.subtype()); + base64::engine::general_purpose::URL_SAFE.encode_string(bytes, &mut data_url); + Ok(Some(Cow::Owned(data_url))) +} + +fn mime_from_upload<'a>(param0: &StmtParam, request: &'a RequestInfo) -> Option<&'a Mime> { + if let StmtParam::UploadedFilePath(name) = param0 { + request.uploaded_files.get(name)?.content_type.as_ref() + } else { + None + } +} + +fn mime_guess_from_filename(filename: &str) -> Mime { + let maybe_mime = mime_guess::from_path(filename).first(); + maybe_mime.unwrap_or(APPLICATION_OCTET_STREAM) +} + pub(super) fn extract_req_param_non_nested<'a>( param: &StmtParam, request: &'a RequestInfo, @@ -210,6 +272,15 @@ pub(super) fn extract_req_param_non_nested<'a>( StmtParam::Literal(x) => Some(Cow::Owned(x.to_string())), StmtParam::AllVariables(get_or_post) => extract_get_or_post(*get_or_post, request), StmtParam::Path => Some(Cow::Borrowed(&request.path)), + StmtParam::UploadedFilePath(x) => request + .uploaded_files + .get(x) + .and_then(|x| x.file.path().to_str()) + .map(Cow::Borrowed), + StmtParam::ReadFileAsText(_) => bail!("Nested read_file_as_text() function not allowed",), + StmtParam::ReadFileAsDataUrl(_) => { + bail!("Nested read_file_as_data_url() function not allowed",) + } }) } diff --git a/src/webserver/database/sql_to_json.rs b/src/webserver/database/sql_to_json.rs index 070d979b..68e7592d 100644 --- a/src/webserver/database/sql_to_json.rs +++ b/src/webserver/database/sql_to_json.rs @@ -1,4 +1,3 @@ -pub use crate::file_cache::FileCache; use crate::utils::add_value_to_map; use serde_json::{self, Map, Value}; use sqlx::any::AnyRow; diff --git a/src/webserver/http.rs b/src/webserver/http.rs index d3cdac75..205080b0 100644 --- a/src/webserver/http.rs +++ b/src/webserver/http.rs @@ -1,37 +1,32 @@ use crate::render::{HeaderContext, PageContext, RenderContext}; use crate::webserver::database::{execute_queries::stream_query_results, DbItem}; +use crate::webserver::http_request_info::extract_request_info; use crate::webserver::ErrorWithStatus; use crate::{AppState, Config, ParsedSqlFile}; use actix_web::dev::{fn_service, ServiceFactory, ServiceRequest}; use actix_web::error::ErrorInternalServerError; use actix_web::http::header::{ContentType, Header, HttpDate, IfModifiedSince, LastModified}; use actix_web::http::{header, StatusCode, Uri}; -use actix_web::web::Form; use actix_web::{ - dev::ServiceResponse, middleware, middleware::Logger, web, web::Bytes, App, FromRequest, - HttpResponse, HttpServer, + dev::ServiceResponse, middleware, middleware::Logger, web, web::Bytes, App, HttpResponse, + HttpServer, }; +use super::static_content; use actix_web::body::{BoxBody, MessageBody}; -use actix_web_httpauth::headers::authorization::{Authorization, Basic}; use anyhow::Context; use chrono::{DateTime, Utc}; use futures_util::stream::Stream; use futures_util::StreamExt; use std::borrow::Cow; -use std::collections::hash_map::Entry; -use std::collections::HashMap; use std::io::Write; use std::mem; -use std::net::IpAddr; use std::path::PathBuf; use std::pin::Pin; use std::sync::Arc; use std::time::SystemTime; use tokio::sync::mpsc; -use super::static_content; - /// If the sending queue exceeds this number of outgoing messages, an error will be thrown /// This prevents a single request from using up all available memory const MAX_PENDING_MESSAGES: usize = 128; @@ -286,9 +281,7 @@ fn send_anyhow_error(e: &anyhow::Error, resp_send: tokio::sync::oneshot::Sender< .unwrap_or_else(|_| log::error!("could not send headers")); } -type ParamMap = HashMap; - -#[derive(Debug, serde::Serialize, serde::Deserialize)] +#[derive(Debug, serde::Serialize, serde::Deserialize, PartialEq)] #[serde(untagged)] pub enum SingleOrVec { Single(String), @@ -296,7 +289,7 @@ pub enum SingleOrVec { } impl SingleOrVec { - fn merge(&mut self, other: Self) { + pub(crate) fn merge(&mut self, other: Self) { match (self, other) { (Self::Single(old), Self::Single(new)) => *old = new, (old, mut new) => { @@ -322,80 +315,6 @@ impl SingleOrVec { } } -#[derive(Debug)] -pub struct RequestInfo { - pub path: String, - pub get_variables: ParamMap, - pub post_variables: ParamMap, - pub headers: ParamMap, - pub client_ip: Option, - pub cookies: ParamMap, - pub basic_auth: Option, - pub app_state: Arc, -} - -fn param_map>(values: PAIRS) -> ParamMap { - values - .into_iter() - .fold(HashMap::new(), |mut map, (mut k, v)| { - let entry = if k.ends_with("[]") { - k.replace_range(k.len() - 2.., ""); - SingleOrVec::Vec(vec![v]) - } else { - SingleOrVec::Single(v) - }; - match map.entry(k) { - Entry::Occupied(mut s) => { - SingleOrVec::merge(s.get_mut(), entry); - } - Entry::Vacant(v) => { - v.insert(entry); - } - } - map - }) -} - -async fn extract_request_info(req: &mut ServiceRequest, app_state: Arc) -> RequestInfo { - let (http_req, payload) = req.parts_mut(); - let post_variables = Form::>::from_request(http_req, payload) - .await - .map(Form::into_inner) - .unwrap_or_default(); - - let headers = req.headers().iter().map(|(name, value)| { - ( - name.to_string(), - String::from_utf8_lossy(value.as_bytes()).to_string(), - ) - }); - let get_variables = web::Query::>::from_query(req.query_string()) - .map(web::Query::into_inner) - .unwrap_or_default(); - let client_ip = req.peer_addr().map(|addr| addr.ip()); - - let raw_cookies = req.cookies(); - let cookies = raw_cookies - .iter() - .flat_map(|c| c.iter()) - .map(|cookie| (cookie.name().to_string(), cookie.value().to_string())); - - let basic_auth = Authorization::::parse(req) - .ok() - .map(Authorization::into_scheme); - - RequestInfo { - path: req.path().to_string(), - headers: param_map(headers), - get_variables: param_map(get_variables), - post_variables: param_map(post_variables), - client_ip, - cookies: param_map(cookies), - basic_auth, - app_state, - } -} - /// Resolves the path in a query to the path to a local SQL file if there is one that matches fn path_to_sql_file(path: &str) -> Option { let mut path = PathBuf::from(path.strip_prefix('/').unwrap_or(path)); diff --git a/src/webserver/http_request_info.rs b/src/webserver/http_request_info.rs new file mode 100644 index 00000000..134aa940 --- /dev/null +++ b/src/webserver/http_request_info.rs @@ -0,0 +1,303 @@ +use super::http::SingleOrVec; +use crate::AppState; +use actix_multipart::form::bytes::Bytes; +use actix_multipart::form::tempfile::TempFile; +use actix_multipart::form::FieldReader; +use actix_multipart::form::Limits; +use actix_multipart::Multipart; +use actix_web::dev::ServiceRequest; +use actix_web::http::header::Header; +use actix_web::http::header::CONTENT_TYPE; +use actix_web::web; +use actix_web::web::Form; +use actix_web::FromRequest; +use actix_web::HttpRequest; +use actix_web_httpauth::headers::authorization::Authorization; +use actix_web_httpauth::headers::authorization::Basic; +use anyhow::anyhow; +use std::collections::hash_map::Entry; +use std::collections::HashMap; +use std::net::IpAddr; +use std::sync::Arc; +use tokio_stream::StreamExt; + +#[derive(Debug)] +pub struct RequestInfo { + pub path: String, + pub get_variables: ParamMap, + pub post_variables: ParamMap, + pub uploaded_files: HashMap, + pub headers: ParamMap, + pub client_ip: Option, + pub cookies: ParamMap, + pub basic_auth: Option, + pub app_state: Arc, +} + +pub(crate) async fn extract_request_info( + req: &mut ServiceRequest, + app_state: Arc, +) -> RequestInfo { + let (http_req, payload) = req.parts_mut(); + let config = &app_state.config; + let (post_variables, uploaded_files) = extract_post_data(http_req, payload, config).await; + + let headers = req.headers().iter().map(|(name, value)| { + ( + name.to_string(), + String::from_utf8_lossy(value.as_bytes()).to_string(), + ) + }); + let get_variables = web::Query::>::from_query(req.query_string()) + .map(web::Query::into_inner) + .unwrap_or_default(); + let client_ip = req.peer_addr().map(|addr| addr.ip()); + + let raw_cookies = req.cookies(); + let cookies = raw_cookies + .iter() + .flat_map(|c| c.iter()) + .map(|cookie| (cookie.name().to_string(), cookie.value().to_string())); + + let basic_auth = Authorization::::parse(req) + .ok() + .map(Authorization::into_scheme); + + RequestInfo { + path: req.path().to_string(), + headers: param_map(headers), + get_variables: param_map(get_variables), + post_variables: param_map(post_variables), + uploaded_files: HashMap::from_iter(uploaded_files), + client_ip, + cookies: param_map(cookies), + basic_auth, + app_state, + } +} + +async fn extract_post_data( + http_req: &mut actix_web::HttpRequest, + payload: &mut actix_web::dev::Payload, + config: &crate::app_config::AppConfig, +) -> (Vec<(String, String)>, Vec<(String, TempFile)>) { + let content_type = http_req + .headers() + .get(&CONTENT_TYPE) + .map(AsRef::as_ref) + .unwrap_or_default(); + if content_type.starts_with(b"application/x-www-form-urlencoded") { + match extract_urlencoded_post_variables(http_req, payload).await { + Ok(post_variables) => (post_variables, Vec::new()), + Err(e) => { + log::error!("Could not read urlencoded POST request data: {}", e); + (Vec::new(), Vec::new()) + } + } + } else if content_type.starts_with(b"multipart/form-data") { + extract_multipart_post_data(http_req, payload, config) + .await + .unwrap_or_else(|e| { + log::error!("Could not read request data: {}", e); + (Vec::new(), Vec::new()) + }) + } else { + let ct_str = String::from_utf8_lossy(content_type); + log::debug!("Not parsing POST data from request without known content type {ct_str}"); + (Vec::new(), Vec::new()) + } +} + +async fn extract_urlencoded_post_variables( + http_req: &mut actix_web::HttpRequest, + payload: &mut actix_web::dev::Payload, +) -> actix_web::Result> { + Form::>::from_request(http_req, payload) + .await + .map(Form::into_inner) +} + +async fn extract_multipart_post_data( + http_req: &mut actix_web::HttpRequest, + payload: &mut actix_web::dev::Payload, + config: &crate::app_config::AppConfig, +) -> anyhow::Result<(Vec<(String, String)>, Vec<(String, TempFile)>)> { + let mut post_variables = Vec::new(); + let mut uploaded_files = Vec::new(); + + let mut multipart = Multipart::from_request(http_req, payload) + .await + .map_err(|e| anyhow!("could not parse request as multipart form data: {e}"))?; + + let mut limits = Limits::new(config.max_uploaded_file_size, config.max_uploaded_file_size); + log::trace!( + "Parsing multipart form data with a {:?} KiB limit", + limits.total_limit_remaining / 1024 + ); + + while let Some(part) = multipart.next().await { + let field = part.map_err(|e| anyhow!("unable to read form field: {e}"))?; + // test if field is a file + let filename = field.content_disposition().get_filename(); + let field_name = field + .content_disposition() + .get_name() + .unwrap_or_default() + .to_string(); + log::trace!("Parsing multipart field: {}", field_name); + if let Some(filename) = filename { + log::debug!("Extracting file: {field_name} ({filename})"); + let extracted = extract_file(http_req, field, &mut limits).await?; + log::trace!("Extracted file {field_name} to {:?}", extracted.file.path()); + uploaded_files.push((field_name, extracted)); + } else { + let text_contents = extract_text(http_req, field, &mut limits).await?; + log::trace!("Extracted field as text: {field_name} = {text_contents:?}"); + post_variables.push((field_name, text_contents)); + } + } + Ok((post_variables, uploaded_files)) +} + +async fn extract_text( + req: &HttpRequest, + field: actix_multipart::Field, + limits: &mut Limits, +) -> anyhow::Result { + // field is an async stream of Result objects, we collect them into a Vec + let data = Bytes::read_field(req, field, limits) + .await + .map(|bytes| bytes.data) + .map_err(|e| anyhow!("failed to read form field data: {e}"))?; + Ok(String::from_utf8(data.to_vec())?) +} + +async fn extract_file( + req: &HttpRequest, + field: actix_multipart::Field, + limits: &mut Limits, +) -> anyhow::Result { + // extract a tempfile from the field + let file = TempFile::read_field(req, field, limits) + .await + .map_err(|e| anyhow!("Failed to save uploaded file: {e}"))?; + Ok(file) +} + +pub type ParamMap = HashMap; + +fn param_map>(values: PAIRS) -> ParamMap { + values + .into_iter() + .fold(HashMap::new(), |mut map, (mut k, v)| { + let entry = if k.ends_with("[]") { + k.replace_range(k.len() - 2.., ""); + SingleOrVec::Vec(vec![v]) + } else { + SingleOrVec::Single(v) + }; + match map.entry(k) { + Entry::Occupied(mut s) => { + SingleOrVec::merge(s.get_mut(), entry); + } + Entry::Vacant(v) => { + v.insert(entry); + } + } + map + }) +} + +#[cfg(test)] +mod test { + use super::*; + use crate::app_config::AppConfig; + use actix_web::{http::header::ContentType, test::TestRequest}; + + #[actix_web::test] + async fn test_extract_empty_request() { + let config = + serde_json::from_str::(r#"{"listen_on": "localhost:1234"}"#).unwrap(); + let mut service_request = TestRequest::default().to_srv_request(); + let app_data = Arc::new(AppState::init(&config).await.unwrap()); + let request_info = extract_request_info(&mut service_request, app_data).await; + assert_eq!(request_info.post_variables.len(), 0); + assert_eq!(request_info.uploaded_files.len(), 0); + assert_eq!(request_info.get_variables.len(), 0); + } + + #[actix_web::test] + async fn test_extract_urlencoded_request() { + let config = + serde_json::from_str::(r#"{"listen_on": "localhost:1234"}"#).unwrap(); + let mut service_request = TestRequest::get() + .uri("/?my_array[]=5") + .insert_header(ContentType::form_url_encoded()) + .set_payload("my_array[]=3&my_array[]=Hello%20World&repeated=1&repeated=2") + .to_srv_request(); + let app_data = Arc::new(AppState::init(&config).await.unwrap()); + let request_info = extract_request_info(&mut service_request, app_data).await; + assert_eq!( + request_info.post_variables, + vec![ + ( + "my_array".to_string(), + SingleOrVec::Vec(vec!["3".to_string(), "Hello World".to_string()]) + ), + ("repeated".to_string(), SingleOrVec::Single("2".to_string())), // without brackets, only the last value is kept + ] + .into_iter() + .collect::() + ); + assert_eq!(request_info.uploaded_files.len(), 0); + assert_eq!( + request_info.get_variables, + vec![( + "my_array".to_string(), + SingleOrVec::Vec(vec!["5".to_string()]) + )] // with brackets, even if there is only one value, it is kept as a vector + .into_iter() + .collect::() + ); + } + + #[actix_web::test] + async fn test_extract_multipart_form_data() { + env_logger::init(); + let config = + serde_json::from_str::(r#"{"listen_on": "localhost:1234"}"#).unwrap(); + let mut service_request = TestRequest::get() + .insert_header(("content-type", "multipart/form-data;boundary=xxx")) + .set_payload( + "--xxx\r\n\ + Content-Disposition: form-data; name=\"my_array[]\"\r\n\ + Content-Type: text/plain\r\n\ + \r\n\ + 3\r\n\ + --xxx\r\n\ + Content-Disposition: form-data; name=\"my_uploaded_file\"; filename=\"test.txt\"\r\n\ + Content-Type: text/plain\r\n\ + \r\n\ + Hello World\r\n\ + --xxx--\r\n" + ) + .to_srv_request(); + let app_data = Arc::new(AppState::init(&config).await.unwrap()); + let request_info = extract_request_info(&mut service_request, app_data).await; + assert_eq!( + request_info.post_variables, + vec![( + "my_array".to_string(), + SingleOrVec::Vec(vec!["3".to_string()]) + ),] + .into_iter() + .collect::() + ); + assert_eq!(request_info.uploaded_files.len(), 1); + let my_upload = &request_info.uploaded_files["my_uploaded_file"]; + assert_eq!(my_upload.file_name.as_ref().unwrap(), "test.txt"); + assert_eq!(request_info.get_variables.len(), 0); + assert_eq!(std::fs::read(&my_upload.file).unwrap(), b"Hello World"); + assert_eq!(request_info.get_variables.len(), 0); + } +} diff --git a/src/webserver/mod.rs b/src/webserver/mod.rs index 6d072671..2a889521 100644 --- a/src/webserver/mod.rs +++ b/src/webserver/mod.rs @@ -1,6 +1,7 @@ pub mod database; pub mod error_with_status; pub mod http; +pub mod http_request_info; pub use database::Database; pub use error_with_status::ErrorWithStatus; diff --git a/tests/index.rs b/tests/index.rs index ffbcf565..4e66ddac 100644 --- a/tests/index.rs +++ b/tests/index.rs @@ -1,7 +1,7 @@ use actix_web::{ body::MessageBody, - http::{self, header::ContentType}, - test, + http::{self, header::ContentType, StatusCode}, + test::{self, TestRequest}, }; use sqlpage::{app_config::AppConfig, webserver::http::main_handler, AppState}; @@ -86,14 +86,43 @@ async fn test_files() { } } -async fn req_path(path: &str) -> Result { +#[actix_web::test] +async fn test_file_upload() -> actix_web::Result<()> { + let req = get_request_to("/tests/upload_file_test.sql") + .await? + .insert_header(("content-type", "multipart/form-data; boundary=1234567890")) + .set_payload( + "--1234567890\r\n\ + Content-Disposition: form-data; name=\"my_file\"; filename=\"testfile.txt\"\r\n\ + Content-Type: text/plain\r\n\ + \r\n\ + Hello, world!\r\n\ + --1234567890--\r\n", + ) + .to_srv_request(); + let resp = main_handler(req).await?; + + assert_eq!(resp.status(), StatusCode::OK); + let body = test::read_body(resp).await; + let body_str = String::from_utf8(body.to_vec()).unwrap(); + assert!( + body_str.contains("Hello, world!"), + "{body_str}\nexpected to contain: Hello, world!" + ); + Ok(()) +} + +async fn get_request_to(path: &str) -> actix_web::Result { init_log(); let config = test_config(); let state = AppState::init(&config).await.unwrap(); let data = actix_web::web::Data::new(state); - let req = test::TestRequest::get() - .uri(path) - .app_data(data) + Ok(test::TestRequest::get().uri(path).app_data(data)) +} + +async fn req_path(path: &str) -> Result { + let req = get_request_to(path) + .await? .insert_header(ContentType::plaintext()) .to_srv_request(); main_handler(req).await diff --git a/tests/it_works.txt b/tests/it_works.txt new file mode 100644 index 00000000..dfbb1c5a --- /dev/null +++ b/tests/it_works.txt @@ -0,0 +1 @@ +It works ! \ No newline at end of file diff --git a/tests/sql_test_files/it_works_read_file_as_data_url.sql b/tests/sql_test_files/it_works_read_file_as_data_url.sql new file mode 100644 index 00000000..21985d7b --- /dev/null +++ b/tests/sql_test_files/it_works_read_file_as_data_url.sql @@ -0,0 +1,12 @@ +set actual = sqlpage.read_file_as_data_url('tests/it_works.txt') +set expected = 'data:text/plain;base64,SXQgd29ya3MgIQ=='; + +select 'text' as component, + case $actual + when $expected + then 'It works !' + else + 'Failed. + Expected: ' || $expected || + 'Got: ' || $actual + end as contents; diff --git a/tests/sql_test_files/it_works_read_file_as_text.sql b/tests/sql_test_files/it_works_read_file_as_text.sql new file mode 100644 index 00000000..18c980e2 --- /dev/null +++ b/tests/sql_test_files/it_works_read_file_as_text.sql @@ -0,0 +1 @@ +select 'text' as component, sqlpage.read_file_as_text('tests/it_works.txt') as contents; diff --git a/tests/sql_test_files/it_works_uploaded_file_is_null.sql b/tests/sql_test_files/it_works_uploaded_file_is_null.sql new file mode 100644 index 00000000..cf3ba9d5 --- /dev/null +++ b/tests/sql_test_files/it_works_uploaded_file_is_null.sql @@ -0,0 +1,7 @@ +-- checks that sqlpage.uploaded_file_path returns null when there is no uploaded_file +set actual = sqlpage.uploaded_file_path('my_file'); +select 'text' as component, + case when $actual is null + then 'It works !' + else 'Failed. Expected: null. Got: ' || $actual + end as contents; diff --git a/tests/upload_file_test.sql b/tests/upload_file_test.sql new file mode 100644 index 00000000..76ec2b91 --- /dev/null +++ b/tests/upload_file_test.sql @@ -0,0 +1,2 @@ +select 'text' as component, + sqlpage.read_file_as_text(sqlpage.uploaded_file_path('my_file')) as contents; \ No newline at end of file