From 4142432483c4e8de44ab7b0d6ac944f3251e03c8 Mon Sep 17 00:00:00 2001 From: David Legrand Date: Mon, 27 Apr 2026 15:49:43 +0200 Subject: [PATCH] net.s3, net.http: add an S3-compatible client and a scheme-handler dispatch in http.fetch (#27008) --- .gitignore | 3 + vlib/net/http/http.v | 40 ++++ vlib/net/s3/README.md | 109 +++++++++ vlib/net/s3/bucket.v | 192 +++++++++++++++ vlib/net/s3/bucket_test.v | 45 ++++ vlib/net/s3/client.v | 405 ++++++++++++++++++++++++++++++++ vlib/net/s3/credentials.v | 204 ++++++++++++++++ vlib/net/s3/credentials_test.v | 104 +++++++++ vlib/net/s3/encoding.v | 103 +++++++++ vlib/net/s3/encoding_test.v | 49 ++++ vlib/net/s3/errors.v | 206 +++++++++++++++++ vlib/net/s3/errors_test.v | 82 +++++++ vlib/net/s3/fetch.v | 162 +++++++++++++ vlib/net/s3/fetch_test.v | 79 +++++++ vlib/net/s3/file.v | 88 +++++++ vlib/net/s3/http_bridge.v | 51 ++++ vlib/net/s3/integration_test.v | 287 +++++++++++++++++++++++ vlib/net/s3/list.v | 121 ++++++++++ vlib/net/s3/list_test.v | 57 +++++ vlib/net/s3/multipart.v | 410 +++++++++++++++++++++++++++++++++ vlib/net/s3/s3.v | 40 ++++ vlib/net/s3/signer.v | 294 +++++++++++++++++++++++ vlib/net/s3/signer_test.v | 224 ++++++++++++++++++ vlib/net/s3/types.v | 169 ++++++++++++++ 24 files changed, 3524 insertions(+) create mode 100644 vlib/net/s3/README.md create mode 100644 vlib/net/s3/bucket.v create mode 100644 vlib/net/s3/bucket_test.v create mode 100644 vlib/net/s3/client.v create mode 100644 vlib/net/s3/credentials.v create mode 100644 vlib/net/s3/credentials_test.v create mode 100644 vlib/net/s3/encoding.v create mode 100644 vlib/net/s3/encoding_test.v create mode 100644 vlib/net/s3/errors.v create mode 100644 vlib/net/s3/errors_test.v create mode 100644 vlib/net/s3/fetch.v create mode 100644 vlib/net/s3/fetch_test.v create mode 100644 vlib/net/s3/file.v create mode 100644 vlib/net/s3/http_bridge.v create mode 100644 vlib/net/s3/integration_test.v create mode 100644 vlib/net/s3/list.v create mode 100644 vlib/net/s3/list_test.v create mode 100644 vlib/net/s3/multipart.v create mode 100644 vlib/net/s3/s3.v create mode 100644 vlib/net/s3/signer.v create mode 100644 vlib/net/s3/signer_test.v create mode 100644 vlib/net/s3/types.v diff --git a/.gitignore b/.gitignore index 0fdcdd5a0..fc5fa41a4 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,9 @@ !vlib/mcp/server.v !vlib/mcp/server_test.v +# unignore vlib/net/s3 +!vlib/net/s3/** + !.github/show_manual_release_cmd_test.v !*/ # Do not add !*.* here; it overrides local excludes from .git/info/exclude. diff --git a/vlib/net/http/http.v b/vlib/net/http/http.v index 4688e6ea0..c0cf0e016 100644 --- a/vlib/net/http/http.v +++ b/vlib/net/http/http.v @@ -1,6 +1,7 @@ // Copyright (c) 2019-2024 Alexander Medvednikov. All rights reserved. // Use of this source code is governed by an MIT license // that can be found in the LICENSE file. +@[has_globals] module http import net.urllib @@ -197,10 +198,49 @@ pub fn prepare(config FetchConfig) !Request { return req } +// SchemeHandlerFn dispatches a `fetch()` call for a non-HTTP scheme. Used by +// out-of-tree-friendly modules like `net.s3` to register themselves at init +// time without forcing `net.http` to know about them statically. +pub type SchemeHandlerFn = fn (config FetchConfig) !Response + +__global scheme_handlers = map[string]SchemeHandlerFn{} + +// register_scheme attaches `handler` as the dispatcher for URLs with the +// given `scheme` (e.g. `'s3'`). Handlers are looked up by `fetch()` before +// the native HTTP path runs. Modules typically call this from `init()`. +pub fn register_scheme(scheme string, handler SchemeHandlerFn) { + scheme_handlers[scheme] = handler +} + +// unregister_scheme removes a previously-registered scheme handler. Mostly +// useful in tests that install a temporary handler. +pub fn unregister_scheme(scheme string) { + scheme_handlers.delete(scheme) +} + +fn scheme_of(url string) string { + colon := url.index(':') or { return '' } + if colon == 0 { + return '' + } + return url[..colon] +} + // TODO: @[noinline] attribute is used for temporary fix the 'get_text()' intermittent segfault / nil value when compiling with GCC 13.2.x and -prod option ( Issue #20506 ) // fetch sends an HTTP request to the `url` with the given method and configuration. +// When `config.url` uses a scheme registered via `register_scheme` (e.g. +// `s3://`), the call is delegated to that handler instead of the native +// HTTP path. @[noinline] pub fn fetch(config FetchConfig) !Response { + if scheme_handlers.len > 0 { + scheme := scheme_of(config.url) + if scheme != '' && scheme != 'http' && scheme != 'https' { + if h := scheme_handlers[scheme] { + return h(config) + } + } + } req := prepare(config)! return req.do()! } diff --git a/vlib/net/s3/README.md b/vlib/net/s3/README.md new file mode 100644 index 000000000..0e2d08cac --- /dev/null +++ b/vlib/net/s3/README.md @@ -0,0 +1,109 @@ +`net.s3` is an S3-compatible client written in pure V. + +It speaks the AWS Signature Version 4 protocol on top of `crypto.hmac` and +`crypto.sha256`, with no third-party dependencies. The same client targets any +S3-compatible endpoint (hosted or self-hosted) by configuring the right +`endpoint`, `region` and credentials. + +## Quick start + +```v ignore +import net.s3 + +c := s3.new_client(s3.Credentials{ + endpoint: 'https://s3.example.com' + access_key_id: '...' + secret_access_key: '...' + bucket: 'my-bucket' +}) + +c.put('hello.txt', 'Hi from V!'.bytes())! +text := c.get_string('hello.txt')! +url := c.presign('hello.txt', expires_in: 3600)! +``` + +## Credentials from the environment + +`Credentials.from_env()` resolves each field from the first non-empty +provider-specific environment variable, so the same code works against AWS, +self-hosted and managed S3 services without reconfiguration: + +```v ignore +import net.s3 + +c := s3.new_client(s3.Credentials.from_env()) +``` + +Supported variables (first non-empty wins per field): + +* key id: `S3_ACCESS_KEY_ID`, `AWS_ACCESS_KEY_ID`, `CELLAR_ADDON_KEY_ID`, + `SCW_ACCESS_KEY`, `B2_APPLICATION_KEY_ID`, `R2_ACCESS_KEY_ID`, `SPACES_KEY` +* secret: `S3_SECRET_ACCESS_KEY`, `AWS_SECRET_ACCESS_KEY`, + `CELLAR_ADDON_KEY_SECRET`, `SCW_SECRET_KEY`, `B2_APPLICATION_KEY`, + `R2_SECRET_ACCESS_KEY`, `SPACES_SECRET` +* session token: `S3_SESSION_TOKEN`, `AWS_SESSION_TOKEN` +* region: `S3_REGION`, `AWS_REGION`, `AWS_DEFAULT_REGION`, + `SCW_DEFAULT_REGION` +* bucket: `S3_BUCKET` +* endpoint: `S3_ENDPOINT`, `AWS_ENDPOINT`, `AWS_ENDPOINT_URL`, + `CELLAR_ADDON_HOST`, `B2_ENDPOINT`, `R2_ENDPOINT`, `SPACES_ENDPOINT` + +## Multipart upload + +`upload_file` automatically picks single-shot or multipart based on file size. +For finer control, `start_multipart` returns a stateful `MultipartUploader` +that streams chunks generated on the fly: + +```v ignore +import net.s3 + +c := s3.new_client(s3.Credentials.from_env()) +c.upload_file('big.bin', '/path/to/big.bin', s3.PutOptions{ + content_type: 'application/octet-stream' +})! +``` + +## `s3://` URLs + +Importing `net.s3` registers an `s3://` scheme handler with `net.http`, so the +generic `http.fetch(url: 's3://...')` route works out of the box. There is +also a direct `s3.fetch` helper: + +```v ignore +import net.s3 + +resp := s3.fetch('s3://my-bucket/hello.txt')! +println(resp.body.bytestr()) +``` + +## File handle + +`Client.file(key)` returns a small `File` reference for ergonomic call sites: + +```v ignore +import net.s3 + +c := s3.new_client(s3.Credentials.from_env()) +f := c.file('hello.txt') +text := f.text()! +url := f.presign(expires_in: 3600)! +``` + +## Tests + +The unit suite is offline and runs by default: + +``` +v test vlib/net/s3/ +``` + +The integration suite is gated on `S3_INTEGRATION=1` and exercises the +client against a live endpoint: + +``` +S3_INTEGRATION=1 \ +S3_HOST=https://s3.example.com \ +S3_KEY_ID=... S3_KEY_SECRET=... \ +S3_BUCKET=v-s3-tests \ +v test vlib/net/s3/integration_test.v +``` diff --git a/vlib/net/s3/bucket.v b/vlib/net/s3/bucket.v new file mode 100644 index 000000000..09dabe6f1 --- /dev/null +++ b/vlib/net/s3/bucket.v @@ -0,0 +1,192 @@ +// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved. +// Use of this source code is governed by an MIT license +// that can be found in the LICENSE file. +module s3 + +// BucketOptions configures bucket-level operations. `region_constraint` +// lets `create_bucket` pin the bucket to a specific region (S3 sends a +// `` body when this is non-empty). +@[params] +pub struct BucketOptions { +pub: + bucket string + acl Acl + region_constraint string +} + +// create_bucket creates a new bucket. Returns: +// - nil on success (HTTP 200) +// - S3Error("BucketAlreadyOwnedByYou") if you already own this bucket +// - S3Error("BucketAlreadyExists") if someone else owns it +// - S3Error("InvalidBucketName") for non-conformant names +// +// The S3 wire response for these states is HTTP 409, parsed from the +// returned XML body. +pub fn (c &Client) create_bucket(opts BucketOptions) ! { + bucket := pick_bucket(c.credentials, opts.bucket)! + validate_bucket_name(bucket)! + creds := c.creds_for(bucket) + path := bucket_path(creds, bucket) + body := if opts.region_constraint != '' { + '${escape_xml(opts.region_constraint)}' + } else { + '' + } + mut headers := map[string]string{} + if opts.acl != .unset { + headers['x-amz-acl'] = opts.acl.to_header_value() + } + if body != '' { + headers['content-type'] = 'application/xml' + } + signed := sign_request(creds, SignRequest{ + method: 'PUT' + path: path + payload_hash: if body == '' { empty_sha256 } else { sha256_hex(body.bytes()) } + extra_headers: headers + })! + resp := c.do_http(signed, body)! + if resp.status_code !in [200, 204] { + return new_http_error(resp.status_code, bucket, resp.body) + } +} + +// delete_bucket removes an empty bucket. S3 returns 409 BucketNotEmpty if it +// still has keys — caller is expected to clean up first or handle the error. +pub fn (c &Client) delete_bucket(opts BucketOptions) ! { + bucket := pick_bucket(c.credentials, opts.bucket)! + creds := c.creds_for(bucket) + path := bucket_path(creds, bucket) + signed := sign_request(creds, SignRequest{ + method: 'DELETE' + path: path + payload_hash: empty_sha256 + })! + resp := c.do_http(signed, '')! + if resp.status_code !in [200, 204] { + return new_http_error(resp.status_code, bucket, resp.body) + } +} + +// bucket_exists checks bucket existence/access. Returns true if accessible (200), +// false on 404 / 403 (no such bucket or no read permission), error on other +// statuses. Uses HEAD under the hood — no body is fetched. +pub fn (c &Client) bucket_exists(opts BucketOptions) !bool { + bucket := pick_bucket(c.credentials, opts.bucket)! + creds := c.creds_for(bucket) + path := bucket_path(creds, bucket) + signed := sign_request(creds, SignRequest{ + method: 'HEAD' + path: path + payload_hash: empty_sha256 + })! + resp := c.do_http(signed, '')! + if resp.status_code == 404 || resp.status_code == 403 { + return false + } + if resp.status_code != 200 { + return new_http_error(resp.status_code, bucket, resp.body) + } + return true +} + +// validate_bucket_name applies the S3 bucket naming rules honoured by most +// providers: +// - 3..63 chars +// - lowercase letters, digits, dots, hyphens only +// - must start/end with letter or digit +// - no consecutive dots, no `.-` / `-.` +// - cannot look like an IPv4 address +// +// Provider-specific reservations (e.g. `xn--`, `sthree-`) are intentionally +// not checked here — they vary, so the server-side error is authoritative. +pub fn validate_bucket_name(name string) ! { + if name.len < 3 || name.len > 63 { + return new_error('InvalidBucketName', 'Bucket name must be 3..63 characters: ${name}') + } + first := name[0] + last := name[name.len - 1] + if !is_lower_alnum(first) || !is_lower_alnum(last) { + return new_error('InvalidBucketName', + 'Bucket name must start and end with a lowercase letter or digit: ${name}') + } + mut prev := u8(0) + for b in name.bytes() { + if !is_bucket_char(b) { + return new_error('InvalidBucketName', 'Bucket name "${name}" contains invalid character: ${[ + b, + ].bytestr()}') + } + if prev == `.` && b == `.` { + return new_error('InvalidBucketName', 'Bucket name "${name}" contains consecutive dots') + } + if (prev == `.` && b == `-`) || (prev == `-` && b == `.`) { + return new_error('InvalidBucketName', + 'Bucket name "${name}" contains adjacent dot/hyphen') + } + prev = b + } + if looks_like_ipv4(name) { + return new_error('InvalidBucketName', 'Bucket name "${name}" looks like an IP address') + } +} + +fn is_lower_alnum(b u8) bool { + return (b >= `a` && b <= `z`) || (b >= `0` && b <= `9`) +} + +fn is_bucket_char(b u8) bool { + return is_lower_alnum(b) || b == `-` || b == `.` +} + +fn looks_like_ipv4(s string) bool { + parts := s.split('.') + if parts.len != 4 { + return false + } + for p in parts { + if p.len == 0 || p.len > 3 { + return false + } + for b in p.bytes() { + if b < `0` || b > `9` { + return false + } + } + } + return true +} + +fn pick_bucket(c Credentials, override string) !string { + b := if override != '' { override } else { c.bucket } + if b == '' { + return new_error('InvalidPath', + 'No bucket given (set Credentials.bucket or pass bucket via options)') + } + return b +} + +fn bucket_path(creds Credentials, bucket string) string { + extra := creds.extra_path() + if creds.virtual_hosted_style { + return if extra == '' { '/' } else { extra } + } + encoded := uri_encode_path(strip_slashes(bucket)) + return '${extra}/${encoded}' +} + +// escape_xml does the minimal XML special-char escape we need for body content. +fn escape_xml(s string) string { + mut out := []u8{cap: s.len} + for b in s.bytes() { + match b { + `<` { out << '<'.bytes() } + `>` { out << '>'.bytes() } + `&` { out << '&'.bytes() } + `"` { out << '"'.bytes() } + `'` { out << '''.bytes() } + else { out << b } + } + } + return out.bytestr() +} diff --git a/vlib/net/s3/bucket_test.v b/vlib/net/s3/bucket_test.v new file mode 100644 index 000000000..8089e4ccb --- /dev/null +++ b/vlib/net/s3/bucket_test.v @@ -0,0 +1,45 @@ +// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved. +// Use of this source code is governed by an MIT license +// that can be found in the LICENSE file. +module s3 + +fn test_validate_bucket_name_accepts_valid() { + validate_bucket_name('my-bucket') or { assert false, 'should pass' } + validate_bucket_name('a.b.c') or { assert false, 'should pass' } + validate_bucket_name('abc') or { assert false, 'should pass' } + validate_bucket_name('a' + 'b'.repeat(61) + 'c') or { assert false, 'should pass' } +} + +fn test_validate_bucket_name_rejects_invalid() { + if _ := validate_bucket_name('Bad-UPPER') { + assert false, 'uppercase must be rejected' + } + if _ := validate_bucket_name('ab') { + assert false, 'shorter than 3 chars must be rejected' + } + if _ := validate_bucket_name('-leading-hyphen') { + assert false, 'leading hyphen must be rejected' + } + if _ := validate_bucket_name('trailing-hyphen-') { + assert false, 'trailing hyphen must be rejected' + } + if _ := validate_bucket_name('192.168.0.1') { + assert false, 'IPv4-shaped name must be rejected' + } + if _ := validate_bucket_name('a..b') { + assert false, 'consecutive dots must be rejected' + } + if _ := validate_bucket_name('a.-b') { + assert false, 'adjacent dot/hyphen must be rejected' + } + if _ := validate_bucket_name('a-.b') { + assert false, 'adjacent hyphen/dot must be rejected' + } + if _ := validate_bucket_name('') { + assert false, 'empty name must be rejected' + } + long := 'a'.repeat(64) + if _ := validate_bucket_name(long) { + assert false, 'longer than 63 chars must be rejected' + } +} diff --git a/vlib/net/s3/client.v b/vlib/net/s3/client.v new file mode 100644 index 000000000..becbd254b --- /dev/null +++ b/vlib/net/s3/client.v @@ -0,0 +1,405 @@ +// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved. +// Use of this source code is governed by an MIT license +// that can be found in the LICENSE file. +module s3 + +import net.http +import time + +// Client is the entry point for S3 operations. It carries default credentials +// and tuneable HTTP behaviour; per-call overrides go through the option params +// of each method. Instantiate once, reuse for many objects. +@[heap] +pub struct Client { +pub: + credentials Credentials + // part_size is the multipart-upload chunk size in bytes (default 5 MiB, + // the S3 minimum). + part_size i64 = 5 * 1024 * 1024 + // queue_size is the intended parallel-upload concurrency for multipart + // (currently sequential; reserved). + queue_size int = 5 + // retry is the number of retry attempts for failed uploads. + retry int = 3 + // read_timeout / write_timeout map onto V's net.http settings. + // Defaults are generous because parts can be 5 MiB+ on slow links. + read_timeout i64 = 5 * 60 * time.second + write_timeout i64 = 5 * 60 * time.second +} + +// new_client builds a Client. Pass an empty Credentials to fall back to env vars. +pub fn new_client(creds Credentials) Client { + resolved := if creds.access_key_id == '' && creds.secret_access_key == '' { + Credentials.from_env().merge(creds) + } else { + creds + } + return Client{ + credentials: resolved + } +} + +// PutOptions configures a put / write call. +@[params] +pub struct PutOptions { +pub: + bucket string + content_type string + content_disposition string + content_encoding string + cache_control string + acl Acl + storage_class StorageClass + request_payer bool + // hash_payload, when true, computes SHA-256 of the body before signing + // instead of using `UNSIGNED-PAYLOAD`. Slightly stronger integrity guarantee + // at the cost of one full body scan. + hash_payload bool +} + +// GetOptions configures a get / read call. +@[params] +pub struct GetOptions { +pub: + bucket string + range string // e.g. 'bytes=0-1023' for partial downloads + version_id string + request_payer bool +} + +// StatOptions configures stat / size / exists checks. +@[params] +pub struct StatOptions { +pub: + bucket string + request_payer bool +} + +// put uploads `data` to `key`. Use `upload_file` / `upload_bytes_multipart` +// for streaming or for files larger than ~100 MiB; `put` keeps everything in +// memory. +pub fn (c &Client) put(key string, data []u8, opts PutOptions) ! { + creds := c.creds_for(opts.bucket) + host := canonical_host(creds, opts.bucket) + if host == '' { + return new_error('InvalidEndpoint', 'cannot determine S3 host (set endpoint or bucket)') + } + path := build_object_path(creds, opts.bucket, key)! + payload_hash := if opts.hash_payload { sha256_hex(data) } else { unsigned_payload } + signed := sign_request(creds, SignRequest{ + method: 'PUT' + path: path + payload_hash: payload_hash + extra_headers: put_object_headers(opts) + })! + resp := c.do_http(signed, data.bytestr())! + if resp.status_code !in [200, 204] { + return new_http_error(resp.status_code, key, resp.body) + } +} + +// put_object_headers maps `PutOptions` to the wire headers used by both the +// single-shot `put` and the multipart `initiate_multipart` paths. Keeping the +// mapping in one place ensures the two upload modes accept the exact same set +// of options without drift. +fn put_object_headers(opts PutOptions) map[string]string { + mut headers := map[string]string{} + if opts.content_type != '' { + headers['content-type'] = opts.content_type + } + if opts.content_disposition != '' { + headers['content-disposition'] = opts.content_disposition + } + if opts.content_encoding != '' { + headers['content-encoding'] = opts.content_encoding + } + if opts.cache_control != '' { + headers['cache-control'] = opts.cache_control + } + if opts.acl != .unset { + headers['x-amz-acl'] = opts.acl.to_header_value() + } + if opts.storage_class != .unset { + headers['x-amz-storage-class'] = opts.storage_class.to_header_value() + } + if opts.request_payer { + headers['x-amz-request-payer'] = 'requester' + } + return headers +} + +// get downloads the entire object body into memory. For large files set a +// `range` and assemble the result yourself, or use a presigned URL with an +// HTTP streaming client. +pub fn (c &Client) get(key string, opts GetOptions) ![]u8 { + creds := c.creds_for(opts.bucket) + path := build_object_path(creds, opts.bucket, key)! + mut headers := map[string]string{} + if opts.range != '' { + headers['range'] = opts.range + } + if opts.request_payer { + headers['x-amz-request-payer'] = 'requester' + } + mut query := '' + if opts.version_id != '' { + query = 'versionId=' + uri_encode_query(opts.version_id) + } + signed := sign_request(creds, SignRequest{ + method: 'GET' + path: path + query: query + payload_hash: empty_sha256 + extra_headers: headers + })! + resp := c.do_http(signed, '')! + if resp.status_code !in [200, 206] { + return new_http_error(resp.status_code, key, resp.body) + } + return resp.body.bytes() +} + +// get_string is a convenience wrapper around `get` that returns the body as a +// V `string`. +pub fn (c &Client) get_string(key string, opts GetOptions) !string { + bytes := c.get(key, opts)! + return bytes.bytestr() +} + +// stat returns the object metadata (size / last_modified / etag / content_type). +// Returns an `S3Error` with code `NoSuchKey` when the object doesn't exist. +pub fn (c &Client) stat(key string, opts StatOptions) !Stat { + creds := c.creds_for(opts.bucket) + path := build_object_path(creds, opts.bucket, key)! + mut headers := map[string]string{} + if opts.request_payer { + headers['x-amz-request-payer'] = 'requester' + } + signed := sign_request(creds, SignRequest{ + method: 'HEAD' + path: path + payload_hash: empty_sha256 + extra_headers: headers + })! + resp := c.do_http(signed, '')! + if resp.status_code == 404 { + return new_error('NoSuchKey', 'No such key: ${key}') + } + if resp.status_code !in [200, 204] { + return new_http_error(resp.status_code, key, resp.body) + } + return Stat{ + size: resp.header.get(.content_length) or { '0' }.i64() + last_modified: resp.header.get(.last_modified) or { '' } + etag: (resp.header.get(.etag) or { '' }).trim('"') + content_type: resp.header.get(.content_type) or { '' } + } +} + +// exists is `stat` + boolean: true on 200, false on 404, error otherwise. +pub fn (c &Client) exists(key string, opts StatOptions) !bool { + c.stat(key, opts) or { + if err is S3Error && err.code == 'NoSuchKey' { + return false + } + return err + } + return true +} + +// size returns just the Content-Length of the object. +pub fn (c &Client) size(key string, opts StatOptions) !i64 { + st := c.stat(key, opts)! + return st.size +} + +// delete removes a single object. S3 returns 204 on success, 204 on +// already-absent (idempotent) — we surface success in both cases. +pub fn (c &Client) delete(key string, opts StatOptions) ! { + creds := c.creds_for(opts.bucket) + path := build_object_path(creds, opts.bucket, key)! + mut headers := map[string]string{} + if opts.request_payer { + headers['x-amz-request-payer'] = 'requester' + } + signed := sign_request(creds, SignRequest{ + method: 'DELETE' + path: path + payload_hash: empty_sha256 + extra_headers: headers + })! + resp := c.do_http(signed, '')! + if resp.status_code !in [200, 204] { + return new_http_error(resp.status_code, key, resp.body) + } +} + +// presign generates a presigned URL — see PresignOptions for tunables. +pub fn (c &Client) presign(key string, opts PresignOptions) !string { + creds := c.creds_for(opts.bucket) + path := build_object_path(creds, opts.bucket, key)! + mut extra := map[string]string{} + if opts.acl != .unset { + extra['X-Amz-Acl'] = opts.acl.to_header_value() + } + if opts.storage_class != .unset { + extra['x-amz-storage-class'] = opts.storage_class.to_header_value() + } + if opts.content_type != '' { + extra['response-content-type'] = opts.content_type + } + if opts.content_disposition != '' { + extra['response-content-disposition'] = opts.content_disposition + } + if opts.request_payer { + extra['x-amz-request-payer'] = 'requester' + } + return presign_url(creds, PresignRequest{ + method: http_method_to_string(opts.method) + path: path + expires_in: opts.expires_in + extra_query: extra + }) +} + +// file returns a File reference for the given key, bound to this client. +// `path` may be `/` if the client has no default bucket and +// `bucket` here is empty. +pub fn (c &Client) file(key string, opts FileOptions) File { + return File{ + client: c + key: key + bucket: opts.bucket + } +} + +// creds_for returns credentials with `bucket` overridden if the caller passed one. +fn (c &Client) creds_for(bucket string) Credentials { + if bucket == '' { + return c.credentials + } + mut copy := c.credentials + copy = Credentials{ + ...copy + bucket: bucket + } + return copy +} + +// build_object_path produces the canonical URI path for a key. +// Path style: `//[+endpoint extra path]` +// Virtual hosted: `/` +// Returns an error if neither bucket nor key is provided. +// The key is forwarded byte-exact (only percent-encoded): S3 treats keys as +// opaque identifiers, so `folder/`, `/x` and `a//b` all designate distinct +// objects and must reach the wire as written. +pub fn build_object_path(creds Credentials, bucket_override string, key string) !string { + bucket := if bucket_override != '' { bucket_override } else { creds.bucket } + if key == '' { + return new_error('InvalidPath', 'Empty object key') + } + encoded_key := uri_encode_path(key) + extra := creds.extra_path() + if creds.virtual_hosted_style { + return '${extra}/${encoded_key}' + } + if bucket == '' { + return new_error('InvalidPath', + 'No bucket given (set Credentials.bucket or pass bucket via options)') + } + encoded_bucket := uri_encode_path(strip_slashes(bucket)) + return '${extra}/${encoded_bucket}/${encoded_key}' +} + +// HttpResponse is the small subset of an HTTP response we surface to callers +// of the lower-level helpers. `body` is the raw response body for non-stream +// requests; `header` keeps V's typed Header for parsing. +pub struct HttpResponse { +pub: + status_code int + body string + header http.Header +} + +// do_http fires off the signed request via V's `net.http`. The HTTP method is +// taken from `signed.method` (set by `sign_request`) so callers cannot drift +// between what was signed and what is sent on the wire. +fn (c &Client) do_http(signed SignedRequest, body string) !HttpResponse { + mut header := http.new_header() + for k, v in signed.headers { + // `Host` is set automatically by V's http client; skip to avoid duplicates. + if k.to_lower() == 'host' { + continue + } + header.add_custom(canonical_header_name(k), v)! + } + method_enum := parse_http_method(signed.method)! + req := http.new_request(method_enum, signed.url, body) + mut req_mut := http.Request{ + ...req + header: header + read_timeout: c.read_timeout + write_timeout: c.write_timeout + } + resp := req_mut.do() or { + return new_error('NetworkError', 'HTTP request failed: ${err.msg()}') + } + $if s3_debug ? { + eprintln('[s3] ${signed.method} ${signed.url}') + for k, v in signed.headers { + eprintln('[s3] > ${k}: ${redacted_value(k, v)}') + } + eprintln('[s3] -> ${resp.status_code}') + eprintln('[s3] body: ${resp.body}') + } + return HttpResponse{ + status_code: resp.status_code + body: resp.body + header: resp.header + } +} + +// redacted_value masks credentials so they cannot leak via s3_debug logs. +fn redacted_value(name string, value string) string { + low := name.to_lower() + if low == 'authorization' || low.starts_with('x-amz-security-token') { + return '' + } + return value +} + +// canonical_header_name turns 'x-amz-content-sha256' into 'X-Amz-Content-Sha256' +// for the wire. HTTP/1.1 says headers are case-insensitive, but some +// intermediaries / mocks aren't, so we emit the canonical Title-Case form. +fn canonical_header_name(s string) string { + mut out := []u8{cap: s.len} + mut upper_next := true + for b in s.bytes() { + if b == `-` { + out << b + upper_next = true + continue + } + if upper_next && b >= `a` && b <= `z` { + out << b - 32 + } else { + out << b + } + upper_next = false + } + return out.bytestr() +} + +// parse_http_method maps a SigV4 canonical method string to V's `http.Method`. +// The signer enforces the supported set up-front, so anything unexpected here +// indicates a programming bug rather than user input. +fn parse_http_method(s string) !http.Method { + return match s.to_upper() { + 'GET' { http.Method.get } + 'PUT' { http.Method.put } + 'POST' { http.Method.post } + 'DELETE' { http.Method.delete } + 'HEAD' { http.Method.head } + else { new_error('InvalidMethod', 'Unsupported HTTP method: ${s}') } + } +} diff --git a/vlib/net/s3/credentials.v b/vlib/net/s3/credentials.v new file mode 100644 index 000000000..2ad126b8b --- /dev/null +++ b/vlib/net/s3/credentials.v @@ -0,0 +1,204 @@ +// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved. +// Use of this source code is governed by an MIT license +// that can be found in the LICENSE file. +module s3 + +import os + +// Credentials carries the authentication material plus endpoint and addressing +// preferences. It is intentionally kept small; defaults (region, endpoint, etc.) +// are derived only when the request is signed, never stored implicitly, so the +// same Credentials value can be reused across regions/endpoints. +// +// Field naming matches V conventions (snake_case). The `from_env` helper +// recognises several provider conventions — see `from_env`. +pub struct Credentials { +pub: + access_key_id string + secret_access_key string + session_token string + region string + bucket string + endpoint string // 'https://s3.fr-par.scw.cloud' or 'host:port' — host part is what gets signed + virtual_hosted_style bool // when true, '.' addressing + insecure_http bool // permit `http://` endpoints (false by default — never silently downgrades) +} + +// Credentials.from_env reads credentials from environment variables, trying several +// provider conventions in order so the same code works against many hosts +// without reconfiguration. Each field is resolved independently; the first +// non-empty value wins. +// +// Lookup order per field: +// key id : S3_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID, +// CELLAR_ADDON_KEY_ID, SCW_ACCESS_KEY, +// B2_APPLICATION_KEY_ID, R2_ACCESS_KEY_ID, SPACES_KEY +// secret : S3_SECRET_ACCESS_KEY, AWS_SECRET_ACCESS_KEY, +// CELLAR_ADDON_KEY_SECRET, SCW_SECRET_KEY, +// B2_APPLICATION_KEY, R2_SECRET_ACCESS_KEY, SPACES_SECRET +// session : S3_SESSION_TOKEN, AWS_SESSION_TOKEN +// region : S3_REGION, AWS_REGION, AWS_DEFAULT_REGION, SCW_DEFAULT_REGION +// bucket : S3_BUCKET +// endpoint : S3_ENDPOINT, AWS_ENDPOINT, AWS_ENDPOINT_URL, +// CELLAR_ADDON_HOST, B2_ENDPOINT, R2_ENDPOINT, SPACES_ENDPOINT +pub fn Credentials.from_env() Credentials { + endpoint := env_first('S3_ENDPOINT', 'AWS_ENDPOINT', 'AWS_ENDPOINT_URL', 'CELLAR_ADDON_HOST', + 'B2_ENDPOINT', 'R2_ENDPOINT', 'SPACES_ENDPOINT') + insecure := endpoint.starts_with('http://') + return Credentials{ + access_key_id: env_first('S3_ACCESS_KEY_ID', 'AWS_ACCESS_KEY_ID', + 'CELLAR_ADDON_KEY_ID', 'SCW_ACCESS_KEY', 'B2_APPLICATION_KEY_ID', 'R2_ACCESS_KEY_ID', + 'SPACES_KEY') + secret_access_key: env_first('S3_SECRET_ACCESS_KEY', 'AWS_SECRET_ACCESS_KEY', + 'CELLAR_ADDON_KEY_SECRET', 'SCW_SECRET_KEY', 'B2_APPLICATION_KEY', + 'R2_SECRET_ACCESS_KEY', 'SPACES_SECRET') + session_token: env_first('S3_SESSION_TOKEN', 'AWS_SESSION_TOKEN') + region: env_first('S3_REGION', 'AWS_REGION', 'AWS_DEFAULT_REGION', + 'SCW_DEFAULT_REGION') + bucket: env_first('S3_BUCKET') + endpoint: endpoint + virtual_hosted_style: false + insecure_http: insecure + } +} + +// merge produces a copy of `c` with non-empty fields from `other` overriding. +// Useful when callers pass per-call overrides while keeping a default Client. +pub fn (c Credentials) merge(other Credentials) Credentials { + return Credentials{ + access_key_id: if other.access_key_id != '' { + other.access_key_id + } else { + c.access_key_id + } + secret_access_key: if other.secret_access_key != '' { + other.secret_access_key + } else { + c.secret_access_key + } + session_token: if other.session_token != '' { + other.session_token + } else { + c.session_token + } + region: if other.region != '' { other.region } else { c.region } + bucket: if other.bucket != '' { other.bucket } else { c.bucket } + endpoint: if other.endpoint != '' { other.endpoint } else { c.endpoint } + virtual_hosted_style: c.virtual_hosted_style || other.virtual_hosted_style + insecure_http: c.insecure_http || other.insecure_http + } +} + +// resolved_region returns the region to use for signing. Order: +// 1. explicit `c.region` +// 2. parsed from `c.endpoint` when it follows the `s3..amazonaws.com` pattern +// 3. `'auto'` for Cloudflare R2 +// 4. `'us-east-1'` (S3 historical default) when no endpoint is set +pub fn (c Credentials) resolved_region() string { + if c.region != '' { + return c.region + } + return guess_region(c.endpoint) +} + +// validate ensures the credentials carry the minimum needed to sign a +// request. Also rejects credentials / region / bucket / endpoint values that +// contain CR or LF — those would let an attacker who controls *any* config +// field smuggle headers into the Authorization line. +pub fn (c Credentials) validate() ! { + if c.access_key_id == '' || c.secret_access_key == '' { + return new_error('MissingCredentials', + "Missing S3 credentials. 'access_key_id' and 'secret_access_key' are required.") + } + for name, value in { + 'access_key_id': c.access_key_id + 'secret_access_key': c.secret_access_key + 'session_token': c.session_token + 'region': c.region + 'bucket': c.bucket + 'endpoint': c.endpoint + } { + if contains_crlf(value) { + return new_error('InvalidCredentials', + 'Credential field "${name}" contains CR or LF — refused (header-injection guard)') + } + } +} + +// host_only returns the bare host[:port] from `c.endpoint`, stripping any +// scheme and trailing path. Returned value is what gets signed in the +// `host` header for SigV4. +pub fn (c Credentials) host_only() string { + mut ep := c.endpoint + if i := ep.index('://') { + ep = ep[i + 3..] + } + if j := ep.index('/') { + ep = ep[..j] + } + return ep +} + +// extra_path returns the path component of `c.endpoint`, including any +// leading '/'. Useful for proxies that mount S3 under a sub-path. +pub fn (c Credentials) extra_path() string { + mut ep := c.endpoint + if i := ep.index('://') { + ep = ep[i + 3..] + } + if j := ep.index('/') { + return ep[j..] + } + return '' +} + +// scheme returns 'http' or 'https' based on the endpoint's explicit scheme +// when present, falling back to `insecure_http`. So all three of these work: +// endpoint: 'https://s3.example.com' → https +// endpoint: 's3.example.com' → https (default) +// endpoint: 'http://localhost:9000' → http (auto-detected) +// endpoint: 'localhost:9000', insecure_http: true → http +pub fn (c Credentials) scheme() string { + if c.endpoint.starts_with('http://') { + return 'http' + } + if c.endpoint.starts_with('https://') { + return 'https' + } + return if c.insecure_http { 'http' } else { 'https' } +} + +// guess_region derives the SigV4 region from an endpoint URL. Public so it +// can be reused by the higher-level Client for log / inspect output. +pub fn guess_region(endpoint string) string { + if endpoint == '' { + return 'us-east-1' + } + if endpoint.ends_with('.r2.cloudflarestorage.com') { + return 'auto' + } + if amz_end := endpoint.index('.amazonaws.com') { + if s3_pos := endpoint.index('s3.') { + start := s3_pos + 3 + if start < amz_end { + return endpoint[start..amz_end] + } + } + // AWS global endpoint (`s3.amazonaws.com`) and legacy forms + // (`s3-external-1.amazonaws.com`) sign with `us-east-1`. SigV4 + // rejects `auto` against AWS, so fall back here instead. + return 'us-east-1' + } + return 'auto' +} + +// env_first returns the first non-empty env var among the names provided. +fn env_first(names ...string) string { + for n in names { + v := os.getenv(n) + if v != '' { + return v + } + } + return '' +} diff --git a/vlib/net/s3/credentials_test.v b/vlib/net/s3/credentials_test.v new file mode 100644 index 000000000..803264012 --- /dev/null +++ b/vlib/net/s3/credentials_test.v @@ -0,0 +1,104 @@ +// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved. +// Use of this source code is governed by an MIT license +// that can be found in the LICENSE file. +module s3 + +fn test_credentials_validate_rejects_crlf_injection() { + c := Credentials{ + access_key_id: 'KEY' + secret_access_key: 'secret\r\nInjected: header' + } + if _ := c.validate() { + assert false, 'should have rejected CRLF' + } +} + +fn test_credentials_from_env_returns_empty_when_unset() { + // Cannot safely set env vars from a test without polluting the parent + // shell, so just verify the call returns a value of the right shape. + c := Credentials.from_env() + _ = c +} + +fn test_endpoint_scheme_is_derived_when_explicit() { + c1 := Credentials{ + access_key_id: 'K' + secret_access_key: 'S' + endpoint: 'https://s3.example.com' + } + assert c1.scheme() == 'https' + c2 := Credentials{ + access_key_id: 'K' + secret_access_key: 'S' + endpoint: 'http://localhost:9000' + } + assert c2.scheme() == 'http', 'http:// endpoint must yield http even when insecure_http is unset' + c3 := Credentials{ + access_key_id: 'K' + secret_access_key: 'S' + endpoint: 'http://localhost:9000' + insecure_http: true + } + assert c3.scheme() == 'http' + // Bare host: defaults to https unless insecure_http is set. + c4 := Credentials{ + access_key_id: 'K' + secret_access_key: 'S' + endpoint: 'localhost:9000' + } + assert c4.scheme() == 'https' + c5 := Credentials{ + access_key_id: 'K' + secret_access_key: 'S' + endpoint: 'localhost:9000' + insecure_http: true + } + assert c5.scheme() == 'http' +} + +fn test_endpoint_host_only_strips_scheme_and_path() { + c := Credentials{ + access_key_id: 'K' + secret_access_key: 'S' + endpoint: 'https://proxy.example.com:8443/s3' + } + assert c.host_only() == 'proxy.example.com:8443' + assert c.extra_path() == '/s3' +} + +fn test_guess_region_paths() { + assert guess_region('') == 'us-east-1' + assert guess_region('https://s3.eu-west-3.amazonaws.com') == 'eu-west-3' + assert guess_region('https://my-acct.r2.cloudflarestorage.com') == 'auto' + assert guess_region('https://s3.example.com') == 'auto' + // AWS global endpoints sign with us-east-1, not 'auto'. + assert guess_region('https://s3.amazonaws.com') == 'us-east-1' + assert guess_region('s3.amazonaws.com') == 'us-east-1' + assert guess_region('https://s3-external-1.amazonaws.com') == 'us-east-1' +} + +fn test_credentials_merge_overrides_only_non_empty() { + base := Credentials{ + access_key_id: 'BASE' + secret_access_key: 'basesecret' + region: 'eu-west-1' + bucket: 'b1' + } + other := Credentials{ + bucket: 'b2' + } + got := base.merge(other) + assert got.access_key_id == 'BASE' + assert got.region == 'eu-west-1' + assert got.bucket == 'b2' +} + +fn test_resolved_region_prefers_explicit() { + c := Credentials{ + access_key_id: 'K' + secret_access_key: 'S' + region: 'eu-central-1' + endpoint: 'https://s3.us-west-2.amazonaws.com' + } + assert c.resolved_region() == 'eu-central-1' +} diff --git a/vlib/net/s3/encoding.v b/vlib/net/s3/encoding.v new file mode 100644 index 000000000..9a4f03b0d --- /dev/null +++ b/vlib/net/s3/encoding.v @@ -0,0 +1,103 @@ +// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved. +// Use of this source code is governed by an MIT license +// that can be found in the LICENSE file. +module s3 + +// uri_encode performs RFC 3986 percent-encoding as required by Signature V4. +// Only the unreserved set A–Z / a–z / 0–9 / '-' / '_' / '.' / '~' is preserved. +// When `encode_slash` is false (used for object keys), '/' is left intact and +// backslashes are normalized to '/' so Windows-style paths produce the same +// canonical key. All other bytes are emitted as %XX with uppercase hex digits. +pub fn uri_encode(input string, encode_slash bool) string { + mut out := []u8{cap: input.len + (input.len >> 2)} // 25% headroom + for b in input.bytes() { + match b { + `A`...`Z`, `a`...`z`, `0`...`9`, `-`, `_`, `.`, `~` { + out << b + } + `/`, `\\` { + if encode_slash { + append_percent(mut out, b) + } else { + out << if b == `\\` { u8(`/`) } else { b } + } + } + else { + append_percent(mut out, b) + } + } + } + return out.bytestr() +} + +// uri_encode_path encodes an S3 object key segment-aware: '/' is preserved +// because S3 paths use it as the segment separator. +@[inline] +pub fn uri_encode_path(path string) string { + return uri_encode(path, false) +} + +// uri_encode_query encodes a value that will appear inside a query string. +// Slashes must be percent-encoded. +@[inline] +pub fn uri_encode_query(value string) string { + return uri_encode(value, true) +} + +// strip_slashes removes leading and trailing '/' or '\\' separators. S3 +// canonical paths must contain a single leading slash, no trailing one. +pub fn strip_slashes(s string) string { + if s == '' { + return s + } + mut start := 0 + mut end := s.len + for start < end && (s[start] == `/` || s[start] == `\\`) { + start++ + } + for end > start && (s[end - 1] == `/` || s[end - 1] == `\\`) { + end-- + } + return s[start..end] +} + +// contains_crlf returns true if `value` contains a CR or LF byte. Header +// values that pass user-provided strings (ACL, content-type, …) MUST be +// checked to prevent HTTP header injection (CRLF smuggling). +@[inline] +pub fn contains_crlf(value string) bool { + for b in value.bytes() { + if b == `\r` || b == `\n` { + return true + } + } + return false +} + +// to_hex_lower formats raw bytes as their lowercase hex string. Used for +// SHA-256 digests inside SigV4 (the spec requires lowercase). +pub fn to_hex_lower(data []u8) string { + mut out := []u8{len: data.len * 2} + for i, b in data { + out[i * 2] = hex_lower_nibble(b >> 4) + out[i * 2 + 1] = hex_lower_nibble(b & 0x0F) + } + return out.bytestr() +} + +@[inline] +fn append_percent(mut out []u8, b u8) { + out << `%` + out << hex_upper_nibble(b >> 4) + out << hex_upper_nibble(b & 0x0F) +} + +@[inline] +fn hex_upper_nibble(n u8) u8 { + return if n < 10 { n + `0` } else { n - 10 + `A` } +} + +@[inline] +fn hex_lower_nibble(n u8) u8 { + return if n < 10 { n + `0` } else { n - 10 + `a` } +} diff --git a/vlib/net/s3/encoding_test.v b/vlib/net/s3/encoding_test.v new file mode 100644 index 000000000..e621c5159 --- /dev/null +++ b/vlib/net/s3/encoding_test.v @@ -0,0 +1,49 @@ +// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved. +// Use of this source code is governed by an MIT license +// that can be found in the LICENSE file. +module s3 + +fn test_uri_encode_path_keeps_slashes() { + assert uri_encode_path('/foo/bar baz/test.txt') == '/foo/bar%20baz/test.txt' + assert uri_encode_path('/folder/file with %.txt') == '/folder/file%20with%20%25.txt' +} + +fn test_uri_encode_path_normalises_backslashes() { + // Windows-style separators must collapse to '/' in the canonical key. + assert uri_encode_path('foo\\bar\\baz.txt') == 'foo/bar/baz.txt' +} + +fn test_uri_encode_query_encodes_slashes() { + assert uri_encode_query('a/b') == 'a%2Fb' + assert uri_encode_query('hello world') == 'hello%20world' +} + +fn test_uri_encode_unreserved_passthrough() { + // RFC 3986 unreserved set must stay intact. + assert uri_encode('AZaz09-_.~', false) == 'AZaz09-_.~' +} + +fn test_uri_encode_percent_uppercase() { + // AWS SigV4 mandates uppercase hex digits. + assert uri_encode('é', true) == '%C3%A9' +} + +fn test_strip_slashes_trims_both_ends() { + assert strip_slashes('/foo/bar/') == 'foo/bar' + assert strip_slashes('//foo//') == 'foo' + assert strip_slashes('foo') == 'foo' + assert strip_slashes('') == '' + assert strip_slashes('///') == '' +} + +fn test_contains_crlf() { + assert contains_crlf('hello\r\nworld') == true + assert contains_crlf('hello\rworld') == true + assert contains_crlf('hello\nworld') == true + assert contains_crlf('plain') == false +} + +fn test_to_hex_lower() { + assert to_hex_lower([u8(0x00), 0x0f, 0xff, 0xa0]) == '000fffa0' + assert to_hex_lower([]) == '' +} diff --git a/vlib/net/s3/errors.v b/vlib/net/s3/errors.v new file mode 100644 index 000000000..e0fa80b89 --- /dev/null +++ b/vlib/net/s3/errors.v @@ -0,0 +1,206 @@ +// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved. +// Use of this source code is governed by an MIT license +// that can be found in the LICENSE file. +module s3 + +// S3Error is the structured error returned for every signing or service +// failure. `code` is stable (e.g. `NoSuchKey`, `MissingCredentials`), +// `message` is human-readable, `status` is the HTTP status (0 for +// client-side errors), and `path` is the offending object key when known. +// +// V's `IError` interface only requires `msg()` and `code()`, so this type can +// be returned via `!T` and inspected with type-assertion / `as`. +pub struct S3Error { +pub: + code string + message string + status int + path string + resource string // S3's field, when present + request_id string // S3 RequestId, useful for support tickets +} + +// msg renders the error in a single line. Includes the S3 error code so users +// can switch on it without parsing the prose. Path is appended when known. +pub fn (e &S3Error) msg() string { + mut buf := '[${e.code}] ${e.message}' + if e.status != 0 { + buf += ' (HTTP ${e.status})' + } + if e.path != '' { + buf += ' — ${e.path}' + } + return buf +} + +// code returns a stable numeric code so callers can use `if err.code() == ...`. +// We map a few well-known S3 codes; everything else returns 0 so callers fall +// back to string comparison on `e.code`. +pub fn (e &S3Error) code() int { + return match e.code { + 'NoSuchKey' { + 404 + } + 'NoSuchBucket' { + 404 + } + 'BucketAlreadyExists', 'BucketAlreadyOwnedByYou' { + 409 + } + 'AccessDenied', 'SignatureDoesNotMatch' { + 403 + } + 'MissingCredentials', 'InvalidEndpoint', 'InvalidPath', 'InvalidMethod', + 'InvalidSessionToken' { + 400 + } + else { + e.status + } + } +} + +// new_error builds an S3Error from a code + message. Use this for client-side +// validation failures (missing creds, invalid path, etc.). +pub fn new_error(code string, message string) IError { + return &S3Error{ + code: code + message: message + } +} + +// new_http_error wraps an HTTP-level failure. Body is the raw response body +// — `parse_xml_error` is responsible for digging out the structured +// `` envelope when the server returns one. +pub fn new_http_error(status int, path string, body string) IError { + parsed := parse_xml_error(body) + if parsed.code != '' { + return &S3Error{ + code: parsed.code + message: parsed.message + status: status + path: path + resource: parsed.resource + request_id: parsed.request_id + } + } + // Body was empty or unparseable — fall back to a generic code keyed by status. + return &S3Error{ + code: fallback_code_for(status) + message: if body.len > 0 { body } else { 'HTTP ${status}' } + status: status + path: path + } +} + +// XmlErrorFields is what we pull out of an `` XML envelope. +struct XmlErrorFields { + code string + message string + resource string + request_id string +} + +// parse_xml_error extracts the standard S3 error XML: +// +// +// ... +// ... +// ... +// ... +// +// +// We use a tiny tag scanner (no DOM) — the format is rigid enough that this +// stays correct, faster than spinning up the full XML parser, and there is no +// attribute parsing to worry about. +fn parse_xml_error(body string) XmlErrorFields { + return XmlErrorFields{ + code: extract_xml_tag(body, 'Code') + message: extract_xml_tag(body, 'Message') + resource: extract_xml_tag(body, 'Resource') + request_id: extract_xml_tag(body, 'RequestId') + } +} + +// extract_xml_tag returns the inner text of `...` (first match, +// case-sensitive) with the five predefined XML entities decoded, or '' if +// the tag is absent. +pub fn extract_xml_tag(body string, tag string) string { + open := '<' + tag + '>' + close := '' + start := body.index(open) or { return '' } + rest_off := start + open.len + end := body.index_after(close, rest_off) or { return '' } + return decode_xml_entities(body[rest_off..end]) +} + +// decode_xml_entities decodes the five predefined XML entities. We don't try +// to handle arbitrary `&#nn;` sequences because S3 only ever uses these five. +pub fn decode_xml_entities(s string) string { + if !s.contains('&') { + return s + } + mut out := []u8{cap: s.len} + mut i := 0 + for i < s.len { + if s[i] != `&` { + out << s[i] + i++ + continue + } + matched := match true { + s.len - i >= 5 && s[i..i + 5] == '&' { + out << `&` + i += 5 + true + } + s.len - i >= 4 && s[i..i + 4] == '<' { + out << `<` + i += 4 + true + } + s.len - i >= 4 && s[i..i + 4] == '>' { + out << `>` + i += 4 + true + } + s.len - i >= 6 && s[i..i + 6] == '"' { + out << `"` + i += 6 + true + } + s.len - i >= 6 && s[i..i + 6] == ''' { + out << `'` + i += 6 + true + } + else { + false + } + } + + if !matched { + out << s[i] + i++ + } + } + return out.bytestr() +} + +fn fallback_code_for(status int) string { + return match status { + 301, 307 { 'PermanentRedirect' } + 400 { 'BadRequest' } + 401, 403 { 'AccessDenied' } + 404 { 'NotFound' } + 405 { 'MethodNotAllowed' } + 409 { 'Conflict' } + 411 { 'LengthRequired' } + 412 { 'PreconditionFailed' } + 416 { 'InvalidRange' } + 429 { 'TooManyRequests' } + 500 { 'InternalError' } + 503 { 'ServiceUnavailable' } + else { 'HTTPError' } + } +} diff --git a/vlib/net/s3/errors_test.v b/vlib/net/s3/errors_test.v new file mode 100644 index 000000000..d52cb9d6e --- /dev/null +++ b/vlib/net/s3/errors_test.v @@ -0,0 +1,82 @@ +// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved. +// Use of this source code is governed by an MIT license +// that can be found in the LICENSE file. +module s3 + +fn test_extract_xml_tag_basic() { + body := 'NoSuchKeyKey not foundR1' + assert extract_xml_tag(body, 'Code') == 'NoSuchKey' + assert extract_xml_tag(body, 'Message') == 'Key not found' + assert extract_xml_tag(body, 'RequestId') == 'R1' + assert extract_xml_tag(body, 'Missing') == '' +} + +fn test_extract_xml_tag_decodes_entities() { + body := 'a & b <tag>' + assert extract_xml_tag(body, 'Message') == 'a & b ' +} + +fn test_decode_xml_entities() { + assert decode_xml_entities('"hello"') == '"hello"' + assert decode_xml_entities('a & b < c > d 'e'') == "a & b < c > d 'e'" + assert decode_xml_entities('no entities here') == 'no entities here' +} + +fn test_s3error_msg_contains_code_and_path() { + e := S3Error{ + code: 'NoSuchKey' + message: 'object missing' + status: 404 + path: 'foo/bar.txt' + } + rendered := e.msg() + assert rendered.contains('NoSuchKey') + assert rendered.contains('object missing') + assert rendered.contains('404') + assert rendered.contains('foo/bar.txt') +} + +fn test_s3error_code_maps_known_strings() { + assert (&S3Error{ + code: 'NoSuchKey' + }).code() == 404 + assert (&S3Error{ + code: 'NoSuchBucket' + }).code() == 404 + assert (&S3Error{ + code: 'AccessDenied' + }).code() == 403 + assert (&S3Error{ + code: 'BucketAlreadyExists' + }).code() == 409 + // Unknown code falls back to the wire status. + assert (&S3Error{ + code: 'WeirdCode' + status: 418 + }).code() == 418 +} + +fn test_new_http_error_parses_xml_body() { + body := 'NoSuchKeyThe specified key does not exist./bucket/missing.txtABC123' + err := new_http_error(404, 'missing.txt', body) + if err is S3Error { + assert err.code == 'NoSuchKey' + assert err.message == 'The specified key does not exist.' + assert err.resource == '/bucket/missing.txt' + assert err.request_id == 'ABC123' + assert err.status == 404 + assert err.path == 'missing.txt' + } else { + assert false, 'expected S3Error' + } +} + +fn test_new_http_error_falls_back_when_body_empty() { + err := new_http_error(503, 'foo', '') + if err is S3Error { + assert err.code == 'ServiceUnavailable' + assert err.status == 503 + } else { + assert false, 'expected S3Error' + } +} diff --git a/vlib/net/s3/fetch.v b/vlib/net/s3/fetch.v new file mode 100644 index 000000000..1aebb2d2c --- /dev/null +++ b/vlib/net/s3/fetch.v @@ -0,0 +1,162 @@ +// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved. +// Use of this source code is governed by an MIT license +// that can be found in the LICENSE file. +module s3 + +import net.http + +// FetchOptions overlays an S3 endpoint over the `fetch` call. All fields +// are optional; bucket and key are taken from the URL. +@[params] +pub struct FetchOptions { +pub: + method http.Method = .get + body []u8 + credentials Credentials + // content_type, acl, etc. are forwarded as-is when method is PUT/POST. + content_type string + content_disposition string + content_encoding string + cache_control string + acl Acl + storage_class StorageClass + request_payer bool + range string + hash_payload bool +} + +// FetchResponse is the simplified return type of `fetch`. It's intentionally +// flat (no streaming yet) — easier to consume than V's `http.Response` and +// surfaces the most useful fields. +pub struct FetchResponse { +pub: + status_code int + body []u8 + headers map[string]string + etag string + content_type string + content_length i64 +} + +// fetch is a `fetch('s3://bucket/key', { ... })`-style helper. +// +// Examples: +// +// resp := s3.fetch('s3://my-bucket/path/to/file.txt')! +// resp := s3.fetch('s3://my-bucket/key', method: .put, body: 'hello'.bytes())! +// resp := s3.fetch('s3://key', method: .get, credentials: s3.Credentials{ bucket: 'b', ... })! +// +// The URL must use the `s3://` scheme. Anything else is rejected outright +// (avoids accidentally calling a real HTTP endpoint with S3 credentials). +pub fn fetch(url string, opts FetchOptions) !FetchResponse { + if !url.starts_with('s3://') { + return new_error('InvalidURL', 's3.fetch only accepts s3:// URLs (got: ${redact_url(url)})') + } + bucket, key := parse_s3_url(url)! + mut creds := if opts.credentials.access_key_id == '' { + Credentials.from_env() + } else { + opts.credentials + } + if bucket != '' { + creds = Credentials{ + ...creds + bucket: bucket + } + } + creds.validate()! + + c := Client{ + credentials: creds + } + match opts.method { + .get { + data := c.get(key, + bucket: bucket + range: opts.range + request_payer: opts.request_payer + )! + return FetchResponse{ + status_code: 200 + body: data + content_length: i64(data.len) + } + } + .head { + st := c.stat(key, + bucket: bucket + request_payer: opts.request_payer + )! + return FetchResponse{ + status_code: 200 + etag: st.etag + content_type: st.content_type + content_length: st.size + } + } + .put { + c.put(key, opts.body, + bucket: bucket + content_type: opts.content_type + content_disposition: opts.content_disposition + content_encoding: opts.content_encoding + cache_control: opts.cache_control + acl: opts.acl + storage_class: opts.storage_class + request_payer: opts.request_payer + hash_payload: opts.hash_payload + )! + return FetchResponse{ + status_code: 200 + } + } + .delete { + c.delete(key, + bucket: bucket + request_payer: opts.request_payer + )! + return FetchResponse{ + status_code: 204 + } + } + else { + return new_error('InvalidMethod', 's3.fetch: unsupported method ${opts.method}') + } + } + + return new_error('InvalidMethod', 's3.fetch: unreachable') +} + +// parse_s3_url splits `s3://bucket/key/with/slashes` into (bucket, key). +// +// Special case: `s3://key` (no second path component) returns ('', 'key') — +// the caller is then expected to provide the bucket via credentials. +pub fn parse_s3_url(url string) !(string, string) { + if !url.starts_with('s3://') { + return new_error('InvalidURL', 'Not an s3:// URL') + } + rest := url[5..] // strip 's3://' + if rest == '' { + return new_error('InvalidURL', 'Empty s3:// URL') + } + if i := rest.index('/') { + bucket := rest[..i] + key := rest[i + 1..] + if key == '' { + return new_error('InvalidURL', 'Empty key in s3:// URL') + } + return bucket, key + } + // no '/' separator — treat the whole rest as the key + return '', rest +} + +// redact_url strips query strings before logging. Used in error messages so +// presigned-URL-style query params (which can contain credentials) aren't +// leaked into logs. +pub fn redact_url(url string) string { + if i := url.index('?') { + return url[..i] + '?' + } + return url +} diff --git a/vlib/net/s3/fetch_test.v b/vlib/net/s3/fetch_test.v new file mode 100644 index 000000000..47ee0275e --- /dev/null +++ b/vlib/net/s3/fetch_test.v @@ -0,0 +1,79 @@ +// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved. +// Use of this source code is governed by an MIT license +// that can be found in the LICENSE file. +module s3 + +fn test_parse_s3_url_full() { + bucket, key := parse_s3_url('s3://my-bucket/path/to/key.txt') or { panic(err) } + assert bucket == 'my-bucket' + assert key == 'path/to/key.txt' +} + +fn test_parse_s3_url_only_key() { + bucket, key := parse_s3_url('s3://standalone-key') or { panic(err) } + assert bucket == '' + assert key == 'standalone-key' +} + +fn test_parse_s3_url_rejects_other_schemes() { + if _, _ := parse_s3_url('https://example.com/x') { + assert false, 'should have errored' + } + if _, _ := parse_s3_url('http://x') { + assert false, 'should have errored' + } + if _, _ := parse_s3_url('') { + assert false, 'should have errored' + } +} + +fn test_parse_s3_url_rejects_empty_key() { + if _, _ := parse_s3_url('s3://my-bucket/') { + assert false, 'should have errored' + } + if _, _ := parse_s3_url('s3://') { + assert false, 'should have errored' + } +} + +fn test_redact_url_strips_query() { + assert redact_url('https://x.y/path?X-Amz-Signature=abc') == 'https://x.y/path?' + assert redact_url('https://x.y/path') == 'https://x.y/path' +} + +fn test_fetch_rejects_non_s3_scheme() { + if _ := fetch('https://example.com/x') { + assert false + } + if _ := fetch('http://example.com/x') { + assert false + } + if _ := fetch('file:///etc/passwd') { + assert false + } +} + +fn test_fetch_rejects_post_method() { + // POST is not part of the s3.fetch contract: object writes use PUT, + // and POST in S3 is reserved for multipart-control endpoints which + // take an XML body that fetch does not generate. Reject explicitly + // rather than silently rewriting to PUT. + if _ := fetch('s3://my-bucket/key.txt', method: .post, body: 'x'.bytes()) { + assert false, 'POST should be rejected' + } +} + +fn test_build_object_path_preserves_leading_and_trailing_slashes() { + creds := Credentials{ + access_key_id: 'K' + secret_access_key: 'S' + bucket: 'b' + } + // S3 keys are byte-exact: `folder/`, `/x` and `a//b` must round-trip + // untouched (only percent-encoded). Stripping slashes would address a + // different object than requested. + assert build_object_path(creds, '', 'folder/')! == '/b/folder/' + assert build_object_path(creds, '', '/x')! == '/b//x' + assert build_object_path(creds, '', 'a//b')! == '/b/a//b' + assert build_object_path(creds, '', 'plain.txt')! == '/b/plain.txt' +} diff --git a/vlib/net/s3/file.v b/vlib/net/s3/file.v new file mode 100644 index 000000000..c11002f21 --- /dev/null +++ b/vlib/net/s3/file.v @@ -0,0 +1,88 @@ +// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved. +// Use of this source code is governed by an MIT license +// that can be found in the LICENSE file. +module s3 + +// FileOptions configures a File handle. +@[params] +pub struct FileOptions { +pub: + bucket string +} + +// File is a reference to one S3 object. It holds no buffer; every method +// round-trips to S3 (or generates a presigned URL for `presign`). +// +// Construct via `Client.file(...)` or directly with `File{ client: &c, key: '...' }`. +pub struct File { +pub: + client &Client + bucket string + key string +} + +// read returns the full object body. For range reads, see `read_range`. +pub fn (f &File) read() ![]u8 { + return f.client.get(f.key, bucket: f.bucket) +} + +// text is a UTF-8 convenience over `read`. +pub fn (f &File) text() !string { + return f.client.get_string(f.key, bucket: f.bucket) +} + +// read_range fetches `bytes=-` (HTTP Range semantics). +// Use `end < 0` to read to end-of-file. +pub fn (f &File) read_range(begin i64, end i64) ![]u8 { + range := if end < 0 { 'bytes=${begin}-' } else { 'bytes=${begin}-${end}' } + return f.client.get(f.key, bucket: f.bucket, range: range) +} + +// write uploads `data` as the entire object body. +pub fn (f &File) write(data []u8, opts PutOptions) ! { + mut o := opts + if o.bucket == '' { + o = PutOptions{ + ...opts + bucket: f.bucket + } + } + f.client.put(f.key, data, o)! +} + +// write_string is a UTF-8 convenience over `write`. +pub fn (f &File) write_string(s string, opts PutOptions) ! { + f.write(s.bytes(), opts)! +} + +// stat returns object metadata (size / etag / last-modified / content-type). +pub fn (f &File) stat() !Stat { + return f.client.stat(f.key, bucket: f.bucket) +} + +// exists is a HEAD that converts 404 into `false`. +pub fn (f &File) exists() !bool { + return f.client.exists(f.key, bucket: f.bucket) +} + +// size returns the Content-Length, in bytes. +pub fn (f &File) size() !i64 { + return f.client.size(f.key, bucket: f.bucket) +} + +// delete removes the object. Idempotent: no error when the object is absent. +pub fn (f &File) delete() ! { + f.client.delete(f.key, bucket: f.bucket)! +} + +// presign returns a presigned URL for this object. See `PresignOptions`. +pub fn (f &File) presign(opts PresignOptions) !string { + mut o := opts + if o.bucket == '' && f.bucket != '' { + o = PresignOptions{ + ...opts + bucket: f.bucket + } + } + return f.client.presign(f.key, o) +} diff --git a/vlib/net/s3/http_bridge.v b/vlib/net/s3/http_bridge.v new file mode 100644 index 000000000..ab409c965 --- /dev/null +++ b/vlib/net/s3/http_bridge.v @@ -0,0 +1,51 @@ +// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved. +// Use of this source code is governed by an MIT license +// that can be found in the LICENSE file. +module s3 + +import net.http + +// init registers an `s3://` scheme handler with `net.http` so that +// `http.fetch(url: 's3://...')` and friends route here transparently. Only +// active when this module is imported. +fn init() { + http.register_scheme('s3', http_fetch_handler) +} + +// http_fetch_handler bridges `http.fetch` into `s3.fetch`. Credentials are +// pulled from the environment (`S3_*`, `AWS_*`, `CELLAR_ADDON_*`, …); for +// explicit credentials, callers should use `s3.fetch` directly. +fn http_fetch_handler(config http.FetchConfig) !http.Response { + resp := fetch(config.url, + method: config.method + body: config.data.bytes() + credentials: Credentials.from_env() + content_type: config.header.get(.content_type) or { '' } + )! + mut header := http.new_header() + for k, v in resp.headers { + header.add_custom(k, v) or {} + } + if resp.content_type != '' { + header.set(.content_type, resp.content_type) + } + if resp.etag != '' { + header.set(.etag, resp.etag) + } + return http.Response{ + status_code: resp.status_code + body: resp.body.bytestr() + header: header + } +} + +fn http_method_to_string(m http.Method) string { + return match m { + .get { 'GET' } + .put { 'PUT' } + .post { 'POST' } + .delete { 'DELETE' } + .head { 'HEAD' } + else { 'GET' } + } +} diff --git a/vlib/net/s3/integration_test.v b/vlib/net/s3/integration_test.v new file mode 100644 index 000000000..8c21687a1 --- /dev/null +++ b/vlib/net/s3/integration_test.v @@ -0,0 +1,287 @@ +// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved. +// Use of this source code is governed by an MIT license +// that can be found in the LICENSE file. +module s3 + +// Integration tests against a live S3-compatible endpoint. They are skipped +// unless the `S3_INTEGRATION` env var is set, and the `S3_HOST`, `S3_KEY_ID` +// and `S3_KEY_SECRET` env vars are populated. `v test .` stays fast and +// offline by default. +// +// Run with: +// S3_INTEGRATION=1 \ +// S3_HOST=s3.example.com \ +// S3_KEY_ID=... S3_KEY_SECRET=... \ +// S3_BUCKET=vs3-tests \ +// v test . +import os +import rand +import time +import net.http + +// http_dispatch_creds_present returns true when the env exposes the AWS-style +// names that http.fetch's bridge will pick up via `Credentials.from_env()`. +// The integration test below relies on this — `S3_HOST` / `S3_KEY_ID` / +// `S3_KEY_SECRET` (used by the rest of this file) aren't recognised by +// `from_env()`, so we map them on the fly when needed. +fn ensure_aws_env_for_bridge() { + if os.getenv('S3_ACCESS_KEY_ID') == '' { + os.setenv('S3_ACCESS_KEY_ID', os.getenv('S3_KEY_ID'), true) + } + if os.getenv('S3_SECRET_ACCESS_KEY') == '' { + os.setenv('S3_SECRET_ACCESS_KEY', os.getenv('S3_KEY_SECRET'), true) + } + if os.getenv('S3_ENDPOINT') == '' { + host := os.getenv('S3_HOST') + endpoint := if host.contains('://') { host } else { 'https://${host}' } + os.setenv('S3_ENDPOINT', endpoint, true) + } +} + +fn integration_enabled() bool { + return os.getenv('S3_INTEGRATION') != '' +} + +fn live_client() ?Client { + if !integration_enabled() { + return none + } + host := os.getenv('S3_HOST') + key_id := os.getenv('S3_KEY_ID') + secret := os.getenv('S3_KEY_SECRET') + if host == '' || key_id == '' || secret == '' { + return none + } + endpoint := if host.contains('://') { host } else { 'https://${host}' } + bucket := os.getenv_opt('S3_BUCKET') or { 'vs3-tests' } + return Client{ + credentials: Credentials{ + endpoint: endpoint + access_key_id: key_id + secret_access_key: secret + bucket: bucket + region: 'us-east-1' + } + } +} + +fn rand_key(prefix string) string { + return '${prefix}/${time.now().unix()}-${rand.u64()}.txt' +} + +fn test_integration_roundtrip() { + c := live_client() or { return } + key := rand_key('it/roundtrip') + defer { + c.delete(key) or {} + } + c.put(key, 'hi'.bytes(), content_type: 'text/plain') or { panic(err) } + got := c.get_string(key) or { panic(err) } + assert got == 'hi' + stat := c.stat(key) or { panic(err) } + assert stat.size == 2 + assert stat.content_type.starts_with('text/plain') +} + +fn test_integration_range_read() { + c := live_client() or { return } + key := rand_key('it/range') + defer { + c.delete(key) or {} + } + c.put(key, 'abcdefghijklmnop'.bytes()) or { panic(err) } + first := c.get(key, range: 'bytes=0-4') or { panic(err) } + assert first.bytestr() == 'abcde', 'got: ${first.bytestr()}' + tail := c.get(key, range: 'bytes=10-') or { panic(err) } + assert tail.bytestr() == 'klmnop', 'got: ${tail.bytestr()}' +} + +fn test_integration_presign_get() { + c := live_client() or { return } + key := rand_key('it/presign') + defer { + c.delete(key) or {} + } + c.put(key, 'PRESIGN_OK'.bytes()) or { panic(err) } + url := c.presign(key, expires_in: 60) or { panic(err) } + body := fetch_url(url) or { panic(err) } + assert body.contains('PRESIGN_OK'), 'got: ${body}' +} + +fn test_integration_list_with_prefix() { + c := live_client() or { return } + prefix := 'it/list-${rand.u64()}/' + defer { + // Best-effort cleanup + if res := c.list(prefix: prefix, max_keys: 100) { + for o in res.objects { + c.delete(o.key) or {} + } + } + } + for i in 0 .. 3 { + c.put('${prefix}item-${i}.txt', 'x'.bytes()) or { panic(err) } + } + res := c.list(prefix: prefix, fetch_owner: false) or { panic(err) } + assert res.objects.len == 3 + for o in res.objects { + assert o.key.starts_with(prefix) + assert o.size == 1 + } +} + +fn test_integration_exists_and_delete() { + c := live_client() or { return } + key := rand_key('it/exists') + c.put(key, 'present'.bytes()) or { panic(err) } + assert c.exists(key) or { panic(err) } + c.delete(key) or { panic(err) } + assert !(c.exists(key) or { panic(err) }) +} + +fn test_integration_fetch_helper() { + c := live_client() or { return } + key := rand_key('it/fetch') + url := 's3://${c.credentials.bucket}/${key}' + defer { + c.delete(key) or {} + } + put := fetch(url, + method: .put + body: 'fetch_works'.bytes() + credentials: c.credentials + ) or { panic(err) } + assert put.status_code == 200 + get := fetch(url, credentials: c.credentials) or { panic(err) } + assert get.body.bytestr() == 'fetch_works' +} + +fn test_integration_multipart_upload_bytes() { + c := live_client() or { return } + // 6 MiB triggers multipart (one full part + a tail). + mut data := []u8{len: 6 * 1024 * 1024} + for i in 0 .. data.len { + data[i] = u8(i & 0xFF) + } + key := rand_key('it/multipart') + defer { + c.delete(key) or {} + } + c.upload_bytes_multipart(key, data) or { panic(err) } + stat := c.stat(key) or { panic(err) } + assert stat.size == data.len + got := c.get(key) or { panic(err) } + assert got.len == data.len + for i in 0 .. data.len { + if got[i] != data[i] { + assert false, 'byte mismatch at ${i}' + break + } + } +} + +fn test_integration_multipart_parallel_many_parts() { + c := live_client() or { return } + // Force 8 parts at the 5 MiB minimum + tail. With queue_size=5 (default) + // we get genuine parallel dispatch and out-of-order completion that the + // dispatcher must re-order before CompleteMultipartUpload. + mut data := []u8{len: 8 * 5 * 1024 * 1024 + 1024} + for i in 0 .. data.len { + data[i] = u8((i * 31 + 7) & 0xFF) + } + key := rand_key('it/parallel') + defer { + c.delete(key) or {} + } + c.upload_bytes_multipart(key, data) or { panic(err) } + stat := c.stat(key) or { panic(err) } + assert stat.size == data.len, 'size mismatch: got ${stat.size}' + // Verify a tail range, where corruption from out-of-order parts would land. + tail := c.get(key, range: 'bytes=${data.len - 1024}-') or { panic(err) } + for i in 0 .. tail.len { + if tail[i] != data[data.len - 1024 + i] { + assert false, 'tail byte mismatch at offset ${i}' + break + } + } +} + +fn test_integration_bucket_lifecycle() { + c := live_client() or { return } + // Random suffix keeps reruns idempotent. + name := 's3v-it-${rand.u64():08x}'.to_lower() + c.create_bucket(bucket: name) or { + // Some providers require region constraint or have different defaults + // — surface but don't fail the suite for that one provider quirk. + eprintln('skip: create_bucket(${name}) failed: ${err}') + return + } + defer { + c.delete_bucket(bucket: name) or {} + } + exists := c.bucket_exists(bucket: name) or { panic(err) } + assert exists, 'bucket ${name} should exist after create' + // Delete once explicitly so we can assert it disappeared. + c.delete_bucket(bucket: name) or { panic(err) } + gone := c.bucket_exists(bucket: name) or { panic(err) } + assert !gone, 'bucket ${name} should be gone after delete' +} + +fn test_integration_invalid_credentials_yields_clean_error() { + if !integration_enabled() { + return + } + host := os.getenv('S3_HOST') + if host == '' { + return + } + c := Client{ + credentials: Credentials{ + endpoint: 'https://${host}' + access_key_id: 'AKIADOESNOTEXIST' + secret_access_key: 'badbadbadbadbadbadbadbadbadbadbadbadbadbad' + bucket: os.getenv_opt('S3_BUCKET') or { 'vs3-tests' } + } + } + if _ := c.stat('any-key') { + assert false, 'expected error' + } else { + assert err is S3Error, 'got non-S3 error: ${typeof(err).name}' + } +} + +// fetch_url is a tiny `http.get`-style helper used only in the presign test. +fn fetch_url(url string) !string { + resp := http.get(url)! + return resp.body +} + +// Exercises the `http.fetch(url: 's3://...')` bridge end-to-end. The s3 +// module's `init()` registers itself with net.http; this test goes through +// that path (no direct `s3.fetch` call) to prove the dispatch works. +fn test_integration_http_fetch_dispatch_to_s3() { + c := live_client() or { return } + ensure_aws_env_for_bridge() + bucket := c.credentials.bucket + key := rand_key('it/http-bridge') + url := 's3://${bucket}/${key}' + + put := http.fetch(url: url, method: .put, data: 'via-http.fetch') or { + assert false, 'PUT through bridge failed: ${err}' + return + } + assert put.status_code == 200, 'unexpected PUT status: ${put.status_code}' + + get := http.fetch(url: url) or { + assert false, 'GET through bridge failed: ${err}' + return + } + assert get.status_code == 200, 'unexpected GET status: ${get.status_code}' + assert get.body == 'via-http.fetch', 'body mismatch: ${get.body}' + + del := http.fetch(url: url, method: .delete) or { + assert false, 'DELETE through bridge failed: ${err}' + return + } + assert del.status_code in [200, 204], 'unexpected DELETE status: ${del.status_code}' +} diff --git a/vlib/net/s3/list.v b/vlib/net/s3/list.v new file mode 100644 index 000000000..cdf1739b1 --- /dev/null +++ b/vlib/net/s3/list.v @@ -0,0 +1,121 @@ +// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved. +// Use of this source code is governed by an MIT license +// that can be found in the LICENSE file. +module s3 + +// list returns up to ~1000 objects matching `opts`. Use the returned +// `next_continuation_token` to page through more. +// +// Speaks the ListObjectsV2 protocol. +pub fn (c &Client) list(opts ListOptions) !ListResult { + bucket := pick_bucket(c.credentials, opts.bucket)! + creds := c.creds_for(bucket) + path := bucket_path(creds, bucket) + mut q := map[string]string{} + q['list-type'] = '2' + if opts.prefix != '' { + q['prefix'] = opts.prefix + } + if opts.continuation_token != '' { + q['continuation-token'] = opts.continuation_token + } + if opts.delimiter != '' { + q['delimiter'] = opts.delimiter + } + if opts.max_keys > 0 { + q['max-keys'] = opts.max_keys.str() + } + if opts.start_after != '' { + q['start-after'] = opts.start_after + } + if opts.encoding_type != '' { + q['encoding-type'] = opts.encoding_type + } + if opts.fetch_owner { + q['fetch-owner'] = 'true' + } + query := canonical_query_string(q) + signed := sign_request(creds, SignRequest{ + method: 'GET' + path: path + query: query + payload_hash: empty_sha256 + })! + resp := c.do_http(signed, '')! + if resp.status_code != 200 { + return new_http_error(resp.status_code, bucket, resp.body) + } + return parse_list_response(resp.body) +} + +// parse_list_response is hand-rolled XML extraction. ListObjectsV2 uses a +// rigid structure with no attributes we care about, so a tag scanner beats +// pulling in the full XML decoder both in code size and predictability +// against malformed input. +pub fn parse_list_response(body string) !ListResult { + objects := parse_contents(body) + common := parse_common_prefixes(body) + max_keys := extract_xml_tag(body, 'MaxKeys').int() + key_count := extract_xml_tag(body, 'KeyCount').int() + is_truncated := extract_xml_tag(body, 'IsTruncated') == 'true' + return ListResult{ + name: extract_xml_tag(body, 'Name') + prefix: extract_xml_tag(body, 'Prefix') + delimiter: extract_xml_tag(body, 'Delimiter') + start_after: extract_xml_tag(body, 'StartAfter') + max_keys: max_keys + key_count: key_count + is_truncated: is_truncated + continuation_token: extract_xml_tag(body, 'ContinuationToken') + next_continuation_token: extract_xml_tag(body, 'NextContinuationToken') + objects: objects + common_prefixes: common + } +} + +fn parse_contents(body string) []ObjectInfo { + mut out := []ObjectInfo{} + mut start := 0 + for { + open := body.index_after('', start) or { break } + close := body.index_after('', open) or { break } + seg := body[open + ''.len..close] + mut owner := ?Owner(none) + owner_open := seg.index('') or { -1 } + if owner_open >= 0 { + owner_close := seg.index('') or { -1 } + if owner_close > owner_open { + owner_seg := seg[owner_open + ''.len..owner_close] + owner = Owner{ + id: extract_xml_tag(owner_seg, 'ID') + display_name: extract_xml_tag(owner_seg, 'DisplayName') + } + } + } + out << ObjectInfo{ + key: extract_xml_tag(seg, 'Key') + last_modified: extract_xml_tag(seg, 'LastModified') + etag: extract_xml_tag(seg, 'ETag').trim('"') + size: extract_xml_tag(seg, 'Size').i64() + storage_class: extract_xml_tag(seg, 'StorageClass') + owner: owner + } + start = close + ''.len + } + return out +} + +fn parse_common_prefixes(body string) []CommonPrefix { + mut out := []CommonPrefix{} + mut start := 0 + for { + open := body.index_after('', start) or { break } + close := body.index_after('', open) or { break } + seg := body[open + ''.len..close] + out << CommonPrefix{ + prefix: extract_xml_tag(seg, 'Prefix') + } + start = close + ''.len + } + return out +} diff --git a/vlib/net/s3/list_test.v b/vlib/net/s3/list_test.v new file mode 100644 index 000000000..bd3678f01 --- /dev/null +++ b/vlib/net/s3/list_test.v @@ -0,0 +1,57 @@ +// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved. +// Use of this source code is governed by an MIT license +// that can be found in the LICENSE file. +module s3 + +// Captured from a real ListObjectsV2 response (S3-compatible endpoint), +// trimmed to the fields we parse. +const list_response_xml = ' + + test-bucket + foo/ + 2 + 1000 + false + + foo/a.txt + 10 + "d41d8cd98f00b204e9800998ecf8427e" + 2024-01-15T00:00:00.000Z + STANDARD + + + foo/b.txt + 20 + "9b8b80d22b1d56a3f9b2c25d6e3e74c5" + 2024-01-16T00:00:00.000Z + +' + +fn test_parse_list_response_basic() { + r := parse_list_response(list_response_xml) or { panic(err) } + assert r.name == 'test-bucket' + assert r.prefix == 'foo/' + assert r.key_count == 2 + assert !r.is_truncated + assert r.objects.len == 2 + assert r.objects[0].key == 'foo/a.txt' + assert r.objects[0].size == 10 + assert r.objects[0].etag == 'd41d8cd98f00b204e9800998ecf8427e' + assert r.objects[1].key == 'foo/b.txt' +} + +fn test_parse_list_response_truncated() { + body := 'btrueopaqueToken==1only.txt1"abc"' + r := parse_list_response(body) or { panic(err) } + assert r.is_truncated + assert r.next_continuation_token == 'opaqueToken==' +} + +fn test_parse_list_response_common_prefixes() { + body := 'b/folder1/folder2/0' + r := parse_list_response(body) or { panic(err) } + assert r.delimiter == '/' + assert r.common_prefixes.len == 2 + assert r.common_prefixes[0].prefix == 'folder1/' + assert r.common_prefixes[1].prefix == 'folder2/' +} diff --git a/vlib/net/s3/multipart.v b/vlib/net/s3/multipart.v new file mode 100644 index 000000000..7b821fa63 --- /dev/null +++ b/vlib/net/s3/multipart.v @@ -0,0 +1,410 @@ +// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved. +// Use of this source code is governed by an MIT license +// that can be found in the LICENSE file. +module s3 + +import os +import strings +import time + +// Multipart upload constants — defaults aligned with the S3 protocol limits. +pub const min_part_size = i64(5 * 1024 * 1024) // 5 MiB +pub const max_part_size = i64(5 * 1024 * 1024 * 1024) // 5 GiB +pub const max_parts = 10000 // hard S3 limit + +// upload_file streams a local file to S3, choosing single-part or multipart +// based on size. Use this for files larger than ~50 MiB or anything you +// don't want to slurp into memory. +// +// `key` is the destination object key. The local file is read in +// `client.part_size` chunks (default 5 MiB). +pub fn (c &Client) upload_file(key string, local_path string, opts PutOptions) ! { + stat_local := os.stat(local_path) or { + return new_error('LocalFileNotFound', + 'Cannot stat local file: ${local_path} (${err.msg()})') + } + size := i64(stat_local.size) + if size <= min_part_size { + data := os.read_bytes(local_path) or { + return new_error('LocalReadFailed', 'Cannot read ${local_path}: ${err.msg()}') + } + return c.put(key, data, opts) + } + c.upload_file_multipart(key, local_path, size, opts)! +} + +// MultipartUploader is a stateful handle to an in-flight multipart upload, +// produced by `Client.start_multipart`. Use it when you want to push parts +// generated on-the-fly (network sources, decompressors, anything you don't +// want to materialise on disk): +// +// mut up := c.start_multipart('key', s3.PutOptions{ content_type: 'application/octet-stream' })! +// for chunk in chunks { up.upload(chunk)! } +// up.complete()! +// +// On error, `complete()` / `upload()` return without aborting. The caller +// must invoke `abort()` themselves so that `defer { up.abort() or {} }` +// remains an explicit, visible cleanup hook. +pub struct MultipartUploader { +mut: + client &Client + key string + upload_id string + opts PutOptions + parts []PartRef + part_number int + completed bool + aborted bool +} + +// start_multipart begins a multipart upload and returns a MultipartUploader. +// Memory cost: zero — each `upload(chunk)` call streams the chunk to S3 and +// returns when it is acknowledged. +pub fn (c &Client) start_multipart(key string, opts PutOptions) !MultipartUploader { + upload_id := c.initiate_multipart(key, opts)! + return MultipartUploader{ + // `unsafe { c }` is required to store the receiver's pointer in the + // returned struct — V's borrow checker forbids this implicitly, even + // though `Client` is `@[heap]` so the lifetime is safe. + client: unsafe { c } + key: key + upload_id: upload_id + opts: opts + parts: []PartRef{} + part_number: 1 + } +} + +// upload pushes one chunk as the next part. Each chunk MUST be at least +// `min_part_size` (5 MiB) except the last one — that's an S3 invariant +// the server enforces; we do not buffer for you. +pub fn (mut u MultipartUploader) upload(data []u8) ! { + if u.completed || u.aborted { + return new_error('InvalidState', 'multipart upload is already finalised') + } + if u.part_number > max_parts { + return new_error('TooManyParts', 'Exceeded ${max_parts} parts — increase part_size') + } + etag := u.client.upload_part(u.key, u.upload_id, u.part_number, data, u.opts)! + u.parts << PartRef{ + part_number: u.part_number + etag: etag + } + u.part_number++ +} + +// complete finalises the upload. After this call the object is visible. +pub fn (mut u MultipartUploader) complete() ! { + if u.completed { + return + } + u.client.complete_multipart(u.key, u.upload_id, u.parts, u.opts)! + u.completed = true +} + +// abort cancels the in-flight upload. Idempotent. +pub fn (mut u MultipartUploader) abort() ! { + if u.aborted || u.completed { + return + } + u.client.abort_multipart(u.key, u.upload_id, u.opts)! + u.aborted = true +} + +// upload_bytes_multipart uploads an in-memory byte buffer using multipart. +// Useful for large generated payloads (test fixtures up to 5 GiB). Uses up +// to `Client.queue_size` parallel part uploads. +pub fn (c &Client) upload_bytes_multipart(key string, data []u8, opts PutOptions) ! { + upload_id := c.initiate_multipart(key, opts)! + parts := c.run_parallel_parts(key, upload_id, opts, fn [data] (offset i64, want int) ![]u8 { + end := if offset + i64(want) > i64(data.len) { i64(data.len) } else { offset + i64(want) } + return data[offset..end] + }, i64(data.len)) or { + c.abort_multipart(key, upload_id, opts) or {} + return err + } + c.complete_multipart(key, upload_id, parts, opts)! +} + +// upload_file_multipart streams a local file part-by-part with up to +// `Client.queue_size` concurrent uploads. Peak memory is roughly +// `queue_size * part_size`. +pub fn (c &Client) upload_file_multipart(key string, local_path string, size i64, opts PutOptions) ! { + mut f := os.open(local_path) or { + return new_error('LocalReadFailed', 'open ${local_path}: ${err.msg()}') + } + defer { + f.close() + } + upload_id := c.initiate_multipart(key, opts)! + parts := c.run_parallel_parts(key, upload_id, opts, fn [mut f] (offset i64, want int) ![]u8 { + mut buf := []u8{len: want} + n := f.read_bytes_into(u64(offset), mut buf) or { + return new_error('LocalReadFailed', 'read_bytes_into: ${err.msg()}') + } + if n <= 0 { + return new_error('LocalReadFailed', 'unexpected EOF at offset ${offset}') + } + return buf[..n] + }, size) or { + c.abort_multipart(key, upload_id, opts) or {} + return err + } + c.complete_multipart(key, upload_id, parts, opts)! +} + +// PartJob is a single upload-part task pushed onto the worker queue. +struct PartJob { + part_number int + data []u8 +} + +// PartResult carries the worker outcome back to the dispatcher. +struct PartResult { + part_number int + etag string + err ?IError +} + +// part_worker pulls jobs off `jobs` and pushes results onto `results`. Exits +// when `jobs` is closed. +fn (c &Client) part_worker(key string, upload_id string, opts PutOptions, jobs chan PartJob, results chan PartResult) { + for { + job := <-jobs or { return } + etag := c.upload_part(key, upload_id, job.part_number, job.data, opts) or { + results <- PartResult{ + part_number: job.part_number + err: err + } + continue + } + results <- PartResult{ + part_number: job.part_number + etag: etag + } + } +} + +// run_parallel_parts dispatches `producer` chunks across a worker pool of +// `Client.queue_size` goroutines and gathers the resulting PartRefs. The +// caller is responsible for issuing `initiate_multipart` / `abort_multipart` +// / `complete_multipart` around this call. `producer(offset, want)` must +// return a freshly-allocated chunk of length ≤ `want` bytes (workers may +// hold the slice past the next call). Returns parts in completion order; +// `complete_multipart` re-sorts by part number before sending the manifest. +fn (c &Client) run_parallel_parts(key string, upload_id string, opts PutOptions, producer fn (offset i64, want int) ![]u8, total i64) ![]PartRef { + workers := if c.queue_size < 1 { 1 } else { c.queue_size } + part_size := if c.part_size < min_part_size { min_part_size } else { c.part_size } + jobs := chan PartJob{cap: workers} + results := chan PartResult{cap: workers} + for _ in 0 .. workers { + spawn c.part_worker(key, upload_id, opts, jobs, results) + } + + mut parts := []PartRef{} + mut first_err := ?IError(none) + mut sent := 0 + mut received := 0 + mut offset := i64(0) + mut part_number := 1 + + for offset < total && first_err == none { + if part_number > max_parts { + first_err = new_error('TooManyParts', + 'Exceeded ${max_parts} parts — increase part_size') + break + } + want := if offset + part_size > total { int(total - offset) } else { int(part_size) } + chunk := producer(offset, want) or { + first_err = err + break + } + if chunk.len == 0 { + break + } + jobs <- PartJob{ + part_number: part_number + data: chunk + } + sent++ + offset += i64(chunk.len) + part_number++ + // Drain ready results without blocking — surfaces errors early so we + // can stop dispatching new parts. + for { + select { + r := <-results { + received++ + if e := r.err { + if first_err == none { + first_err = e + } + } else { + parts << PartRef{ + part_number: r.part_number + etag: r.etag + } + } + } + else { + break + } + } + } + } + jobs.close() + for received < sent { + r := <-results + received++ + if e := r.err { + if first_err == none { + first_err = e + } + } else { + parts << PartRef{ + part_number: r.part_number + etag: r.etag + } + } + } + results.close() + if e := first_err { + return e + } + return parts +} + +// PartRef is one ETag/PartNumber pair recorded during multipart upload. +pub struct PartRef { +pub: + part_number int + etag string +} + +// initiate_multipart starts an upload and returns the UploadId. ACL, +// content-type, and friends from `opts` are sent here — they apply to the +// final object. +pub fn (c &Client) initiate_multipart(key string, opts PutOptions) !string { + creds := c.creds_for(opts.bucket) + path := build_object_path(creds, opts.bucket, key)! + signed := sign_request(creds, SignRequest{ + method: 'POST' + path: path + query: 'uploads=' + payload_hash: empty_sha256 + extra_headers: put_object_headers(opts) + })! + resp := c.do_http(signed, '')! + if resp.status_code != 200 { + return new_http_error(resp.status_code, key, resp.body) + } + upload_id := extract_xml_tag(resp.body, 'UploadId') + if upload_id == '' { + return new_error('InvalidResponse', 'CreateMultipartUploadResult missing UploadId') + } + return upload_id +} + +// upload_part uploads a single chunk and returns the server-side ETag. Up to +// `Client.retry` attempts with exponential backoff (200ms, 400ms, 800ms, …) +// on transient failures. +// +// The payload is always SHA-256-signed (not `UNSIGNED-PAYLOAD`) so the +// service validates byte-for-byte integrity end-to-end. Multipart uploads +// don't carry a Content-MD5 header by default; without payload signing a +// flipped bit in transit would silently produce a corrupt object that +// passes the multipart ETag check. +pub fn (c &Client) upload_part(key string, upload_id string, part_number int, data []u8, opts PutOptions) !string { + creds := c.creds_for(opts.bucket) + path := build_object_path(creds, opts.bucket, key)! + query := canonical_query_string({ + 'partNumber': part_number.str() + 'uploadId': upload_id + }) + // Hash the payload once — multi-MiB chunks should not be re-hashed per attempt. + // The signature itself still has to be recomputed on each retry because the + // `x-amz-date` header advances. + payload_hash := sha256_hex(data) + body := data.bytestr() + max_attempts := if c.retry < 1 { 1 } else { c.retry } + mut last_err := IError(new_error('NetworkError', 'upload_part: no attempt was made')) + for attempt in 0 .. max_attempts { + signed := sign_request(creds, SignRequest{ + method: 'PUT' + path: path + query: query + payload_hash: payload_hash + })! + resp := c.do_http(signed, body) or { + last_err = err + if attempt + 1 < max_attempts { + time.sleep((1 << attempt) * 200 * time.millisecond) + } + continue + } + if resp.status_code == 200 { + etag := resp.header.get(.etag) or { '' } + if etag == '' { + return new_error('InvalidResponse', 'UploadPart returned no ETag') + } + return etag.trim('"') + } + last_err = new_http_error(resp.status_code, key, resp.body) + if attempt + 1 < max_attempts { + time.sleep((1 << attempt) * 200 * time.millisecond) + } + } + return last_err +} + +// complete_multipart finalizes a multipart upload. Parts must be in ascending +// `part_number` order; we sort defensively. +pub fn (c &Client) complete_multipart(key string, upload_id string, parts []PartRef, opts PutOptions) ! { + mut sorted := parts.clone() + sorted.sort(a.part_number < b.part_number) + mut body := strings.new_builder(1024) + body.write_string('') + for p in sorted { + body.write_string('${p.part_number}"${p.etag}"') + } + body.write_string('') + xml := body.str() + creds := c.creds_for(opts.bucket) + path := build_object_path(creds, opts.bucket, key)! + q := 'uploadId=' + uri_encode_query(upload_id) + signed := sign_request(creds, SignRequest{ + method: 'POST' + path: path + query: q + payload_hash: sha256_hex(xml.bytes()) + extra_headers: { + 'content-type': 'application/xml' + } + })! + resp := c.do_http(signed, xml)! + if resp.status_code != 200 { + return new_http_error(resp.status_code, key, resp.body) + } + // CompleteMultipartUpload may return HTTP 200 with an body — surface those. + if resp.body.contains('') { + return new_http_error(200, key, resp.body) + } +} + +// abort_multipart cancels an in-flight upload. Best-effort — callers usually +// invoke it inside an error path and don't care about the result. +pub fn (c &Client) abort_multipart(key string, upload_id string, opts PutOptions) ! { + creds := c.creds_for(opts.bucket) + path := build_object_path(creds, opts.bucket, key)! + q := 'uploadId=' + uri_encode_query(upload_id) + signed := sign_request(creds, SignRequest{ + method: 'DELETE' + path: path + query: q + payload_hash: empty_sha256 + })! + resp := c.do_http(signed, '')! + if resp.status_code !in [204, 200, 404] { + return new_http_error(resp.status_code, key, resp.body) + } +} diff --git a/vlib/net/s3/s3.v b/vlib/net/s3/s3.v new file mode 100644 index 000000000..0aa082d74 --- /dev/null +++ b/vlib/net/s3/s3.v @@ -0,0 +1,40 @@ +// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved. +// Use of this source code is governed by an MIT license +// that can be found in the LICENSE file. +// Module s3 is an S3-compatible client for V. +// +// Quick start: +// +// import s3 +// +// client := s3.new_client(s3.Credentials{ +// endpoint: 'https://s3.example.com' +// access_key_id: '...' +// secret_access_key: '...' +// bucket: 'my-bucket' +// }) +// client.put('hello.txt', 'Hi from V!'.bytes())! +// text := client.get_string('hello.txt')! +// url := client.presign('hello.txt', expires_in: 3600)! +// +// // s3:// fetch helper: +// resp := s3.fetch('s3://my-bucket/hello.txt')! +// +// Construct credentials from the environment (S3_*, AWS_*, CELLAR_ADDON_*, +// SCW_*, B2_*, R2_*, SPACES_* — first non-empty wins per field): +// +// client := s3.new_client(s3.Credentials.from_env()) +// +// Multipart for large files: +// +// client.upload_file('big.bin', '/path/to/big.bin', s3.PutOptions{ content_type: 'application/octet-stream' })! +module s3 + +// version is the module version, kept in sync with v.mod. +pub const version = '0.1.0' + +// info_string returns a one-line build-time identification, useful in +// User-Agent strings or `--version` output. +pub fn info_string() string { + return 's3/${version}' +} diff --git a/vlib/net/s3/signer.v b/vlib/net/s3/signer.v new file mode 100644 index 000000000..85ba89e81 --- /dev/null +++ b/vlib/net/s3/signer.v @@ -0,0 +1,294 @@ +// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved. +// Use of this source code is governed by an MIT license +// that can be found in the LICENSE file. +module s3 + +import crypto.hmac +import crypto.sha256 +import time + +// service_name is the SigV4 service identifier baked into the signing key. +pub const service_name = 's3' + +// algo is the SigV4 algorithm marker S3 expects. +pub const algo = 'AWS4-HMAC-SHA256' + +// unsigned_payload is the magic string used in `x-amz-content-sha256` when +// the payload is not pre-hashed. Used by the single-shot put path to avoid +// buffering / re-scanning the entire body just to sign it. +pub const unsigned_payload = 'UNSIGNED-PAYLOAD' + +// empty_sha256 is `sha256("")` precomputed. Used for HEAD/GET/DELETE where +// there is no body and we want a real hash for stricter S3 endpoints. +pub const empty_sha256 = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' + +// SignRequest describes the request to sign. The signer is intentionally +// agnostic of HTTP option semantics (ACL, storage class, …) — the caller +// pre-fills `extra_headers` with whatever it intends to send. That keeps the +// signer pure and easy to test against published SigV4 reference vectors. +pub struct SignRequest { +pub: + method string @[required] // GET, PUT, POST, DELETE, HEAD + path string @[required] // canonical URI (already URI-encoded, kept as-is) + query string // canonical query string (sorted, encoded; no leading '?') + payload_hash string // hex SHA-256 of body or `unsigned_payload` + extra_headers map[string]string // lowercase keys, raw values — will be added to canonical/signed set + sign_time time.Time // when omitted (Time{}) we use time.utc() +} + +// SignedRequest is the output of header signing. +pub struct SignedRequest { +pub: + method string // canonical HTTP method that was signed (GET, PUT, …) + url string // scheme://host//? + host string // host[:port], suitable for the Host header + amz_date string // YYYYMMDDTHHMMSSZ + authorization string // ready-to-send Authorization header value + headers map[string]string // ALL headers that MUST be sent for the signature to verify +} + +// sign_request builds the SigV4 Authorization header for an HTTP request. +// +// The returned `SignedRequest.headers` is the full set the caller must send +// (excluding `Content-Length`, which the HTTP client adds itself). Adding, +// removing, or mutating a header after signing will break the signature. +pub fn sign_request(creds Credentials, req SignRequest) !SignedRequest { + creds.validate()! + method := normalize_method(req.method) or { + return new_error('InvalidMethod', + 'Method must be GET, PUT, POST, DELETE or HEAD; got: ${req.method}') + } + host := canonical_host(creds, '') + if host == '' { + return new_error('InvalidEndpoint', + 'No endpoint and no bucket provided — cannot determine host') + } + region := creds.resolved_region() + now := if req.sign_time.year == 0 { time.utc() } else { req.sign_time } + amz_date := format_amz_date(now) + amz_day := amz_date[..8] + payload := if req.payload_hash == '' { unsigned_payload } else { req.payload_hash } + + // Build the headers set. We always sign host + amz date + content sha + // because S3 requires them. Reject CRLF before they reach the wire. + mut headers := map[string]string{} + for k, v in req.extra_headers { + if contains_crlf(v) || contains_crlf(k) { + return new_error('InvalidHeader', + 'Header "${k}" contains CR/LF — refused (header injection guard)') + } + headers[k.to_lower()] = v + } + headers['host'] = host + headers['x-amz-content-sha256'] = payload + headers['x-amz-date'] = amz_date + if creds.session_token != '' { + headers['x-amz-security-token'] = creds.session_token + } + + signed_header_names := sorted_keys(headers).join(';') + canonical := build_canonical_request(method, req.path, req.query, headers, signed_header_names, + payload) + credential_scope := '${amz_day}/${region}/${service_name}/aws4_request' + string_to_sign := '${algo}\n${amz_date}\n${credential_scope}\n${sha256_hex(canonical.bytes())}' + signature := to_hex_lower(hmac_sha256(derive_signing_key(creds.secret_access_key, amz_day, + region, service_name), string_to_sign.bytes())) + authorization := '${algo} Credential=${creds.access_key_id}/${credential_scope}, SignedHeaders=${signed_header_names}, Signature=${signature}' + headers['authorization'] = authorization + + scheme := creds.scheme() + mut url := '${scheme}://${host}${req.path}' + if req.query != '' { + url += '?' + req.query + } + return SignedRequest{ + method: method + url: url + host: host + amz_date: amz_date + authorization: authorization + headers: headers + } +} + +// PresignRequest describes a presigned URL to generate. The output is a +// self-contained URL — no extra headers are required at request time. +pub struct PresignRequest { +pub: + method string @[required] + path string @[required] // canonical URI (already URI-encoded) + expires_in int = 86400 // 1..604800 seconds (SigV4 hard limit is 7 days) + extra_query map[string]string // additional signed query params (e.g. response-content-type, x-amz-acl) + sign_time time.Time +} + +// presign_url returns a fully-formed `https://...` URL signed via SigV4 +// query-string parameters. The URL is valid until `now + expires_in`. +pub fn presign_url(creds Credentials, req PresignRequest) !string { + creds.validate()! + method := normalize_method(req.method) or { + return new_error('InvalidMethod', + 'Method must be GET, PUT, POST, DELETE or HEAD; got: ${req.method}') + } + if req.expires_in < 1 || req.expires_in > 604800 { + return new_error('InvalidExpiry', 'expires_in must be between 1 and 604800 seconds') + } + host := canonical_host(creds, '') + if host == '' { + return new_error('InvalidEndpoint', + 'No endpoint and no bucket provided — cannot determine host') + } + region := creds.resolved_region() + now := if req.sign_time.year == 0 { time.utc() } else { req.sign_time } + amz_date := format_amz_date(now) + amz_day := amz_date[..8] + credential := '${creds.access_key_id}/${amz_day}/${region}/${service_name}/aws4_request' + + // Required signed query parameters; we then merge in extras from the caller. + mut params := map[string]string{} + params['X-Amz-Algorithm'] = algo + params['X-Amz-Credential'] = credential + params['X-Amz-Date'] = amz_date + params['X-Amz-Expires'] = req.expires_in.str() + params['X-Amz-SignedHeaders'] = 'host' + if creds.session_token != '' { + params['X-Amz-Security-Token'] = creds.session_token + } + for k, v in req.extra_query { + if contains_crlf(k) || contains_crlf(v) { + return new_error('InvalidQueryParam', 'Query param "${k}" contains CR/LF — refused') + } + params[k] = v + } + + canonical_query := canonical_query_string(params) + headers := { + 'host': host + } + canonical := build_canonical_request(method, req.path, canonical_query, headers, 'host', + unsigned_payload) + credential_scope := '${amz_day}/${region}/${service_name}/aws4_request' + string_to_sign := '${algo}\n${amz_date}\n${credential_scope}\n${sha256_hex(canonical.bytes())}' + signature := to_hex_lower(hmac_sha256(derive_signing_key(creds.secret_access_key, amz_day, + region, service_name), string_to_sign.bytes())) + scheme := creds.scheme() + return '${scheme}://${host}${req.path}?${canonical_query}&X-Amz-Signature=${signature}' +} + +// build_canonical_request assembles the canonical request string per +// SigV4 §3.2. `path` and `query` MUST already be URI-encoded. +pub fn build_canonical_request(method string, path string, query string, headers map[string]string, + signed_headers string, payload_hash string) string { + sorted := sorted_keys(headers) + mut lines := []string{cap: sorted.len} + for k in sorted { + lines << '${k}:${normalize_header_value(headers[k])}' + } + return '${method}\n${path}\n${query}\n${lines.join('\n')}\n\n${signed_headers}\n${payload_hash}' +} + +// canonical_query_string sorts query params by key (then by value if the same +// key appears twice — we don't here) and joins them as `k=v&k=v` with values +// already URI-encoded. +pub fn canonical_query_string(params map[string]string) string { + keys := sorted_keys(params) + mut parts := []string{cap: keys.len} + for k in keys { + parts << '${uri_encode_query(k)}=${uri_encode_query(params[k])}' + } + return parts.join('&') +} + +// derive_signing_key implements the four-step HMAC chain that produces the +// SigV4 signing key. The result is cacheable per (secret, date, region, +// service) tuple — left to the caller to memoize if needed. +pub fn derive_signing_key(secret string, date string, region string, service string) []u8 { + k_date := hmac_sha256(('AWS4' + secret).bytes(), date.bytes()) + k_region := hmac_sha256(k_date, region.bytes()) + k_service := hmac_sha256(k_region, service.bytes()) + return hmac_sha256(k_service, 'aws4_request'.bytes()) +} + +// format_amz_date returns the basic ISO-8601 timestamp used by SigV4 +// (`YYYYMMDDTHHMMSSZ`). `t` is assumed to be in UTC. +pub fn format_amz_date(t time.Time) string { + return '${t.year:04d}${t.month:02d}${t.day:02d}T${t.hour:02d}${t.minute:02d}${t.second:02d}Z' +} + +// canonical_host resolves the on-the-wire host. Path-style addressing is +// the default; virtual-hosted style uses `.`. If no +// endpoint is configured we fall back to `s3..amazonaws.com` (the +// canonical default for clients pointed at AWS S3 itself). +pub fn canonical_host(creds Credentials, bucket_override string) string { + bucket := if bucket_override != '' { bucket_override } else { creds.bucket } + host := creds.host_only() + if host == '' { + region := creds.resolved_region() + if creds.virtual_hosted_style { + if bucket == '' { + return '' + } + return '${bucket}.s3.${region}.amazonaws.com' + } + return 's3.${region}.amazonaws.com' + } + if creds.virtual_hosted_style && bucket != '' { + return '${bucket}.${host}' + } + return host +} + +// normalize_method uppercases and validates the HTTP method. +pub fn normalize_method(m string) ?string { + up := m.to_upper() + return match up { + 'GET', 'PUT', 'POST', 'DELETE', 'HEAD' { up } + else { none } + } +} + +// normalize_header_value collapses runs of internal whitespace to a single +// space and trims leading/trailing whitespace, per SigV4 §3.2.2 step 3. +fn normalize_header_value(v string) string { + if v == '' { + return v + } + mut out := []u8{cap: v.len} + mut prev_space := false + mut started := false + for b in v.bytes() { + if b == ` ` || b == `\t` { + if started { + prev_space = true + } + continue + } + if prev_space { + out << ` ` + prev_space = false + } + out << b + started = true + } + return out.bytestr() +} + +// sha256_hex returns the lowercase hex digest of `data`. +@[inline] +pub fn sha256_hex(data []u8) string { + return to_hex_lower(sha256.sum256(data)) +} + +// hmac_sha256 wraps `crypto.hmac.new` for HMAC-SHA-256 with the V stdlib's +// type signature (a hash function that returns `[]u8`). +@[inline] +pub fn hmac_sha256(key []u8, data []u8) []u8 { + return hmac.new(key, data, sha256.sum256, sha256.block_size) +} + +// sorted_keys returns the keys of `m` in lexical order. +fn sorted_keys(m map[string]string) []string { + mut keys := m.keys() + keys.sort() + return keys +} diff --git a/vlib/net/s3/signer_test.v b/vlib/net/s3/signer_test.v new file mode 100644 index 000000000..010406088 --- /dev/null +++ b/vlib/net/s3/signer_test.v @@ -0,0 +1,224 @@ +// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved. +// Use of this source code is governed by an MIT license +// that can be found in the LICENSE file. +module s3 + +import time + +// Test vectors come from the AWS-published Signature V4 reference test +// suite ("Examples of the complete version 4 signing process"). +// These secret/key pairs are the documented example values, not real +// credentials. +const aws_secret = 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' +const aws_key_id = 'AKIAIOSFODNN7EXAMPLE' + +// SigV4 vector for `s3-get-object` (examplebucket / test.txt with a Range +// header). Source: AWS docs (Signature V4 examples). +// +// GET /test.txt HTTP/1.1 +// Host: examplebucket.s3.amazonaws.com +// Range: bytes=0-9 +// X-Amz-Date: 20130524T000000Z +// X-Amz-Content-SHA256: +fn test_aws_sigv4_get_object_vector() { + t := time.parse_iso8601('2013-05-24T00:00:00.000Z') or { panic(err) } + creds := Credentials{ + access_key_id: aws_key_id + secret_access_key: aws_secret + region: 'us-east-1' + endpoint: 'examplebucket.s3.amazonaws.com' + virtual_hosted_style: false // host already carries the bucket + } + req := SignRequest{ + method: 'GET' + path: '/test.txt' + query: '' + payload_hash: empty_sha256 + extra_headers: { + 'range': 'bytes=0-9' + } + sign_time: t + } + signed := sign_request(creds, req) or { panic(err) } + expected := 'AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, SignedHeaders=host;range;x-amz-content-sha256;x-amz-date, Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41' + assert signed.authorization == expected, 'Got:\n${signed.authorization}\nExpected:\n${expected}' +} + +// SigV4 vector for `s3-put-object` (examplebucket / test$file.text with body +// "Welcome to Amazon S3."). +// +// PUT /test%24file.text HTTP/1.1 +// Host: examplebucket.s3.amazonaws.com +// Date: Fri, 24 May 2013 00:00:00 GMT +// x-amz-date: 20130524T000000Z +// x-amz-storage-class: REDUCED_REDUNDANCY +fn test_aws_sigv4_put_object_vector() { + t := time.parse_iso8601('2013-05-24T00:00:00.000Z') or { panic(err) } + creds := Credentials{ + access_key_id: aws_key_id + secret_access_key: aws_secret + region: 'us-east-1' + endpoint: 'examplebucket.s3.amazonaws.com' + virtual_hosted_style: false + } + body := 'Welcome to Amazon S3.' + body_hash := sha256_hex(body.bytes()) + req := SignRequest{ + method: 'PUT' + path: '/test%24file.text' + query: '' + payload_hash: body_hash + extra_headers: { + 'date': 'Fri, 24 May 2013 00:00:00 GMT' + 'x-amz-storage-class': 'REDUCED_REDUNDANCY' + } + sign_time: t + } + signed := sign_request(creds, req) or { panic(err) } + expected := 'AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, SignedHeaders=date;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class, Signature=98ad721746da40c64f1a55b78f14c238d841ea1380cd77a1b5971af0ece108bd' + assert signed.authorization == expected, 'Got:\n${signed.authorization}\nExpected:\n${expected}' +} + +// Asserts the canonical request string for a ListObjectsV2 GET. The +// canonical request is the most stable interop point — Authorization line +// formatting has wobbled across AWS doc revisions, but the canonical hash +// is always the same. +fn test_aws_sigv4_list_objects_canonical_request() { + headers := { + 'host': 'examplebucket.s3.amazonaws.com' + 'x-amz-content-sha256': empty_sha256 + 'x-amz-date': '20130524T000000Z' + } + canonical := build_canonical_request('GET', '/', 'list-type=2&prefix=foo', headers, + 'host;x-amz-content-sha256;x-amz-date', empty_sha256) + expected := 'GET\n/\nlist-type=2&prefix=foo\nhost:examplebucket.s3.amazonaws.com\nx-amz-content-sha256:${empty_sha256}\nx-amz-date:20130524T000000Z\n\nhost;x-amz-content-sha256;x-amz-date\n${empty_sha256}' + assert canonical == expected, 'canonical mismatch:\n${canonical}' +} + +// Cross-checked manually with openssl on the AWS docs example +// (date 20120215, region us-east-1, service iam): +// +// echo -n 'aws4_request' | openssl dgst -sha256 -mac HMAC -macopt hexkey: +fn test_signing_key_chain() { + key := derive_signing_key(aws_secret, '20120215', 'us-east-1', 'iam') + assert to_hex_lower(key) == '004aa806e13dae88b9032d9261bcb04c67d023afadd221e6b0d206e1760e0b5e', 'Got: ${to_hex_lower(key)}' +} + +fn test_normalize_header_value_collapses_whitespace() { + assert normalize_header_value(' hello world ') == 'hello world' + assert normalize_header_value('a\tb\tc') == 'a b c' + assert normalize_header_value('') == '' +} + +fn test_canonical_query_string_sorted_and_encoded() { + q := canonical_query_string({ + 'b': '2' + 'a': '1' + 'c': 'hello world' + }) + assert q == 'a=1&b=2&c=hello%20world' +} + +fn test_format_amz_date() { + t := time.parse_iso8601('2024-01-15T12:34:56.000Z') or { panic(err) } + assert format_amz_date(t) == '20240115T123456Z' +} + +fn test_signer_rejects_unsupported_method() { + creds := Credentials{ + access_key_id: 'KEY' + secret_access_key: 'secret' + region: 'us-east-1' + endpoint: 'examplebucket.s3.amazonaws.com' + } + if _ := sign_request(creds, SignRequest{ method: 'PATCH', path: '/x', payload_hash: empty_sha256 }) { + assert false + } +} + +fn test_signer_rejects_crlf_in_extra_headers() { + creds := Credentials{ + access_key_id: 'KEY' + secret_access_key: 'secret' + region: 'us-east-1' + endpoint: 'examplebucket.s3.amazonaws.com' + } + bad := SignRequest{ + method: 'GET' + path: '/foo' + payload_hash: empty_sha256 + extra_headers: { + 'x-evil': 'value\r\nInjected: yes' + } + } + if _ := sign_request(creds, bad) { + assert false, 'expected CRLF rejection' + } else { + if err is S3Error { + assert err.code == 'InvalidHeader' || err.code == 'InvalidCredentials', 'unexpected code: ${err.code}' + } else { + assert false, 'wrong error type' + } + } +} + +fn test_presign_rejects_crlf_in_extra_query() { + creds := Credentials{ + access_key_id: 'KEY' + secret_access_key: 'secret' + region: 'us-east-1' + endpoint: 'examplebucket.s3.amazonaws.com' + } + bad := PresignRequest{ + method: 'GET' + path: '/foo' + extra_query: { + 'X-Evil': 'value\r\nInjected: yes' + } + } + if _ := presign_url(creds, bad) { + assert false, 'expected CRLF rejection' + } else { + assert err is S3Error + } +} + +fn test_presign_rejects_invalid_expiry() { + creds := Credentials{ + access_key_id: 'KEY' + secret_access_key: 'secret' + region: 'us-east-1' + endpoint: 'examplebucket.s3.amazonaws.com' + } + if _ := presign_url(creds, PresignRequest{ method: 'GET', path: '/x', expires_in: 0 }) { + assert false, 'expires_in=0 must be rejected' + } + if _ := presign_url(creds, PresignRequest{ + method: 'GET' + path: '/x' + expires_in: 700_000 + }) + { + assert false, 'expires_in > 7 days must be rejected' + } +} + +fn test_presign_url_contains_required_query_params() { + creds := Credentials{ + access_key_id: 'KEY' + secret_access_key: 'secret' + region: 'us-east-1' + endpoint: 'examplebucket.s3.amazonaws.com' + } + url := presign_url(creds, PresignRequest{ + method: 'GET' + path: '/foo.txt' + expires_in: 300 + }) or { panic(err) } + assert url.contains('X-Amz-Algorithm=AWS4-HMAC-SHA256') + assert url.contains('X-Amz-Credential=') + assert url.contains('X-Amz-Date=') + assert url.contains('X-Amz-Expires=300') + assert url.contains('X-Amz-SignedHeaders=host') + assert url.contains('X-Amz-Signature=') +} diff --git a/vlib/net/s3/types.v b/vlib/net/s3/types.v new file mode 100644 index 000000000..e1d273435 --- /dev/null +++ b/vlib/net/s3/types.v @@ -0,0 +1,169 @@ +// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved. +// Use of this source code is governed by an MIT license +// that can be found in the LICENSE file. +module s3 + +import net.http + +// ACL is the canned Access Control List applied to a stored object or bucket. +// Values match the S3 wire protocol. +pub enum Acl { + unset + private + public_read + public_read_write + aws_exec_read + authenticated_read + bucket_owner_read + bucket_owner_full_control + log_delivery_write +} + +// to_header_value renders the ACL as the canonical S3 string used in headers +// and presigned query parameters. `.unset` returns an empty string so callers +// can skip the header. +pub fn (a Acl) to_header_value() string { + return match a { + .unset { '' } + .private { 'private' } + .public_read { 'public-read' } + .public_read_write { 'public-read-write' } + .aws_exec_read { 'aws-exec-read' } + .authenticated_read { 'authenticated-read' } + .bucket_owner_read { 'bucket-owner-read' } + .bucket_owner_full_control { 'bucket-owner-full-control' } + .log_delivery_write { 'log-delivery-write' } + } +} + +// parse_acl returns the matching Acl from a wire string. +// Returns `.unset` when the input is empty or unknown so the caller can ignore it. +pub fn parse_acl(s string) Acl { + return match s { + 'private' { Acl.private } + 'public-read' { Acl.public_read } + 'public-read-write' { Acl.public_read_write } + 'aws-exec-read' { Acl.aws_exec_read } + 'authenticated-read' { Acl.authenticated_read } + 'bucket-owner-read' { Acl.bucket_owner_read } + 'bucket-owner-full-control' { Acl.bucket_owner_full_control } + 'log-delivery-write' { Acl.log_delivery_write } + else { Acl.unset } + } +} + +// StorageClass enumerates the standard S3 storage tiers. `.unset` means the +// server default (typically STANDARD) and produces no `x-amz-storage-class` +// header. Providers vary on which tiers they actually honour. +pub enum StorageClass { + unset + standard + deep_archive + express_onezone + glacier + glacier_ir + intelligent_tiering + onezone_ia + outposts + reduced_redundancy + snow + standard_ia +} + +// to_header_value renders the storage class as the on-wire string. +pub fn (sc StorageClass) to_header_value() string { + return match sc { + .unset { '' } + .standard { 'STANDARD' } + .deep_archive { 'DEEP_ARCHIVE' } + .express_onezone { 'EXPRESS_ONEZONE' } + .glacier { 'GLACIER' } + .glacier_ir { 'GLACIER_IR' } + .intelligent_tiering { 'INTELLIGENT_TIERING' } + .onezone_ia { 'ONEZONE_IA' } + .outposts { 'OUTPOSTS' } + .reduced_redundancy { 'REDUCED_REDUNDANCY' } + .snow { 'SNOW' } + .standard_ia { 'STANDARD_IA' } + } +} + +// Stat is the result of HEAD-ing an object. +pub struct Stat { +pub: + size i64 // Content-Length in bytes + last_modified string // RFC 1123 date as returned by S3 + etag string // unquoted ETag (server returns it wrapped in quotes; we strip them) + content_type string // MIME type +} + +// Owner identifies an S3 object owner — only populated when `fetch_owner` is true. +pub struct Owner { +pub: + id string + display_name string +} + +// ObjectInfo is one entry from a ListObjectsV2 response. +pub struct ObjectInfo { +pub: + key string + last_modified string + etag string + size i64 + storage_class string + owner ?Owner +} + +// CommonPrefix represents a "directory-like" prefix in a list result when a +// delimiter is provided. +pub struct CommonPrefix { +pub: + prefix string +} + +// ListResult aggregates a ListObjectsV2 response. +// NextContinuationToken should be passed back as `continuation_token` to fetch the next page. +pub struct ListResult { +pub: + name string + prefix string + delimiter string + start_after string + max_keys int + key_count int + is_truncated bool + continuation_token string + next_continuation_token string + objects []ObjectInfo + common_prefixes []CommonPrefix +} + +// ListOptions configures a ListObjectsV2 call. `bucket` overrides the client's +// default; leave empty to use the client's bound bucket. +@[params] +pub struct ListOptions { +pub: + bucket string + prefix string + continuation_token string + delimiter string + max_keys int // 0 means default (server picks 1000) + start_after string + encoding_type string // 'url' or empty + fetch_owner bool +} + +// PresignOptions controls presigned URL generation. +@[params] +pub struct PresignOptions { +pub: + bucket string // overrides the client's default bucket for this call + method http.Method = .get + expires_in int = 86400 // seconds, 1..604800 + acl Acl + storage_class StorageClass + content_type string + content_disposition string + request_payer bool +} -- 2.39.5