From 19d2c622078c21ad9a14cfa7a30fdd42f67cd01c Mon Sep 17 00:00:00 2001 From: Richard Wheeler Date: Fri, 26 Jun 2026 17:32:45 -0400 Subject: [PATCH] ci: add h2spec RFC 9113/7541 conformance gate for net.http HTTP/2 (#27563) --- .github/workflows/h2spec.yml | 62 +++++++++ vlib/net/http/h2spec/.gitignore | 6 + vlib/net/http/h2spec/README.md | 66 +++++++++ .../http/h2spec/h2spec_expected_failures.txt | 59 +++++++++ vlib/net/http/h2spec/h2spec_server.v | 39 ++++++ vlib/net/http/h2spec/run_h2spec.sh | 125 ++++++++++++++++++ 6 files changed, 357 insertions(+) create mode 100644 .github/workflows/h2spec.yml create mode 100644 vlib/net/http/h2spec/.gitignore create mode 100644 vlib/net/http/h2spec/README.md create mode 100644 vlib/net/http/h2spec/h2spec_expected_failures.txt create mode 100644 vlib/net/http/h2spec/h2spec_server.v create mode 100644 vlib/net/http/h2spec/run_h2spec.sh diff --git a/.github/workflows/h2spec.yml b/.github/workflows/h2spec.yml new file mode 100644 index 000000000..cd8bb7d0b --- /dev/null +++ b/.github/workflows/h2spec.yml @@ -0,0 +1,62 @@ +# Runs the h2spec RFC 9113 / 7541 conformance suite against the net.http HTTP/2 +# server on every change to the h2 code. Uses a PINNED h2spec release binary +# (no `go install`, no @latest) so results are reproducible. The gate compares +# the failing cases against vlib/net/http/h2spec/h2spec_expected_failures.txt and +# fails only on a regression (a case that newly fails). +name: h2spec conformance + +on: + push: + paths: &h2paths + - 'vlib/net/http/h2_*.v' + - 'vlib/net/http/server*.v' + - 'vlib/net/http/http*.v' + - 'vlib/net/http/h2spec/**' + # `**` (not `*.v`): mbedtls.c.v #inserts mbedtls_helpers.h / mbedtls_threading.h, + # so TLS/ALPN behavior can change via the headers too (Codex P2). + - 'vlib/net/mbedtls/**' + - '.github/workflows/h2spec.yml' + pull_request: + paths: *h2paths + +concurrency: + group: h2spec-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + h2spec: + runs-on: ubuntu-latest + env: + H2SPEC_VERSION: v2.6.0 + H2SPEC_SHA256: 157ee0de702e01ad40e752dbf074b366027e550c8e7504f9450da2809e279318 + steps: + - uses: actions/checkout@v4 + + - name: Build the V compiler + run: | + make + ./v -o vnew cmd/v + + - name: Fetch pinned h2spec release + run: | + url="https://github.com/summerwind/h2spec/releases/download/${H2SPEC_VERSION}/h2spec_linux_amd64.tar.gz" + curl -fsSL "$url" -o h2spec.tar.gz + echo "${H2SPEC_SHA256} h2spec.tar.gz" | sha256sum -c - + tar -xzf h2spec.tar.gz + sudo install -m 0755 h2spec /usr/local/bin/h2spec + h2spec --version + + - name: Run h2spec conformance gate + env: + # VEXE: ./vnew + VFLAGS_CC: -cc gcc # mbedtls needs a real C compiler, not tcc + run: bash vlib/net/http/h2spec/run_h2spec.sh + + - name: Upload h2spec report + if: always() + uses: actions/upload-artifact@v4 + with: + name: h2spec-report + path: | + vlib/net/http/h2spec/h2spec_report.xml + vlib/net/http/h2spec/h2spec_failed_now.txt diff --git a/vlib/net/http/h2spec/.gitignore b/vlib/net/http/h2spec/.gitignore new file mode 100644 index 000000000..179762b85 --- /dev/null +++ b/vlib/net/http/h2spec/.gitignore @@ -0,0 +1,6 @@ +# Artifacts generated by run_h2spec.sh +h2spec_server.bin +h2spec_report.xml +h2spec_failed_now.txt +h2spec_expected.norm.txt +h2spec*.tar.gz diff --git a/vlib/net/http/h2spec/README.md b/vlib/net/http/h2spec/README.md new file mode 100644 index 000000000..026474e41 --- /dev/null +++ b/vlib/net/http/h2spec/README.md @@ -0,0 +1,66 @@ +# h2spec conformance harness + +Independent RFC 9113 (HTTP/2) and RFC 7541 (HPACK) conformance testing for the +`net.http` HTTP/2 server, using [h2spec](https://github.com/summerwind/h2spec) — +the reference conformance suite (~146 cases). This complements the in-repo unit +tests, which exercise the implementation against its *own* encoder/decoder; h2spec +is an *external* client, so it catches bugs shared between V's encode and decode +sides that internal round-trip tests cannot. + +## Files + +| File | Purpose | +|------|---------| +| `h2spec_server.v` | Minimal HTTP/2 target server (TLS + ALPN `h2`, answers 200). The thing h2spec connects to. | +| `run_h2spec.sh` | Builds the target, starts it, runs a **pinned** h2spec, diffs failures against the baseline. | +| `h2spec_expected_failures.txt` | Known-failure baseline (37 cases). CI fails only on a *regression* (a new failure not listed). | +| `.github/workflows/h2spec.yml` | CI job: fetches a pinned h2spec release (sha256-verified) and runs the gate. | + +## Running locally + +```sh +# 1. obtain a PINNED h2spec (no `go install` / @latest — pin a release): +# https://github.com/summerwind/h2spec/releases -> h2spec__amd64.{tar.gz,zip} +# 2. point the harness at it and at the V compiler: +H2SPEC_BIN=/path/to/h2spec VEXE=./vnew VFLAGS_CC='-cc gcc' \ + bash vlib/net/http/h2spec/run_h2spec.sh +``` + +It builds `h2spec_server.v`, starts it on `127.0.0.1:$H2SPEC_PORT` (default 18443), +runs `h2spec -t -k` against it, and writes `h2spec_report.xml` + +`h2spec_failed_now.txt`. Exit 0 = no regressions vs the baseline. + +## Why pinned, not `go install` + +`go install …@latest` is unpinned (results can change with an upstream release), +needs a Go toolchain in the job, and adds supply-chain surface. The CI workflow +downloads a **specific** h2spec release tarball and verifies its `sha256`. The +runner never installs anything — it expects `h2spec` to be provided. + +## Status / what is validated + +- ✅ `h2spec_server.v` builds (gcc), listens, and negotiates ALPN `h2` (the same + server driver `test_server_tls_h2_negotiation` exercises end-to-end). +- ✅ **The harness was run end-to-end with a pinned h2spec v2.6.0**: 146 tests, + **109 pass / 37 fail** against the server at this branch's base. The 37-failure + set was **identical across four runs** (`--timeout 2` and `--timeout 5`, two + each) and is recorded in `h2spec_expected_failures.txt` as the gate baseline. + The JUnit parser is validated against those real reports. +- ⚠️ **Timing flakiness — handled.** A *separate* handful of h2spec cases + (CONTINUATION sequencing, unknown error codes) wait for the server's GOAWAY/close + and flake at h2spec's 2s default under load. The runner therefore uses + `--timeout 5` (`H2SPEC_TIMEOUT`). The lesson: always confirm the failure set is + stable across repeated runs before trusting a per-case h2spec gate — a single + run can mis-attribute a flaky timeout as a real failure (or regression). +- The baseline reflects genuine server-side conformance gaps (see the file's + header). Each is a tracked item; remove a line as the server is hardened. +- The baseline was generated locally; the first CI run on Linux validates it. If + a case drifts by platform, the harness's "now passing" / regression messages + make the one-line fix obvious. + +## Expected value + +h2spec independently covers the areas the internal tests cover only piecemeal — +the **stream state machine** (§5.1 transitions, illegal-frame `STREAM_CLOSED`), +frame sequencing, and error semantics — turning "internally tested" into +"externally RFC-validated." It is the missing Stage-3 of the HTTP/2 test plan. diff --git a/vlib/net/http/h2spec/h2spec_expected_failures.txt b/vlib/net/http/h2spec/h2spec_expected_failures.txt new file mode 100644 index 000000000..73a8be3e7 --- /dev/null +++ b/vlib/net/http/h2spec/h2spec_expected_failures.txt @@ -0,0 +1,59 @@ +# h2spec known-failure baseline for the net.http HTTP/2 server. +# +# One case ID per line, format " :: " exactly as emitted by +# the JUnit parser in run_h2spec.sh. Lines starting with # are comments. +# +# This is the set of h2spec cases the V server does NOT yet pass. CI fails only on +# a REGRESSION (a case not listed here that newly fails). When a listed case starts +# passing, run_h2spec.sh warns so the line can be removed and the gate tightened. +# +# Generated with h2spec v2.6.0 against the net.http server at this branch's base +# (146 tests: 109 pass, 37 fail). The set was identical across four runs (h2spec +# --timeout 2 and --timeout 5, two each) — stable. NOTE: a handful of OTHER h2spec +# cases (CONTINUATION sequencing, unknown error codes) are timing-flaky at h2spec's +# 2s default — they wait for the server's GOAWAY/close — so run_h2spec.sh uses +# --timeout 5 (H2SPEC_TIMEOUT) to keep the gate stable. Do not add a flaky case to +# this list; raise the timeout instead. These 37 are genuine server-side conformance +# gaps: stricter request +# validation (uppercase/duplicate/missing pseudo-headers, connection-specific & TE +# headers, content-length vs DATA), HPACK decode error SCOPE (must be a +# connection-level COMPRESSION_ERROR, not RST_STREAM), the stream state machine +# (frames on idle/closed streams), CONTINUATION sequencing, and flow-control +# overflow/zero-increment handling. Remove a line when the server is fixed to pass. +generic/4 :: Sends a POST request with trailers +hpack/2.3.3 :: Sends a indexed header field representation with invalid index +hpack/2.3.3 :: Sends a literal header field representation with invalid index +hpack/4.2 :: Sends a dynamic table size update at the end of header block +hpack/5.2 :: Sends a Huffman-encoded string literal representation containing the EOS symbol +hpack/5.2 :: Sends a Huffman-encoded string literal representation padded by zero +hpack/5.2 :: Sends a Huffman-encoded string literal representation with padding longer than 7 bits +hpack/6.1 :: Sends a indexed header field representation with index 0 +hpack/6.3 :: Sends a dynamic table size update larger than the value of SETTINGS_HEADER_TABLE_SIZE +http2/4.3 :: Sends invalid header block fragment +http2/5.1 :: closed: Sends a DATA frame +http2/5.1 :: closed: Sends a DATA frame after sending RST_STREAM frame +http2/5.1 :: half closed (remote): Sends a DATA frame +http2/5.1 :: idle: Sends a DATA frame +http2/5.1 :: idle: Sends a RST_STREAM frame +http2/5.1 :: idle: Sends a WINDOW_UPDATE frame +http2/5.1.2 :: Sends HEADERS frames that causes their advertised concurrent stream limit to be exceeded +http2/5.3.1 :: Sends HEADERS frame that depends on itself +http2/5.3.1 :: Sends PRIORITY frame that depend on itself +http2/6.1 :: Sends a DATA frame on the stream that is not in "open" or "half-closed (local)" state +http2/6.4 :: Sends a RST_STREAM frame on a idle stream +http2/6.5.2 :: SETTINGS_ENABLE_PUSH (0x2): Sends the value other than 0 or 1 +http2/6.9 :: Sends a WINDOW_UPDATE frame with a flow control window increment of 0 +http2/6.9 :: Sends a WINDOW_UPDATE frame with a flow control window increment of 0 on a stream +http2/6.9.1 :: Sends multiple WINDOW_UPDATE frames increasing the flow control window to above 2^31-1 +http2/6.9.1 :: Sends multiple WINDOW_UPDATE frames increasing the flow control window to above 2^31-1 on a stream +http2/6.9.2 :: Sends a SETTINGS frame for window size to be negative +http2/8.1.2 :: Sends a HEADERS frame that contains the header field name in uppercase letters +http2/8.1.2.1 :: Sends a HEADERS frame that contains a pseudo-header field that appears in a header block after a regular header field +http2/8.1.2.2 :: Sends a HEADERS frame that contains the TE header field with any value other than "trailers" +http2/8.1.2.2 :: Sends a HEADERS frame that contains the connection-specific header field +http2/8.1.2.3 :: Sends a HEADERS frame that omits ":scheme" pseudo-header field +http2/8.1.2.3 :: Sends a HEADERS frame with duplicated ":method" pseudo-header field +http2/8.1.2.3 :: Sends a HEADERS frame with duplicated ":path" pseudo-header field +http2/8.1.2.3 :: Sends a HEADERS frame with duplicated ":scheme" pseudo-header field +http2/8.1.2.6 :: Sends a HEADERS frame with the "content-length" header field which does not equal the DATA frame payload length +http2/8.1.2.6 :: Sends a HEADERS frame with the "content-length" header field which does not equal the sum of the multiple DATA frames payload length diff --git a/vlib/net/http/h2spec/h2spec_server.v b/vlib/net/http/h2spec/h2spec_server.v new file mode 100644 index 000000000..b6ba00cbc --- /dev/null +++ b/vlib/net/http/h2spec/h2spec_server.v @@ -0,0 +1,39 @@ +module main + +import os +import net.http + +// h2spec target: a minimal HTTP/2 server for RFC 9113 / 7541 conformance testing +// with h2spec (https://github.com/summerwind/h2spec). Build and run it, then: +// h2spec -t -k -h 127.0.0.1 -p $H2SPEC_PORT +// It terminates TLS with a throwaway in-memory cert (the same self-signed pair +// the net.http TLS tests use) and answers every request with 200, so h2spec can +// exercise the framing / HPACK / flow-control / error layers against the real +// net.http HTTP/2 server driver. See run_h2spec.sh for the CI harness. + +struct H2SpecHandler {} + +fn (mut h H2SpecHandler) handle(req http.Request) http.Response { + return http.Response{ + status_code: 200 + body: 'ok' + } +} + +fn main() { + port := (os.getenv_opt('H2SPEC_PORT') or { '8443' }).int() + mut srv := &http.Server{ + addr: '127.0.0.1:${port}' + cert: h2spec_cert + cert_key: h2spec_key + in_memory_verification: true + enable_http2: true + handler: H2SpecHandler{} + show_startup_message: false + } + println('h2spec target listening on 127.0.0.1:${port} (h2 over TLS)') + srv.listen_and_serve() +} + +const h2spec_cert = '-----BEGIN CERTIFICATE-----\nMIIEOTCCAyECFG64Q2g46jZb3kRbDOJWX/BwjSp6MA0GCSqGSIb3DQEBCwUAMEUx\nCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl\ncm5ldCBXaWRnaXRzIFB0eSBMdGQwIBcNMjMwODAyMTcyOTQyWhgPMjA1MDEyMTcx\nNzI5NDJaMGsxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRQwEgYD\nVQQHDAtMb3MgQW5nZWxlczEdMBsGA1UECgwUQ2F0YWx5c3QgRGV2ZWxvcG1lbnQx\nEjAQBgNVBAMMCWxvY2FsaG9zdDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoC\nggIBALqAI4fqUi+QBVWcsXglouLdOML5+w0+1hSR1KdO0Q5XPdQAs/yYWJ+KUkDw\nG++rfy9DUPq7FNRBVurXQkcAtn6gXdllGUSjwUiDo/N4mMOyS/2sufBuaeww7jVi\nrppH+zwP1tUnjRd6khl6bi1Ian9VSzr3Iy9CkXIg1GU4CPXkOydLeoQfepXxWoK1\nOUNwT3VKC/stAfY3j/NIIeiJYkyuRGFCkxn/BUjN+AsXiTugRcYKEFHdIPkOuCXp\nYbhf+lLsczpxCs3rdZG9b/N6mEDCzXTmeHkmsjdPTf+1k5DZZvKzVBBrgdxCgBb7\n5RwjF5v9WmnIc33wWgfJC6FaUzj9NYxYUbPHD+jTz0rJB/jj4u/xJlM/e5NRmXdW\n70pOMKXtWjRSolLOFIPKLY1qs3KMTAZxKKWPDDF7WlMJxMRt7nnnks5yw43Nog4C\njDLk1ZgETnPpLgo3jbmJdIv+OHKTJrBlVvDq7VTyixCoS5G8KoOmyQJhaXG6NwE2\niVhH5JIKgzgCfetfDsnjxqJ/qtrFXPa8FF2TsomD0NK/GZmIcs+9OeVB75Jn5uhF\nfLHScpiTbuu5w3P/LI/MqihLRB6RRNnRzPH8fIg5bYC9b770ta/8GcFRuYE8t+UR\nGtqXJoIKixbDlqV54kal8FQzYzhETf9+NM6Kb/lKEfG/pslvAgMBAAEwDQYJKoZI\nhvcNAQELBQADggEBALI3uNiNO0QE1brA3QYFK+d9ZroB72NrJ0UNkzYHDg2Fc6xg\n4aVVfaxY08+TmKc0JlMOW+pUxeCW/+UBSngdQiR9EE9xm0k0XIrAsy9RXxRvEtPu\nM1VI2h7ayp1Y2BrnQinevTSgtqLRyS1VbOFRl1FiyVvinw2I0KsDdAMNevAPXcOa\nQ8pUgUq6f56DkhocQaj+hxD/uV8HryNxuoSXnPhvfTN3z4YRGzsaWevJ9EYJliOM\n+XugcqfFJ+W7/QCEcAHCL+Bw6OydG5NFORr3p57PXjjcL/uKmxPBrWg2Bz6uT4uR\nMhj0zttiFHLAt9jGfyk6W57UNUja1e1ggftJJhs=\n-----END CERTIFICATE-----\n' +const h2spec_key = '-----BEGIN RSA PRIVATE KEY-----\nMIIJKQIBAAKCAgEAuoAjh+pSL5AFVZyxeCWi4t04wvn7DT7WFJHUp07RDlc91ACz\n/JhYn4pSQPAb76t/L0NQ+rsU1EFW6tdCRwC2fqBd2WUZRKPBSIOj83iYw7JL/ay5\n8G5p7DDuNWKumkf7PA/W1SeNF3qSGXpuLUhqf1VLOvcjL0KRciDUZTgI9eQ7J0t6\nhB96lfFagrU5Q3BPdUoL+y0B9jeP80gh6IliTK5EYUKTGf8FSM34CxeJO6BFxgoQ\nUd0g+Q64JelhuF/6UuxzOnEKzet1kb1v83qYQMLNdOZ4eSayN09N/7WTkNlm8rNU\nEGuB3EKAFvvlHCMXm/1aachzffBaB8kLoVpTOP01jFhRs8cP6NPPSskH+OPi7/Em\nUz97k1GZd1bvSk4wpe1aNFKiUs4Ug8otjWqzcoxMBnEopY8MMXtaUwnExG3ueeeS\nznLDjc2iDgKMMuTVmAROc+kuCjeNuYl0i/44cpMmsGVW8OrtVPKLEKhLkbwqg6bJ\nAmFpcbo3ATaJWEfkkgqDOAJ9618OyePGon+q2sVc9rwUXZOyiYPQ0r8ZmYhyz705\n5UHvkmfm6EV8sdJymJNu67nDc/8sj8yqKEtEHpFE2dHM8fx8iDltgL1vvvS1r/wZ\nwVG5gTy35REa2pcmggqLFsOWpXniRqXwVDNjOERN/340zopv+UoR8b+myW8CAwEA\nAQKCAgEAkcoffF0JOBMOiHlAJhrNtSiX+ZruzNDlCxlgshUjyWEbfQG7sWbqSHUZ\njZflTrqyZqDpyca7Jp2ZM2Vocxa0klIMayfj08trCaOWY3pPeROE4d3HUJMPjEpH\nvEXTFdnVJIOBPgl3+vWfBfm17QIh9j4X3BVbVNNl3WCaiDGAl699Kl+Pe38cFeCh\nD3JZPEWsZ5SlvwjU8sNGbThjAWN8C1NjMuCXG4hGej5Ae3M/nPPR91jgnw4Me4Ut\nIL3K3RVyGqaqAPJjLsu0kWQUArJAGMfvUkXjwVklkaUV5SHtJBs+pdTXjyprTmJR\nvSXWWON5zkAEEJNY7QcZaeKYi96PFLUFI+ciEdnXn74CfSKhgZCBo+OyFZjDWW5R\nNmgAbZTN2RW0z+V54Lg36JfJrmiGs8TN06KwNjFo+iOJCdQnoUSIhTlmMfVbXPah\ntRfQvwqtfqVS9W/jkiGq9yDDqyXx093R/QTM/XqDlWJ2iOJFppOJefGFCWF6Fwll\nVT9povTAGQmXFiAxwFZxWtbFa0i8fP5QG80X6l/gRklSd6ZXAVvcLkaFGqxunDAe\nrYC2jBwHWRpVmbxw880SWRzlAsJXc7M8PQnBTlyX1mFZNnwAJgqplz0BQHQhQh4V\nqNfisUm9smtda+Hr9GBBUxs09ulery3I0lQjsArVxPqPVgUbFPECggEBANqLA5fH\n2LupOBoFH/fK5jixyGdSB8eJvU+XuS8RBBexnzTQApmDHiU7Axa/cKvxAfUgwBpU\n6OIsL6Lq6wowVInBgo7GraACwspGMIP8Z7+A8qDgSWIcpXP21Ny2RW+nukdH8ZnV\nTFtiFxLYU9GRfzSUcqvE0miKfMGP/S9Cqbew00K6CQ2xurLTR2AchfUQZJJIg7eF\nRBoftthXLQ+s1JoiLJX2gqCliFy32RMAUP+pKvKVJmVQh8bxEkoEzTV2eY7eTxsH\nJDH5hD66EZ5bW/nVAMruJ3iKjy3WvjDbnddNAz9IFKrd1RMP9dgSEKuSv/HhqwPe\n1q9Wm6LWZo8BlYcCggEBANp3M14QMcMxRlZE0TiSopi1CaE8OG0C9apToS1dol2s\n4lCsWHVPIC516LMPGU0bmCdtwJey1mgXQEKVxCWHkVhhoCKT/tN53o5qkptrhrXL\npbqmRfoMXI7LwJU+Vqi5fwSPGrSR/IzHwCUL7pHTbYN7wT5rr2rcC84XYSX31TFm\nNfMnbDuUk33ycAo07Vqts5A5FN+xViEUMFSDmfA2XmOAV77awz0l/3n3qOg9lQYe\nU4Av2nT19lGELirLInkB1ndLirWAcLaCBXKOLW4bzpNm9Bt8aiziVzcUzlJlLa+1\nnb/7//xzKi0eM/BhyJfhsmOz5B8AQ6Ca/keDk8M7JtkCggEARl8DDinE6VCpBv/l\ndlX4YgMlQ9fPN3pr4ig58iTpi3Ofj1L3s1TcLSLecMG+Vy9o8PTVxuTWhJWz1SMO\nAh7j6ePM1Yq2N9MLxDRrxOROyASOnCz8lEIjKL8vdc6fdz+sJO3OpzleuAJS6beM\n7euK6XRvpE3hbtZBK9bgsQonOkYPEOp0pds4AgM0dYdZvzrDF7OP7lVUQ5E4wFr5\n4JVHdEZS0wsoru/+g9STaqHscxaXBLvwPCl9Pxs7R2haZ7+5jr6Y/FwFVK5C3ivu\nJm7GpCDpe27KeO8tAZancXYWUlCzHfpo5Ug/Jz85a5UNlyHO+uUuuzVTLeyWew3M\nwnnBGwKCAQEAqGTBP3wUH3TX1p9s9cJxemvxZEra44woeIXF8wX9pV8hgzWVabb4\nA1f3ai31Pq5KdfnvPf8nrUxex/RRIOyCaDG4EW8qOS/zEKutHgef6nly4ZBQ2BC3\nN4pug5ttiNiSw5za5NyyYoGF5ghweA8UlwjJR6gRqri6kL0MsQt7VXyHkUmN787y\ncV5yZiut2PuTMVQOdu5miVDagAqAmdwOnXvMJtzRKU0kw4rWs0zklbbCfkhkh0sf\n9m2AeJPjmoqEGags3wKF3ugR8t8MvZbJgG0XNCiOXtKIj3iGIJTExm+jjNxd0OWk\nWOqy9lMpH4lky91ZtVuqxR0za0RMnWv24QKCAQBe8l0w9AYVNGDLv1jyPcbsncty\nNYI81yqe2mL+TC00sMCeil7C7WCP7kRklY01rH5q5gJ9Q1UV+bOj2fQdXDmQ5Bgo\n41jseh44gkbuXAeWcSDrDkJCrfvlNqFobTmUb8cdb9aQlHYfOJ31367LJspiw2SY\nmCbnLQ5sMnyBiMkcn0GfBV6IAkZVN73DPa8a1m/0Qrrv1GmBJFVbuZd9d/hAWpHa\nekhXPq0Sta+RNDfBR3aI5lAmVA17qRGiubQYJ+Ldq0aRJ40fGE51ctoSU/5RMcmh\n6+Qro+jSC94L46xMFp+1J5atgB1p/jVzTT/Ws7SLyotYUSL8zU7tcLiycQXs\n-----END RSA PRIVATE KEY-----\n' diff --git a/vlib/net/http/h2spec/run_h2spec.sh b/vlib/net/http/h2spec/run_h2spec.sh new file mode 100644 index 000000000..76961dfcd --- /dev/null +++ b/vlib/net/http/h2spec/run_h2spec.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +# run_h2spec.sh — run the h2spec HTTP/2 conformance suite against the net.http +# HTTP/2 server and gate on a known-failures baseline. +# +# h2spec (https://github.com/summerwind/h2spec) is the reference RFC 9113 / 7541 +# conformance tester. It connects as a client and runs ~146 cases. This harness: +# 1. builds the V h2spec target server (h2spec_server.v), +# 2. starts it on a free port (TLS + ALPN h2), +# 3. runs a PINNED h2spec binary against it, +# 4. compares the set of FAILED case IDs against h2spec_expected_failures.txt, +# 5. fails only on a regression (a case that newly fails, or a recorded failure +# that now passes — so the baseline is kept honest and shrinks over time). +# +# h2spec is NOT installed here. Provide it via $H2SPEC_BIN or on PATH; CI fetches +# a pinned release (see .github/workflows/h2spec.yml). This script never runs +# `go install` — the version must be pinned and reproducible. +# +# Env: +# VEXE path to the V compiler (default: ./v then ./vnew) +# H2SPEC_BIN path to the h2spec binary (default: h2spec on PATH) +# H2SPEC_PORT port for the target server (default: 18443) +# H2SPEC_TIMEOUT h2spec per-case timeout, seconds (default: 5). Several h2spec +# cases wait for the server's GOAWAY/close and FLAKE at h2spec's +# 2s default under load; 5s is comfortably stable here. Raise it +# if a timing case still flakes on a slow runner. +# VFLAGS_CC extra V flags, e.g. '-cc gcc' (mbedtls needs a real C compiler) + +set -uo pipefail +orig_pwd="$(pwd)" # the caller's cwd, captured BEFORE we cd into the script dir +cd "$(dirname "$0")" +here="$(pwd)" +repo_root="$(cd ../../../.. && pwd)" + +# resolve a caller-supplied path argument: an absolute path or a bare command name +# (PATH lookup, e.g. "h2spec") is used as-is; a relative path is resolved against +# the caller's original cwd, since we cd'd into the script dir above. Without this, +# a documented value like `VEXE=./vnew` (relative to the repo root) would wrongly +# resolve to vlib/net/http/h2spec/vnew and the build would fail. (Codex P1/P3.) +resolve_path() { + case "$1" in + /*) printf '%s' "$1" ;; # absolute path + */*) printf '%s' "$orig_pwd/$1" ;; # relative path containing a slash + *) printf '%s' "$1" ;; # bare name -> PATH lookup + esac +} + +VEXE="${VEXE:-}" +if [ -z "$VEXE" ]; then + if [ -x "$repo_root/v" ]; then VEXE="$repo_root/v"; elif [ -x "$repo_root/vnew" ]; then VEXE="$repo_root/vnew"; else VEXE="v"; fi +else + VEXE="$(resolve_path "$VEXE")" +fi +H2SPEC_BIN="$(resolve_path "${H2SPEC_BIN:-h2spec}")" +PORT="${H2SPEC_PORT:-18443}" +TIMEOUT="${H2SPEC_TIMEOUT:-5}" +SERVER_BIN="$here/h2spec_server.bin" +EXPECTED="$here/h2spec_expected_failures.txt" + +if ! command -v "$H2SPEC_BIN" >/dev/null 2>&1 && [ ! -x "$H2SPEC_BIN" ]; then + echo "ERROR: h2spec not found (set H2SPEC_BIN or put it on PATH; CI fetches a pinned release)." >&2 + exit 2 +fi + +echo "==> building h2spec target server with $VEXE ${VFLAGS_CC:-}" +# shellcheck disable=SC2086 +"$VEXE" ${VFLAGS_CC:-} -o "$SERVER_BIN" "$here/h2spec_server.v" || { echo "build failed" >&2; exit 1; } + +cleanup() { [ -n "${SRV_PID:-}" ] && kill "$SRV_PID" 2>/dev/null; rm -f "$SERVER_BIN"; } +trap cleanup EXIT + +echo "==> starting target on 127.0.0.1:$PORT" +H2SPEC_PORT="$PORT" "$SERVER_BIN" & +SRV_PID=$! + +# Wait for the port to accept connections (up to ~10s). +for _ in $(seq 1 100); do + if (exec 3<>"/dev/tcp/127.0.0.1/$PORT") 2>/dev/null; then exec 3>&- 3<&-; break; fi + kill -0 "$SRV_PID" 2>/dev/null || { echo "server exited early" >&2; exit 1; } + sleep 0.1 +done + +echo "==> running h2spec (TLS, insecure cert OK)" +# -t TLS, -k skip cert verification (self-signed). Parse the JUnit XML report — +# its identifiers are h2spec's stable +# contract, unlike the pretty console output. A failing case has a +# (or ) child; we use "classname :: name" as the stable case ID. +report_xml="$here/h2spec_report.xml" +"$H2SPEC_BIN" -t -k -h 127.0.0.1 -p "$PORT" --timeout "$TIMEOUT" --junit-report "$report_xml" 2>&1 | tail -40 || true +if [ ! -s "$report_xml" ]; then + echo "ERROR: h2spec produced no JUnit report ($report_xml); check the flags/version." >&2 + exit 2 +fi +# One ... per line is not guaranteed, so track state. +awk ' + //{ if (pkg!="" && fail) print pkg " :: " cls; pkg=""; cls=""; fail=0 } +' "$report_xml" | sed 's/"/"/g; s/"/"/g; s/&/\&/g; s/<//g' \ + | sort -u > "$here/h2spec_failed_now.txt" + +grep -vE '^\s*#|^\s*$' "$EXPECTED" 2>/dev/null | sort -u > "$here/h2spec_expected.norm.txt" || true + +new_failures="$(comm -23 "$here/h2spec_failed_now.txt" "$here/h2spec_expected.norm.txt" | grep -vE '^\s*$' || true)" +now_passing="$(comm -13 "$here/h2spec_failed_now.txt" "$here/h2spec_expected.norm.txt" | grep -vE '^\s*$' || true)" + +rc=0 +if [ -n "$new_failures" ]; then + echo "::error:: h2spec REGRESSION — these cases newly fail:" >&2 + echo "$new_failures" >&2 + rc=1 +fi +if [ -n "$now_passing" ]; then + # Fail too (not just warn): a recorded failure that now passes must be removed + # from the baseline, otherwise the gate keeps allowing that case and a later + # PR can reintroduce the failure unnoticed. (Codex P2.) The 37 baseline cases + # are deterministic at H2SPEC_TIMEOUT=5, so this does not flap. + echo "::error:: these recorded failures now PASS — remove them from h2spec_expected_failures.txt to keep the gate honest:" >&2 + echo "$now_passing" >&2 + rc=1 +fi +[ "$rc" -eq 0 ] && echo "==> h2spec: no regressions vs baseline" +exit "$rc" -- 2.39.5