master
1//! HTTP(S) Client implementation.
2//!
3//! Connections are opened in a thread-safe manner, but individual Requests are not.
4//!
5//! TLS support may be disabled via `std.options.http_disable_tls`.
6
7const std = @import("../std.zig");
8const builtin = @import("builtin");
9const testing = std.testing;
10const http = std.http;
11const mem = std.mem;
12const Uri = std.Uri;
13const Allocator = mem.Allocator;
14const assert = std.debug.assert;
15const Io = std.Io;
16const Writer = std.Io.Writer;
17const Reader = std.Io.Reader;
18const HostName = std.Io.net.HostName;
19
20const Client = @This();
21
22pub const disable_tls = std.options.http_disable_tls;
23
24/// Used for all client allocations. Must be thread-safe.
25allocator: Allocator,
26/// Used for opening TCP connections.
27io: Io,
28
29ca_bundle: if (disable_tls) void else std.crypto.Certificate.Bundle = if (disable_tls) {} else .{},
30ca_bundle_mutex: std.Thread.Mutex = .{},
31/// Used both for the reader and writer buffers.
32tls_buffer_size: if (disable_tls) u0 else usize = if (disable_tls) 0 else std.crypto.tls.Client.min_buffer_len,
33/// If non-null, ssl secrets are logged to a stream. Creating such a stream
34/// allows other processes with access to that stream to decrypt all
35/// traffic over connections created with this `Client`.
36ssl_key_log: ?*std.crypto.tls.Client.SslKeyLog = null,
37
38/// The time used to decide whether certificates are expired.
39///
40/// When this is `null`, the next time this client performs an HTTPS request,
41/// it will first check the time and rescan the system for root certificates.
42now: ?Io.Timestamp = null,
43
44/// The pool of connections that can be reused (and currently in use).
45connection_pool: ConnectionPool = .{},
46/// Each `Connection` allocates this amount for the reader buffer.
47///
48/// If the entire HTTP header cannot fit in this amount of bytes,
49/// `error.HttpHeadersOversize` will be returned from `Request.wait`.
50read_buffer_size: usize = 8192,
51/// Each `Connection` allocates this amount for the writer buffer.
52write_buffer_size: usize = 1024,
53
54/// If populated, all http traffic travels through this third party.
55/// This field cannot be modified while the client has active connections.
56/// Pointer to externally-owned memory.
57http_proxy: ?*Proxy = null,
58/// If populated, all https traffic travels through this third party.
59/// This field cannot be modified while the client has active connections.
60/// Pointer to externally-owned memory.
61https_proxy: ?*Proxy = null,
62
63/// A Least-Recently-Used cache of open connections to be reused.
64pub const ConnectionPool = struct {
65 mutex: std.Thread.Mutex = .{},
66 /// Open connections that are currently in use.
67 used: std.DoublyLinkedList = .{},
68 /// Open connections that are not currently in use.
69 free: std.DoublyLinkedList = .{},
70 free_len: usize = 0,
71 free_size: usize = 32,
72
73 /// The criteria for a connection to be considered a match.
74 pub const Criteria = struct {
75 host: HostName,
76 port: u16,
77 protocol: Protocol,
78 };
79
80 /// Finds and acquires a connection from the connection pool matching the criteria.
81 /// If no connection is found, null is returned.
82 ///
83 /// Threadsafe.
84 pub fn findConnection(pool: *ConnectionPool, criteria: Criteria) ?*Connection {
85 pool.mutex.lock();
86 defer pool.mutex.unlock();
87
88 var next = pool.free.last;
89 while (next) |node| : (next = node.prev) {
90 const connection: *Connection = @alignCast(@fieldParentPtr("pool_node", node));
91 if (connection.protocol != criteria.protocol) continue;
92 if (connection.port != criteria.port) continue;
93
94 // Domain names are case-insensitive (RFC 5890, Section 2.3.2.4)
95 if (!connection.host().eql(criteria.host)) continue;
96
97 pool.acquireUnsafe(connection);
98 return connection;
99 }
100
101 return null;
102 }
103
104 /// Acquires an existing connection from the connection pool. This function is not threadsafe.
105 pub fn acquireUnsafe(pool: *ConnectionPool, connection: *Connection) void {
106 pool.free.remove(&connection.pool_node);
107 pool.free_len -= 1;
108
109 pool.used.append(&connection.pool_node);
110 }
111
112 /// Acquires an existing connection from the connection pool. This function is threadsafe.
113 pub fn acquire(pool: *ConnectionPool, connection: *Connection) void {
114 pool.mutex.lock();
115 defer pool.mutex.unlock();
116
117 return pool.acquireUnsafe(connection);
118 }
119
120 /// Tries to release a connection back to the connection pool.
121 /// If the connection is marked as closing, it will be closed instead.
122 ///
123 /// Threadsafe.
124 pub fn release(pool: *ConnectionPool, connection: *Connection, io: Io) void {
125 pool.mutex.lock();
126 defer pool.mutex.unlock();
127
128 pool.used.remove(&connection.pool_node);
129
130 if (connection.closing or pool.free_size == 0) return connection.destroy(io);
131
132 if (pool.free_len >= pool.free_size) {
133 const popped: *Connection = @alignCast(@fieldParentPtr("pool_node", pool.free.popFirst().?));
134 pool.free_len -= 1;
135
136 popped.destroy(io);
137 }
138
139 if (connection.proxied) {
140 // proxied connections go to the end of the queue, always try direct connections first
141 pool.free.prepend(&connection.pool_node);
142 } else {
143 pool.free.append(&connection.pool_node);
144 }
145
146 pool.free_len += 1;
147 }
148
149 /// Adds a newly created node to the pool of used connections. This function is threadsafe.
150 pub fn addUsed(pool: *ConnectionPool, connection: *Connection) void {
151 pool.mutex.lock();
152 defer pool.mutex.unlock();
153
154 pool.used.append(&connection.pool_node);
155 }
156
157 /// Resizes the connection pool.
158 ///
159 /// If the new size is smaller than the current size, then idle connections will be closed until the pool is the new size.
160 ///
161 /// Threadsafe.
162 pub fn resize(pool: *ConnectionPool, allocator: Allocator, new_size: usize) void {
163 pool.mutex.lock();
164 defer pool.mutex.unlock();
165
166 const next = pool.free.first;
167 _ = next;
168 while (pool.free_len > new_size) {
169 const popped = pool.free.popFirst() orelse unreachable;
170 pool.free_len -= 1;
171
172 popped.data.close(allocator);
173 allocator.destroy(popped);
174 }
175
176 pool.free_size = new_size;
177 }
178
179 /// Frees the connection pool and closes all connections within.
180 ///
181 /// All future operations on the connection pool will deadlock.
182 ///
183 /// Threadsafe.
184 pub fn deinit(pool: *ConnectionPool, io: Io) void {
185 pool.mutex.lock();
186
187 var next = pool.free.first;
188 while (next) |node| {
189 const connection: *Connection = @alignCast(@fieldParentPtr("pool_node", node));
190 next = node.next;
191 connection.destroy(io);
192 }
193
194 next = pool.used.first;
195 while (next) |node| {
196 const connection: *Connection = @alignCast(@fieldParentPtr("pool_node", node));
197 next = node.next;
198 connection.destroy(io);
199 }
200
201 pool.* = undefined;
202 }
203};
204
205pub const Protocol = enum {
206 plain,
207 tls,
208
209 fn port(protocol: Protocol) u16 {
210 return switch (protocol) {
211 .plain => 80,
212 .tls => 443,
213 };
214 }
215
216 pub fn fromScheme(scheme: []const u8) ?Protocol {
217 const protocol_map = std.StaticStringMap(Protocol).initComptime(.{
218 .{ "http", .plain },
219 .{ "ws", .plain },
220 .{ "https", .tls },
221 .{ "wss", .tls },
222 });
223 return protocol_map.get(scheme);
224 }
225
226 pub fn fromUri(uri: Uri) ?Protocol {
227 return fromScheme(uri.scheme);
228 }
229};
230
231pub const Connection = struct {
232 client: *Client,
233 stream_writer: Io.net.Stream.Writer,
234 stream_reader: Io.net.Stream.Reader,
235 /// Entry in `ConnectionPool.used` or `ConnectionPool.free`.
236 pool_node: std.DoublyLinkedList.Node,
237 port: u16,
238 host_len: u8,
239 proxied: bool,
240 closing: bool,
241 protocol: Protocol,
242
243 const Plain = struct {
244 connection: Connection,
245
246 fn create(
247 client: *Client,
248 remote_host: HostName,
249 port: u16,
250 stream: Io.net.Stream,
251 ) error{OutOfMemory}!*Plain {
252 const io = client.io;
253 const gpa = client.allocator;
254 const alloc_len = allocLen(client, remote_host.bytes.len);
255 const base = try gpa.alignedAlloc(u8, .of(Plain), alloc_len);
256 errdefer gpa.free(base);
257 const host_buffer = base[@sizeOf(Plain)..][0..remote_host.bytes.len];
258 const socket_read_buffer = host_buffer.ptr[host_buffer.len..][0..client.read_buffer_size];
259 const socket_write_buffer = socket_read_buffer.ptr[socket_read_buffer.len..][0..client.write_buffer_size];
260 assert(base.ptr + alloc_len == socket_write_buffer.ptr + socket_write_buffer.len);
261 @memcpy(host_buffer, remote_host.bytes);
262 const plain: *Plain = @ptrCast(base);
263 plain.* = .{
264 .connection = .{
265 .client = client,
266 .stream_writer = stream.writer(io, socket_write_buffer),
267 .stream_reader = stream.reader(io, socket_read_buffer),
268 .pool_node = .{},
269 .port = port,
270 .host_len = @intCast(remote_host.bytes.len),
271 .proxied = false,
272 .closing = false,
273 .protocol = .plain,
274 },
275 };
276 return plain;
277 }
278
279 fn destroy(plain: *Plain) void {
280 const c = &plain.connection;
281 const gpa = c.client.allocator;
282 const base: [*]align(@alignOf(Plain)) u8 = @ptrCast(plain);
283 gpa.free(base[0..allocLen(c.client, c.host_len)]);
284 }
285
286 fn allocLen(client: *Client, host_len: usize) usize {
287 return @sizeOf(Plain) + host_len + client.read_buffer_size + client.write_buffer_size;
288 }
289
290 fn host(plain: *Plain) HostName {
291 const base: [*]u8 = @ptrCast(plain);
292 return .{ .bytes = base[@sizeOf(Plain)..][0..plain.connection.host_len] };
293 }
294 };
295
296 const Tls = struct {
297 client: std.crypto.tls.Client,
298 connection: Connection,
299
300 /// Asserts that `client.now` is non-null.
301 fn create(
302 client: *Client,
303 remote_host: HostName,
304 port: u16,
305 stream: Io.net.Stream,
306 ) !*Tls {
307 const io = client.io;
308 const gpa = client.allocator;
309 const alloc_len = allocLen(client, remote_host.bytes.len);
310 const base = try gpa.alignedAlloc(u8, .of(Tls), alloc_len);
311 errdefer gpa.free(base);
312 const host_buffer = base[@sizeOf(Tls)..][0..remote_host.bytes.len];
313 // The TLS client wants enough buffer for the max encrypted frame
314 // size, and the HTTP body reader wants enough buffer for the
315 // entire HTTP header. This means we need a combined upper bound.
316 const tls_read_buffer_len = client.tls_buffer_size + client.read_buffer_size;
317 const tls_read_buffer = host_buffer.ptr[host_buffer.len..][0..tls_read_buffer_len];
318 const tls_write_buffer = tls_read_buffer.ptr[tls_read_buffer.len..][0..client.tls_buffer_size];
319 const socket_write_buffer = tls_write_buffer.ptr[tls_write_buffer.len..][0..client.write_buffer_size];
320 const socket_read_buffer = socket_write_buffer.ptr[socket_write_buffer.len..][0..client.tls_buffer_size];
321 assert(base.ptr + alloc_len == socket_read_buffer.ptr + socket_read_buffer.len);
322 @memcpy(host_buffer, remote_host.bytes);
323 const tls: *Tls = @ptrCast(base);
324 var random_buffer: [176]u8 = undefined;
325 std.crypto.random.bytes(&random_buffer);
326 tls.* = .{
327 .connection = .{
328 .client = client,
329 .stream_writer = stream.writer(io, tls_write_buffer),
330 .stream_reader = stream.reader(io, socket_read_buffer),
331 .pool_node = .{},
332 .port = port,
333 .host_len = @intCast(remote_host.bytes.len),
334 .proxied = false,
335 .closing = false,
336 .protocol = .tls,
337 },
338 // TODO data race here on ca_bundle if the user sets `now` to null
339 .client = std.crypto.tls.Client.init(
340 &tls.connection.stream_reader.interface,
341 &tls.connection.stream_writer.interface,
342 .{
343 .host = .{ .explicit = remote_host.bytes },
344 .ca = .{ .bundle = client.ca_bundle },
345 .ssl_key_log = client.ssl_key_log,
346 .read_buffer = tls_read_buffer,
347 .write_buffer = socket_write_buffer,
348 .entropy = &random_buffer,
349 .realtime_now_seconds = client.now.?.toSeconds(),
350 // This is appropriate for HTTPS because the HTTP headers contain
351 // the content length which is used to detect truncation attacks.
352 .allow_truncation_attacks = true,
353 },
354 ) catch |err| switch (err) {
355 error.WriteFailed => return tls.connection.stream_writer.err.?,
356 error.ReadFailed => return tls.connection.stream_reader.err.?,
357 else => |e| return e,
358 },
359 };
360 return tls;
361 }
362
363 fn destroy(tls: *Tls) void {
364 const c = &tls.connection;
365 const gpa = c.client.allocator;
366 const base: [*]align(@alignOf(Tls)) u8 = @ptrCast(tls);
367 gpa.free(base[0..allocLen(c.client, c.host_len)]);
368 }
369
370 fn allocLen(client: *Client, host_len: usize) usize {
371 const tls_read_buffer_len = client.tls_buffer_size + client.read_buffer_size;
372 return @sizeOf(Tls) + host_len + tls_read_buffer_len + client.tls_buffer_size +
373 client.write_buffer_size + client.tls_buffer_size;
374 }
375
376 fn host(tls: *Tls) HostName {
377 const base: [*]u8 = @ptrCast(tls);
378 return .{ .bytes = base[@sizeOf(Tls)..][0..tls.connection.host_len] };
379 }
380 };
381
382 pub const ReadError = std.crypto.tls.Client.ReadError || Io.net.Stream.Reader.Error;
383
384 pub fn getReadError(c: *const Connection) ?ReadError {
385 return switch (c.protocol) {
386 .tls => {
387 if (disable_tls) unreachable;
388 const tls: *const Tls = @alignCast(@fieldParentPtr("connection", c));
389 return tls.client.read_err orelse c.stream_reader.err.?;
390 },
391 .plain => {
392 return c.stream_reader.err.?;
393 },
394 };
395 }
396
397 fn getStream(c: *Connection) Io.net.Stream {
398 return c.stream_reader.stream;
399 }
400
401 pub fn host(c: *Connection) HostName {
402 return switch (c.protocol) {
403 .tls => {
404 if (disable_tls) unreachable;
405 const tls: *Tls = @alignCast(@fieldParentPtr("connection", c));
406 return tls.host();
407 },
408 .plain => {
409 const plain: *Plain = @alignCast(@fieldParentPtr("connection", c));
410 return plain.host();
411 },
412 };
413 }
414
415 /// If this is called without calling `flush` or `end`, data will be
416 /// dropped unsent.
417 pub fn destroy(c: *Connection, io: Io) void {
418 c.stream_reader.stream.close(io);
419 switch (c.protocol) {
420 .tls => {
421 if (disable_tls) unreachable;
422 const tls: *Tls = @alignCast(@fieldParentPtr("connection", c));
423 tls.destroy();
424 },
425 .plain => {
426 const plain: *Plain = @alignCast(@fieldParentPtr("connection", c));
427 plain.destroy();
428 },
429 }
430 }
431
432 /// HTTP protocol from client to server.
433 /// This either goes directly to `stream_writer`, or to a TLS client.
434 pub fn writer(c: *Connection) *Writer {
435 return switch (c.protocol) {
436 .tls => {
437 if (disable_tls) unreachable;
438 const tls: *Tls = @alignCast(@fieldParentPtr("connection", c));
439 return &tls.client.writer;
440 },
441 .plain => &c.stream_writer.interface,
442 };
443 }
444
445 /// HTTP protocol from server to client.
446 /// This either comes directly from `stream_reader`, or from a TLS client.
447 pub fn reader(c: *Connection) *Reader {
448 return switch (c.protocol) {
449 .tls => {
450 if (disable_tls) unreachable;
451 const tls: *Tls = @alignCast(@fieldParentPtr("connection", c));
452 return &tls.client.reader;
453 },
454 .plain => &c.stream_reader.interface,
455 };
456 }
457
458 pub fn flush(c: *Connection) Writer.Error!void {
459 if (c.protocol == .tls) {
460 if (disable_tls) unreachable;
461 const tls: *Tls = @alignCast(@fieldParentPtr("connection", c));
462 try tls.client.writer.flush();
463 }
464 try c.stream_writer.interface.flush();
465 }
466
467 /// If the connection is a TLS connection, sends the close_notify alert.
468 ///
469 /// Flushes all buffers.
470 pub fn end(c: *Connection) Writer.Error!void {
471 if (c.protocol == .tls) {
472 if (disable_tls) unreachable;
473 const tls: *Tls = @alignCast(@fieldParentPtr("connection", c));
474 try tls.client.end();
475 }
476 try c.stream_writer.interface.flush();
477 }
478};
479
480pub const Response = struct {
481 request: *Request,
482 /// Pointers in this struct are invalidated when the response body stream
483 /// is initialized.
484 head: Head,
485
486 pub const Head = struct {
487 bytes: []const u8,
488 version: http.Version,
489 status: http.Status,
490 reason: []const u8,
491 location: ?[]const u8 = null,
492 content_type: ?[]const u8 = null,
493 content_disposition: ?[]const u8 = null,
494
495 keep_alive: bool,
496
497 /// If present, the number of bytes in the response body.
498 content_length: ?u64 = null,
499
500 transfer_encoding: http.TransferEncoding = .none,
501 content_encoding: http.ContentEncoding = .identity,
502
503 pub const ParseError = error{
504 HttpConnectionHeaderUnsupported,
505 HttpContentEncodingUnsupported,
506 HttpHeaderContinuationsUnsupported,
507 HttpHeadersInvalid,
508 HttpTransferEncodingUnsupported,
509 InvalidContentLength,
510 };
511
512 pub fn parse(bytes: []const u8) ParseError!Head {
513 var res: Head = .{
514 .bytes = bytes,
515 .status = undefined,
516 .reason = undefined,
517 .version = undefined,
518 .keep_alive = false,
519 };
520 var it = mem.splitSequence(u8, bytes, "\r\n");
521
522 const first_line = it.first();
523 if (first_line.len < 12) return error.HttpHeadersInvalid;
524
525 const version: http.Version = switch (int64(first_line[0..8])) {
526 int64("HTTP/1.0") => .@"HTTP/1.0",
527 int64("HTTP/1.1") => .@"HTTP/1.1",
528 else => return error.HttpHeadersInvalid,
529 };
530 if (first_line[8] != ' ') return error.HttpHeadersInvalid;
531 const status: http.Status = @enumFromInt(parseInt3(first_line[9..12]));
532 const reason = mem.trimLeft(u8, first_line[12..], " ");
533
534 res.version = version;
535 res.status = status;
536 res.reason = reason;
537 res.keep_alive = switch (version) {
538 .@"HTTP/1.0" => false,
539 .@"HTTP/1.1" => true,
540 };
541
542 while (it.next()) |line| {
543 if (line.len == 0) return res;
544 switch (line[0]) {
545 ' ', '\t' => return error.HttpHeaderContinuationsUnsupported,
546 else => {},
547 }
548
549 var line_it = mem.splitScalar(u8, line, ':');
550 const header_name = line_it.next().?;
551 const header_value = mem.trim(u8, line_it.rest(), " \t");
552 if (header_name.len == 0) return error.HttpHeadersInvalid;
553
554 if (std.ascii.eqlIgnoreCase(header_name, "connection")) {
555 res.keep_alive = !std.ascii.eqlIgnoreCase(header_value, "close");
556 } else if (std.ascii.eqlIgnoreCase(header_name, "content-type")) {
557 res.content_type = header_value;
558 } else if (std.ascii.eqlIgnoreCase(header_name, "location")) {
559 res.location = header_value;
560 } else if (std.ascii.eqlIgnoreCase(header_name, "content-disposition")) {
561 res.content_disposition = header_value;
562 } else if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) {
563 // Transfer-Encoding: second, first
564 // Transfer-Encoding: deflate, chunked
565 var iter = mem.splitBackwardsScalar(u8, header_value, ',');
566
567 const first = iter.first();
568 const trimmed_first = mem.trim(u8, first, " ");
569
570 var next: ?[]const u8 = first;
571 if (std.meta.stringToEnum(http.TransferEncoding, trimmed_first)) |transfer| {
572 if (res.transfer_encoding != .none) return error.HttpHeadersInvalid; // we already have a transfer encoding
573 res.transfer_encoding = transfer;
574
575 next = iter.next();
576 }
577
578 if (next) |second| {
579 const trimmed_second = mem.trim(u8, second, " ");
580
581 if (http.ContentEncoding.fromString(trimmed_second)) |transfer| {
582 if (res.content_encoding != .identity) return error.HttpHeadersInvalid; // double compression is not supported
583 res.content_encoding = transfer;
584 } else {
585 return error.HttpTransferEncodingUnsupported;
586 }
587 }
588
589 if (iter.next()) |_| return error.HttpTransferEncodingUnsupported;
590 } else if (std.ascii.eqlIgnoreCase(header_name, "content-length")) {
591 const content_length = std.fmt.parseInt(u64, header_value, 10) catch return error.InvalidContentLength;
592
593 if (res.content_length != null and res.content_length != content_length) return error.HttpHeadersInvalid;
594
595 res.content_length = content_length;
596 } else if (std.ascii.eqlIgnoreCase(header_name, "content-encoding")) {
597 if (res.content_encoding != .identity) return error.HttpHeadersInvalid;
598
599 const trimmed = mem.trim(u8, header_value, " ");
600
601 if (http.ContentEncoding.fromString(trimmed)) |ce| {
602 res.content_encoding = ce;
603 } else {
604 return error.HttpContentEncodingUnsupported;
605 }
606 }
607 }
608 return error.HttpHeadersInvalid; // missing empty line
609 }
610
611 test parse {
612 const response_bytes = "HTTP/1.1 200 OK\r\n" ++
613 "LOcation:url\r\n" ++
614 "content-tYpe: text/plain\r\n" ++
615 "content-disposition:attachment; filename=example.txt \r\n" ++
616 "content-Length:10\r\n" ++
617 "TRansfer-encoding:\tdeflate, chunked \r\n" ++
618 "connectioN:\t keep-alive \r\n\r\n";
619
620 const head = try Head.parse(response_bytes);
621
622 try testing.expectEqual(.@"HTTP/1.1", head.version);
623 try testing.expectEqualStrings("OK", head.reason);
624 try testing.expectEqual(.ok, head.status);
625
626 try testing.expectEqualStrings("url", head.location.?);
627 try testing.expectEqualStrings("text/plain", head.content_type.?);
628 try testing.expectEqualStrings("attachment; filename=example.txt", head.content_disposition.?);
629
630 try testing.expectEqual(true, head.keep_alive);
631 try testing.expectEqual(10, head.content_length.?);
632 try testing.expectEqual(.chunked, head.transfer_encoding);
633 try testing.expectEqual(.deflate, head.content_encoding);
634 }
635
636 pub fn iterateHeaders(h: Head) http.HeaderIterator {
637 return .init(h.bytes);
638 }
639
640 test iterateHeaders {
641 const response_bytes = "HTTP/1.1 200 OK\r\n" ++
642 "LOcation:url\r\n" ++
643 "content-tYpe: text/plain\r\n" ++
644 "content-disposition:attachment; filename=example.txt \r\n" ++
645 "content-Length:10\r\n" ++
646 "TRansfer-encoding:\tdeflate, chunked \r\n" ++
647 "connectioN:\t keep-alive \r\n\r\n";
648
649 const head = try Head.parse(response_bytes);
650 var it = head.iterateHeaders();
651 {
652 const header = it.next().?;
653 try testing.expectEqualStrings("LOcation", header.name);
654 try testing.expectEqualStrings("url", header.value);
655 try testing.expect(!it.is_trailer);
656 }
657 {
658 const header = it.next().?;
659 try testing.expectEqualStrings("content-tYpe", header.name);
660 try testing.expectEqualStrings("text/plain", header.value);
661 try testing.expect(!it.is_trailer);
662 }
663 {
664 const header = it.next().?;
665 try testing.expectEqualStrings("content-disposition", header.name);
666 try testing.expectEqualStrings("attachment; filename=example.txt", header.value);
667 try testing.expect(!it.is_trailer);
668 }
669 {
670 const header = it.next().?;
671 try testing.expectEqualStrings("content-Length", header.name);
672 try testing.expectEqualStrings("10", header.value);
673 try testing.expect(!it.is_trailer);
674 }
675 {
676 const header = it.next().?;
677 try testing.expectEqualStrings("TRansfer-encoding", header.name);
678 try testing.expectEqualStrings("deflate, chunked", header.value);
679 try testing.expect(!it.is_trailer);
680 }
681 {
682 const header = it.next().?;
683 try testing.expectEqualStrings("connectioN", header.name);
684 try testing.expectEqualStrings("keep-alive", header.value);
685 try testing.expect(!it.is_trailer);
686 }
687 try testing.expectEqual(null, it.next());
688 }
689
690 inline fn int64(array: *const [8]u8) u64 {
691 return @bitCast(array.*);
692 }
693
694 fn parseInt3(text: *const [3]u8) u10 {
695 const nnn: @Vector(3, u8) = text.*;
696 const zero: @Vector(3, u8) = .{ '0', '0', '0' };
697 const mmm: @Vector(3, u10) = .{ 100, 10, 1 };
698 return @reduce(.Add, (nnn -% zero) *% mmm);
699 }
700
701 test parseInt3 {
702 const expectEqual = testing.expectEqual;
703 try expectEqual(@as(u10, 0), parseInt3("000"));
704 try expectEqual(@as(u10, 418), parseInt3("418"));
705 try expectEqual(@as(u10, 999), parseInt3("999"));
706 }
707
708 /// Help the programmer avoid bugs by calling this when the string
709 /// memory of `Head` becomes invalidated.
710 fn invalidateStrings(h: *Head) void {
711 h.bytes = undefined;
712 h.reason = undefined;
713 if (h.location) |*s| s.* = undefined;
714 if (h.content_type) |*s| s.* = undefined;
715 if (h.content_disposition) |*s| s.* = undefined;
716 }
717 };
718
719 /// If compressed body has been negotiated this will return compressed bytes.
720 ///
721 /// If the returned `Reader` returns `error.ReadFailed` the error is
722 /// available via `bodyErr`.
723 ///
724 /// Asserts that this function is only called once.
725 ///
726 /// See also:
727 /// * `readerDecompressing`
728 pub fn reader(response: *Response, transfer_buffer: []u8) *Reader {
729 response.head.invalidateStrings();
730 const req = response.request;
731 if (!req.method.responseHasBody()) return .ending;
732 const head = &response.head;
733 return req.reader.bodyReader(transfer_buffer, head.transfer_encoding, head.content_length);
734 }
735
736 /// If compressed body has been negotiated this will return decompressed bytes.
737 ///
738 /// If the returned `Reader` returns `error.ReadFailed` the error is
739 /// available via `bodyErr`.
740 ///
741 /// Asserts that this function is only called once.
742 ///
743 /// See also:
744 /// * `reader`
745 pub fn readerDecompressing(
746 response: *Response,
747 transfer_buffer: []u8,
748 decompress: *http.Decompress,
749 decompress_buffer: []u8,
750 ) *Reader {
751 response.head.invalidateStrings();
752 const head = &response.head;
753 return response.request.reader.bodyReaderDecompressing(
754 transfer_buffer,
755 head.transfer_encoding,
756 head.content_length,
757 head.content_encoding,
758 decompress,
759 decompress_buffer,
760 );
761 }
762
763 /// After receiving `error.ReadFailed` from the `Reader` returned by
764 /// `reader` or `readerDecompressing`, this function accesses the
765 /// more specific error code.
766 pub fn bodyErr(response: *const Response) ?http.Reader.BodyError {
767 return response.request.reader.body_err;
768 }
769
770 pub fn iterateTrailers(response: *const Response) http.HeaderIterator {
771 const r = &response.request.reader;
772 assert(r.state == .ready);
773 return .{
774 .bytes = r.trailers,
775 .index = 0,
776 .is_trailer = true,
777 };
778 }
779};
780
781pub const Request = struct {
782 /// This field is provided so that clients can observe redirected URIs.
783 ///
784 /// Its backing memory is externally provided by API users when creating a
785 /// request, and then again provided externally via `redirect_buffer` to
786 /// `receiveHead`.
787 uri: Uri,
788 client: *Client,
789 /// This is null when the connection is released.
790 connection: ?*Connection,
791 reader: http.Reader,
792 keep_alive: bool,
793
794 method: http.Method,
795 version: http.Version = .@"HTTP/1.1",
796 transfer_encoding: TransferEncoding,
797 redirect_behavior: RedirectBehavior,
798 accept_encoding: @TypeOf(default_accept_encoding) = default_accept_encoding,
799
800 /// Whether the request should handle a 100-continue response before sending the request body.
801 handle_continue: bool,
802
803 /// Standard headers that have default, but overridable, behavior.
804 headers: Headers,
805
806 /// Populated in `receiveHead`; used in `deinit` to determine whether to
807 /// discard the body to reuse the connection.
808 response_content_length: ?u64 = null,
809 /// Populated in `receiveHead`; used in `deinit` to determine whether to
810 /// discard the body to reuse the connection.
811 response_transfer_encoding: http.TransferEncoding = .none,
812
813 /// These headers are kept including when following a redirect to a
814 /// different domain.
815 /// Externally-owned; must outlive the Request.
816 extra_headers: []const http.Header,
817
818 /// These headers are stripped when following a redirect to a different
819 /// domain.
820 /// Externally-owned; must outlive the Request.
821 privileged_headers: []const http.Header,
822
823 pub const default_accept_encoding: [@typeInfo(http.ContentEncoding).@"enum".fields.len]bool = b: {
824 var result: [@typeInfo(http.ContentEncoding).@"enum".fields.len]bool = @splat(false);
825 result[@intFromEnum(http.ContentEncoding.gzip)] = true;
826 result[@intFromEnum(http.ContentEncoding.deflate)] = true;
827 result[@intFromEnum(http.ContentEncoding.identity)] = true;
828 break :b result;
829 };
830
831 pub const TransferEncoding = union(enum) {
832 content_length: u64,
833 chunked: void,
834 none: void,
835 };
836
837 pub const Headers = struct {
838 host: Value = .default,
839 authorization: Value = .default,
840 user_agent: Value = .default,
841 connection: Value = .default,
842 accept_encoding: Value = .default,
843 content_type: Value = .default,
844
845 pub const Value = union(enum) {
846 default,
847 omit,
848 override: []const u8,
849 };
850 };
851
852 /// Any value other than `not_allowed` or `unhandled` means that integer represents
853 /// how many remaining redirects are allowed.
854 pub const RedirectBehavior = enum(u16) {
855 /// The next redirect will cause an error.
856 not_allowed = 0,
857 /// Redirects are passed to the client to analyze the redirect response
858 /// directly.
859 unhandled = std.math.maxInt(u16),
860 _,
861
862 pub fn init(n: u16) RedirectBehavior {
863 assert(n != std.math.maxInt(u16));
864 return @enumFromInt(n);
865 }
866
867 pub fn subtractOne(rb: *RedirectBehavior) void {
868 switch (rb.*) {
869 .not_allowed => unreachable,
870 .unhandled => unreachable,
871 _ => rb.* = @enumFromInt(@intFromEnum(rb.*) - 1),
872 }
873 }
874
875 pub fn remaining(rb: RedirectBehavior) u16 {
876 assert(rb != .unhandled);
877 return @intFromEnum(rb);
878 }
879 };
880
881 /// Returns the request's `Connection` back to the pool of the `Client`.
882 pub fn deinit(r: *Request) void {
883 const io = r.client.io;
884 if (r.connection) |connection| {
885 connection.closing = connection.closing or switch (r.reader.state) {
886 .ready => false,
887 .received_head => c: {
888 if (r.method.requestHasBody()) break :c true;
889 if (!r.method.responseHasBody()) break :c false;
890 const reader = r.reader.bodyReader(&.{}, r.response_transfer_encoding, r.response_content_length);
891 _ = reader.discardRemaining() catch |err| switch (err) {
892 error.ReadFailed => break :c true,
893 };
894 break :c r.reader.state != .ready;
895 },
896 else => true,
897 };
898 r.client.connection_pool.release(connection, io);
899 }
900 r.* = undefined;
901 }
902
903 /// Sends and flushes a complete request as only HTTP head, no body.
904 pub fn sendBodiless(r: *Request) Writer.Error!void {
905 try sendBodilessUnflushed(r);
906 try r.connection.?.flush();
907 }
908
909 /// Sends but does not flush a complete request as only HTTP head, no body.
910 pub fn sendBodilessUnflushed(r: *Request) Writer.Error!void {
911 assert(r.transfer_encoding == .none);
912 assert(!r.method.requestHasBody());
913 try sendHead(r);
914 }
915
916 /// Transfers the HTTP head over the connection and flushes.
917 ///
918 /// See also:
919 /// * `sendBodyUnflushed`
920 pub fn sendBody(r: *Request, buffer: []u8) Writer.Error!http.BodyWriter {
921 const result = try sendBodyUnflushed(r, buffer);
922 try r.connection.?.flush();
923 return result;
924 }
925
926 /// Transfers the HTTP head and body over the connection and flushes.
927 pub fn sendBodyComplete(r: *Request, body: []u8) Writer.Error!void {
928 r.transfer_encoding = .{ .content_length = body.len };
929 var bw = try sendBodyUnflushed(r, body);
930 bw.writer.end = body.len;
931 try bw.end();
932 try r.connection.?.flush();
933 }
934
935 /// Transfers the HTTP head over the connection, which is not flushed until
936 /// `BodyWriter.flush` or `BodyWriter.end` is called.
937 ///
938 /// See also:
939 /// * `sendBody`
940 pub fn sendBodyUnflushed(r: *Request, buffer: []u8) Writer.Error!http.BodyWriter {
941 assert(r.method.requestHasBody());
942 try sendHead(r);
943 const http_protocol_output = r.connection.?.writer();
944 return switch (r.transfer_encoding) {
945 .chunked => .{
946 .http_protocol_output = http_protocol_output,
947 .state = .init_chunked,
948 .writer = .{
949 .buffer = buffer,
950 .vtable = &.{
951 .drain = http.BodyWriter.chunkedDrain,
952 .sendFile = http.BodyWriter.chunkedSendFile,
953 },
954 },
955 },
956 .content_length => |len| .{
957 .http_protocol_output = http_protocol_output,
958 .state = .{ .content_length = len },
959 .writer = .{
960 .buffer = buffer,
961 .vtable = &.{
962 .drain = http.BodyWriter.contentLengthDrain,
963 .sendFile = http.BodyWriter.contentLengthSendFile,
964 },
965 },
966 },
967 .none => .{
968 .http_protocol_output = http_protocol_output,
969 .state = .none,
970 .writer = .{
971 .buffer = buffer,
972 .vtable = &.{
973 .drain = http.BodyWriter.noneDrain,
974 .sendFile = http.BodyWriter.noneSendFile,
975 },
976 },
977 },
978 };
979 }
980
981 /// Sends HTTP headers without flushing.
982 fn sendHead(r: *Request) Writer.Error!void {
983 const uri = r.uri;
984 const connection = r.connection.?;
985 const w = connection.writer();
986
987 try w.writeAll(@tagName(r.method));
988 try w.writeByte(' ');
989
990 if (r.method == .CONNECT) {
991 try uri.writeToStream(w, .{ .authority = true });
992 } else {
993 try uri.writeToStream(w, .{
994 .scheme = connection.proxied,
995 .authentication = connection.proxied,
996 .authority = connection.proxied,
997 .path = true,
998 .query = true,
999 });
1000 }
1001 try w.writeByte(' ');
1002 try w.writeAll(@tagName(r.version));
1003 try w.writeAll("\r\n");
1004
1005 if (try emitOverridableHeader("host: ", r.headers.host, w)) {
1006 try w.writeAll("host: ");
1007 try uri.writeToStream(w, .{ .authority = true });
1008 try w.writeAll("\r\n");
1009 }
1010
1011 if (try emitOverridableHeader("authorization: ", r.headers.authorization, w)) {
1012 if (uri.user != null or uri.password != null) {
1013 try w.writeAll("authorization: ");
1014 try basic_authorization.write(uri, w);
1015 try w.writeAll("\r\n");
1016 }
1017 }
1018
1019 if (try emitOverridableHeader("user-agent: ", r.headers.user_agent, w)) {
1020 try w.writeAll("user-agent: zig/");
1021 try w.writeAll(builtin.zig_version_string);
1022 try w.writeAll(" (std.http)\r\n");
1023 }
1024
1025 if (try emitOverridableHeader("connection: ", r.headers.connection, w)) {
1026 if (r.keep_alive) {
1027 try w.writeAll("connection: keep-alive\r\n");
1028 } else {
1029 try w.writeAll("connection: close\r\n");
1030 }
1031 }
1032
1033 if (try emitOverridableHeader("accept-encoding: ", r.headers.accept_encoding, w)) {
1034 try w.writeAll("accept-encoding: ");
1035 for (r.accept_encoding, 0..) |enabled, i| {
1036 if (!enabled) continue;
1037 const tag: http.ContentEncoding = @enumFromInt(i);
1038 if (tag == .identity) continue;
1039 const tag_name = @tagName(tag);
1040 try w.ensureUnusedCapacity(tag_name.len + 2);
1041 try w.writeAll(tag_name);
1042 try w.writeAll(", ");
1043 }
1044 w.undo(2);
1045 try w.writeAll("\r\n");
1046 }
1047
1048 switch (r.transfer_encoding) {
1049 .chunked => try w.writeAll("transfer-encoding: chunked\r\n"),
1050 .content_length => |len| try w.print("content-length: {d}\r\n", .{len}),
1051 .none => {},
1052 }
1053
1054 if (try emitOverridableHeader("content-type: ", r.headers.content_type, w)) {
1055 // The default is to omit content-type if not provided because
1056 // "application/octet-stream" is redundant.
1057 }
1058
1059 for (r.extra_headers) |header| {
1060 assert(header.name.len != 0);
1061
1062 try w.writeAll(header.name);
1063 try w.writeAll(": ");
1064 try w.writeAll(header.value);
1065 try w.writeAll("\r\n");
1066 }
1067
1068 if (connection.proxied) proxy: {
1069 const proxy = switch (connection.protocol) {
1070 .plain => r.client.http_proxy,
1071 .tls => r.client.https_proxy,
1072 } orelse break :proxy;
1073
1074 const authorization = proxy.authorization orelse break :proxy;
1075 try w.writeAll("proxy-authorization: ");
1076 try w.writeAll(authorization);
1077 try w.writeAll("\r\n");
1078 }
1079
1080 try w.writeAll("\r\n");
1081 }
1082
1083 pub const ReceiveHeadError = http.Reader.HeadError || ConnectError || error{
1084 /// Server sent headers that did not conform to the HTTP protocol.
1085 ///
1086 /// To find out more detailed diagnostics, `http.Reader.head_buffer` can be
1087 /// passed directly to `Request.Head.parse`.
1088 HttpHeadersInvalid,
1089 TooManyHttpRedirects,
1090 /// This can be avoided by calling `receiveHead` before sending the
1091 /// request body.
1092 RedirectRequiresResend,
1093 HttpRedirectLocationMissing,
1094 HttpRedirectLocationOversize,
1095 HttpRedirectLocationInvalid,
1096 HttpContentEncodingUnsupported,
1097 HttpChunkInvalid,
1098 HttpChunkTruncated,
1099 HttpHeadersOversize,
1100 UnsupportedUriScheme,
1101
1102 /// Sending the request failed. Error code can be found on the
1103 /// `Connection` object.
1104 WriteFailed,
1105 };
1106
1107 /// If handling redirects and the request has no payload, then this
1108 /// function will automatically follow redirects.
1109 ///
1110 /// If a request payload is present, then this function will error with
1111 /// `error.RedirectRequiresResend`.
1112 ///
1113 /// This function takes an auxiliary buffer to store the arbitrarily large
1114 /// URI which may need to be merged with the previous URI, and that data
1115 /// needs to survive across different connections, which is where the input
1116 /// buffer lives.
1117 ///
1118 /// `redirect_buffer` must outlive accesses to `Request.uri`. If this
1119 /// buffer capacity would be exceeded, `error.HttpRedirectLocationOversize`
1120 /// is returned instead. This buffer may be empty if no redirects are to be
1121 /// handled.
1122 ///
1123 /// If this fails with `error.ReadFailed` then the `Connection.getReadError`
1124 /// method of `r.connection` can be used to get more detailed information.
1125 pub fn receiveHead(r: *Request, redirect_buffer: []u8) ReceiveHeadError!Response {
1126 var aux_buf = redirect_buffer;
1127 while (true) {
1128 const head_buffer = try r.reader.receiveHead();
1129 const response: Response = .{
1130 .request = r,
1131 .head = Response.Head.parse(head_buffer) catch return error.HttpHeadersInvalid,
1132 };
1133 const head = &response.head;
1134
1135 if (head.status == .@"continue") {
1136 if (r.handle_continue) continue;
1137 r.response_transfer_encoding = head.transfer_encoding;
1138 r.response_content_length = head.content_length;
1139 return response; // we're not handling the 100-continue
1140 }
1141
1142 // This while loop is for handling redirects, which means the request's
1143 // connection may be different than the previous iteration. However, it
1144 // is still guaranteed to be non-null with each iteration of this loop.
1145 const connection = r.connection.?;
1146
1147 if (r.method == .CONNECT and head.status.class() == .success) {
1148 // This connection is no longer doing HTTP.
1149 connection.closing = false;
1150 r.response_transfer_encoding = head.transfer_encoding;
1151 r.response_content_length = head.content_length;
1152 return response;
1153 }
1154
1155 connection.closing = !head.keep_alive or !r.keep_alive;
1156
1157 // Any response to a HEAD request and any response with a 1xx
1158 // (Informational), 204 (No Content), or 304 (Not Modified) status
1159 // code is always terminated by the first empty line after the
1160 // header fields, regardless of the header fields present in the
1161 // message.
1162 if (r.method == .HEAD or head.status.class() == .informational or
1163 head.status == .no_content or head.status == .not_modified)
1164 {
1165 r.response_transfer_encoding = head.transfer_encoding;
1166 r.response_content_length = head.content_length;
1167 return response;
1168 }
1169
1170 if (head.status.class() == .redirect and r.redirect_behavior != .unhandled) {
1171 if (r.redirect_behavior == .not_allowed) {
1172 // Connection can still be reused by skipping the body.
1173 const reader = r.reader.bodyReader(&.{}, head.transfer_encoding, head.content_length);
1174 _ = reader.discardRemaining() catch |err| switch (err) {
1175 error.ReadFailed => connection.closing = true,
1176 };
1177 return error.TooManyHttpRedirects;
1178 }
1179 try r.redirect(head, &aux_buf);
1180 try r.sendBodiless();
1181 continue;
1182 }
1183
1184 if (!r.accept_encoding[@intFromEnum(head.content_encoding)])
1185 return error.HttpContentEncodingUnsupported;
1186
1187 r.response_transfer_encoding = head.transfer_encoding;
1188 r.response_content_length = head.content_length;
1189 return response;
1190 }
1191 }
1192
1193 /// This function takes an auxiliary buffer to store the arbitrarily large
1194 /// URI which may need to be merged with the previous URI, and that data
1195 /// needs to survive across different connections, which is where the input
1196 /// buffer lives.
1197 ///
1198 /// `aux_buf` must outlive accesses to `Request.uri`.
1199 fn redirect(r: *Request, head: *const Response.Head, aux_buf: *[]u8) !void {
1200 const io = r.client.io;
1201 const new_location = head.location orelse return error.HttpRedirectLocationMissing;
1202 if (new_location.len > aux_buf.*.len) return error.HttpRedirectLocationOversize;
1203 const location = aux_buf.*[0..new_location.len];
1204 @memcpy(location, new_location);
1205 {
1206 // Skip the body of the redirect response to leave the connection in
1207 // the correct state. This causes `new_location` to be invalidated.
1208 const reader = r.reader.bodyReader(&.{}, head.transfer_encoding, head.content_length);
1209 _ = reader.discardRemaining() catch |err| switch (err) {
1210 error.ReadFailed => return r.reader.body_err.?,
1211 };
1212 }
1213 const new_uri = r.uri.resolveInPlace(location.len, aux_buf) catch |err| switch (err) {
1214 error.UnexpectedCharacter => return error.HttpRedirectLocationInvalid,
1215 error.InvalidFormat => return error.HttpRedirectLocationInvalid,
1216 error.InvalidPort => return error.HttpRedirectLocationInvalid,
1217 error.InvalidHostName => return error.HttpRedirectLocationInvalid,
1218 error.NoSpaceLeft => return error.HttpRedirectLocationOversize,
1219 };
1220
1221 const protocol = Protocol.fromUri(new_uri) orelse return error.UnsupportedUriScheme;
1222 const old_connection = r.connection.?;
1223 const old_host = old_connection.host();
1224 var new_host_name_buffer: [HostName.max_len]u8 = undefined;
1225 const new_host = try new_uri.getHost(&new_host_name_buffer);
1226 const keep_privileged_headers =
1227 std.ascii.eqlIgnoreCase(r.uri.scheme, new_uri.scheme) and
1228 old_host.sameParentDomain(new_host);
1229
1230 r.client.connection_pool.release(old_connection, io);
1231 r.connection = null;
1232
1233 if (!keep_privileged_headers) {
1234 // When redirecting to a different domain, strip privileged headers.
1235 r.privileged_headers = &.{};
1236 }
1237
1238 if (switch (head.status) {
1239 .see_other => true,
1240 .moved_permanently, .found => r.method == .POST,
1241 else => false,
1242 }) {
1243 // A redirect to a GET must change the method and remove the body.
1244 r.method = .GET;
1245 r.transfer_encoding = .none;
1246 r.headers.content_type = .omit;
1247 }
1248
1249 if (r.transfer_encoding != .none) {
1250 // The request body has already been sent. The request is
1251 // still in a valid state, but the redirect must be handled
1252 // manually.
1253 return error.RedirectRequiresResend;
1254 }
1255
1256 const new_connection = try r.client.connect(new_host, uriPort(new_uri, protocol), protocol);
1257 r.uri = new_uri;
1258 r.connection = new_connection;
1259 r.reader = .{
1260 .in = new_connection.reader(),
1261 .state = .ready,
1262 // Populated when `http.Reader.bodyReader` is called.
1263 .interface = undefined,
1264 .max_head_len = r.client.read_buffer_size,
1265 };
1266 r.redirect_behavior.subtractOne();
1267 }
1268
1269 /// Returns true if the default behavior is required, otherwise handles
1270 /// writing (or not writing) the header.
1271 fn emitOverridableHeader(prefix: []const u8, v: Headers.Value, bw: *Writer) Writer.Error!bool {
1272 switch (v) {
1273 .default => return true,
1274 .omit => return false,
1275 .override => |x| {
1276 var vecs: [3][]const u8 = .{ prefix, x, "\r\n" };
1277 try bw.writeVecAll(&vecs);
1278 return false;
1279 },
1280 }
1281 }
1282};
1283
1284pub const Proxy = struct {
1285 protocol: Protocol,
1286 host: HostName,
1287 authorization: ?[]const u8,
1288 port: u16,
1289 supports_connect: bool,
1290};
1291
1292/// Release all associated resources with the client.
1293///
1294/// All pending requests must be de-initialized and all active connections released
1295/// before calling this function.
1296pub fn deinit(client: *Client) void {
1297 const io = client.io;
1298 assert(client.connection_pool.used.first == null); // There are still active requests.
1299
1300 client.connection_pool.deinit(io);
1301 if (!disable_tls) client.ca_bundle.deinit(client.allocator);
1302
1303 client.* = undefined;
1304}
1305
1306/// Populates `http_proxy` and `https_proxy` via standard proxy environment variables.
1307/// Asserts the client has no active connections.
1308/// Uses `arena` for a few small allocations that must outlive the client, or
1309/// at least until those fields are set to different values.
1310pub fn initDefaultProxies(client: *Client, arena: Allocator) !void {
1311 // Prevent any new connections from being created.
1312 client.connection_pool.mutex.lock();
1313 defer client.connection_pool.mutex.unlock();
1314
1315 assert(client.connection_pool.used.first == null); // There are active requests.
1316
1317 if (client.http_proxy == null) {
1318 client.http_proxy = try createProxyFromEnvVar(arena, &.{
1319 "http_proxy", "HTTP_PROXY", "all_proxy", "ALL_PROXY",
1320 });
1321 }
1322
1323 if (client.https_proxy == null) {
1324 client.https_proxy = try createProxyFromEnvVar(arena, &.{
1325 "https_proxy", "HTTPS_PROXY", "all_proxy", "ALL_PROXY",
1326 });
1327 }
1328}
1329
1330fn createProxyFromEnvVar(arena: Allocator, env_var_names: []const []const u8) !?*Proxy {
1331 const content = for (env_var_names) |name| {
1332 const content = std.process.getEnvVarOwned(arena, name) catch |err| switch (err) {
1333 error.EnvironmentVariableNotFound => continue,
1334 else => |e| return e,
1335 };
1336
1337 if (content.len == 0) continue;
1338
1339 break content;
1340 } else return null;
1341
1342 const uri = Uri.parse(content) catch try Uri.parseAfterScheme("http", content);
1343 const protocol = Protocol.fromUri(uri) orelse return null;
1344 const raw_host = try uri.getHostAlloc(arena);
1345
1346 const authorization: ?[]const u8 = if (uri.user != null or uri.password != null) a: {
1347 const authorization = try arena.alloc(u8, basic_authorization.valueLengthFromUri(uri));
1348 assert(basic_authorization.value(uri, authorization).len == authorization.len);
1349 break :a authorization;
1350 } else null;
1351
1352 const proxy = try arena.create(Proxy);
1353 proxy.* = .{
1354 .protocol = protocol,
1355 .host = raw_host,
1356 .authorization = authorization,
1357 .port = uriPort(uri, protocol),
1358 .supports_connect = true,
1359 };
1360 return proxy;
1361}
1362
1363pub const basic_authorization = struct {
1364 pub const max_user_len = 255;
1365 pub const max_password_len = 255;
1366 pub const max_value_len = valueLength(max_user_len, max_password_len);
1367
1368 pub fn valueLength(user_len: usize, password_len: usize) usize {
1369 return "Basic ".len + std.base64.standard.Encoder.calcSize(user_len + 1 + password_len);
1370 }
1371
1372 pub fn valueLengthFromUri(uri: Uri) usize {
1373 const user: Uri.Component = uri.user orelse .empty;
1374 const password: Uri.Component = uri.password orelse .empty;
1375
1376 var dw: Writer.Discarding = .init(&.{});
1377 user.formatUser(&dw.writer) catch unreachable; // discarding
1378 const user_len = dw.count + dw.writer.end;
1379
1380 dw.count = 0;
1381 dw.writer.end = 0;
1382 password.formatPassword(&dw.writer) catch unreachable; // discarding
1383 const password_len = dw.count + dw.writer.end;
1384
1385 return valueLength(@intCast(user_len), @intCast(password_len));
1386 }
1387
1388 pub fn value(uri: Uri, out: []u8) []u8 {
1389 var bw: Writer = .fixed(out);
1390 write(uri, &bw) catch unreachable;
1391 return bw.buffered();
1392 }
1393
1394 pub fn write(uri: Uri, out: *Writer) Writer.Error!void {
1395 var buf: [max_user_len + 1 + max_password_len]u8 = undefined;
1396 var w: Writer = .fixed(&buf);
1397 const user: Uri.Component = uri.user orelse .empty;
1398 const password: Uri.Component = uri.password orelse .empty;
1399 user.formatUser(&w) catch unreachable;
1400 w.writeByte(':') catch unreachable;
1401 password.formatPassword(&w) catch unreachable;
1402 try out.print("Basic {b64}", .{w.buffered()});
1403 }
1404};
1405
1406pub const ConnectTcpError = error{
1407 TlsInitializationFailed,
1408} || Allocator.Error || HostName.ConnectError;
1409
1410/// Reuses a `Connection` if one matching `host` and `port` is already open.
1411///
1412/// Threadsafe.
1413pub fn connectTcp(
1414 client: *Client,
1415 host: HostName,
1416 port: u16,
1417 protocol: Protocol,
1418) ConnectTcpError!*Connection {
1419 return connectTcpOptions(client, .{ .host = host, .port = port, .protocol = protocol });
1420}
1421
1422pub const ConnectTcpOptions = struct {
1423 host: HostName,
1424 port: u16,
1425 protocol: Protocol,
1426
1427 proxied_host: ?HostName = null,
1428 proxied_port: ?u16 = null,
1429 timeout: Io.Timeout = .none,
1430};
1431
1432pub fn connectTcpOptions(client: *Client, options: ConnectTcpOptions) ConnectTcpError!*Connection {
1433 const io = client.io;
1434 const host = options.host;
1435 const port = options.port;
1436 const protocol = options.protocol;
1437
1438 const proxied_host = options.proxied_host orelse host;
1439 const proxied_port = options.proxied_port orelse port;
1440
1441 if (client.connection_pool.findConnection(.{
1442 .host = proxied_host,
1443 .port = proxied_port,
1444 .protocol = protocol,
1445 })) |conn| return conn;
1446
1447 var stream = try host.connect(io, port, .{ .mode = .stream });
1448 errdefer stream.close(io);
1449
1450 switch (protocol) {
1451 .tls => {
1452 if (disable_tls) return error.TlsInitializationFailed;
1453 const tc = Connection.Tls.create(client, proxied_host, proxied_port, stream) catch |err| switch (err) {
1454 error.OutOfMemory => |e| return e,
1455 error.Unexpected => |e| return e,
1456 error.Canceled => |e| return e,
1457 else => return error.TlsInitializationFailed,
1458 };
1459 client.connection_pool.addUsed(&tc.connection);
1460 return &tc.connection;
1461 },
1462 .plain => {
1463 const pc = try Connection.Plain.create(client, proxied_host, proxied_port, stream);
1464 client.connection_pool.addUsed(&pc.connection);
1465 return &pc.connection;
1466 },
1467 }
1468}
1469
1470pub const ConnectUnixError = Allocator.Error || std.posix.SocketError || error{NameTooLong} || std.posix.ConnectError;
1471
1472/// Connect to `path` as a unix domain socket. This will reuse a connection if one is already open.
1473///
1474/// This function is threadsafe.
1475pub fn connectUnix(client: *Client, path: []const u8) ConnectUnixError!*Connection {
1476 if (client.connection_pool.findConnection(.{
1477 .host = path,
1478 .port = 0,
1479 .protocol = .plain,
1480 })) |node|
1481 return node;
1482
1483 const conn = try client.allocator.create(ConnectionPool.Node);
1484 errdefer client.allocator.destroy(conn);
1485 conn.* = .{ .data = undefined };
1486
1487 const stream = try Io.net.connectUnixSocket(path);
1488 errdefer stream.close();
1489
1490 conn.data = .{
1491 .stream = stream,
1492 .tls_client = undefined,
1493 .protocol = .plain,
1494
1495 .host = try client.allocator.dupe(u8, path),
1496 .port = 0,
1497 };
1498 errdefer client.allocator.free(conn.data.host);
1499
1500 client.connection_pool.addUsed(conn);
1501
1502 return &conn.data;
1503}
1504
1505/// Connect to `proxied_host:proxied_port` using the specified proxy with HTTP
1506/// CONNECT. This will reuse a connection if one is already open.
1507///
1508/// This function is threadsafe.
1509pub fn connectProxied(
1510 client: *Client,
1511 proxy: *Proxy,
1512 proxied_host: HostName,
1513 proxied_port: u16,
1514) !*Connection {
1515 const io = client.io;
1516 if (!proxy.supports_connect) return error.TunnelNotSupported;
1517
1518 if (client.connection_pool.findConnection(.{
1519 .host = proxied_host,
1520 .port = proxied_port,
1521 .protocol = proxy.protocol,
1522 })) |node| return node;
1523
1524 var maybe_valid = false;
1525 (tunnel: {
1526 const connection = try client.connectTcpOptions(.{
1527 .host = proxy.host,
1528 .port = proxy.port,
1529 .protocol = proxy.protocol,
1530 .proxied_host = proxied_host,
1531 .proxied_port = proxied_port,
1532 });
1533 errdefer {
1534 connection.closing = true;
1535 client.connection_pool.release(connection, io);
1536 }
1537
1538 var req = client.request(.CONNECT, .{
1539 .scheme = "http",
1540 .host = .{ .raw = proxied_host.bytes },
1541 .port = proxied_port,
1542 }, .{
1543 .redirect_behavior = .unhandled,
1544 .connection = connection,
1545 }) catch |err| {
1546 break :tunnel err;
1547 };
1548 defer req.deinit();
1549
1550 req.sendBodiless() catch |err| break :tunnel err;
1551 const response = req.receiveHead(&.{}) catch |err| break :tunnel err;
1552
1553 if (response.head.status.class() == .server_error) {
1554 maybe_valid = true;
1555 break :tunnel error.ServerError;
1556 }
1557
1558 if (response.head.status != .ok) break :tunnel error.ConnectionRefused;
1559
1560 // this connection is now a tunnel, so we can't use it for anything
1561 // else, it will only be released when the client is de-initialized.
1562 req.connection = null;
1563
1564 connection.closing = false;
1565
1566 return connection;
1567 }) catch {
1568 // something went wrong with the tunnel
1569 proxy.supports_connect = maybe_valid;
1570 return error.TunnelNotSupported;
1571 };
1572}
1573
1574pub const ConnectError = ConnectTcpError || RequestError;
1575
1576/// Connect to `host:port` using the specified protocol. This will reuse a
1577/// connection if one is already open.
1578///
1579/// If a proxy is configured for the client, then the proxy will be used to
1580/// connect to the host.
1581///
1582/// This function is threadsafe.
1583pub fn connect(
1584 client: *Client,
1585 host: HostName,
1586 port: u16,
1587 protocol: Protocol,
1588) ConnectError!*Connection {
1589 const proxy = switch (protocol) {
1590 .plain => client.http_proxy,
1591 .tls => client.https_proxy,
1592 } orelse return client.connectTcp(host, port, protocol);
1593
1594 // Prevent proxying through itself.
1595 if (proxy.host.eql(host) and proxy.port == port and proxy.protocol == protocol) {
1596 return client.connectTcp(host, port, protocol);
1597 }
1598
1599 if (proxy.supports_connect) tunnel: {
1600 return connectProxied(client, proxy, host, port) catch |err| switch (err) {
1601 error.TunnelNotSupported => break :tunnel,
1602 else => |e| return e,
1603 };
1604 }
1605
1606 // fall back to using the proxy as a normal http proxy
1607 const connection = try client.connectTcp(proxy.host, proxy.port, proxy.protocol);
1608 connection.proxied = true;
1609 return connection;
1610}
1611
1612pub const RequestError = ConnectTcpError || error{
1613 UnsupportedUriScheme,
1614 UriMissingHost,
1615 CertificateBundleLoadFailure,
1616};
1617
1618pub const RequestOptions = struct {
1619 version: http.Version = .@"HTTP/1.1",
1620
1621 /// Automatically ignore 100 Continue responses. This assumes you don't
1622 /// care, and will have sent the body before you wait for the response.
1623 ///
1624 /// If this is not the case AND you know the server will send a 100
1625 /// Continue, set this to false and wait for a response before sending the
1626 /// body. If you wait AND the server does not send a 100 Continue before
1627 /// you finish the request, then the request *will* deadlock.
1628 handle_continue: bool = true,
1629
1630 /// If false, close the connection after the one request. If true,
1631 /// participate in the client connection pool.
1632 keep_alive: bool = true,
1633
1634 /// This field specifies whether to automatically follow redirects, and if
1635 /// so, how many redirects to follow before returning an error.
1636 ///
1637 /// This will only follow redirects for repeatable requests (ie. with no
1638 /// payload or the server has acknowledged the payload).
1639 redirect_behavior: Request.RedirectBehavior = @enumFromInt(3),
1640
1641 /// Must be an already acquired connection.
1642 connection: ?*Connection = null,
1643
1644 /// Standard headers that have default, but overridable, behavior.
1645 headers: Request.Headers = .{},
1646 /// These headers are kept including when following a redirect to a
1647 /// different domain.
1648 /// Externally-owned; must outlive the Request.
1649 extra_headers: []const http.Header = &.{},
1650 /// These headers are stripped when following a redirect to a different
1651 /// domain.
1652 /// Externally-owned; must outlive the Request.
1653 privileged_headers: []const http.Header = &.{},
1654};
1655
1656fn uriPort(uri: Uri, protocol: Protocol) u16 {
1657 return uri.port orelse protocol.port();
1658}
1659
1660/// Open a connection to the host specified by `uri` and prepare to send a HTTP request.
1661///
1662/// The caller is responsible for calling `deinit()` on the `Request`.
1663/// This function is threadsafe.
1664///
1665/// Asserts that "\r\n" does not occur in any header name or value.
1666pub fn request(
1667 client: *Client,
1668 method: http.Method,
1669 uri: Uri,
1670 options: RequestOptions,
1671) RequestError!Request {
1672 const io = client.io;
1673
1674 if (std.debug.runtime_safety) {
1675 for (options.extra_headers) |header| {
1676 assert(header.name.len != 0);
1677 assert(std.mem.indexOfScalar(u8, header.name, ':') == null);
1678 assert(std.mem.indexOfPosLinear(u8, header.name, 0, "\r\n") == null);
1679 assert(std.mem.indexOfPosLinear(u8, header.value, 0, "\r\n") == null);
1680 }
1681 for (options.privileged_headers) |header| {
1682 assert(header.name.len != 0);
1683 assert(std.mem.indexOfPosLinear(u8, header.name, 0, "\r\n") == null);
1684 assert(std.mem.indexOfPosLinear(u8, header.value, 0, "\r\n") == null);
1685 }
1686 }
1687
1688 const protocol = Protocol.fromUri(uri) orelse return error.UnsupportedUriScheme;
1689
1690 if (protocol == .tls) {
1691 if (disable_tls) unreachable;
1692 {
1693 client.ca_bundle_mutex.lock();
1694 defer client.ca_bundle_mutex.unlock();
1695
1696 if (client.now == null) {
1697 const now = try Io.Clock.real.now(io);
1698 client.now = now;
1699 client.ca_bundle.rescan(client.allocator, io, now) catch
1700 return error.CertificateBundleLoadFailure;
1701 }
1702 }
1703 }
1704
1705 const connection = options.connection orelse c: {
1706 var host_name_buffer: [HostName.max_len]u8 = undefined;
1707 const host_name = try uri.getHost(&host_name_buffer);
1708 break :c try client.connect(host_name, uriPort(uri, protocol), protocol);
1709 };
1710
1711 return .{
1712 .uri = uri,
1713 .client = client,
1714 .connection = connection,
1715 .reader = .{
1716 .in = connection.reader(),
1717 .state = .ready,
1718 // Populated when `http.Reader.bodyReader` is called.
1719 .interface = undefined,
1720 .max_head_len = client.read_buffer_size,
1721 },
1722 .keep_alive = options.keep_alive,
1723 .method = method,
1724 .version = options.version,
1725 .transfer_encoding = .none,
1726 .redirect_behavior = options.redirect_behavior,
1727 .handle_continue = options.handle_continue,
1728 .headers = options.headers,
1729 .extra_headers = options.extra_headers,
1730 .privileged_headers = options.privileged_headers,
1731 };
1732}
1733
1734pub const FetchOptions = struct {
1735 /// `null` means it will be heap-allocated.
1736 redirect_buffer: ?[]u8 = null,
1737 /// `null` means it will be heap-allocated.
1738 decompress_buffer: ?[]u8 = null,
1739 redirect_behavior: ?Request.RedirectBehavior = null,
1740 /// If the server sends a body, it will be written here.
1741 response_writer: ?*Writer = null,
1742
1743 location: Location,
1744 method: ?http.Method = null,
1745 payload: ?[]const u8 = null,
1746 raw_uri: bool = false,
1747 keep_alive: bool = true,
1748
1749 /// Standard headers that have default, but overridable, behavior.
1750 headers: Request.Headers = .{},
1751 /// These headers are kept including when following a redirect to a
1752 /// different domain.
1753 /// Externally-owned; must outlive the Request.
1754 extra_headers: []const http.Header = &.{},
1755 /// These headers are stripped when following a redirect to a different
1756 /// domain.
1757 /// Externally-owned; must outlive the Request.
1758 privileged_headers: []const http.Header = &.{},
1759
1760 pub const Location = union(enum) {
1761 url: []const u8,
1762 uri: Uri,
1763 };
1764};
1765
1766pub const FetchResult = struct {
1767 status: http.Status,
1768};
1769
1770pub const FetchError = Uri.ParseError || RequestError || Request.ReceiveHeadError || error{
1771 StreamTooLong,
1772 /// TODO provide optional diagnostics when this occurs or break into more error codes
1773 WriteFailed,
1774 UnsupportedCompressionMethod,
1775};
1776
1777/// Perform a one-shot HTTP request with the provided options.
1778///
1779/// This function is threadsafe.
1780pub fn fetch(client: *Client, options: FetchOptions) FetchError!FetchResult {
1781 const uri = switch (options.location) {
1782 .url => |u| try Uri.parse(u),
1783 .uri => |u| u,
1784 };
1785 const method: http.Method = options.method orelse
1786 if (options.payload != null) .POST else .GET;
1787
1788 const redirect_behavior: Request.RedirectBehavior = options.redirect_behavior orelse
1789 if (options.payload == null) @enumFromInt(3) else .unhandled;
1790
1791 var req = try request(client, method, uri, .{
1792 .redirect_behavior = redirect_behavior,
1793 .headers = options.headers,
1794 .extra_headers = options.extra_headers,
1795 .privileged_headers = options.privileged_headers,
1796 .keep_alive = options.keep_alive,
1797 });
1798 defer req.deinit();
1799
1800 if (options.payload) |payload| {
1801 req.transfer_encoding = .{ .content_length = payload.len };
1802 var body = try req.sendBodyUnflushed(&.{});
1803 try body.writer.writeAll(payload);
1804 try body.end();
1805 try req.connection.?.flush();
1806 } else {
1807 try req.sendBodiless();
1808 }
1809
1810 const redirect_buffer: []u8 = if (redirect_behavior == .unhandled) &.{} else options.redirect_buffer orelse
1811 try client.allocator.alloc(u8, 8 * 1024);
1812 defer if (options.redirect_buffer == null) client.allocator.free(redirect_buffer);
1813
1814 var response = try req.receiveHead(redirect_buffer);
1815
1816 const response_writer = options.response_writer orelse {
1817 const reader = response.reader(&.{});
1818 _ = reader.discardRemaining() catch |err| switch (err) {
1819 error.ReadFailed => return response.bodyErr().?,
1820 };
1821 return .{ .status = response.head.status };
1822 };
1823
1824 const decompress_buffer: []u8 = switch (response.head.content_encoding) {
1825 .identity => &.{},
1826 .zstd => options.decompress_buffer orelse try client.allocator.alloc(u8, std.compress.zstd.default_window_len),
1827 .deflate, .gzip => options.decompress_buffer orelse try client.allocator.alloc(u8, std.compress.flate.max_window_len),
1828 .compress => return error.UnsupportedCompressionMethod,
1829 };
1830 defer if (options.decompress_buffer == null) client.allocator.free(decompress_buffer);
1831
1832 var transfer_buffer: [64]u8 = undefined;
1833 var decompress: http.Decompress = undefined;
1834 const reader = response.readerDecompressing(&transfer_buffer, &decompress, decompress_buffer);
1835
1836 _ = reader.streamRemaining(response_writer) catch |err| switch (err) {
1837 error.ReadFailed => return response.bodyErr().?,
1838 else => |e| return e,
1839 };
1840
1841 return .{ .status = response.head.status };
1842}
1843
1844test {
1845 _ = Response;
1846}