Issue with HTTPS Proxy Configuration in WebKit WebView

Hello,

I am trying to apply ProxyConfiguration on the WebKit webview. I referred to the following sources:

import WebKit

class WebKitViewModel: ObservableObject {
    let webView: WKWebView
    @Published var urlString: String = "https://example.com"

    init() {
        webView = WKWebView(frame: .zero)
    }
    
    func loadUrl() {
        guard let url = URL(string: urlString) else {
                    return
                }
        var request = URLRequest(url: url)
        let endpoint = NWEndpoint.hostPort(host: "127.0.0.1", port: 9077)
        let proxyConfig = ProxyConfiguration.init(httpCONNECTProxy: endpoint)
        let websiteDataStore = WKWebsiteDataStore.default()
        websiteDataStore.proxyConfigurations = [proxyConfig]
        webView.configuration.websiteDataStore = websiteDataStore
        webView.load(request)
    }
}

However, this configuration only works for HTTP proxies. When I try to use an HTTPS proxy, it does not work.

When I use NWConnection to connect to the proxy, it works successfully:

import Foundation
import Network

public class HTTPProxy {
  private let proxyHost: NWEndpoint.Host
  private let proxyPort: NWEndpoint.Port
  private var connection: NWConnection?

  public init(proxyHost: String, proxyPort: UInt16) {
    self.proxyHost = NWEndpoint.Host(proxyHost)
    self.proxyPort = NWEndpoint.Port(rawValue: proxyPort)!
  }

  public func sendHTTPRequest(completion: @escaping (Result<String, Error>) -> Void) {
    let tlsOptions = NWProtocolTLS.Options()
    let parameters = NWParameters(tls: tlsOptions)

    connection = NWConnection(host: proxyHost, port: proxyPort, using: parameters)

    connection?.stateUpdateHandler = { [weak self] state in
      switch state {
      case .ready:
        self?.sendConnectRequest(completion: completion)
      case .failed(let error):
        completion(.failure(error))
      default:
        break
      }
    }

      connection?.start(queue: .global())
    }

  private func sendConnectRequest(completion: @escaping (Result<String, Error>) -> Void) {
    guard let connection = connection else {
        completion(.failure(NSError(domain: "Connection not available", code: -1, userInfo: nil)))
        return
    }

    let username = "xxxx"
    let password = "xxxx"

    let credentials = "\(username):\(password)"

    guard let credentialsData = credentials.data(using: .utf8) else {
        print("Error encoding credentials")
        fatalError()
    }

    let base64Credentials = credentialsData.base64EncodedString()

    let proxyAuthorization = "Basic \(base64Credentials)"


    let connectString = "CONNECT api.ipify.org:80 HTTP/1.1\r\n" +
    "Host: api.ipify.org:80\r\n" +
    "Proxy-Authorization: \(proxyAuthorization)\r\n" +
    "Connection: keep-alive\r\n" +
    "\r\n"

    if let connectData = connectString.data(using: .utf8) {
      connection.send(content: connectData, completion: .contentProcessed { error in
        if let error = error {
          completion(.failure(error))
        } else {
          self.receiveConnectResponse(completion: completion)
        }
      })
    }
  }

  private func receiveConnectResponse(completion: @escaping (Result<String, Error>) -> Void) {
    connection?.receive(minimumIncompleteLength: 1, maximumLength: 65536) { data, context, isComplete, error in
      if let data = data, let responseString = String(data: data, encoding: .utf8) {
        if responseString.contains("200 OK") {
          self.sendRequest(completion: completion)
        } else {
          completion(.failure(NSError(domain: "Failed to establish connection", code: -1, userInfo: nil)))
        }
      } else if let error = error {
        completion(.failure(error))
      }
    }
  }

  private func sendRequest(completion: @escaping (Result<String, Error>) -> Void) {
    guard let connection = connection else {
      completion(.failure(NSError(domain: "Connection not available", code: -1, userInfo: nil)))
      return
    }

    let requestString = "GET /?format=json HTTP/1.1\r\n" +
    "Host: api.ipify.org\r\n" +
    // "Proxy-Authorization: Basic xxxxxxxx\r\n" +
    "Connection: keep-alive\r\n" +
    "\r\n"

    print("Sending HTTP request:\n\(requestString)")

    if let requestData = requestString.data(using: .utf8) {
      connection.send(content: requestData, completion: .contentProcessed { error in
        if let error = error {
          completion(.failure(error))
        } else {
          self.receiveResponse(completion: completion)
        }
      })
    }
  }

  private func receiveResponse(completion: @escaping (Result<String, Error>) -> Void) {
    connection?.receive(minimumIncompleteLength: 1, maximumLength: 65536) { data, context, isComplete, error in
      if let data = data, !data.isEmpty {
        print ("Data: \(data)")
        if let responseString = String(data: data, encoding: .utf8) {
          print("Received response:\n\(responseString)")
          completion(.success(responseString))
        } else {
          completion(.failure(NSError(domain: "Invalid response data", code: -1, userInfo: nil)))
        }
      } else if let error = error {
        completion(.failure(error))
      }

      if isComplete {
        self.connection?.cancel()
        self.connection = nil
      } else {
        self.receiveResponse(completion: completion)
      }
    }
  }
}

This approach works for connecting to the proxy, but it does not help with configuring the proxy for WebKit.

Could someone please assist me in configuring a proxy for WebKit WebView to work with HTTPS proxies?

Thank you!

However, this configuration only works for HTTP proxies. When I try to use an HTTPS proxy, it does not work.

Are you sure you got that the right way around?

I’m aware of a bug (r. 126265139) where HTTP CONNECT style proxies don’t work for HTTP. This affects both URLSession and WKWebView. The specific symptoms are:

  • If you issue an http: request, it’ll go through the proxy.

  • But if you issue an http: request, it’ll always go direct.

This bug is fixed in the iOS 18.0b3 (and similar) builds that we’re currently seeding.

I think you can work around this using a SOCK5 proxy.

Share and Enjoy

Quinn “The Eskimo!” @ Developer Technical Support @ Apple
let myEmail = "eskimo" + "1" + "@" + "apple.com"

Thanks for your support!

Are you sure you got that the right way around?

I have set up a local proxy server using Squid, configured with http_port 3128 and https_port 3129 for testing purposes.

  • if I connect to http_port 3128 -> then I check log on server see that request go through proxy
  • if I I connect to https_port 3129 -> then I check log on server see that a error when make a request.

I think you can work around this using a SOCK5 proxy.

I can not use SOCK5 proxy because the proxy server that I want to connect just support HTTP CONNECT proxy require Basic Authentication via HTTPS:

curl -v https://sg.http-proxy.privateinternetbrowsing.com:443
*   Trying 156.146.57.8:443...
* Connected to sg.http-proxy.privateinternetbrowsing.com (156.146.57.8) port 443
* ALPN: curl offers h2,http/1.1
* (304) (OUT), TLS handshake, Client hello (1):
*  CAfile: /etc/ssl/cert.pem
*  CApath: none
* (304) (IN), TLS handshake, Server hello (2):
* (304) (IN), TLS handshake, Unknown (8):
* (304) (IN), TLS handshake, Certificate (11):
* (304) (IN), TLS handshake, CERT verify (15):
* (304) (IN), TLS handshake, Finished (20):
* (304) (OUT), TLS handshake, Finished (20):
* SSL connection using TLSv1.3 / AEAD-CHACHA20-POLY1305-SHA256
* ALPN: server accepted http/1.1
* Server certificate:
*  subject: CN=sg.http-proxy.privateinternetbrowsing.com
*  start date: Jun 25 12:31:20 2024 GMT
*  expire date: Sep 23 12:31:19 2024 GMT
*  subjectAltName: host "sg.http-proxy.privateinternetbrowsing.com" matched cert's "sg.http-proxy.privateinternetbrowsing.com"
*  issuer: C=US; O=Let's Encrypt; CN=E6
*  SSL certificate verify ok.
* using HTTP/1.1
> GET / HTTP/1.1
> Host: sg.http-proxy.privateinternetbrowsing.com
> User-Agent: curl/8.4.0
> Accept: */*
> 
< HTTP/1.1 407 Proxy Authentication Required
< Content-Type: text/plain; charset=utf-8
< Proxy-Authenticate: Basic
< X-Content-Type-Options: nosniff
< Date: Mon, 15 Jul 2024 17:47:20 GMT
< Content-Length: 35
< 
This proxy requires authentication
* Connection #0 to host sg.http-proxy.privateinternetbrowsing.com left intact

If you issue an http: request, it’ll go through the proxy. But if you issue an http: request, it’ll always go direct.

you mean that?:

  • If you issue an https: request, it’ll go through the proxy.
  • But if you issue an http: request, it’ll always go direct

I am not sure if it is related to https://developer.apple.com/forums/thread/734679 or not

I am not sure if it is related to https://developer.apple.com/forums/thread/734679 or not

you mean that … if you issue an http: request, it’ll always go direct

Yes. That’s the behaviour I see on currently released systems.

Note In the following I’m using URLSession, but my experience is that this issue is rooted in Network framework and thus affects all APIs layered on top of that, including URLSession and WKWebView.

Consider this code to set up a URLSession:

let config = URLSessionConfiguration.default
config.requestCachePolicy = .reloadIgnoringLocalCacheData
var proxy = ProxyConfiguration(httpCONNECTProxy: .hostPort(host: "lefty.local", port: 12345))
proxy.allowFailover = false
config.proxyConfigurations = [proxy]
session = URLSession(configuration: config, delegate: nil, delegateQueue: .main)

And this code to run a request:

let url = URL(string: "http://example.com")!
print("will start task, url: \(url)")
let request = URLRequest(url: url, cachePolicy: .reloadIgnoringLocalCacheData, timeoutInterval: 60.0)
session.dataTask(with: request) { (data, response, error) in
    if let error = error as NSError? {
        print("task did fail, error: \(error.domain) / \(error.code)")
        return
    }
    let response = response as! HTTPURLResponse
    let data = data!
    print("task finished, status: \(response.statusCode), bytes: \(data.count)")
}.resume()
print("did start task")

where lefty.local is my Mac, which is on the same Wi-Fi network as my iOS test device. If I run a dummy HTTP CONNECT server on my Mac like so:

% nc -l 12345

and then run the above code on my iOS device, I see the CONNECT request coming through:

% nc -l 12345
CONNECT example.com:443 HTTP/1.1
Host: example.com
Proxy-Connection: keep-alive
Connection: keep-alive
…

OTOH, if I repeat the test with http://example.com then my proxy doesn’t see the CONNECT request and instead the session fetches the data directly:

will start task, url: http://example.com
did start task
task finished, status: 200, bytes: 1256

I’m using Xcode 15.4 on macOS 14.5 targeting iOS 17.5.1.

Share and Enjoy

Quinn “The Eskimo!” @ Developer Technical Support @ Apple
let myEmail = "eskimo" + "1" + "@" + "apple.com"

Yes, I checked on my side, and the behavior is as you described. Thank you.

I see that the CONNECT request does not include the authentication information:

"Proxy-Authorization: Basic xxxxxxxx\r\n"

even though I added applyCredential:

proxyConfig.applyCredential(username: "***", password: "***")

I think this is the cause of my issue, making WKWebView unable to connect through the proxy server.

Could this be related to an issue with applyCredential? It seems to be mentioned here: https://developer.apple.com/forums/thread/734679

I'm using Xcode 15.2 on macOS 14.3 targeting iOS 17.2.

I see that the CONNECT request does not include the authentication information:

That is not, in and of itself, a problem. If the proxy requires credentials it’s supposed to respond with an HTTP 407 status code. Once it does that, the client should retry with the credential.

Now, there may still be bugs here — proxy authentication is an ongoing source of grief on our platforms — but it’s important that you check the above before coming to any conclusions. Given that this is plaintext, you can check the behaviour using a simple packet trace.

Oh, and it’s worth continuing to test with URLSession rather than the web view. That simplifies things considerably and, if you’re able to get it work with URLSession and still have problems with the web view, that’s clearly a web view issue.

Share and Enjoy

Quinn “The Eskimo!” @ Developer Technical Support @ Apple
let myEmail = "eskimo" + "1" + "@" + "apple.com"

Issue with HTTPS Proxy Configuration in WebKit WebView
 
 
Q