//
// Copyright (c) 2008, Brian Frank and Andy Frank
// Licensed under the Academic Free License version 3.0
//
// History:
//   24 Dec 08  Brian Frank       Almost Christmas!
//   15 Jan 13  Nicholas Harker   Added SSL Support
//   21 Jan 13  Nicholas Harker   Added Proxy Exclusion Support
//   03 Aug 15  Matthew Giannini  RFC6265
//

using inet

**
** The 'WebClient' class is used to manage client side HTTP requests
** and responses.  The basic lifecycle of WebClient:
**   1. configure request fields such as 'reqUri', 'reqMethod', and 'reqHeaders'
**   2. send request headers via 'writeReq'
**   3. optionally write request body via 'reqOut'
**   4. read response status and headers via 'readRes'
**   5. process response fields such as 'resCode' and 'resHeaders'
**   6. optionally read response body via 'resIn'
**
** Using the low level methods 'writeReq' and 'readRes' enables HTTP
** pipelining (multiple requests and responses on the same TCP socket
** connection).  There are also a series of convenience methods which
** make common cases easier.
**
** See [pod doc]`pod-doc#webClient` and [examples]`examples::web-client`.
**
class WebClient
{

//////////////////////////////////////////////////////////////////////////
// Constructor
//////////////////////////////////////////////////////////////////////////

  **
  ** Construct with optional request URI.
  **
  new make(Uri? reqUri := null)
  {
    if (reqUri != null) this.reqUri = reqUri

    // default headers
    reqHeaders["Accept-Encoding"] = "gzip"
  }

  ** The `inet::SocketConfig` to use for creating sockets
  SocketConfig socketConfig := SocketConfig.cur

//////////////////////////////////////////////////////////////////////////
// Request
//////////////////////////////////////////////////////////////////////////

  **
  ** The absolute URI of request.
  **
  Uri reqUri := ``
  {
    set
    {
      if (!it.isAbs) throw ArgErr("Request URI not absolute: `$it`")
      if (it.scheme != "http" && it.scheme != "https") throw ArgErr("Request URI is not http or https: `$it`")
      &reqUri = it
    }
  }

  **
  ** The HTTP method for the request.  Defaults to "GET".
  **
  Str reqMethod := "GET" { set { &reqMethod = it.upper } }

  **
  ** HTTP version to use for request must be 1.0 or 1.1.
  ** Default is 1.1.
  **
  Version reqVersion := ver11

  **
  ** The HTTP headers to use for the next request.  This map uses
  ** case insensitive keys.  The "Host" header is implicitly defined
  ** by 'reqUri' and must not be defined in this map.
  **
  Str:Str reqHeaders := Str:Str[:] { caseInsensitive = true }

  **
  ** Get the output stream used to write the request body.  This
  ** stream is only available if the request headers included a
  ** "Content-Type" header.  If an explicit "Content-Length" was
  ** specified then this is a fixed length output stream, otherwise
  ** the request is automatically configured to use a chunked
  ** transfer encoding.  This stream should be closed once the
  ** content has been fully written.
  **
  OutStream reqOut()
  {
    if (reqOutStream == null) throw IOErr("No output stream for request")
    return reqOutStream
  }

//////////////////////////////////////////////////////////////////////////
// Response
//////////////////////////////////////////////////////////////////////////

  **
  ** HTTP version returned by response.
  **
  Version resVersion := ver11

  **
  ** HTTP status code returned by response.
  **
  Int resCode

  **
  ** HTTP status reason phrase returned by response.
  **
  Str resPhrase := ""

  **
  ** HTTP headers returned by response.
  **
  Str:Str resHeaders := noHeaders

  **
  ** Get a response header.  If not found and checked
  ** is false then return true, otherwise throw Err.
  **
  Str? resHeader(Str key, Bool checked := true)
  {
    val := resHeaders[key]
    if (val != null || !checked) return val
    throw Err("Missing HTTP header '$key'")
  }

  **
  ** Input stream to read response content.  The input stream
  ** will correctly handle end of stream when the content has been
  ** fully read.  If the "Content-Length" header was specified the
  ** end of stream is based on the fixed number of bytes.  If the
  ** "Transfer-Encoding" header defines a chunked encoding, then
  ** chunks are automatically handled.  If the response has no
  ** content body, then throw IOErr.
  **
  ** The response input stream is automatically configured with
  ** the correct character encoding if one is specified in the
  ** "Content-Type" response header.
  **
  ** Also see convenience methods: `resStr` and `resBuf`.
  **
  InStream resIn()
  {
    if (resInStream == null) throw IOErr("No input stream for response $resCode")
    return resInStream
  }

  **
  ** Return the entire response back as an in-memory string.
  ** Convenience for 'resIn.readAllStr'.
  **
  Str resStr()
  {
    return resIn.readAllStr
  }

  **
  ** Return the entire response back as an in-memory byte buffer.
  ** Convenience for 'resIn.readAllBuf'.
  **
  Buf resBuf()
  {
    return resIn.readAllBuf
  }

//////////////////////////////////////////////////////////////////////////
// Cookies
//////////////////////////////////////////////////////////////////////////

  **
  ** Cookies to pass for "Cookie" request header.  If set to an empty
  ** list then the "Cookie" request header is removed.  After a request
  ** has been completed if the "Set-Cookie" response header specified
  ** one or more cookies then this field is automatically updated with
  ** the server specified cookies.
  **
  Cookie[] cookies := Cookie#.emptyList
  {
    set
    {
      // save field
      &cookies = it

      // set reqHeaders (RFC 6265 § 4.2.1)
      if (it.isEmpty) { reqHeaders.remove("Cookie"); return }
      reqHeaders["Cookie"] =
        it.size == 1 ?
        it.first.toNameValStr :
        it.join("; ") |c| { c.toNameValStr }
    }
  }

//////////////////////////////////////////////////////////////////////////
// Networking
//////////////////////////////////////////////////////////////////////////

  @Deprecated { msg = "Use WebClient.socketConfig to configure sockets" }
  once SocketOptions socketOptions()
  {
    TcpSocket().options
  }

  **
  ** When set to true a 3xx response with a Location header
  ** will automatically update the `reqUri` field and retry the
  ** request using the alternate URI.  Redirects are not followed
  ** if the request has a content body.
  **
  Bool followRedirects := true

//////////////////////////////////////////////////////////////////////////
// Proxy Support
//////////////////////////////////////////////////////////////////////////

  **
  ** If non-null, then all requests are routed through this
  ** proxy address (host and port).  Default is configured
  ** in "etc/web/config.props" with the key "proxy".  Proxy
  ** exceptions can be configured with the "proxy.exceptions"
  ** config key as comma separated list of Regex globs.
  **
  Uri? proxy := proxyDef

  private static Uri? proxyDef()
  {
    try
      return WebClient#.pod.config("proxy")?.toUri
    catch (Err e)
      e.trace
    return null
  }

  private Bool isProxy(Uri uri)
  {
    proxy != null && !proxyExceptions.any |re| { re.matches(uri.host.toStr) }
  }

  private once Regex[] proxyExceptions()
  {
    try
      return WebClient#.pod.config("proxy.exceptions")?.split(',')?.map |tok->Regex| { Regex.glob(tok) } ?: Regex[,]
    catch (Err e)
      e.trace
    return Regex[,]
  }

//////////////////////////////////////////////////////////////////////////
// Authentication
//////////////////////////////////////////////////////////////////////////

  **
  ** Authenticate request using HTTP Basic with given username
  ** and password.
  **
  This authBasic(Str username, Str password)
  {
    enc := "${username}:${password}".toBuf.toBase64
    reqHeaders["Authorization"] = "Basic ${enc}"
    return this
  }

//////////////////////////////////////////////////////////////////////////
// Get
//////////////////////////////////////////////////////////////////////////

  **
  ** Make a GET request and return the response content as
  ** an in-memory string.  The web client is automatically closed.
  ** Throw IOErr is response is not 200.
  **
  Str getStr()
  {
    try
      return getIn.readAllStr
    finally
      close
  }

  **
  ** Make a GET request and return the response content as
  ** an in-memory byte buffer.  The web client is automatically closed.
  ** Throw IOErr is response is not 200.
  **
  Buf getBuf()
  {
    try
      return getIn.readAllBuf
    finally
      close
  }

  **
  ** Make a GET request and return the input stream to the
  ** response or throw IOErr if response is not 200.  It is the
  ** caller's responsibility to close this web client.
  **
  InStream getIn()
  {
    reqMethod = "GET"
    writeReq
    readRes
    if (resCode != 200) throw IOErr("Bad HTTP response $resCode $resPhrase")
    return resIn
  }

//////////////////////////////////////////////////////////////////////////
// Post/Patch
//////////////////////////////////////////////////////////////////////////

  **
  ** Convenience for 'writeForm("POST", form).readRes'
  **
  This postForm(Str:Str form)
  {
    writeForm("POST", form).readRes
  }

  **
  ** Convenience for 'writeStr("POST", content).readRes'
  **
  This postStr(Str content)
  {
    writeStr("POST", content).readRes
  }

  **
  ** Convenience for 'writeBuf("POST", content).readRes'
  **
  This postBuf(Buf buf)
  {
    writeBuf("POST", buf).readRes
  }

  **
  ** Convenience for 'writeFile("POST", file).readRes'
  **
  This postFile(File file)
  {
    writeFile("POST", file).readRes
  }

  **
  ** Make a request with the given HTTP method to the URI with the given form data.
  ** Set the Content-Type to application/x-www-form-urlencoded.
  ** This method does not support the ["Expect" header]`pod-doc#expectContinue` (it
  ** writes all form data before reading response). Should primarily be used for POST
  ** and PATCH requests.
  **
  This writeForm(Str method, Str:Str form)
  {
    if (reqHeaders["Expect"] != null) throw UnsupportedErr("'Expect' header")
    body := Uri.encodeQuery(form)
    reqMethod = method
    reqHeaders["Content-Type"] = "application/x-www-form-urlencoded"
    reqHeaders["Content-Length"] = body.size.toStr // encoded form is ASCII
    writeReq
    reqOut.print(body).close
    return this
  }

  **
  ** Make a request with the given HTTP method to the URI using UTF-8 encoding of given
  ** string.  If Content-Type is not already set, then set it
  ** to "text/plain; charset=utf-8".  This method does not support the
  ** ["Expect" header]`pod-doc#expectContinue` (it writes full string
  ** before reading response). Should primarily be used for "POST" and "PATCH"
  ** requests.
  **
  This writeStr(Str method, Str content)
  {
    if (reqHeaders["Expect"] != null) throw UnsupportedErr("'Expect' header")
    body := Buf().print(content).flip
    reqMethod = method
    ct := reqHeaders["Content-Type"]
    if (ct == null)
      reqHeaders["Content-Type"] = "text/plain; charset=utf-8"
    reqHeaders["Content-Length"] = body.size.toStr
    writeReq
    reqOut.writeBuf(body).close
    return this
  }

  **
  ** Write a binary buffer using the given HTTP method to the URI.  If Content-Type
  ** header is not already set, then it is set as ""application/octet-stream". This
  ** method does not support the ["Expect" header]`pod-doc#expectContinue`
  **
  This writeBuf(Str method, Buf content)
  {
    if (reqHeaders["Expect"] != null) throw UnsupportedErr("'Expect' header")
    reqMethod = method
    ct := reqHeaders["Content-Type"]
    if (ct == null)
      reqHeaders["Content-Type"] = "application/octet-stream"
    reqHeaders["Content-Length"] = content.size.toStr
    writeReq
    reqOut.writeBuf(content.seek(0))
    reqOut.close
    return this
  }

  **
  ** Write a file using the given HTTP method to the URI.  If Content-Type header is not already
  ** set, then it is set from the file extension's MIME type. This method does
  ** not support the ["Expect" header]`pod-doc#expectContinue` (it
  ** writes full file before reading response). Should primarily be used for "POST" and
  ** "PATCH" requests.
  **
  This writeFile(Str method, File file)
  {
    if (reqHeaders["Expect"] != null) throw UnsupportedErr("'Expect' header")
    reqMethod = method
    ct := reqHeaders["Content-Type"]
    if (ct == null)
      reqHeaders["Content-Type"] = file.mimeType?.toStr ?: "application/octet-stream"
    if (file.size != null)
      reqHeaders["Content-Length"] = file.size.toStr
    writeReq
    file.in.pipe(reqOut, file.size)
    reqOut.close
    return this
  }

//////////////////////////////////////////////////////////////////////////
// Service
//////////////////////////////////////////////////////////////////////////

  **
  ** Write the request line and request headers.  Once this method
  ** completes the request body may be written via `reqOut`, or the
  ** response may be immediately read via `readRes`.  Throw IOErr
  ** if there is a network or protocol error.  Return this.
  **
  This writeReq()
  {
    // sanity checks
    if (!reqUri.isAbs || reqUri.scheme == null || reqUri.host == null) throw Err("reqUri is not absolute: `$reqUri`")
    if (!reqHeaders.caseInsensitive) throw Err("reqHeaders must be case insensitive")
    if (reqHeaders.containsKey("Host")) throw Err("reqHeaders must not define 'Host'")

    // connect to the host:port if we aren't already connected
    isHttps := reqUri.scheme == "https"
    defPort := isHttps ? 443 : 80
    usingProxy := isProxy(reqUri)
    isTunnel := usingProxy && isHttps
    if (!isConnected)
    {
      if (isTunnel) socket = openHttpsTunnel
      else
      {
        // make https or http socket
        socket = TcpSocket(socketConfig)
        if (isHttps) socket = socket.upgradeTls

        // connect to proxy or directly to request host
        connUri := usingProxy ? proxy : reqUri
        socket.connect(IpAddr(connUri.host), connUri.port ?: defPort)
      }
    }

    // request uri is absolute if proxy, relative otherwise
    reqPath := (usingProxy ? reqUri : reqUri.relToAuth).encode

    // host authority header
    host := reqUri.host
    if (reqUri.port != null && reqUri.port != defPort) host += ":$reqUri.port"

    // figure out if/how we are streaming out content body
    out := socket.out
    reqOutStream = WebUtil.makeContentOutStream(reqHeaders, out)

    // send request
    out.print(reqMethod).print(" ").print(reqPath)
       .print(" HTTP/").print(reqVersion).print("\r\n")
    out.print("Host: ").print(host).print("\r\n")
    WebUtil.writeHeaders(out, reqHeaders)
    out.print("\r\n")
    out.flush

    return this
  }

  ** Open an https tunnel through our proxy server.
  private TcpSocket openHttpsTunnel()
  {
    socket = TcpSocket(socketConfig)

    // make CONNECT request to proxy server on http port
    socket.connect(IpAddr(proxy.host), proxy.port ?: 80)
    out := socket.out
    out.print("CONNECT ${reqUri.host}:${reqUri.port ?: 443} HTTP/${reqVersion}").print("\r\n")
       .print("Host: ${reqUri.host}:${reqUri.port ?: 443}").print("\r\n")
       .print("\r\n")
    out.flush

    // expect a 200 response code
    readRes
    if (resCode != 200) throw IOErr("Could not open tunnel: bad HTTP response $resCode $resPhrase")

    // upgrade to SSL socket now
    return socket.upgradeTls(IpAddr(reqUri.host), reqUri.port ?: 443)
  }

  **
  ** Read the response status line and response headers.  This method
  ** may be called after the request has been written via `writeReq`
  ** and `reqOut`.  Once this method completes the response status and
  ** headers are available.  If there is a response body, it is available
  ** for reading via `resIn`.  Throw IOErr if there is a network or
  ** protocol error.  Return this.
  **
  This readRes()
  {
    // read response
    if (!isConnected) throw IOErr("Not connected")
    in := socket.in
    res := ""
    try
    {
      // parse status-line
      res = WebUtil.readLine(in)
      if (res.startsWith("HTTP/1.1")) resVersion = ver11
      else if (res.startsWith("HTTP/1.0")) resVersion = ver10
      else throw Err("Not HTTP")
      resCode = res[9..11].toInt
      resPhrase = res[13..-1]

      // parse response headers
      setCookies := Cookie[,]
      resHeaders = WebUtil.doParseHeaders(in, setCookies)
      if (!setCookies.isEmpty) cookies = setCookies
    }
    catch (Err e) throw IOErr("Invalid HTTP response: $res", e)

    // check for redirect
    if (checkFollowRedirect) return this

    // if there is response content, then create wrap the raw socket
    // input stream with the appropiate chunked input stream
    resInStream = WebUtil.makeContentInStream(resHeaders, socket.in)

    return this
  }

  **
  ** If we have a 3xx statu code with a location header,
  ** then check for an automate redirect.
  **
  private Bool checkFollowRedirect()
  {
    // only redirect on 3xx status code
    if (resCode / 100 != 3) return false

    // must be explicitly configured for redirects
    if (!followRedirects) return false

    // only redirect when there is no request content
    if (reqOutStream != null) return false

    // only redirect if a location header was given
    loc := resHeaders["Location"]
    if (loc == null) return false

    // redirect
    try
    {
      ++numRedirects
      close
      newUri := Uri.decode(loc)
      if (!newUri.isAbs) newUri = reqUri + newUri
      if (reqUri == newUri && numRedirects > 20) throw Err("Cyclical redirect: $newUri")
      reqUri = newUri
      writeReq
      readRes
      return true
    }
    finally
    {
      --numRedirects
    }
  }

  **
  ** Return if this web client is currently connected to the remote host.
  **
  Bool isConnected()
  {
    return socket != null && socket.isConnected
  }

  **
  ** Close the HTTP request and the underlying socket.  Return this.
  **
  This close()
  {
    if (socket != null) socket.close
    socket = null
    return this
  }

//////////////////////////////////////////////////////////////////////////
// Fields
//////////////////////////////////////////////////////////////////////////

  private static const Version ver10 := Version("1.0")
  private static const Version ver11 := Version("1.1")
  private static const Str:Str noHeaders := Str:Str[:]

  private InStream? resInStream
  private OutStream? reqOutStream
  internal TcpSocket? socket
  private Int numRedirects := 0

}