local base64 = require "base64"
local comm = require "comm"
local coroutine = require "coroutine"
local math = require "math"
local nmap = require "nmap"
local os = require "os"
local sasl = require "sasl"
local shortport = require "shortport"
local slaxml = require "slaxml"
local stdnse = require "stdnse"
local string = require "string"
local stringaux = require "stringaux"
local table = require "table"
local tableaux = require "tableaux"
local url = require "url"
local smbauth = require "smbauth"
local unicode = require "unicode"
_ENV = stdnse.module("http", stdnse.seeall)
local have_ssl, openssl = pcall(require,'openssl')
local have_zlib, zlib = pcall(require,'zlib')
USER_AGENT = stdnse.get_script_args('http.useragent') or "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)"
local host_header = stdnse.get_script_args('http.host')
local MAX_REDIRECT_COUNT = 5
local MAX_BODY_SIZE = tonumber(stdnse.get_script_args('http.max-body-size')) or 2*1024*1024
local TRUNCATED_OK = string.lower(stdnse.get_script_args('http.truncated-ok') or "false") ~= "false"
local ERR_OVERSIZED_BODY = "response body too large"
local function table_augment(to, from)
  for k, v in pairs(from) do
    if type( to[k] ) == 'table' then
      table_augment(to[k], from[k])
    else
      to[k] = from[k]
    end
  end
end
local get_default_port = url.get_default_port
local function get_host_field(host, port, scheme)
  if host_header then return host_header end
  if not host then return nil end
  if type(port) == "number" then
    port = {number=port, protocol="tcp"}
  end
  local number = port.number
  if scheme then
    if number == get_default_port(scheme) then
      return stdnse.get_hostname(host)
    end
  else
    scheme = url.get_default_scheme(port)
    if scheme then
      local ssl_port = shortport.ssl(host, port)
      if (ssl_port and scheme == 'https') or
        (not ssl_port and scheme == 'http') then
        return stdnse.get_hostname(host)
      end
    end
  end
  return stdnse.get_hostname(host) .. ":" .. number
end
local function skip_space(s, offset)
  local _, i, space = s:find("^([ \t]*)", offset)
  return i + 1, space
end
local function get_token(s, offset)
  local _, i, token = s:find("^([^()<>@,;:\\\"/%[%]?={} \0\001-\031\127]+)", offset)
  if i then
    return i + 1, token
  else
    return nil
  end
end
local function get_quoted_string(s, offset, crlf)
  local result = {}
  local i = offset
  assert(s:sub(i, i) == "\"")
  i = i + 1
  while i <= s:len() do
    local c = s:sub(i, i)
    if c == "\"" then
      return i + 1, table.concat(result)
    elseif c == "\\" then
      i = i + 1
      c = s:sub(i, i)
      if c == "" then
        error("\\ escape at end of input while parsing quoted-string.")
      end
      if c:byte(1) > 127 then
        error(string.format("Unexpected character with value > 127 (0x%02X) in quoted-string.", c:byte(1)))
      end
    else
      c = s:sub(i, i)
      if c ~= "\t" and c:match("^[\0\001-\031\127]$") then
        error(string.format("Unexpected control character in quoted-string: 0x%02X.", c:byte(1)))
      end
    end
    result[#result + 1] = c
    i = i + 1
  end
  return nil
end
local function skip_lws(s, pos)
  local _, e
  while true do
    while string.match(s, "^[ \t]", pos) do
      pos = pos + 1
    end
    _, e = string.find(s, "^\r?\n[ \t]", pos)
    if not e then
      return pos
    end
    pos = e + 1
  end
end
local function validate_options(options)
  local bad = false
  if(options == nil) then
    return true
  end
  for key, value in pairs(options) do
    if(key == 'timeout') then
      if(type(tonumber(value)) ~= 'number') then
        stdnse.debug1('http: options.timeout contains a non-numeric value')
        bad = true
      end
    elseif(key == 'header') then
      if(type(value) ~= 'table') then
        stdnse.debug1("http: options.header should be a table")
        bad = true
      end
    elseif(key == 'content') then
      if(type(value) ~= 'string' and type(value) ~= 'table') then
        stdnse.debug1("http: options.content should be a string or a table")
        bad = true
      end
    elseif(key == 'cookies') then
      if(type(value) == 'table') then
        for _, cookie in ipairs(value) do
          for cookie_key, cookie_value in pairs(cookie) do
            if(cookie_key == 'name') then
              if(type(cookie_value) ~= 'string') then
                stdnse.debug1("http: options.cookies[i].name should be a string")
                bad = true
              end
            elseif(cookie_key == 'value') then
              if(type(cookie_value) ~= 'string') then
                stdnse.debug1("http: options.cookies[i].value should be a string")
                bad = true
              end
            elseif(cookie_key == 'path') then
              if(type(cookie_value) ~= 'string') then
                stdnse.debug1("http: options.cookies[i].path should be a string")
                bad = true
              end
            elseif(cookie_key == 'expires') then
              if(type(cookie_value) ~= 'string') then
                stdnse.debug1("http: options.cookies[i].expires should be a string")
                bad = true
              end
            elseif(cookie_key == 'max-age') then
              if(type(cookie_value) ~= 'string') then
                stdnse.debug1("http: options.cookies[i].max-age should be a string")
                bad = true
              end
            elseif not (cookie_key == 'httponly' or cookie_key == 'secure') then
              stdnse.debug1("http: Unknown field in cookie table: %s", cookie_key)
            end
          end
        end
      elseif(type(value) ~= 'string') then
        stdnse.debug1("http: options.cookies should be a table or a string")
        bad = true
      end
    elseif(key == 'auth') then
      if(type(value) == 'table') then
        if(value['username'] == nil or value['password'] == nil) then
          stdnse.debug1("http: options.auth should contain both a 'username' and a 'password' key")
          bad = true
        end
      else
        stdnse.debug1("http: options.auth should be a table")
        bad = true
      end
    elseif (key == 'digestauth') then
      if(type(value) == 'table') then
        local req_keys = {"username","realm","nonce","digest-uri","response"}
        for _,k in ipairs(req_keys) do
          if not value[k] then
            stdnse.debug1("http: options.digestauth missing key: %s",k)
            bad = true
            break
          end
        end
      else
        bad = true
        stdnse.debug1("http: options.digestauth should be a table")
      end
    elseif (key == 'ntlmauth') then
      stdnse.debug1("Proceeding with ntlm message")
    elseif(key == 'bypass_cache' or key == 'no_cache' or key == 'no_cache_body'
           or key == 'any_af' or key == "truncated_ok") then
      if(type(value) ~= 'boolean') then
        stdnse.debug1("http: options.%s must be a boolean value", key)
        bad = true
      end
    elseif(key == 'redirect_ok') then
      if(type(value)~= 'function' and type(value)~='boolean' and type(value) ~= 'number') then
        stdnse.debug1("http: options.redirect_ok must be a function or boolean or number")
        bad = true
      end
    elseif(key == 'scheme') then
      if type(value) ~= 'string' then
        stdnse.debug1("http: options.scheme must be a string")
        bad = true
      end
    elseif(key == 'max_body_size') then
      if type(value) ~= 'number' then
        stdnse.debug1("http: options.max_body_size must be a number")
        bad = true
      end
    else
      stdnse.debug1("http: Unknown key in the options table: %s", key)
    end
  end
  return not(bad)
end
local function recv_line(s, partial)
  local _, e
  local status, data
  local pos
  partial = partial or ""
  pos = 1
  while true do
    _, e = string.find(partial, "\n", pos, true)
    if e then
      break
    end
    status, data = s:receive()
    if not status then
      return status, data
    end
    pos = #partial
    partial = partial .. data
  end
  return string.sub(partial, 1, e), string.sub(partial, e + 1)
end
local function line_is_empty(line)
  return line == "\r\n" or line == "\n"
end
local function recv_header(s, partial)
  local lines = {}
  partial = partial or ""
  while true do
    local line
    line, partial = recv_line(s, partial)
    if not line then
      return line, partial
    end
    if line_is_empty(line) then
      break
    end
    lines[#lines + 1] = line
  end
  return table.concat(lines), partial
end
local function recv_all(s, partial, maxlen)
  local parts = {}
  local part = partial or ""
  repeat
    if maxlen then
      maxlen = maxlen - #part
      if maxlen < 0 then
        table.insert(parts, part:sub(1, maxlen - 1))
        return nil, ERR_OVERSIZED_BODY, table.concat(parts)
      end
    end
    table.insert(parts, part)
    local status
    status, part = s:receive()
  until not status
  return table.concat(parts), ""
end
local function recv_length(s, recvlen, partial, maxlen)
  local parts = {}
  local part = partial or ""
  partial = ""
  repeat
    if #part > recvlen then
      partial = part:sub(recvlen + 1)
      part = part:sub(1, recvlen)
    end
    if maxlen then
      maxlen = maxlen - #part
      if maxlen < 0 then
        table.insert(parts, part:sub(1, maxlen - 1))
        return nil, ERR_OVERSIZED_BODY, table.concat(parts)
      end
    end
    table.insert(parts, part)
    recvlen = recvlen - #part
    if recvlen == 0 then
      return table.concat(parts), partial
    end
    local status
    status, part = s:receive()
  until not status
  return nil, part, table.concat(parts)
end
local function recv_chunked(s, partial, maxlen)
  local chunks = {}
  repeat
    local line
    line, partial = recv_line(s, partial)
    if not line then
      return nil, "Chunk size not received; " .. partial, table.concat(chunks)
    end
    local pos = skip_space(line)
    local chunklen = line:match("^%x+", pos)
    if not chunklen then
      return nil,
            ("Chunked encoding didn't find hex; got %q."):format(line:sub(pos, pos + 10)),
            table.concat(chunks)
    end
    chunklen = tonumber(chunklen, 16)
    local chunk, fragment
    chunk, partial, fragment = recv_length(s, chunklen, partial, maxlen)
    if not chunk then
      if partial ~= ERR_OVERSIZED_BODY then
        partial = "Incomplete chunk; " .. partial
      end
      table.insert(chunks, fragment)
      return nil, partial, table.concat(chunks)
    end
    table.insert(chunks, chunk)
    if maxlen then
      maxlen = maxlen - chunklen
    end
    line, partial = recv_line(s, partial)
    if not line then
      stdnse.debug2("Didn't find CRLF after chunk-data.")
    elseif not string.match(line, "^\r?\n") then
      return nil,
             ("Didn't find CRLF after chunk-data; got %q."):format(line),
             table.concat(chunks)
    end
  until chunklen == 0
  return table.concat(chunks), partial
end
local function recv_body(s, response, method, partial, maxlen)
  local connection_close, connection_keepalive
  partial = partial or ""
  connection_close = false
  connection_keepalive = false
  if response.header.connection then
    local offset, token
    offset = 0
    while true do
      offset, token = get_token(response.header.connection, offset + 1)
      if not offset then
        break
      end
      if string.lower(token) == "close" then
        connection_close = true
      elseif string.lower(token) == "keep-alive" then
        connection_keepalive = true
      end
    end
  end
  if string.upper(method) == "HEAD"
    or (response.status >= 100 and response.status <= 199)
    or response.status == 204 or response.status == 304 then
    if connection_close or (response.version == "1.0" and not connection_keepalive) then
      return recv_all(s, partial, maxlen)
    else
      return "", partial
    end
  end
  if response.header["transfer-encoding"]
    and response.header["transfer-encoding"] ~= "identity" then
    return recv_chunked(s, partial, maxlen)
  end
  if response.header["transfer-coding"]
    and response.header["transfer-coding"] ~= "identity" then
    return recv_chunked(s, partial, maxlen)
  end
  if response.header["content-length"]  and not response.header["transfer-encoding"] then
    local content_length = tonumber(response.header["content-length"])
    if not content_length then
      return nil, string.format("Content-Length %q is non-numeric", response.header["content-length"])
    end
    return recv_length(s, content_length, partial, maxlen)
  end
  return recv_all(s, partial, maxlen)
end
local function parse_status_line(status_line, response)
  response["status-line"] = status_line
  local version, status, reason_phrase = string.match(status_line,
    "^HTTP/(%d+%.%d+) +(%d+)%f[ \r\n] *(.-)\r?\n$")
  if not version then
    return nil, string.format("Error parsing status-line %q.", status_line)
  end
  response.version = version
  response.status = tonumber(status)
  if not response.status then
    return nil, string.format("Status code is not numeric: %s", status)
  end
  return true
end
local parse_set_cookie -- defined farther down
local function parse_header(header, response)
  local pos
  local name, words
  local s, e
  response.header = {}
  response.rawheader = stringaux.strsplit("\r?\n", header)
  pos = 1
  while pos <= #header do
    e, name = get_token(header, pos)
    if e then
      if header:sub(e, e) ~= ":" then
        name = nil
      end
      pos = e + 1
    end
    pos = skip_lws(header, pos)
    words = {}
    while pos <= #header and not string.match(header, "^\r?\n", pos) do
      s = pos
      while not string.match(header, "^[ \t]", pos) and
        not string.match(header, "^\r?\n", pos) do
        pos = pos + 1
      end
      words[#words + 1] = string.sub(header, s, pos - 1)
      pos = skip_lws(header, pos)
    end
    if name then
      name = string.lower(name)
      local value = table.concat(words, " ")
      if response.header[name] then
        response.header[name] = response.header[name] .. ", " .. value
      else
        response.header[name] = value
      end
      if name == "set-cookie" then
        local cookie, err = parse_set_cookie(value)
        if cookie then
          response.cookies[#response.cookies + 1] = cookie
        else
        end
      end
    end
    s, e = string.find(header, "^\r?\n", pos)
    if not e then
      return nil, string.format("Header field named %q didn't end with CRLF", name)
    end
    pos = e + 1
  end
  return true
end
parse_set_cookie = function (s)
  local name, value
  local _, pos
  local cookie = {}
  s = s:gsub(";", "; ")
  _, pos, cookie.name, cookie.value = s:find("^[ \t]*(.-)[ \t]*=[ \t]*(.-)[ \t]*%f[;\0]")
  if not (cookie.name or ""):find("^[^;]+$") then
    return nil, "Can't get cookie name."
  end
  pos = pos + 1
  while s:sub(pos, pos) == ";" do
    _, pos, name = s:find("[ \t]*(.-)[ \t]*%f[=;\0]", pos + 1)
    pos = pos + 1
    if s:sub(pos, pos) == "=" then
      _, pos, value = s:find("[ \t]*(.-)[ \t]*%f[;\0]", pos + 1)
      pos = pos + 1
    else
      value = ""
    end
    name = name:lower()
    if not (name == "" or name == "name" or name == "value") then
      cookie[name] = value
    end
  end
  return cookie
end
local decode_body = function (body, encodings, maxlen)
  if not encodings then return body end
  if type(encodings) == "string" then
    encodings = stringaux.strsplit("%W+", encodings)
  end
  assert(type(encodings) == "table", "Invalid encoding specification")
  local decoded = {}
  local undecoded = tableaux.tcopy(encodings)
  while #undecoded > 0 do
    local enc = undecoded[1]:lower()
    if enc == "identity" then
      table.insert(decoded, table.remove(undecoded, 1))
    elseif enc == "gzip" and have_zlib then
      local stream = zlib.inflate(body)
      local status, newbody = pcall(stream.read, stream,
                                   maxlen and (maxlen + 1) or "*a")
      stream:close()
      if not status then
        return nil, decoded, undecoded,
               ("Corrupted Content-Encoding: " .. enc), body
      end
      table.insert(decoded, table.remove(undecoded, 1))
      newbody = newbody or ""
      if maxlen and #newbody > maxlen then
        return nil, decoded, undecoded, ERR_OVERSIZED_BODY, newbody:sub(1, maxlen)
      end
      body = newbody
    else
      stdnse.debug1("Unsupported Content-Encoding: %s", enc)
      break
    end
  end
  return body, decoded, undecoded
end
local function next_response(s, method, partial, options)
  local response
  local status_line, header, body, fragment
  local status, err
  partial = partial or ""
  response = {
    status=nil,
    ["status-line"]=nil,
    header={},
    rawheader={},
    cookies={},
    rawbody="",
    body="",
    truncated = nil
  }
  status_line, partial = recv_line(s, partial)
  if not status_line then
    return nil, partial, response
  end
  status, err = parse_status_line(status_line, response)
  if not status then
    return nil, err, response
  end
  header, partial = recv_header(s, partial)
  if not header then
    return nil, partial, response
  end
  status, err = parse_header(header, response)
  if not status then
    return nil, err, response
  end
  options = options or {}
  local maxlen = math.floor(options.max_body_size or MAX_BODY_SIZE)
  if maxlen < 0 then
    maxlen = nil
  end
  local truncated_ok = options.truncated_ok
  if truncated_ok == nil then
    truncated_ok = TRUNCATED_OK
  end
  body, partial, fragment = recv_body(s, response, method, partial, maxlen)
  response.rawbody = body or fragment
  response.body = response.rawbody
  if not body then
    if partial ~= ERR_OVERSIZED_BODY then
      return nil, partial, response
    end
    response.truncated = true
    if not truncated_ok then
      return nil, ("Received " .. ERR_OVERSIZED_BODY), response
    end
  end
  if response.header["content-encoding"] then
    local dcd, undcd
    body, dcd, undcd, err, fragment = decode_body(body, response.header["content-encoding"], maxlen)
    response.body = body or fragment
    response.decoded = dcd
    response.undecoded = undcd
    if not body then
      if err ~= ERR_OVERSIZED_BODY then
        return nil, err, response
      end
      response.truncated = true
      if not truncated_ok then
        return nil, ("Decoded " .. ERR_OVERSIZED_BODY), response
      end
    else
      if response.header["content-length"] then
        response.header["content-length"] = tostring(#body)
      end
    end
    response.header["content-encoding"] = #undcd > 0 and table.concat(undcd, ", ") or nil
  end
  return response, partial
end
local function get_pipeline_limit(response)
  local pipeline = tonumber(stdnse.get_script_args({'http.pipeline', 'pipeline'}))
  if pipeline then
    return pipeline
  end
  if response then
    local hdr = response.header or {}
    local opts = stringaux.strsplit("[,%s]+", (hdr.connection or ""):lower())
    if tableaux.contains(opts, "close") then return 1 end
    if response.version >= "1.1" or tableaux.contains(opts, "keep-alive") then
      return 1 + (tonumber((hdr["keep-alive"] or ""):match("max=(%d+)")) or 39)
    end
  end
  return 1
end
local function buildCookies(cookies, path)
  if type(cookies) == 'string' then return cookies end
  local cookie = {}
  for _, ck in ipairs(cookies or {}) do
    local ckpath = ck["path"]
    if not path or not ckpath
      or ckpath == path
      or ckpath:sub(-1) == "/" and ckpath == path:sub(1, ckpath:len())
      or ckpath .. "/" == path:sub(1, ckpath:len()+1)
      then
        cookie[#cookie+1] = ck["name"] .. "=" .. ck["value"]
      end
    end
  return table.concat(cookie, "; ")
end
local cache = {size = 0};
local function check_size (cache)
  local max_size = tonumber(stdnse.get_script_args({'http.max-cache-size', 'http-max-cache-size'}) or 1e6);
  local size = cache.size;
  if size > max_size then
    stdnse.debug1(
        "Current http cache size (%d bytes) exceeds max size of %d",
        size, max_size);
    table.sort(cache, function(r1, r2)
      return (r1.last_used or 0) < (r2.last_used or 0);
    end);
    for i, record in ipairs(cache) do
      if size <= max_size then break end
      local result = record.result;
      if type(result.body) == "string" then
        size = size - record.size;
        record.size, record.get, result.body = 0, false, "";
      end
    end
    cache.size = size;
  end
  stdnse.debug2("Final http cache size (%d bytes) of max size of %d",
      size, max_size);
  return size;
end
local WORKING = setmetatable({}, {__mode = "v"});
local function lookup_cache (method, host, port, path, options)
  if(not(validate_options(options))) then
    return nil
  end
  options = options or {};
  local bypass_cache = options.bypass_cache; -- do not lookup
  local no_cache = options.no_cache; -- do not save result
  local no_cache_body = options.no_cache_body; -- do not save body
  if type(port) == "table" then port = port.number end
  local key = stdnse.get_hostname(host)..":"..port..":"..path;
  local mutex = nmap.mutex(tostring(lookup_cache)..key);
  local state = {
    mutex = mutex,
    key = key,
    method = method,
    bypass_cache = bypass_cache,
    no_cache = no_cache,
    no_cache_body = no_cache_body,
  };
  while true do
    mutex "lock";
    local record = cache[key];
    if bypass_cache or record == nil or method ~= record.method then
      WORKING[mutex] = coroutine.running();
      cache[key], state.old_record = WORKING, record;
      return nil, state;
    elseif record == WORKING then
      local working = WORKING[mutex];
      if working == nil or coroutine.status(working) == "dead" then
        cache[key] = nil; -- reset
      end
      mutex "done";
    else
      mutex "done";
      record.last_used = os.time();
      return tableaux.tcopy(record.result), state;
    end
  end
end
local function response_is_cacheable(response)
  if not response.status then
    return false
  end
  if response.status == 206 then
    return false
  end
  if response.status == 401 then
    return false
  end
  if response.truncated then
    return false
  end
  return true
end
local function insert_cache (state, response)
  local key = assert(state.key);
  local mutex = assert(state.mutex);
  if response == nil or state.no_cache or not response_is_cacheable(response) then
    cache[key] = state.old_record;
  else
    local record = {
      result = tableaux.tcopy(response),
      last_used = os.time(),
      method = state.method,
      size = type(response.body) == "string" and #response.body or 0,
    };
    response = record.result; -- only modify copy
    cache[key], cache[#cache+1] = record, record;
    if state.no_cache_body then
      response.body = "";
    end
    if type(response.body) == "string" then
      cache.size = cache.size + #response.body;
      check_size(cache);
    end
  end
  mutex "done";
end
local function request_method_needs_content_length(method)
  return method == "POST"
end
local function http_error(status_line, response)
  stdnse.debug2("HTTP response error: %s", status_line)
  return {
    status = nil,
    ["status-line"] = status_line,
    header = {},
    rawheader = {},
    body = nil,
    rawbody = nil,
    truncated = nil,
    incomplete = response
  }
end
local function build_request(host, port, method, path, options)
  if(not(validate_options(options))) then
    return nil
  end
  options = options or {}
  local mod_options = {
    header = {
      Connection = "close",
      Host = get_host_field(host, port, options.scheme),
      ["User-Agent"]  = USER_AGENT
    }
  }
  if options.cookies then
    local cookies = buildCookies(options.cookies, path)
    if #cookies > 0 then
      mod_options.header["Cookie"] = cookies
    end
  end
  if options.auth and not (options.auth.digest or options.auth.ntlm) then
    local username = options.auth.username
    local password = options.auth.password
    local credentials = "Basic " .. base64.enc(username .. ":" .. password)
    mod_options.header["Authorization"] = credentials
  end
  if options.digestauth then
    local order = {"username", "realm", "nonce", "digest-uri", "algorithm", "response", "qop", "nc", "cnonce"}
    local no_quote = {algorithm=true, qop=true, nc=true}
    local creds = {}
    for _,k in ipairs(order) do
      local v = options.digestauth[k]
      if v then
        if no_quote[k] then
          table.insert(creds, ("%s=%s"):format(k,v))
        else
          if k == "digest-uri" then
            table.insert(creds, ('%s="%s"'):format("uri",v))
          else
            table.insert(creds, ('%s="%s"'):format(k,v))
          end
        end
      end
    end
    local credentials = "Digest "..table.concat(creds, ", ")
    mod_options.header["Authorization"] = credentials
  end
  if options.ntlmauth then
    mod_options.header["Authorization"] = "NTLM " .. base64.enc(options.ntlmauth)
  end
  local body
  if type(options.content) == "table" then
    local parts = {}
    for k, v in pairs(options.content) do
      parts[#parts + 1] = url.escape(k) .. "=" .. url.escape(v)
    end
    body = table.concat(parts, "&")
    mod_options.header["Content-Type"] = "application/x-www-form-urlencoded"
  elseif options.content then
    body = options.content
  elseif request_method_needs_content_length(method) then
    body = ""
  end
  if body then
    mod_options.header["Content-Length"] = #body
  end
  table_augment(mod_options, options)
  local request_line = method.." "..path.." HTTP/1.1"
  local header = {}
  for name, value in pairs(mod_options.header) do
    header[#header + 1] = name..": "..value
  end
  return request_line .. "\r\n" .. table.concat(header, "\r\n") .. "\r\n\r\n" .. (body or "")
end
local function do_connect(host, port, data, options)
  if options.scheme == "https" or options.scheme == "http" then
    return comm.opencon(host, port, data, {
        timeout = options.timeout,
        any_af = options.any_af,
        proto = (options.scheme == "https" and "ssl" or "tcp"),
        })
  end
  return comm.tryssl(host, port, data, {timeout = options.timeout, any_af = options.any_af})
end
local function request(host, port, data, options)
  if(not(validate_options(options))) then
    return http_error("Options failed to validate.")
  end
  local method
  local response
  options = options or {}
  if type(port) == 'table' then
    if port.protocol and port.protocol ~= 'tcp' then
      stdnse.debug1("http.request() supports the TCP protocol only, your request to %s cannot be completed.", host)
      return http_error("Unsupported protocol.")
    end
  end
  method = string.match(data, "^(%S+)")
  local socket, partial, opts = do_connect(host, port, data, options)
  if not socket then
    stdnse.debug1("http.request socket error: %s", partial)
    return http_error("Error creating socket.")
  end
  repeat
    local incomplete
    response, partial, incomplete = next_response(socket, method, partial, options)
    if not response then
      return http_error("Error in next_response function; " .. partial, incomplete)
    end
  until not (response.status >= 100 and response.status <= 199)
  socket:close()
  response.ssl = ( opts == 'ssl' )
  return response
end
function generic_request(host, port, method, path, options)
  if(not(validate_options(options))) then
    return http_error("Options failed to validate.")
  end
  local digest_auth = options and options.auth and options.auth.digest
  local ntlm_auth = options and options.auth and options.auth.ntlm
  if (digest_auth or ntlm_auth) and not have_ssl then
    stdnse.debug1("http: digest and ntlm auth require openssl.")
  end
  if digest_auth and have_ssl then
    local options_with_auth_removed = tableaux.tcopy(options)
    options_with_auth_removed["auth"] = nil
    local r = generic_request(host, port, method, path, options_with_auth_removed)
    local h = r.header['www-authenticate']
    if not r.status or (h and not string.find(h:lower(), "digest.-realm")) then
      stdnse.debug1("http: the target doesn't support digest auth or there was an error during request.")
      return http_error("The target doesn't support digest auth or there was an error during request.")
    end
    local dmd5 = sasl.DigestMD5:new(h, options.auth.username, options.auth.password, method, path)
    local _, digest_table = dmd5:calcDigest()
    options.digestauth = digest_table
  end
  if ntlm_auth and have_ssl then
    local custom_options = tableaux.tcopy(options) -- to be sent with the type 1 request
    custom_options["auth"] = nil -- removing the auth options
    custom_options.timeout = nil
    local response = generic_request(host, port, method, path, custom_options)
    local authentication_header = response.header['www-authenticate']
    custom_options.timeout = options.timeout
    custom_options.truncated_ok = false
    custom_options.header = options.header or {}
    custom_options.header["Connection"] = "Keep-Alive" -- Keep-Alive headers are needed for authentication.
    if (not authentication_header) or (not response.status) or (not string.find(authentication_header:lower(), "ntlm")) then
      stdnse.debug1("http: the target doesn't support NTLM or there was an error during request.")
      return http_error("The target doesn't support NTLM or there was an error during request.")
    end
    local hostname = options.auth.hostname or "localhost" -- the hostname to be sent
    local workstation_name = options.auth.workstation_name or "NMAP" -- the workstation name to be sent
    local username = options.auth.username -- the username as specified
    local auth_blob = "NTLMSSP\x00" .. -- NTLM signature
    "\x01\x00\x00\x00" .. -- NTLM Type 1 message
    string.pack("<I4", 0xa208b207) .. -- flags 56, 128, Version, Extended Security, Always Sign, Workstation supplied, Domain Supplied, NTLM Key, OEM, Unicode
    string.pack("<I2I2I4 I2I2I4",#workstation_name, #workstation_name, 40 + #hostname, #hostname, #hostname, 40) .. -- Supplied Domain and Workstation
    string.pack("BB<I2", -- OS version info
    5, 1, 2600) .. -- 5.1.2600
    "\x00\x00\x00\x0f" .. -- OS version info end (static 0x0000000f)
    hostname.. -- HOST NAME
    workstation_name --WORKSTATION name
    custom_options.ntlmauth = auth_blob
    if type(port) == 'table' then
      if port.protocol and port.protocol ~= 'tcp' then
        stdnse.debug1("NTLM authentication supports the TCP protocol only, your request to %s cannot be completed.", host)
        return http_error("Unsupported protocol.")
      end
    end
    local socket, partial, opts = do_connect(host, port, build_request(host, port, method, path, custom_options), options)
    if not socket then
      return http_error("Could not create socket to send type 1 message.")
    end
    repeat
      local incomplete
      response, partial, incomplete = next_response(socket, method, partial, custom_options)
      if not response then
        return http_error("There was error in receiving response of type 1 message; " .. partial, incomplete)
      end
    until not (response.status >= 100 and response.status <= 199)
    authentication_header = response.header['www-authenticate']
    local type2_response = authentication_header:sub(authentication_header:find(' ')+1, -1)
    local _, message_type, _, _, _, flags_received, challenge= string.unpack("<c8 I4 I2I2I4 I4 c8", base64.dec(type2_response))
    if message_type ~= 0x02 then
      stdnse.debug1("Expected type 2 message as response.")
      return
    end
    local is_unicode  = ((flags_received & 0x00000001) == 0x00000001) -- 0x00000001 UNICODE Flag
    local is_extended = ((flags_received & 0x00080000) == 0x00080000) -- 0x00080000 Extended Security Flag
    local type_3_flags = 0xa2888206 -- flags 56, 128, Version, Target Info, Extended Security, Always Sign, NTLM Key, OEM
    local lanman, ntlm
    if is_extended then
      lanman, ntlm = smbauth.get_password_response(nil, username, "", options.auth.password, nil, "ntlmv2_session", challenge, true)
    else
      lanman, ntlm = smbauth.get_password_response(nil, username, "", options.auth.password, nil, "ntlm", challenge, false)
      type_3_flags = type_3_flags - 0x00080000 -- Removing the Extended Security Flag as server doesn't support it.
    end
    local domain = ""
    local session_key = ""
    if is_unicode then
      username = unicode.utf8to16(username)
      hostname = unicode.utf8to16(hostname)
      type_3_flags = type_3_flags - 0x00000001 -- OEM flag is 0x00000002. removing 0x00000001 results in UNICODE flag.
    end
    local BASE_OFFSET = 72 -- Version 3 -- The Session Key<empty in our case>, flags, and OS Version structure are all present.
    auth_blob = string.pack("<z I4 I2I2I4 I2I2I4 I2I2I4 I2I2I4 I2I2I4 I2I2I4 I4 BBI2",
      "NTLMSSP",
      0x00000003,
      #lanman,
      #lanman,
      BASE_OFFSET + #username + #hostname,
      ( #ntlm ),
      ( #ntlm ),
      BASE_OFFSET + #username + #hostname + #lanman,
      #domain,
      #domain,
      BASE_OFFSET,
      #username,
      #username,
      BASE_OFFSET,
      #hostname,
      #hostname,
      BASE_OFFSET + #username,
      #session_key,
      #session_key,
      BASE_OFFSET + #username + #hostname + #lanman + #ntlm,
      type_3_flags,
      5,
      1,
      2600)
    .. "\x00\x00\x00\x0f"
    .. username
    .. hostname
    .. lanman
    .. ntlm
    custom_options.ntlmauth = auth_blob
    socket:send(build_request(host, port, method, path, custom_options))
    repeat
      local incomplete
      response, partial, incomplete = next_response(socket, method, partial, options)
      if not response then
        return http_error("There was error in receiving response of type 3 message; " .. partial, incomplete)
      end
    until not (response.status >= 100 and response.status <= 199)
    socket:close()
    response.ssl = ( opts == 'ssl' )
    return response
  end
  return request(host, port, build_request(host, port, method, path, options), options)
end
function put(host, port, path, options, putdata)
  if(not(validate_options(options))) then
    return http_error("Options failed to validate.")
  end
  if ( not(putdata) ) then
    return http_error("No file to PUT.")
  end
  local mod_options = {
    content = putdata,
  }
  table_augment(mod_options, options or {})
  return generic_request(host, port, "PUT", path, mod_options)
end
local redirect_ok_rules = {
  function (url, host, port)
    return not url.userinfo
  end,
  function (url, host, port)
    local hostname = stdnse.get_hostname(host)
    if hostname == host.ip then
      return url.host == hostname
    end
    local domain = function (h) return (h:match("%..+%..+") or h):lower() end
    return domain(hostname) == domain(url.host)
  end,
  function (url, host, port)
    local url_port = url.port or get_default_port(url.scheme)
    if not url_port or url_port == port.number then
      return true
    end
    return false
  end,
  function (url, host, port)
    if url.scheme and url.port then return true end
    if url.scheme and url.scheme ~= port.service then return false end
    return true
  end,
  function (url, host, port)
    if not url.path then return false end
    if url.path == "/" and url.host == (host.targetname or host.ip) then return false end
    return true
  end,
}
function redirect_ok(host, port, counter)
  if ( "number" == type(port) ) then
    port = { number = port }
  end
  return function(url)
    if ( counter == 0 ) then return false end
    counter = counter - 1
    for i, rule in ipairs( redirect_ok_rules ) do
      if ( not(rule( url, host, port )) ) then
        return false
      end
    end
    return true
  end
end
function parse_redirect(host, port, path, response)
  if ( not(tostring(response.status):match("^30[01237]$")) or
       not(response.header) or
       not(response.header.location) ) then
    return nil
  end
  port = ( "number" == type(port) ) and { number = port } or port
  local u = url.parse(response.header.location)
  if ( not(u.host) ) then
    u.host = stdnse.get_hostname(host)
  end
  u.port = u.port or get_default_port(u.scheme) or port.number
  if ( not(u.path) ) then
    u.path = "/"
  end
  u.path = url.absolute(path, u.path)
  if ( u.query ) then
    u.path = ("%s?%s"):format( u.path, u.query )
  end
  return u
end
local function get_redirect_ok(host, port, options)
  if ( options ) then
    if ( options.redirect_ok == false ) then
      return function() return false end
    elseif( "function" == type(options.redirect_ok) ) then
      return options.redirect_ok(host, port)
    elseif( type(options.redirect_ok) == "number") then
      return redirect_ok(host, port, options.redirect_ok)
    else
      return redirect_ok(host, port, MAX_REDIRECT_COUNT)
    end
  else
    return redirect_ok(host, port, MAX_REDIRECT_COUNT)
  end
end
function get(host, port, path, options)
  if(not(validate_options(options))) then
    return http_error("Options failed to validate.")
  end
  options = options or {}
  local redir_check = get_redirect_ok(host, port, options)
  local response, state, location
  local u = { host = host, port = port, path = path }
  repeat
    response, state = lookup_cache("GET", u.host, u.port, u.path, options);
    if ( response == nil ) then
      response = generic_request(u.host, u.port, "GET", u.path, options)
      insert_cache(state, response);
    end
    u = parse_redirect(host, port, path, response)
    if ( not(u) ) then
      break
    end
    options.scheme = u.scheme or options.scheme
    location = location or {}
    table.insert(location, response.header.location)
  until( not(redir_check(u)) )
  response.location = location
  return response
end
function get_url( u, options )
  if(not(validate_options(options))) then
    return http_error("Options failed to validate.")
  end
  local parsed = url.parse( u )
  local port = {}
  port.service = parsed.scheme
  port.number = parsed.port or get_default_port(parsed.scheme) or 80
  options.scheme = options.scheme or parsed.scheme
  local path = parsed.path or "/"
  if parsed.query then
    path = path .. "?" .. parsed.query
  end
  return get( parsed.host, port, path, options )
end
function head(host, port, path, options)
  if(not(validate_options(options))) then
    return http_error("Options failed to validate.")
  end
  options = options or {}
  local redir_check = get_redirect_ok(host, port, options)
  local response, state, location
  local u = { host = host, port = port, path = path }
  repeat
    response, state = lookup_cache("HEAD", u.host, u.port, u.path, options);
    if response == nil then
      response = generic_request(u.host, u.port, "HEAD", u.path, options)
      insert_cache(state, response);
    end
    u = parse_redirect(host, port, path, response)
    if ( not(u) ) then
      break
    end
    options.scheme = u.scheme or options.scheme
    location = location or {}
    table.insert(location, response.header.location)
  until( not(redir_check(u)) )
  response.location = location
  return response
end
function post( host, port, path, options, ignored, postdata )
  if(not(validate_options(options))) then
    return http_error("Options failed to validate.")
  end
  local mod_options = {
    content = postdata,
  }
  table_augment(mod_options, options or {})
  return generic_request(host, port, "POST", path, mod_options)
end
function pGet( host, port, path, options, ignored, allReqs )
  stdnse.debug1("WARNING: pGet() is deprecated. Use pipeline_add() instead.")
  return pipeline_add(path, options, allReqs, 'GET')
end
function pHead( host, port, path, options, ignored, allReqs )
  stdnse.debug1("WARNING: pHead() is deprecated. Use pipeline_add instead.")
  return pipeline_add(path, options, allReqs, 'HEAD')
end
function addPipeline(host, port, path, options, ignored, allReqs, method)
  stdnse.debug1("WARNING: addPipeline() is deprecated! Use pipeline_add instead.")
  return pipeline_add(path, options, allReqs, method)
end
function pipeline(host, port, allReqs)
  stdnse.debug1("WARNING: pipeline() is deprecated. Use pipeline_go() instead.")
  return pipeline_go(host, port, allReqs)
end
function pipeline_add(path, options, all_requests, method)
  if not validate_options(options) then
    return nil
  end
  options = tableaux.tcopy(options or {})
  method = method or 'GET'
  all_requests = all_requests or {}
  table.insert(all_requests, {method=method, path=path, options=options})
  return all_requests
end
local function force_header (headers, header, value)
  local headers = headers or {}
  local header_lc = header:lower()
  for h in pairs(headers) do
    if h:lower() == header_lc then
      headers[h] = nil
    end
  end
  headers[header] = value
  return headers
end
function pipeline_go(host, port, all_requests)
  local responses = {}
  if (not all_requests or #all_requests == 0) then
    stdnse.debug1("Warning: empty set of requests passed to http.pipeline_go()")
    return responses
  end
  stdnse.debug1("HTTP pipeline: Total number of requests: " .. #all_requests)
  local req = all_requests[1]
  req.options.header = force_header(req.options.header, "Connection", "keep-alive")
  local reqstr = build_request(host, port, req.method, req.path, req.options)
  local socket, partial, bopt = comm.tryssl(host, port, reqstr, {recv_before=false, request_timeout=10000})
  if not socket then
    return nil
  end
  local resp
  resp, partial = next_response(socket, req.method, partial, req.options)
  if not resp then
    return responses
  end
  table.insert(responses, resp)
  local connsent = 1
  local connlimit = get_pipeline_limit(resp)
  local batchlimit = tonumber(stdnse.get_script_args("http.max-pipeline")) or connlimit
  stdnse.debug3("HTTP pipeline: connlimit=%d, batchlimit=%d", connlimit, batchlimit)
  while #responses < #all_requests do
    if connsent >= connlimit or resp.truncated or not socket:get_info() then
      socket:close()
      stdnse.debug3("HTTP pipeline: reconnecting")
      socket:connect(host, port, bopt)
      if not socket then
        return nil
      end
      socket:set_timeout(10000)
      partial = ""
      connsent = 0
    end
    if connlimit > connsent + #all_requests - #responses then
      connlimit = connsent + #all_requests - #responses
    end
    local batchsize = connlimit - connsent
    if batchsize > batchlimit then
      batchsize = batchlimit
    end
    stdnse.debug3("HTTP pipeline: batch=%d, conn=%d/%d, resp=%d/%d", batchsize, connsent, connlimit, #responses, #all_requests)
    local requests = {}
    for i = 1, batchsize do
      local req = all_requests[#responses + i]
      local connmode = connsent + i < connlimit and "keep-alive" or "close"
      req.options.header = force_header(req.options.header, "Connection", connmode)
      table.insert(requests, build_request(host, port, req.method, req.path, req.options))
    end
    socket:send(table.concat(requests))
    for i = 1, batchsize do
      local req = all_requests[#responses + 1]
      resp, partial = next_response(socket, req.method, partial, req.options)
      if not resp then
        stdnse.debug3("HTTP pipeline: response[%d]: %s", #responses + 1, partial)
        connlimit = connsent + i - 1
        if connlimit == 0 then
          stdnse.debug1("HTTP pipeline: First request on a new connection failed; giving up.");
          return responses
        end
        stdnse.debug1("HTTP pipeline: Received only %d of %d batch responses.\nDecreasing connection limit to %d.", i - 1, batchsize, connlimit)
        break
      end
      table.insert(responses, resp)
      if resp.truncated then break end
    end
    connsent = connsent + batchsize
  end
  socket:close()
  stdnse.debug1("HTTP pipeline: Number of received responses: %d", #responses)
  return responses
end
local function skip_space(s, pos)
  local _
  _, pos = string.find(s, "^[ \t]*", pos)
  return pos + 1
end
local function read_token(s, pos)
  local _, token
  pos = skip_space(s, pos)
  _, pos, token = string.find(s, "^([^\0\001-\031()<>@,;:\\\"/?={} \t%[%]\127-\255]+)", pos)
  if token then
    return pos + 1, token
  else
    return nil
  end
end
local function read_quoted_string(s, pos)
  local chars = {}
  if string.sub(s, pos, pos) ~= "\"" then
    return nil
  end
  pos = pos + 1
  pos = skip_space(s, pos)
  while pos <= #s and string.sub(s, pos, pos) ~= "\"" do
    local c
    c = string.sub(s, pos, pos)
    if c == "\\" then
      if pos < #s then
        pos = pos + 1
        c = string.sub(s, pos, pos)
      else
        return nil
      end
    end
    chars[#chars + 1] = c
    pos = pos + 1
  end
  if pos > #s or string.sub(s, pos, pos) ~= "\"" then
    return nil
  end
  return pos + 1, table.concat(chars)
end
local function read_token_or_quoted_string(s, pos)
  pos = skip_space(s, pos)
  if string.sub(s, pos, pos) == "\"" then
    return read_quoted_string(s, pos)
  else
    return read_token(s, pos)
  end
end
function tag_pattern(tag, endtag)
  if endtag then
    return "</%s*" .. stringaux.ipattern(tag) .. "%f[%s>].->"
  else
    return "<%s*" .. stringaux.ipattern(tag) .. "%f[%s/>].->"
  end
end
function grab_forms(body)
  local forms = {}
  if not body then return forms end
  local form_start_expr = tag_pattern("form")
  local form_end_expr = tag_pattern("form", true)
  local form_opening = string.find(body, form_start_expr)
  while form_opening do
    local form_closing = string.find(body, form_end_expr, form_opening+1)
    if form_closing == nil then --html code contains errors
      break
    end
    forms[#forms+1] = string.sub(body, form_opening, form_closing-1)
    if form_closing+1 <= #body then
      form_opening = string.find(body, form_start_expr, form_closing+1)
    else
      break
    end
  end
  return forms
end
local function get_attr (html, name)
  local lhtml = html:lower()
  local lname = name:lower()
  local _, pos = lhtml:find('%s' .. lname .. '%s*=%s*[^%s]')
  if not pos then
    return lhtml:match('[^%s=]%s+' .. lname .. '[%s/>]') and "" or nil
  end
  local value
  _, value = html:match('^([\'"])(.-)%1', pos)
  if not value then
    value = html:match('^[^%s<>=\'"`]+', pos)
  end
  return slaxml.parser.unescape(value)
end
function parse_form(form)
  local parsed = {}
  local fields = {}
  local form_action = get_attr(form, "action")
  if form_action then
    parsed["action"] = form_action
  end
  local form_method = get_attr(form, "method")
  if form_method then
    parsed["method"] = string.lower(form_method)
  end
  local form_id = get_attr(form, "id")
  if form_id then
    parsed["id"] = string.lower(form_id)
  end
  local input_type
  local input_name
  local input_value
  for f in string.gmatch(form, tag_pattern("input")) do
    input_type = get_attr(f, "type")
    input_name = get_attr(f, "name")
    input_value = get_attr(f, "value")
    local next_field_index = #fields+1
    if input_name then
      fields[next_field_index] = {}
      fields[next_field_index]["name"] = input_name
      if input_type then
        fields[next_field_index]["type"] = string.lower(input_type)
      end
      if input_value then
        fields[next_field_index]["value"] = input_value
      end
    end
  end
  for f in string.gmatch(form, tag_pattern("textarea")) do
    input_name = get_attr(f, "name")
    local next_field_index = #fields+1
    if input_name then
      fields[next_field_index] = {}
      fields[next_field_index]["name"] = input_name
      fields[next_field_index]["type"] = "textarea"
    end
  end
  parsed["fields"] = fields
  return parsed
end
local MONTH_MAP = {
  Jan = 1, Feb = 2, Mar = 3, Apr = 4, May = 5, Jun = 6,
  Jul = 7, Aug = 8, Sep = 9, Oct = 10, Nov = 11, Dec = 12
}
function parse_date(s)
  local day, month, year, hour, min, sec, tz, month_name
  day, month_name, year, hour, min, sec, tz = s:match("^%w+, (%d+)[- ](%w+)[- ](%d+) (%d+):(%d+):(%d+) (%w+)$")
  if not day then
    month_name, day, hour, min, sec, year = s:match("%w+ (%w+)  ?(%d+) (%d+):(%d+):(%d+) (%d+)")
    tz = "GMT"
  end
  if not day then
    stdnse.debug1("http.parse_date: can't parse date \"%s\": unknown format.", s)
    return nil
  end
  month = MONTH_MAP[month_name]
  if not month then
    stdnse.debug1("http.parse_date: unknown month name \"%s\".", month_name)
    return nil
  end
  if tz ~= "GMT" then
    stdnse.debug1("http.parse_date: don't know time zone \"%s\", only \"GMT\".", tz)
    return nil
  end
  day = tonumber(day)
  year = tonumber(year)
  hour = tonumber(hour)
  min = tonumber(min)
  sec = tonumber(sec)
  if year < 100 then
    if year < 70 then
      year = year + 2000
    else
      year = year + 1900
    end
  end
  return { year = year, month = month, day = day, hour = hour, min = min, sec = sec, isdst = false }
end
local function read_auth_challenge(s, pos)
  local scheme, params
  pos, scheme = read_token(s, pos)
  if not scheme then
    return nil
  end
  params = {}
  pos = skip_space(s, pos)
  while pos < #s do
    local name, val
    local tmp_pos
    tmp_pos = pos
    tmp_pos, name = read_token(s, tmp_pos)
    if not name then
      pos = skip_space(s, pos + 1)
      return pos, { scheme = scheme, params = nil }
    end
    tmp_pos = skip_space(s, tmp_pos)
    if string.sub(s, tmp_pos, tmp_pos) ~= "=" then
      break
    end
    tmp_pos = tmp_pos + 1
    pos = tmp_pos
    pos, val = read_token_or_quoted_string(s, pos)
    if not val then
      return nil
    end
    if params[name] then
      return nil
    end
    params[name] = val
    pos = skip_space(s, pos)
    if string.sub(s, pos, pos) == "," then
      pos = skip_space(s, pos + 1)
      if pos > #s then
        return nil
      end
    end
  end
  return pos, { scheme = scheme, params = params }
end
function parse_www_authenticate(s)
  local challenges = {}
  local pos
  pos = 1
  while pos <= #s do
    local challenge
    pos, challenge = read_auth_challenge(s, pos)
    if not challenge then
      return nil
    end
    challenges[#challenges + 1] = challenge
  end
  return challenges
end
function get_status_string(data)
  if(data == nil) then
    return "<unknown status>"
  elseif(data['status-line'] == nil) then
    if(data['status'] ~= nil) then
      return data['status']
    end
    return "<unknown status>"
  end
  local space = string.find(data['status-line'], ' ')
  if(space == nil) then
    return data['status-line']
  else
    return (string.sub(data['status-line'], space + 1)):gsub('\r?\n', '')
  end
end
function can_use_head(host, port, result_404, path)
  if(result_404 == 200) then
    return false
  end
  if(path == nil) then
    path = '/'
  end
  local data = head( host, port, path )
  if data then
    if data.status and data.status == 302 and data.header and data.header.location then
      stdnse.debug1("HTTP: Warning: Host returned 302 and not 200 when performing HEAD.")
      return false
    end
    if data.status and data.status == 200 and data.header then
      if #data.body > 0 then
        stdnse.debug1("HTTP: Warning: Host returned data when performing HEAD.")
        return false
      end
      stdnse.debug1("HTTP: Host supports HEAD.")
      return true, data
    end
    stdnse.debug1("HTTP: Didn't receive expected response to HEAD request (got %s).", get_status_string(data))
    return false
  end
  stdnse.debug1("HTTP: HEAD request completely failed.")
  return false
end
function clean_404(body)
  if ( not(body) ) then
    return
  end
  body = string.gsub(body, '%d?%d:%d%d:%d%d', "")
  body = string.gsub(body, '%d%d:%d%d', "")
  body = string.gsub(body, 'AM', "")
  body = string.gsub(body, 'am', "")
  body = string.gsub(body, 'PM', "")
  body = string.gsub(body, 'pm', "")
  body = string.gsub(body, '%d%d%d%d%d%d%d%d', "") -- 4-digit year (has to go first, because it overlaps 2-digit year)
  body = string.gsub(body, '%d%d%d%d%-%d%d%-%d%d', "")
  body = string.gsub(body, '%d%d%d%d/%d%d/%d%d', "")
  body = string.gsub(body, '%d%d%-%d%d%-%d%d%d%d', "")
  body = string.gsub(body, '%d%d%/%d%d%/%d%d%d%d', "")
  body = string.gsub(body, '%d%d%d%d%d%d', "") -- 2-digit year
  body = string.gsub(body, '%d%d%-%d%d%-%d%d', "")
  body = string.gsub(body, '%d%d%/%d%d%/%d%d', "")
  body = string.gsub(body, "/[^ ]+", "") -- Unix - remove everything from a slash till the next space
  body = string.gsub(body, "[a-zA-Z]:\\[^ ]+", "") -- Windows - remove everything from a "x:\" pattern till the next space
  if(have_ssl and nmap.debugging() == 0) then
    return openssl.md5(body)
  end
  return body
end
local function cache_404_response(host, port, response)
  if type(host) == "table" and host.registry then
    host.registry.http_404 = host.registry.http_404 or {}
    local portnum = port
    if type(port) == "table" then
      portnum = port.number
    end
    host.registry.http_404[portnum] = response
  end
  return table.unpack(response)
end
function identify_404(host, port)
  if type(host) == "table" and host.registry and host.registry.http_404 then
    local portnum = port
    if type(port) == "table" then
      portnum = port.number
    end
    local result = host.registry.http_404[portnum]
    if result then
      return table.unpack(result)
    end
  end
  local data
  local bad_responses = { 301, 302, 400, 401, 403, 499, 501, 503 }
  local URL_404_1 = '/nmaplowercheck' .. os.time(os.date('*t'))
  local URL_404_2 = '/NmapUpperCheck' .. os.time(os.date('*t'))
  local URL_404_3 = '/Nmap/folder/check' .. os.time(os.date('*t'))
  data = get(host, port, URL_404_1,{redirect_ok=false})
  if(data == nil) then
    stdnse.debug1("HTTP: Failed while testing for 404 status code")
    return false, "Failed while testing for 404 error message"
  end
  if(data.status and data.status == 404) then
    stdnse.debug1("HTTP: Host returns proper 404 result.")
    return cache_404_response(host, port, {true, 404})
  end
  if(data.status and data.status == 200) then
    stdnse.debug1("HTTP: Host returns 200 instead of 404.")
    if(data.body) then
      local data2 = get(host, port, URL_404_2)
      local data3 = get(host, port, URL_404_3)
      if(data2 == nil or data3 == nil) then
        stdnse.debug1("HTTP: Failed while testing for extra 404 error messages")
        return false, "Failed while testing for extra 404 error messages"
      end
      if(data2.status ~= 200) then
        if(type(data2.status) ~= "number") then
          data2.status = -1
        end
        stdnse.debug1("HTTP: HTTP 404 status changed for second request (became %d).", data2.status)
        return cache_404_response(host, port, {false,
            string.format("HTTP 404 status changed for second request (became %d).", data2.status)
          })
      end
      if(data3.status ~= 200) then
        if(type(data3.status) ~= "number") then
          data3.status = -1
        end
        stdnse.debug1("HTTP: HTTP 404 status changed for third request (became %d).", data3.status)
        return cache_404_response(host, port, {false,
            string.format("HTTP 404 status changed for third request (became %d).", data3.status)
          })
      end
      local clean_body  = clean_404(data.body)
      local clean_body2 = clean_404(data2.body)
      local clean_body3 = clean_404(data3.body)
      if(clean_body ~= clean_body2) then
        stdnse.debug1("HTTP: Two known 404 pages returned valid and different pages; unable to identify valid response.")
        stdnse.debug1("HTTP: If you investigate the server and it's possible to clean up the pages, please post to nmap-dev mailing list.")
        return cache_404_response(host, port, {false,
            "Two known 404 pages returned valid and different pages; unable to identify valid response."
          })
      end
      if(clean_body ~= clean_body3) then
        stdnse.debug1("HTTP: Two known 404 pages returned valid and different pages; unable to identify valid response (happened when checking a folder).")
        stdnse.debug1("HTTP: If you investigate the server and it's possible to clean up the pages, please post to nmap-dev mailing list.")
        return cache_404_response(host, port, {false,
            "Two known 404 pages returned valid and different pages; unable to identify valid response (happened when checking a folder)."
          })
      end
      cache_404_response(host, port, {true, 200, clean_body})
      return true, 200, clean_body
    end
    stdnse.debug1("HTTP: The 200 response didn't contain a body.")
    return cache_404_response(host, port, {true, 200})
  end
  for _,code in pairs(bad_responses) do
    if(data.status and data.status == code) then
      stdnse.debug1("HTTP: Host returns %s instead of 404 File Not Found.", get_status_string(data))
      return cache_404_response(host, port, {true, code})
    end
  end
  stdnse.debug1("Unexpected response returned for 404 check: %s", get_status_string(data))
  return cache_404_response(host, port, {true, data.status})
end
function page_exists(data, result_404, known_404, page, displayall)
  if(data and data.status) then
    if(data.status == 200) then
      if(result_404 == 200) then
        if(#data.body == 0) then
          stdnse.debug1("HTTP: Page returned a totally empty body; page likely doesn't exist")
          return false
        elseif(clean_404(data.body) ~= known_404) then
          stdnse.debug1("HTTP: Page returned a body that doesn't match known 404 body, therefore it exists (%s)", page)
          return true
        else
          return false
        end
      else
        stdnse.debug1("HTTP: Page was '%s', it exists! (%s)", get_status_string(data), page)
        return true
      end
    else
      if(data.status ~= 404 and data.status ~= result_404) then
        stdnse.debug1("HTTP: Page didn't match the 404 response (%s) (%s)", get_status_string(data), page)
        if(data.status == 401) then -- "Authentication Required"
          return true
        elseif(displayall) then
          return true
        end
        return false
      else
        return false
      end
    end
  else
    stdnse.debug1("HTTP: HTTP request failed (is the host still up?)")
    return false
  end
end
function response_contains(response, pattern, case_sensitive)
  local m
  if(pattern == '' or pattern == nil) then
    return true
  end
  local case = function(pattern) return string.lower(pattern or '') end
  if(case_sensitive == true) then
    case = function(pattern) return (pattern or '') end
  end
  pattern = case(pattern)
  m = {string.match(case(response['status-line']), pattern)};
  if(m and #m > 0) then
    return true, m
  end
  for _, header in pairs(response['rawheader']) do
    m = {string.match(case(header), pattern)}
    if(m and #m > 0) then
      return true, m
    end
  end
  m = {string.match(case(response['body']), pattern)}
  if(m and #m > 0) then
    return true, m
  end
  return false
end
function save_path(host, port, path, status, links_to, linked_from, contenttype)
  host = stdnse.get_hostname(host)
  if(type(port) == 'table') then
    port = port['number']
  end
  local parsed = url.parse(path)
  parsed['raw_querystring'] = parsed['query']
  if parsed['fragment'] then
    parsed['raw_querystring'] = ( parsed['raw_querystring'] or "" ) .. '#' .. parsed['fragment']
  end
  if parsed['raw_querystring'] then
    parsed['path_query'] = parsed['path'] .. '?' .. parsed['raw_querystring']
  else
    parsed['path_query'] = parsed['path']
  end
  if(parsed['raw_querystring']) then
    parsed['querystring'] = {}
    local values = stringaux.strsplit('&', parsed['raw_querystring'])
    for i, v in ipairs(values) do
      local name, value = table.unpack(stringaux.strsplit('=', v))
      parsed['querystring'][name] = value
    end
  end
  stdnse.registry_add_array({parsed['host'] or host, 'www', parsed['port'] or port, 'all_pages'}, parsed['path'])
  stdnse.registry_add_array({parsed['host'] or host, 'www', parsed['port'] or port, 'all_pages_full_query'}, parsed['path_query'])
  if(status) then
    stdnse.registry_add_array({parsed['host'] or host, 'www', parsed['port'] or port, 'status_codes', status}, parsed['path'])
  end
  if(parsed['is_folder']) then
    stdnse.registry_add_array({parsed['host'] or host, 'www', parsed['port'] or port, 'directories'}, parsed['path'])
  else
    stdnse.registry_add_array({parsed['host'] or host, 'www', parsed['port'] or port, 'files'}, parsed['path'])
  end
  if(parsed['extension']) then
    stdnse.registry_add_array({parsed['host'] or host, 'www', parsed['port'] or port, 'extensions', parsed['extension']}, parsed['path'])
  end
  if(parsed['querystring']) then
    stdnse.registry_add_array({parsed['host'] or host, 'www', parsed['port'] or port, 'cgi'}, parsed['path'])
    stdnse.registry_add_array({parsed['host'] or host, 'www', parsed['port'] or port, 'cgi_full_query'}, parsed['path_query'])
    stdnse.registry_add_array({parsed['host'] or host, 'www', parsed['port'] or port, 'cgi_querystring', parsed['path'] }, parsed['raw_querystring'])
    for key, value in pairs(parsed['querystring']) do
      stdnse.registry_add_array({parsed['host'] or host, 'www', parsed['port'] or port, 'cgi_args', parsed['path']}, parsed['querystring'])
    end
  end
  if(links_to) then
    if(type(links_to) == 'string') then
      links_to = {links_to}
    end
    for _, v in ipairs(links_to) do
      stdnse.registry_add_array({parsed['host'] or host, 'www', parsed['port'] or port, 'links_to', parsed['path_query']}, v)
    end
  end
  if(linked_from) then
    if(type(linked_from) == 'string') then
      linked_from = {linked_from}
    end
    for _, v in ipairs(linked_from) do
      stdnse.registry_add_array({parsed['host'] or host, 'www', parsed['port'] or port, 'links_to', v}, parsed['path_query'])
    end
  end
  if(contenttype) then
    stdnse.registry_add_array({parsed['host'] or host, 'www', parsed['port'] or port, 'content-type', contenttype}, parsed['path_query'])
  end
end
local unittest = require "unittest"
if not unittest.testing() then
  return _ENV
end
test_suite = unittest.TestSuite:new()
do
  local cookie_tests = {
    { name = "#1198: conflicting attribute name",
      cookie = "JSESSIONID=aaa; name=bbb; value=ccc; attr=ddd",
      parsed = {
        name = "JSESSIONID",
        value = "aaa",
        attr = "ddd",
      }
    },
    { name = "#1171: empty attribute value",
      cookie = "JSESSIONID=aaa; attr1; attr2=; attr3=",
      parsed = {
        name = "JSESSIONID",
        value = "aaa",
        attr1 = "",
        attr2 = "",
        attr3 = "",
      }
    },
    { name = "#1170: quotes present",
      cookie = "aaa=\"b\\\"bb\"; pATH = \"ddd eee\" fff",
      parsed = {
        name = "aaa",
        value = "\"b\\\"bb\"",
        path = "\"ddd eee\" fff"
      }
    },
    { name = "#1169: empty attributes",
      cookie = "JSESSIONID=aaa; ; Path=/;;Secure;",
      parsed = {
        name = "JSESSIONID",
        value = "aaa",
        path = "/",
        secure = ""
      }
    },
    { name = "#844: space in a cookie value",
      cookie = " SESSIONID = IgAAABjN8b3xxxNsLRIiSpHLPn1lE=&IgAAAxxxMT6Bw==&Huawei USG6320&langfrombrows=en-US&copyright=2014 ;secure",
      parsed = {
        name = "SESSIONID",
        value = "IgAAABjN8b3xxxNsLRIiSpHLPn1lE=&IgAAAxxxMT6Bw==&Huawei USG6320&langfrombrows=en-US&copyright=2014",
        secure = ""
      }
    },
    { name = "#866: unexpected attribute",
      cookie = " SID=c98fefa3ad659caa20b89582419bb14f; Max-Age=1200; Version=1",
      parsed = {
        name = "SID",
        value = "c98fefa3ad659caa20b89582419bb14f",
        ["max-age"] = "1200",
        version = "1"
      }
    },
    { name = "#731: trailing semicolon",
      cookie = "session_id=76ca8bc8c19;",
      parsed = {
        name = "session_id",
        value = "76ca8bc8c19"
      }
    },
    { name = "#229: comma is not a delimiter",
      cookie = "c1=aaa; path=/bbb/ccc,ddd/eee",
      parsed = {
        name = "c1",
        value = "aaa",
        path = "/bbb/ccc,ddd/eee"
      }
    },
  }
  for _, test in ipairs(cookie_tests) do
    local parsed = parse_set_cookie(test.cookie)
    test_suite:add_test(unittest.not_nil(parsed), test.name)
    if parsed then
      test_suite:add_test(unittest.keys_equal(parsed, test.parsed), test.name)
    end
  end
  local status_line_tests = {
    { name = "valid status line",
      line = "HTTP/1.0 200 OK\r\n",
      result = true,
      parsed = {
        version = "1.0",
        status = 200,
      }
    },
    { name = "malformed version in status line",
      line = "HTTP/1. 200 OK\r\n",
      result = false,
      parsed = {
        version = nil,
        status = nil,
      }
    },
    { name = "non-integer status code in status line",
      line = "HTTP/1.0 20A OK\r\n",
      result = false,
      parsed = {
        version = "1.0",
        status = nil,
      }
    },
    { name = "missing reason phrase in status line",
      line = "HTTP/1.0 200\r\n",
      result = true,
      parsed = {
        version = "1.0",
        status = 200,
      }
    },
  }
  for _, test in ipairs(status_line_tests) do
    local response = {}
    local result, error = parse_status_line(test.line, response)
    if test.result then
      test_suite:add_test(unittest.not_nil(result), test.name)
    else
      test_suite:add_test(unittest.is_nil(result), test.name)
      test_suite:add_test(unittest.not_nil(error), test.name)
    end
    test_suite:add_test(unittest.equal(response["status-line"], test.line), test.name)
    if result then
      test_suite:add_test(unittest.equal(response.status, test.parsed.status), test.name)
      test_suite:add_test(unittest.equal(response.version, test.parsed.version), test.name)
    end
  end
  local content_encoding_tests = {}
  table.insert(content_encoding_tests,
    { name = "nil encoding list",
      encoding = nil,
      source = "SomePlaintextBody",
      target = "SomePlaintextBody",
      decoded = nil,
      undecoded = nil
    })
  table.insert(content_encoding_tests,
    { name = "no encoding",
      encoding = {},
      source = "SomePlaintextBody",
      target = "SomePlaintextBody",
      decoded = {},
      undecoded = {}
    })
  table.insert(content_encoding_tests,
    { name = "identity encoding",
      encoding = "identity",
      source = "SomePlaintextBody",
      target = "SomePlaintextBody",
      decoded = {"identity"},
      undecoded = {}
    })
  if have_zlib then
    table.insert(content_encoding_tests,
      { name = "gzip encoding",
        encoding = "gzip",
        source = stdnse.fromhex("1f8b0800000000000000f348cdc9c9d75108cf2fca49510400d0c34aec0d000000"),
        target = "Hello, World!",
        decoded = {"gzip"},
        undecoded = {}
      })
    table.insert(content_encoding_tests,
      { name = "corrupted gzip encoding",
        encoding = "gzip",
        source = stdnse.fromhex("2f8b0800000000000000f348cdc9c9d75108cf2fca49510400d0c34aec0d000000"),
        target = nil,
        decoded = {},
        undecoded = {"gzip"},
        err = "Corrupted Content-Encoding: gzip",
        fragment = stdnse.fromhex("2f8b0800000000000000f348cdc9c9d75108cf2fca49510400d0c34aec0d000000")
      })
    table.insert(content_encoding_tests,
      { name = "gzip encoding with maxlen",
        encoding = "gzip",
        source = stdnse.fromhex("1f8b0800000000000000f348cdc9c9d75108cf2fca49510400d0c34aec0d000000"),
        target = "Hello, World!",
        decoded = {"gzip"},
        undecoded = {},
        maxlen = 999
      })
    table.insert(content_encoding_tests,
      { name = "gzip encoding with small maxlen",
        encoding = "gzip",
        source = stdnse.fromhex("1f8b0800000000000000f348cdc9c9d75108cf2fca49510400d0c34aec0d000000"),
        target = nil,
        decoded = {"gzip"},
        undecoded = {},
        maxlen = 4,
        err = ERR_OVERSIZED_BODY,
        fragment = "Hell"
      })
    table.insert(content_encoding_tests,
      { name = "gzip encoding with exact maxlen",
        encoding = "gzip",
        source = stdnse.fromhex("1f8b0800000000000000f348cdc9c9d75108cf2fca49510400d0c34aec0d000000"),
        target = "Hello, World!",
        decoded = {"gzip"},
        undecoded = {},
        maxlen = 13
      })
    table.insert(content_encoding_tests,
      { name = "gzip-encoded empty body",
        encoding = "gzip",
        source = "",
        target = "",
        decoded = {"gzip"},
        undecoded = {},
        maxlen = 999
      })
  end
  table.insert(content_encoding_tests,
    { name = "unknown encoding",
      encoding = "identity, mystery, miracle",
      source = stdnse.fromhex("1f8b0800000000000000f348cdc9c9d75108cf2fca49510400d0c34aec0d000000"),
      target = stdnse.fromhex("1f8b0800000000000000f348cdc9c9d75108cf2fca49510400d0c34aec0d000000"),
      decoded = {"identity"},
      undecoded = {"mystery", "miracle"}
    })
  if have_zlib then
    table.insert(content_encoding_tests,
      { name = "stacked encoding",
        encoding = "identity, gzip, identity",
        source = stdnse.fromhex("1f8b0800000000000000f348cdc9c9d75108cf2fca49510400d0c34aec0d000000"),
        target = "Hello, World!",
        decoded = {"identity", "gzip", "identity"},
        undecoded = {}
      })
  else
    table.insert(content_encoding_tests,
      { name = "stacked encoding",
        encoding = "identity, gzip, identity",
        source = stdnse.fromhex("1f8b0800000000000000f348cdc9c9d75108cf2fca49510400d0c34aec0d000000"),
        target = stdnse.fromhex("1f8b0800000000000000f348cdc9c9d75108cf2fca49510400d0c34aec0d000000"),
        decoded = {"identity"},
        undecoded = {"gzip", "identity"}
      })
  end
  for _, test in ipairs(content_encoding_tests) do
    local body, dcd, undcd, err, fragment = decode_body(test.source, test.encoding, test.maxlen)
    test_suite:add_test(unittest.equal(body, test.target), test.name .. " (body)")
    test_suite:add_test(unittest.identical(dcd, test.decoded), test.name .. " (decoded)")
    test_suite:add_test(unittest.identical(undcd, test.undecoded), test.name .. " (undecoded)")
    test_suite:add_test(unittest.equal(err, test.err), test.name .. " (err)")
    test_suite:add_test(unittest.equal(fragment, test.fragment), test.name .. " (fragment)")
  end
end
return _ENV;
