local _VERSION = _VERSION;
local MAJOR, MINOR = assert(_VERSION:match "^Lua (%d+).(%d+)$");
if tonumber(MAJOR.."."..MINOR) < 5.3 then
  error "NSE requires Lua 5.3 or newer. It looks like you're using an older version of nmap."
end
local NAME = "NSE";
local NSE_PRE_SCAN  = "NSE_PRE_SCAN";
local NSE_SCAN      = "NSE_SCAN";
local NSE_POST_SCAN = "NSE_POST_SCAN";
local YIELD = "NSE_YIELD";
local BASE = "NSE_BASE";
local WAITING_TO_RUNNING = "NSE_WAITING_TO_RUNNING";
local DESTRUCTOR = "NSE_DESTRUCTOR";
local SELECTED_BY_NAME = "NSE_SELECTED_BY_NAME";
local FORMAT_TABLE = "NSE_FORMAT_TABLE";
local FORMAT_XML = "NSE_FORMAT_XML";
local PARALLELISM = "NSE_PARALLELISM";
local ACTION_STARTING = {};
local CONCURRENCY_LIMIT = 1000;
local NSE_SCRIPT_RULES = {
  prerule = "prerule",
  hostrule = "hostrule",
  portrule = "portrule",
  postrule = "postrule",
};
local cnse, rules = ...; -- The NSE C library and Script Rules
local _G = _G;
local assert = assert;
local collectgarbage = collectgarbage;
local error = error;
local ipairs = ipairs;
local load = load;
local loadfile = loadfile;
local next = next;
local pairs = pairs;
local pcall = pcall;
local rawget = rawget;
local rawset = rawset;
local require = require;
local select = select;
local setmetatable = setmetatable;
local tonumber = tonumber;
local tostring = tostring;
local type = type;
local coroutine = require "coroutine";
local create = coroutine.create;
local resume = coroutine.resume;
local status = coroutine.status;
local yield = coroutine.yield;
local wrap = coroutine.wrap;
local debug = require "debug";
local traceback = debug.traceback;
local _R = debug.getregistry();
local io = require "io";
local lines = io.lines;
local open = io.open;
local math = require "math";
local max = math.max;
local package = require "package";
local string = require "string";
local byte = string.byte;
local find = string.find;
local format = string.format;
local gsub = string.gsub;
local lower = string.lower;
local match = string.match;
local sub = string.sub;
local upper = string.upper;
local table = require "table";
local concat = table.concat;
local insert = table.insert;
local pack = table.pack;
local remove = table.remove;
local sort = table.sort;
local unpack = table.unpack;
local os = require "os"
local time = os.time
local difftime = os.difftime
do -- Add loader to look in nselib/?.lua (nselib/ can be in multiple places)
  local function loader (lib)
    lib = lib:gsub("%.", "/"); -- change Lua "module separator" to directory separator
    local name = "nselib/"..lib..".lua";
    local type, path = cnse.fetchfile_absolute(name);
    if type == "file" then
      return assert(loadfile(path));
    else
      return "\n\tNSE failed to find "..name.." in search paths.";
    end
  end
  insert(package.searchers, 1, loader);
end
local lpeg = require "lpeg";
local U = require "lpeg-utility"
local locale = lpeg.locale;
local P = lpeg.P;
local R = lpeg.R;
local S = lpeg.S;
local V = lpeg.V;
local C = lpeg.C;
local Cb = lpeg.Cb;
local Cc = lpeg.Cc;
local Cf = lpeg.Cf;
local Cg = lpeg.Cg;
local Ct = lpeg.Ct;
local nmap = require "nmap";
local lfs = require "lfs";
local socket = require "nmap.socket";
local loop = socket.loop;
local stdnse = require "stdnse";
local strict = require "strict";
assert(_ENV == _G);
strict(_ENV);
local script_database_type, script_database_path =
    cnse.fetchfile_absolute(cnse.script_dbpath);
local script_database_update = cnse.scriptupdatedb;
local script_help = cnse.scripthelp;
local NSE_YIELD_VALUE = {};
do
  local function handle (co, status, ...)
    if status and NSE_YIELD_VALUE == ... then -- NSE has yielded the thread
      return handle(co, resume(co, yield(NSE_YIELD_VALUE)));
    else
      return status, ...;
    end
  end
  function coroutine.resume (co, ...)
    return handle(co, resume(co, ...));
  end
  local resume = coroutine.resume; -- local reference to new coroutine.resume
  local function aux_wrap (status, ...)
    if not status then
      return error(..., 2);
    else
      return ...;
    end
  end
  function coroutine.wrap (f)
    local co = create(f);
    return function (...)
      return aux_wrap(resume(co, ...));
    end
  end
end
local log_write, verbosity, debugging =
    nmap.log_write, nmap.verbosity, nmap.debugging;
local log_write_raw = cnse.log_write;
local function print_verbose (level, fmt, ...)
  if verbosity() >= assert(tonumber(level)) or debugging() > 0 then
    log_write("stdout", format(fmt, ...));
  end
end
local function print_debug (level, fmt, ...)
  if debugging() >= assert(tonumber(level)) then
    log_write("stdout", format(fmt, ...));
  end
end
local function log_error (fmt, ...)
  log_write("stderr", format(fmt, ...));
end
if ("test"):gsub(".*$", "x") == "xx" then
  log_error("Known bug in string.gsub in Lua 5.3 before 5.3.3 will cause bugs in NSE scripts.")
end
local function table_size (t)
  local n = 0; for _ in pairs(t) do n = n + 1; end return n;
end
local function loadscript (filename)
  local source = "@"..filename;
  local function ld ()
    yield [[return function (_ENV) return function (...)]];
    for line in lines(filename, 2^15) do
      yield(line);
    end
    yield [[ end end]];
    return nil;
  end
  return assert(load(wrap(ld), source, "t"))();
end
local tcopy = require "tableaux".tcopy
local function host_copy(t)
  local h = tcopy(t)
  h.registry = t.registry
  return h
end
local function K (a)
  local insensitize = Cf((P(1) / function (a) return S(lower(a)..upper(a)) end)^1, function (a, b) return a * b end);
  return assert(insensitize:match(a)) * #(V "space" + S"()," + P(-1)); -- "keyword" token
end
local REQUIRE_ERROR = {};
rawset(stdnse, "silent_require", function (...)
  local status, mod = pcall(require, ...);
  if not status then
    print_debug(1, "%s", traceback(mod));
    error(REQUIRE_ERROR)
  else
    return mod;
  end
end);
local function against_name(host, port)
  local targetname, ip, portno, ipport, against;
  if host then
    targetname = host.targetname;
    ip = host.ip;
  end
  if port then
    portno = port.number;
  end
  if ip and portno then
    ipport = ip..":"..portno;
  elseif ip then
    ipport = ip;
  end
  if targetname and ipport then
    against = targetname.." ("..ipport..")";
  elseif targetname then
    against = targetname;
  elseif ipport then
    against = ipport;
  end
  if against then
    return " against "..against
  else
    return ""
  end
end
local Script = {};
local Thread = {};
local Worker = {};
do
  function Worker:__index (key)
    return Worker[key] or self.parent[key]
  end
  function Thread:d (fmt, ...)
    local against = against_name(self.host, self.port);
    local function replace(fmt, pattern, repl)
      local r = gsub(repl, "%%", "%%%%%%%%")
      return gsub(fmt, pattern, r);
    end
    if debugging() > 1 then
      fmt = replace(fmt, "%%THREAD_AGAINST", self.info..against);
      fmt = replace(fmt, "%%THREAD", self.info);
    else
      fmt = replace(fmt, "%%THREAD_AGAINST", self.short_basename..against);
      fmt = replace(fmt, "%%THREAD", self.short_basename);
    end
    print_debug(1, fmt, ...);
  end
  function Thread:set_output(r1, r2)
    if not self.worker then
      local tab, str
      if r2 then
        tab, str = r1, tostring(r2);
      elseif type(r1) == "string" then
        tab, str = nil, r1;
      elseif r1 == nil then
        return
      else
        tab, str = r1, nil;
      end
        cnse.host_set_output(self.host, self.id, tab, str);
    end
  end
  function Thread:timed_out ()
    local host_timeout, script_timeout = false, false;
    if cnse.script_timeout and cnse.script_timeout > 0 then
      script_timeout = cnse.script_timeout < difftime(time(), self.start_time)
    end
    if self.type == "hostrule" or self.type == "portrule" then
      host_timeout = cnse.timedOut(self.host);
    end
    return script_timeout or host_timeout;
  end
  function Thread:start_time_out_clock ()
    if self.type == "hostrule" or self.type == "portrule" then
      cnse.startTimeOutClock(self.host);
    end
  end
  function Thread:stop_time_out_clock ()
    if self.type == "hostrule" or self.type == "portrule" then
      cnse.stopTimeOutClock(self.host);
    end
  end
  function Thread:start (timeouts)
    if self.host then
      timeouts[self.host] = timeouts[self.host] or {};
      timeouts[self.host][self.co] = true;
    end
    if self.worker then
      self.start_time = self.parent.start_time
    else
      self.start_time = time()
    end
  end
  function Thread:close (timeouts, result)
    self.error = result;
    if self.host then
      timeouts[self.host][self.co] = nil;
      if not next(timeouts[self.host]) then
        self:stop_time_out_clock();
        timeouts[self.host] = nil;
      end
    end
    local ch = self.close_handlers;
    for key, destructor_t in pairs(ch) do
      destructor_t.destructor(destructor_t.thread, key);
      ch[key] = nil;
    end
  end
  function Script:new_thread (rule, ...)
    local script_type = assert(NSE_SCRIPT_RULES[rule]);
    if not self[rule] then return nil end -- No rule for this script?
    local env = {
        SCRIPT_PATH = self.filename,
        SCRIPT_NAME = self.short_basename,
        SCRIPT_TYPE = script_type,
    };
    setmetatable(env, {__index = _G});
    local forced = self.forced_to_run;
    local script_closure_generator = self.script_closure_generator;
    local function main (...)
      local _ENV = env; -- change the environment
      script_closure_generator(_ENV)();
      if forced or _ENV[rule](...) then
        yield(ACTION_STARTING)
        return action(...)
      end
    end
    local co = create(main);
    local thread = {
      action_started = false,
      args = pack(...),
      close_handlers = {},
      co = co,
      env = env,
      identifier = tostring(co),
      info = format("%s M:%s", self.id, match(tostring(co), "^thread: 0?[xX]?(.*)"));
      parent = nil, -- placeholder
      script = self,
      type = script_type,
      worker = false,
      start_time = 0, --for script timeout
    };
    thread.parent = thread;
    setmetatable(thread, Thread)
    return thread;
  end
  function Thread:new_worker (main, ...)
    local co = create(main);
    print_debug(2, "%s spawning new thread (%s).", self.parent.info, tostring(co));
    local thread = {
      args = pack(...),
      close_handlers = {},
      co = co,
      info = format("%s W:%s", self.id, match(tostring(co), "^thread: 0?[xX]?(.*)"));
      parent = self,
      worker = true,
      start_time = 0,
    };
    setmetatable(thread, Worker)
    local function info ()
      return status(co), rawget(thread, "error");
    end
    return thread, info;
  end
  function Thread:resume (timeouts)
    local ok, r1, r2 = resume(self.co, unpack(self.args, 1, self.args.n));
    local status = status(self.co);
    if ok and r1 == ACTION_STARTING then
      self:d("Starting %THREAD_AGAINST.");
      self.action_started = true
      return self:resume(timeouts);
    elseif not ok then
      if debugging() > 0 then
        self:d("%THREAD_AGAINST threw an error!\n%s\n", traceback(self.co, tostring(r1)));
      else
        self:set_output("ERROR: Script execution failed (use -d to debug)");
      end
      self:close(timeouts, r1);
      return false
    elseif status == "suspended" then
      if r1 == NSE_YIELD_VALUE then
        return true
      else
        self:d("%THREAD yielded unexpectedly and cannot be resumed.");
        self:close(timeouts, "yielded unexpectedly and cannot be resumed");
        return false
      end
    elseif status == "dead" then
      if self.action_started then
        self:set_output(r1, r2);
        self:d("Finished %THREAD_AGAINST.");
      end
      self:close(timeouts);
    end
  end
  function Thread:__index (key)
    return Thread[key] or self.script[key]
  end
  local required_fields = {
    action = "function",
    categories = "table",
    dependencies = "table",
  };
  local quiet_errors = {
    [REQUIRE_ERROR] = true,
  }
  function Script.new (filename, script_params)
    local script_params = script_params or {};
    assert(type(filename) == "string", "string expected");
    if not find(filename, "%.nse$") then
      log_error(
          "Warning: Loading '%s' -- the recommended file extension is '.nse'.",
          filename);
    end
    local basename = match(filename, "([^/\\]+)$") or filename;
    local short_basename = match(filename, "([^/\\]+)%.nse$") or
        match(filename, "([^/\\]+)%.[^.]*$") or filename;
    print_debug(2, "Script %s was selected by %s%s.",
        basename,
        script_params.selection or "(unknown)",
        script_params.forced and " and forced to run" or "");
    local script_closure_generator = loadscript(filename);
    local env = {
      SCRIPT_PATH = filename,
      SCRIPT_NAME = short_basename,
      categories = {},
      dependencies = {},
    };
    setmetatable(env, {__index = _G});
    local script_closure = script_closure_generator(env);
    local co = create(script_closure); -- Create a garbage thread
    local status, e = resume(co); -- Get the globals it loads in env
    if not status then
      if quiet_errors[e] then
        print_verbose(1, "Failed to load '%s'.", filename);
        return nil;
      else
        log_error("Failed to load %s:\n%s", filename, traceback(co, e));
        error("could not load script");
      end
    end
    for f, t in pairs(required_fields) do
      local field = rawget(env, f);
      if field == nil then
        error(filename.." is missing required field: '"..f.."'");
      elseif type(field) ~= t then
        error(filename.." field '"..f.."' is of improper type '"..
            type(field).."', expected type '"..t.."'");
      end
    end
    local rules = {}
    for rule in pairs(NSE_SCRIPT_RULES) do
      local rulef = rawget(env, rule);
      assert(type(rulef) == "function" or rulef == nil,
          rule.." must be a function!");
      rules[rule] = rulef;
    end
    assert(next(rules), filename.." is missing required function: 'rule'");
    local prerule = rules.prerule;
    local hostrule = rules.hostrule;
    local portrule = rules.portrule;
    local postrule = rules.postrule;
    for i, category in ipairs(rawget(env, "categories")) do
      assert(type(category) == "string",
        filename.." has non-string entries in the 'categories' array");
    end
    for i, dependency in ipairs(rawget(env, "dependencies")) do
      assert(type(dependency) == "string",
        filename.." has non-string entries in the 'dependencies' array");
    end
    local script = {
      filename = filename,
      basename = basename,
      short_basename = short_basename,
      id = match(filename, "^.-[/\\]([^\\/]-)%.nse$") or short_basename,
      script_closure_generator = script_closure_generator,
      prerule = prerule,
      hostrule = hostrule,
      portrule = portrule,
      postrule = postrule,
      args = {n = 0};
      description = rawget(env, "description"),
      categories = rawget(env, "categories"),
      author = rawget(env, "author"),
      license = rawget(env, "license"),
      dependencies = rawget(env, "dependencies"),
      threads = {},
      selected_by_name = not not script_params.verbosity,
      forced_to_run = not not script_params.forced,
    };
    return setmetatable(script, Script)
  end
  Script.__index = Script;
end
local function check_rules (rules)
  if cnse.default and #rules == 0 then rules[1] = "default" end
  if cnse.scriptversion then rules[#rules+1] = "version" end
end
local function get_chosen_scripts (rules)
  check_rules(rules);
  local db_env = {Entry = nil};
  local db_closure = assert(loadfile(script_database_path, "t", db_env),
    "database appears to be corrupt or out of date;\n"..
    "\tplease update using: nmap --script-updatedb");
  local chosen_scripts, files_loaded = {}, {};
  local used_rules, forced_rules = {}, {};
  local function is_forced_set (str)
    local specification = match(str, "^%+(.*)$");
    if specification then
      return true, specification;
    else
      return false, str;
    end
  end
  for i, rule in ipairs(rules) do
    rule = match(rule, "^%s*(.-)%s*$"); -- strip surrounding whitespace
    local forced, rule = is_forced_set(rule);
    if rule and rule ~= "" then
      used_rules[rule] = false; -- has not been used yet
      forced_rules[rule] = forced;
      rule = gsub(rule, "\\([^\\])", "\\\\%1");
      rules[i] = rule;
    end
  end
  function db_env.Entry (script_entry)
    local categories = rawget(script_entry, "categories");
    local filename = rawget(script_entry, "filename");
    assert(type(categories) == "table" and type(filename) == "string", "script database appears corrupt, try `nmap --script-updatedb`");
    local escaped_basename = match(filename, "([^/\\]-)%.nse$") or match(filename, "([^/\\]-)$");
    local selected_by_name = false;
    local script_params = {};
    local function match_script (path)
      path = gsub(path, "%.nse$", ""); -- remove optional extension
      path = gsub(path, "[%^%$%(%)%%%.%[%]%+%-%?]", "%%%1"); -- esc magic
      path = gsub(path, "%*", ".*"); -- change to Lua wildcard
      path = "^"..path.."$"; -- anchor to beginning and end
      local found = not not find(escaped_basename, path);
      selected_by_name = selected_by_name or found;
      return found;
    end
    local T = locale {
      V "space"^0 * V "expression" * V "space"^0 * P(-1);
      expression = V "disjunct" + V "conjunct" + V "value";
      disjunct = (V "conjunct" + V "value") * V "space"^0 * K "or" * V "space"^0 * V "expression" / function (a, b) return a or b end;
      conjunct = V "value" * V "space"^0 * K "and" * V "space"^0 * V "expression" / function (a, b) return a and b end;
      value = K "not" * V "space"^0 * V "value" / function (a) return not a end +
              P "(" * V "space"^0 * V "expression" * V "space"^0 * P ")" +
              K "true" * Cc(true) +
              K "false" * Cc(false) +
              V "category" +
              V "path";
      category = K "all" * Cc(true); -- pseudo-category "all" matches everything
      path = R("\033\039", "\042\126")^1 / match_script; -- all graphical characters not '(', ')'
    };
    for i, category in ipairs(categories) do
      assert(type(category) == "string", "bad entry in script database");
      T.category = T.category + K(category) * Cc(true);
    end
    T = P(T);
    for i, rule in ipairs(rules) do
      selected_by_name = false;
      if T:match(rule) then
        used_rules[rule] = true;
        script_params.forced = not not forced_rules[rule];
        if selected_by_name then
          script_params.selection = "name"
          script_params.verbosity = true
        else
          script_params.selection = "category"
        end
        local t, path = cnse.fetchscript(filename);
        if t == "file" then
          if not files_loaded[path] then
            local script = Script.new(path, script_params)
            chosen_scripts[#chosen_scripts+1] = script;
            files_loaded[path] = true;
          end
        else
          log_error("Warning: Could not load '%s': %s", filename, path);
          break;
        end
      end
    end
  end
  db_closure(); -- Load the scripts
  for rule, loaded in pairs(used_rules) do
    if not loaded then -- attempt to load the file/directory
      local script_params = {};
      script_params.forced = not not forced_rules[rule];
      local t, path = cnse.fetchscript(rule);
      if t == nil then -- perhaps omitted the extension?
        t, path = cnse.fetchscript(rule..".nse");
      end
      if t == nil then
        if not (cnse.scriptversion and rule == "version") then
          error("'"..rule.."' did not match a category, filename, or directory");
        end
      elseif t == "file" and not files_loaded[path] then
        script_params.selection = "file path";
        script_params.verbosity = true;
        local script = Script.new(path, script_params);
        chosen_scripts[#chosen_scripts+1] = script;
        files_loaded[path] = true;
      elseif t == "directory" then
        for f in lfs.dir(path) do
          local file = path .."/".. f
          if find(file, "%.nse$") and not files_loaded[file] then
            script_params.selection = "directory";
            local script = Script.new(file, script_params);
            chosen_scripts[#chosen_scripts+1] = script;
            files_loaded[file] = true;
          end
        end
      end
    end
  end
  local name_script = {};
  for i, script in ipairs(chosen_scripts) do
    assert(name_script[script.short_basename] == nil,
      ("duplicate script ID: '%s'"):format(script.short_basename));
    name_script[script.short_basename] = script;
  end
  local chain = {}; -- chain of script names
  local function calculate_runlevel (script)
    chain[#chain+1] = script.short_basename;
    if script.runlevel == false then -- circular dependency
      error("circular dependency in chain `"..concat(chain, "->").."`");
    else
      script.runlevel = false; -- placeholder
    end
    local runlevel = 1;
    for i, dependency in ipairs(script.dependencies) do
      local s = rawget(name_script, dependency);
      if s then
        local r = tonumber(s.runlevel) or calculate_runlevel(s);
        runlevel = max(runlevel, r+1);
      end
    end
    chain[#chain] = nil;
    script.runlevel = runlevel;
    return runlevel;
  end
  for i, script in ipairs(chosen_scripts) do
    local _ = script.runlevel or calculate_runlevel(script);
  end
  return chosen_scripts;
end
local function run (threads_iter, hosts)
  local running, waiting, pending = {}, {}, {};
  local all = setmetatable({}, {__mode = "kv"}); -- base coroutine to Thread
  local current; -- The currently running Thread.
  local total = 0; -- Number of threads, for record keeping.
  local timeouts = {}; -- A list to save and to track scripts timeout.
  local num_threads = 0; -- Number of script instances currently running.
  local yielded_base = setmetatable({}, {__mode = "kv"});
  _R[YIELD] = function (co)
    yielded_base[co] = current; -- set base
    return NSE_YIELD_VALUE; -- return NSE_YIELD_VALUE
  end
  _R[BASE] = function ()
    return current and current.co;
  end
  _R[WAITING_TO_RUNNING] = function (co, ...)
    local base = yielded_base[co] or all[co]; -- translate to base thread
    if base then
      co = base.co;
      if waiting[co] then -- ignore a thread not waiting
        pending[co], waiting[co] = waiting[co], nil;
        pending[co].args = pack(...);
      end
    end
  end
  _R[DESTRUCTOR] = function (what, co, key, destructor)
    local thread = yielded_base[co] or all[co] or current;
    if thread then
      local ch = thread.close_handlers;
      if what == "add" then
        ch[key] = {
          thread = co,
          destructor = destructor
        };
      elseif what == "remove" then
        ch[key] = nil;
      end
    end
  end
  _R[SELECTED_BY_NAME] = function()
    return current and current.selected_by_name;
  end
  rawset(stdnse, "new_thread", function (main, ...)
    assert(type(main) == "function", "function expected");
    if current == nil then
      error "stdnse.new_thread can only be run from an active script"
    end
    local worker, info = current:new_worker(main, ...);
    total, all[worker.co], pending[worker.co], num_threads = total+1, worker, worker, num_threads+1;
    worker:start(timeouts);
    return worker.co, info;
  end);
  rawset(stdnse, "base", function ()
    return current and current.co;
  end);
  rawset(stdnse, "gettid", function ()
    return current and current.identifier;
  end);
  rawset(stdnse, "getid", function ()
    return current and current.id;
  end);
  rawset(stdnse, "getinfo", function ()
    return current and current.info;
  end);
  rawset(stdnse, "gethostport", function ()
    if current then
        return current.host, current.port;
    end
  end);
  rawset(stdnse, "isworker", function ()
    return current and current.worker;
  end);
  local progress = cnse.scan_progress_meter(NAME);
  while next(running) or next(waiting) or threads_iter do
    while threads_iter and num_threads < CONCURRENCY_LIMIT do
      local thread = threads_iter()
      if not thread then
        threads_iter = nil;
        break;
      end
      all[thread.co], running[thread.co], total = thread, thread, total+1;
      num_threads = num_threads + 1;
      thread:start(timeouts);
    end
    local nr, nw = table_size(running), table_size(waiting);
    if total > 0 and cnse.key_was_pressed() then
      print_verbose(1, "Active NSE Script Threads: %d (%d waiting)",
          nr+nw, nw);
      progress("printStats", 1-(nr+nw)/total);
      if debugging() >= 2 then
        for co, thread in pairs(running) do
          thread:d("Running: %THREAD_AGAINST\n\t%s",
              (gsub(traceback(co), "\n", "\n\t")));
        end
        for co, thread in pairs(waiting) do
          thread:d("Waiting: %THREAD_AGAINST\n\t%s",
              (gsub(traceback(co), "\n", "\n\t")));
        end
      elseif debugging() >= 1 then
        local display = {}
        local limit = 0
        for co, thread in pairs(running) do
          local this = display[thread.short_basename]
          if not this then
            this = {}
            limit = limit + 1
            if limit > 5 then
              break
            end
          end
          this[1] = (this[1] or 0) + 1
          display[thread.short_basename] = this
        end
        for co, thread in pairs(waiting) do
          local this = display[thread.short_basename]
          if not this then
            this = {}
            limit = limit + 1
            if limit > 5 then
              break
            end
          end
          this[2] = (this[2] or 0) + 1
          display[thread.short_basename] = this
        end
        if limit <= 5 then
          for name, stats in pairs(display) do
            print_debug(1, "Script %s: %d threads running, %d threads waiting",
              name, stats[1] or 0, stats[2] or 0)
          end
        end
      end
    elseif total > 0 and progress "mayBePrinted" then
      if verbosity() > 1 or debugging() > 0 then
        progress("printStats", 1-(nr+nw)/total);
      else
        progress("printStatsIfNecessary", 1-(nr+nw)/total);
      end
    end
    for co, thread in pairs(waiting) do
      if thread:timed_out() then
        waiting[co], all[co], num_threads = nil, nil, num_threads-1;
        thread:d("%THREAD %stimed out", thread.host
            and format("%s%s ", thread.host.ip,
                    thread.port and ":"..thread.port.number or "")
            or "");
        thread:close(timeouts, "timed out");
      end
    end
    for co, thread in pairs(running) do
      current, running[co] = thread, nil;
      thread:start_time_out_clock();
      if thread:resume(timeouts) then
        waiting[co] = thread;
      else
        all[co], num_threads = nil, num_threads-1;
      end
      current = nil;
    end
    loop(50); -- Allow nsock to perform any pending callbacks
    for co, thread in pairs(pending) do
      pending[co], running[co] = nil, thread;
    end
    collectgarbage "step";
  end
  progress "endTask";
end
local function format_table(obj, indent)
  indent = indent or "  ";
  if type(obj) == "table" then
    local mt = getmetatable(obj)
    if mt and mt["__tostring"] then
      return tostring(obj)
    end
    local lines = {};
    for _, v in ipairs(obj) do
      lines[#lines + 1] = indent .. format_table(v, indent .. "  ");
    end
    for k, v in pairs(obj) do
      if type(k) == "string" then
        lines[#lines + 1] = indent .. k .. ": " .. format_table(v, indent .. "  ");
      end
    end
    return "\n" .. concat(lines, "\n");
  else
    return tostring(obj);
  end
end
_R[FORMAT_TABLE] = format_table
local format_xml
local function format_xml_elem(obj, key)
  if key then
    key = cnse.protect_xml(tostring(key));
  end
  if type(obj) == "table" then
    cnse.xml_start_tag("table", {key=key});
    cnse.xml_newline();
  else
    cnse.xml_start_tag("elem", {key=key});
  end
  format_xml(obj);
  cnse.xml_end_tag();
  cnse.xml_newline();
end
function format_xml(obj, key)
  if type(obj) == "table" then
    for _, v in ipairs(obj) do
      format_xml_elem(v);
    end
    for k, v in pairs(obj) do
      if type(k) == "string" then
        format_xml_elem(v, k);
      end
    end
  else
    cnse.xml_write_escaped(cnse.protect_xml(tostring(obj)));
  end
end
_R[FORMAT_XML] = format_xml
local function format_nsedoc(nsedoc, indent)
  indent = indent or ""
  return gsub(nsedoc, "([^\n]+)", indent .. "%1")
end
local function nsedoc_url(id)
  return format("%s/nsedoc/scripts/%s.html", cnse.NMAP_URL, id)
end
local function script_help_normal(chosen_scripts)
  for i, script in ipairs(chosen_scripts) do
    log_write_raw("stdout", "\n");
    log_write_raw("stdout", format("%s\n", script.id));
    log_write_raw("stdout", format("Categories: %s\n", concat(script.categories, " ")));
    log_write_raw("stdout", format("%s\n", nsedoc_url(script.id)));
    if script.description then
      log_write_raw("stdout", format_nsedoc(script.description, "  "));
    end
  end
end
local function script_help_xml(chosen_scripts)
  cnse.xml_start_tag("nse-scripts");
  cnse.xml_newline();
  local t, scripts_dir, nselib_dir
  t, scripts_dir = cnse.fetchfile_absolute("scripts/")
  assert(t == 'directory', 'could not locate scripts directory');
  t, nselib_dir = cnse.fetchfile_absolute("nselib/")
  assert(t == 'directory', 'could not locate nselib directory');
  cnse.xml_start_tag("directory", { name = "scripts", path = scripts_dir });
  cnse.xml_end_tag();
  cnse.xml_newline();
  cnse.xml_start_tag("directory", { name = "nselib", path = nselib_dir });
  cnse.xml_end_tag();
  cnse.xml_newline();
  for i, script in ipairs(chosen_scripts) do
    cnse.xml_start_tag("script", { filename = script.filename });
    cnse.xml_newline();
    cnse.xml_start_tag("categories");
    for _, category in ipairs(script.categories) do
      cnse.xml_start_tag("category");
      cnse.xml_write_escaped(category);
      cnse.xml_end_tag();
    end
    cnse.xml_end_tag();
    cnse.xml_newline();
    if script.description then
      cnse.xml_start_tag("description");
      cnse.xml_write_escaped(script.description);
      cnse.xml_end_tag();
      cnse.xml_newline();
    end
    cnse.xml_end_tag();
    cnse.xml_newline();
  end
  cnse.xml_end_tag();
  cnse.xml_newline();
end
nmap.registry.args = {};
do
  local args = {};
  if cnse.scriptargsfile then
    local t, path = cnse.fetchfile_absolute(cnse.scriptargsfile)
    assert(t == 'file', format("%s is not a file", path))
    print_debug(1, "Loading script-args from file `%s'", cnse.scriptargsfile);
    args[#args+1] = assert(assert(open(path, 'r')):read "*a"):gsub(",*$", "");
  end
  if cnse.scriptargs then -- Load script arguments (--script-args)
    print_debug(1, "Arguments from CLI: %s", cnse.scriptargs);
    args[#args+1] = cnse.scriptargs;
  end
  if cnse.script_timeout and cnse.script_timeout > 0 then
    print_debug(1, "Set script-timeout as: %d seconds", cnse.script_timeout);
  end
  args = concat(args, ",");
  if #args > 0 then
    print_debug(1, "Arguments parsed: %s", args);
    local function set (t, a, b)
      if b == nil then
        insert(t, a);
        return t;
       else
        return rawset(t, a, b);
      end
    end
    local parser = locale {
      V "space"^0 * V "table" * V "space"^0,
      table = Cf(Ct "" * P "{" * V "space"^0 * (V "fieldlst")^-1 * V "space"^0 * P "}", set);
      hws = V "space" - P "\n", -- horizontal whitespace
      fieldlst = V "field" * (V "hws"^0 * S "\n," * V "space"^0 * V "field")^0;
      field = V "kv" + V "av";
      kv = Cg(V "string" * V "hws"^0 * P "=" * V "hws"^0 * V "value");
      av = Cg(V "value");
      value = V "table" + V "string";
      string = V "qstring" + V "uqstring";
      qstring = U.escaped_quote('"') + U.escaped_quote("'");
      uqstring = V "hws"^0 * C((P(1) - V "hws"^0 * S "\n,{}=")^0) * V "hws"^0; -- everything but '\n,{}=', do not capture final space
    };
    parser = assert(P(parser));
    nmap.registry.args = parser:match("{"..args.."}");
    if not nmap.registry.args then
      log_write("stdout", "args = "..args);
      error "arguments did not parse!"
    end
    if debugging() >= 2 then
      local out = {}
      rawget(stdnse, "pretty_printer")(nmap.registry.args, function (s) out[#out+1] = s end)
      print_debug(2, "%s", concat(out))
    end
  end
end
if script_database_type ~= "file" then
  print_verbose(1, "Script Database missing, will create new one.");
  script_database_update = true; -- force update
end
if script_database_update then
  log_write("stdout", "Updating rule database.");
  local t, path = cnse.fetchfile_absolute('scripts/'); -- fetch script directory
  assert(t == 'directory', 'could not locate scripts directory');
  script_database_path = path.."script.db";
  local db = assert(open(script_database_path, 'w'));
  local scripts = {};
  for f in lfs.dir(path) do
    if match(f, '%.nse$') then
      scripts[#scripts+1] = path.."/"..f;
    end
  end
  sort(scripts);
  for i, script in ipairs(scripts) do
    script = Script.new(script);
    if ( script ) then
      sort(script.categories);
      db:write('Entry { filename = "', script.basename, '", ');
      db:write('categories = {');
      for j, category in ipairs(script.categories) do
        db:write(' "', lower(category), '",');
      end
      db:write(' } }\n');
    end
  end
  db:close();
  log_write("stdout", "Script Database updated successfully.");
end
local chosen_scripts = get_chosen_scripts(rules);
print_verbose(1, "Loaded %d scripts for scanning.", #chosen_scripts);
for i, script in ipairs(chosen_scripts) do
  print_debug(2, "Loaded '%s'.", script.filename);
end
if script_help then
  script_help_normal(chosen_scripts);
  script_help_xml(chosen_scripts);
end
local function main (hosts, scantype)
  local threads, runlevels = {}, {};
  local runlevels = {};
  for i, script in ipairs(chosen_scripts) do
    runlevels[script.runlevel] = runlevels[script.runlevel] or {};
    insert(runlevels[script.runlevel], script);
  end
  if _R[PARALLELISM] > CONCURRENCY_LIMIT then
    CONCURRENCY_LIMIT = _R[PARALLELISM];
  end
  if scantype == NSE_PRE_SCAN then
    print_verbose(1, "Script Pre-scanning.");
  elseif scantype == NSE_SCAN then
    if #hosts > 1 then
      print_verbose(1, "Script scanning %d hosts.", #hosts);
    elseif #hosts == 1 then
      print_verbose(1, "Script scanning %s.", hosts[1].ip);
    end
  elseif scantype == NSE_POST_SCAN then
    print_verbose(1, "Script Post-scanning.");
  end
  for runlevel, scripts in ipairs(runlevels) do
    local function threads_iter ()
      if scantype == NSE_PRE_SCAN then
        for _, script in ipairs(scripts) do
           local thread = script:new_thread("prerule");
           if thread then
             yield(thread)
           end
        end
      elseif scantype == NSE_SCAN then
        for j, host in ipairs(hosts) do
          for _, script in ipairs(scripts) do
            local thread = script:new_thread("hostrule", host_copy(host));
            if thread then
              thread.host = host;
              yield(thread);
            end
          end
          for port in cnse.ports(host) do
            for _, script in ipairs(scripts) do
              local thread = script:new_thread("portrule", host_copy(host), tcopy(port));
              if thread then
                thread.host, thread.port = host, port;
                yield(thread);
              end
            end
          end
        end
      elseif scantype == NSE_POST_SCAN then
        for _, script in ipairs(scripts) do
          local thread = script:new_thread("postrule");
          if thread then
            yield(thread);
          end
        end
      end
    end
    print_verbose(2, "Starting runlevel %u (of %u) scan.", runlevel, #runlevels);
    run(wrap(threads_iter), hosts)
  end
  collectgarbage "collect";
end
return main;
