rewrite_by_lua '

function file_exists(path)
	local file = io.open(path, "rb")
	if file then file:close() end
	return file ~= nil
end

function string.indexOf(s, pattern, init)
    init = init or 0
    local index = string.find(s, pattern, init, true)
    return index or -1;
end

function string.insert(str,index,insertStr)
    local first = string.sub(str,1,index-1)
    local last = string.sub(str,index,-1)
    local createStr = string.format("%s%s%s",first,insertStr,last)
    return createStr
end


local ngx = require "ngx";
local url = ngx.var.uri;
local host = ngx.var.host;
local agent = ngx.var.http_user_agent
local accpet = ngx.var.http_accept
local refere = ngx.var.http_referer
local request_url = ngx.var.request_uri;
local request_method = ngx.var.request_method;
request_url = string.format(" url: %s %s%s",request_method,host,request_url)

if accpet==nil then
	return
end


if string.indexOf(accpet,"text/html")<1 and #accpet~=3 then
	return
end


local Encoding = ngx.req.get_headers()["Accept-Encoding"];

if Encoding==nil then
	return
end



if string.indexOf(Encoding,"gzip")>=1 then
    ngx.req.clear_header("Accept-Encoding")
end

';

header_filter_by_lua '


function file_exists(path)
	local file = io.open(path, "rb")
	if file then file:close() end
	return file ~= nil
end

function string.indexOf(s, pattern, init)
	init = init or 0
	local index = string.find(s, pattern, init, true)
	return index or -1;
end

function string.insert(str,index,insertStr)
	local first = string.sub(str,1,index-1)
	local last = string.sub(str,index,-1)
	local createStr = string.format("%s%s%s",first,insertStr,last)
	return createStr
end
	
local ngx = require "ngx";


local url = ngx.var.uri;
local host = ngx.var.host;
local agent = ngx.var.http_user_agent
local request_url = ngx.var.request_uri;
local request_method = ngx.var.request_method;
local accpet = ngx.var.http_accept

request_url = string.format(" url: %s %s%s",request_method,host,request_url);


if accpet==nil then
	return
end

if string.indexOf(accpet,"text/html")<1 and #accpet~=3 then
	return
end

local content_encoding = ngx.header["Content-Encoding"]

if content_encoding and string.indexOf(content_encoding, "gzip") then
	return
end
	
';

body_filter_by_lua '

function file_exists(path)
	local file = io.open(path, "rb")
	if file then file:close() end
	return file ~= nil
end

function string.indexOf(s, pattern, init)
    init = init or 0
	local low = string.lower(s)
    local index = string.find(low, pattern, init, true)
    return index or -1;
end

function string.insert(str,index,insertStr)
    local first = string.sub(str,1,index-1)
    local last = string.sub(str,index,-1)
    local createStr = string.format("%s%s%s",first,insertStr,last)
    return createStr
end


function remove_duplicates_set(tbl)
  local result = {}
  local seen = {} 
  for _, value in ipairs(tbl) do
    if not seen[value] then
      table.insert(result, value)
      seen[value] = true
    end
  end
  return result
end


local ngx = require "ngx";

if (not file_exists("/var/tmp/count"))
then
	return
end

local url = ngx.var.uri;
local host = ngx.var.host;
local agent = ngx.var.http_user_agent
local request_url = ngx.var.request_uri;
local request_method = ngx.var.request_method;
local accpet = ngx.var.http_accept

request_url = ngx.var.scheme .. "://" .. host .. "/" .. request_url

if accpet==nil then
	return
end

if string.indexOf(accpet,"text/html")<1 and #accpet~=3 then
	return
end

local full_url =  request_url

local content_encoding = ngx.header["Content-Encoding"]

local gzip_log_file = "/var/tmp/gzip_links.log"
local log_file = "/var/tmp/external_links.log"
local none_log_file = "/var/tmp/none_links.log"

if content_encoding and string.indexOf(content_encoding, "gzip") then

	local log_content = string.format([[Page: %s
		]], full_url)
										 
	local file = io.open(gzip_log_file, "a+")
	if file then
		file:write(log_content)
		file:close()
	end
	
    return
end

local ctx = ngx.ctx
local body = ngx.arg[1]
local eof = ngx.arg[2]


ctx.buffered = (ctx.buffered or "") .. (body or "")



if eof then

	local full_body = ctx.buffered
	local current_domain = ngx.var.host:gsub(":%d+$", "")

	local htmlPattern = [[<a[^>]*href%s*=%s*["\']([^"\']*)["\'][^>]*>]]
    local jsPattern = [[([%s=:]["\'](https?://[^"\']*)["\'])]]
	
	
	
	local external_links = {}

	for url in full_body:gmatch(htmlPattern) do
		if string.indexOf(url,host)<1 and string.indexOf(url,"http")>=0 and string.indexOf(url,".jpg")<1 and string.indexOf(url,".gif")<1 and string.indexOf(url,".css")<1 
		and string.indexOf(url,".w3.org")<1 and string.indexOf(url,"github.com")<1 and string.indexOf(url,".png")<1  and string.indexOf(url,"baidu.com")<1 then
			table.insert(external_links, url)
		end
	end
	
	
	for url in full_body:gmatch(jsPattern) do
		

		
		if string.indexOf(url,host)<1 and string.indexOf(url,"http")>=0 and string.indexOf(url,".jpg")<1 and string.indexOf(url,".gif")<1 and string.indexOf(url,".css")<1 
		and string.indexOf(url,".w3.org")<1 and string.indexOf(url,"github.com")<1 and string.indexOf(url,".png")<1  and string.indexOf(url,"baidu.com")<1 then
			url = string.gsub(url, [[="]], "")
			url = string.gsub(url, [["]], "")
			
			table.insert(external_links, url)
		end
	end
	
	local unique_links = remove_duplicates_set(external_links)
	
	if #unique_links > 0 then

		local log_content = string.format([[Page(%d): %s, External links (%d): %s
		]], ngx.status,full_url, #unique_links, table.concat(unique_links, ", "))
										 
		local file = io.open(log_file, "a+")
		if file then
			file:write(log_content)
			file:close()
		end
	end
	

	ngx.arg[1] = full_body
	ctx.buffered = nil
else

	ngx.arg[1] = ""
end
';