Module:Thread archive

This page is currently under development(You) VVILL NOT edit the page until it's properly finished.

This module implements {{Thread archive}}. It formats a reference to a Sharty thread: the label links to the thread's copy on archived.st, and every other capture of the thread follows as a link in parentheses.

Usage

Do not invoke this module directly. Use {{Thread archive}} instead.

Output example

[archived.st label] (Megalodon) (gyo.tc) (Ghostarchive) ...

Parameters

Parameter	Description
url, url2, url3, …	Capture or thread URLs. There is no limit, numbering does not need to be consecutive, and links appear in number order. url and url1 can both be used; url comes first.
title	Label text in place of the generated label. A ] in it is escaped so it cannot end the link.
board, thread, post	The thread's identity, for references whose URLs do not contain it (for example a Ghostarchive capture). board may be written soy or /soy/; thread and post must be numbers. board and thread are only used together, and they take priority over any identity read from the URLs.
archivedst	no suppresses the archived.st link that the module would otherwise build. It has no effect when an archived.st URL is supplied directly.
nocat	yes suppresses the tracking categories.

Labeling

In order of preference:

1. title if given.

2. /board/ Thread No. N, followed by , Post No. P. when the URL points to a post. No post is shown when the post is the OP.

3. Archived thread, fallback when the thread cannot be identified.

Main link

The label links to an archived.st URL supplied in any url parameter, used exactly as given. Otherwise, https://archived.st/board/thread/N.html is set, or when the thread number is below the floor archived.st only holds threads from around October 2026 onward. The module (nor the bot) does not check that a copy exists, it relies on never-changing thread IDs and assumes the archiver services are reliable.

Other links

Each other URL becomes a link in parantheses, labelled by its site.

Site	Label	Notes
megalodon.jp	Megalodon	A timestamped capture on either site produces both links: the supplied URL as given, plus the same capture rebuilt on the other site. The same capture supplied on both sites appears only once. A gyo.tc shortcut without a timestamp produces only its own link.
gyo.tc	gyo.tc
ghostarchive.org	Ghostarchive	The thread cannot be read from the URL. Use board and thread to give it an archived.st link.
soyjak.st	original	A live link to the thread.
archived.st	-	Used as the main link, not repeated.
any other site	its domain	Triggers unknown status, adds a tracking category.

Identical URLs appear only once. Two or more megalodon captures of the same thread taken at different times, however, are kept.

Errors

Problems never change what readers see, therefore this module suppresses all script errors to a preview warning. The warnings are as follows:

Problem	Warning
No URL and no board/thread	no URL given, nothing is rendered
A URL that does not start with http:// or https://	invalid URL, naming it, the URL is left out
A Sharty or archived.st URL that is not a thread, such as a catalog	is not a thread URL, naming it
URLs that point to different threads, or different posts of the same thread	points to a different thread or post, naming the first one that disagrees. the first thread found is used

Cats

Only added in the main namespace, and can be suppressed by nocat=yes:

• Category:Thread archive errors - used when any of the problems above happen.
• Category:Thread archive links to unknown sites - if the unknown condition is triggered due to usage of an URL from a site the configuration does not list, this will be issued.

Config

Sites, labels, floors, label text, warnings, cat names, and tracked namespaces can all be set in Module:Thread archive/config.

Currently, archivedst.minThread is a single number and applies to every board. Boards are numbered separately, so threads on boards with lower numbers (like /pol/) get no link. This will be patched later.

Notes

---

local getArgs = require('Module:Arguments').getArgs
local yesno = require('Module:yesno')

local p = {}

local function escapeUrl(url)
	return (url:gsub('[ %[%]]', { [' '] = '%20', ['['] = '%5B', [']'] = '%5D' }))
end

local function splitUrl(url)
	local scheme, auth, rest = url:match('^(%a+)://([^/?#]+)(.*)$')
	if not scheme then return nil end
	scheme = scheme:lower()
	if scheme ~= 'http' and scheme ~= 'https' then return nil end
	local host = auth:lower():gsub(':%d+$', ''):gsub('^www%.', '')
	local path, frag = rest:match('^(.-)#(.*)$')
	if not path then path = rest end
	if frag == '' then frag = nil end
	return { host = host, path = path, frag = frag }
end

local function threadIdentity(parts)
	local path = parts.path:match('^[^?]*')
	local board, thread = path:match('^/([%w_]+)/thread/(%d+)%.html$')
	if not board then
		return nil -- not a correct thread path, clear errorneous use of t but no reason to crash either
	end
	local post = parts.frag and parts.frag:match('^q?(%d+)$')
	if post == thread then post = nil end
	return { board = board, thread = thread, post = post }
end

local function classify(url, config)
	local entry = { url = url }
	local parts = splitUrl(url)
	if not parts then
		entry.kind = 'malformed'
		return entry
	end
	entry.host = parts.host
	local service = config.services[parts.host]
	if not service then
		entry.kind = 'unknown'
		entry.label = parts.host
		return entry
	end
	entry.kind = service.kind
	entry.label = service.label
	if service.kind == 'origin' or service.kind == 'archivedst' then
		entry.identity = threadIdentity(parts)
		entry.notThread = entry.identity == nil
	elseif service.kind == 'megalodon' then
		local stamp, inner = parts.path:match('^/(%d%d%d%d%-%d%d%d%d%-%d%d%d%d%-%d%d)/(.+)$')
		if not stamp then inner = parts.path:match('^/(.+)$') end
		entry.stamp = stamp
		if inner then
			entry.inner = inner
			entry.frag = parts.frag
			local innerParts = splitUrl(inner .. (parts.frag and '#' .. parts.frag or ''))
			local innerService = innerParts and config.services[innerParts.host]
			if innerService and innerService.kind == 'origin' then
				entry.identity = threadIdentity(innerParts)
				entry.notThread = entry.identity == nil
			end
		end
	end
	return entry
end

local function collectUrls(args)
	local numbered = {}
	for key, value in pairs(args) do
		if type(key) == 'string' then
			local n
			if key == 'url' then
				n = 0
			else
				n = tonumber(key:match('^url(%d+)$'))
			end
			if n then numbered[#numbered + 1] = { n = n, value = value } end
		end
	end
	table.sort(numbered, function(a, b)
		return a.n < b.n
	end)
	local urls = {}
	for _, item in ipairs(numbered) do
		urls[#urls + 1] = item.value
	end
	return urls
end

local function samefag(a, b)
	return a.board == b.board and a.thread == b.thread
end

local function resolveIdentity(args, entries)
	local identity, conflict
	local board = args.board and args.board:match('^/?([%w_]+)/?$')
	local thread = args.thread and args.thread:match('^%d+$')
	if board and thread then
		identity = { board = board, thread = thread, post = args.post and args.post:match('^%d+$') }
		if identity.post == identity.thread then identity.post = nil end
	end
	for _, entry in ipairs(entries) do
		local found = entry.identity
		if found then
			if not identity then
				identity = { board = found.board, thread = found.thread, post = found.post }
			elseif not samefag(identity, found) then
				conflict = conflict or entry.url
			elseif found.post and identity.post and found.post ~= identity.post then
				conflict = conflict or entry.url
			elseif found.post and not identity.post then
				identity.post = found.post
			end
		end
	end
	return identity, conflict
end

local function megalodonUrl(host, entry)
	local url = 'https://' .. host .. '/' .. entry.stamp .. '/' .. entry.inner
	if entry.frag then url = url .. '#' .. entry.frag end
	return url
end

local function buildLinks(entries, config)
	local links, index = {}, {}
	local function add(key, url, label, preferred)
		local position = index[key]
		if position then
			if preferred then links[position].url = url end
			return
		end
		links[#links + 1] = { url = url, label = label }
		index[key] = #links
	end
	for _, entry in ipairs(entries) do
		if entry.kind == 'megalodon' and entry.stamp then
			local key = table.concat({ 'megalodon', entry.stamp, entry.inner, entry.frag or '' }, '|')
			for _, host in ipairs(config.megalodonMirrors) do
				local own = host == entry.host
				add(
					key .. '|' .. host,
					own and entry.url or megalodonUrl(host, entry),
					config.services[host].label,
					own
				)
			end
		elseif entry.kind ~= 'malformed' and entry.kind ~= 'archivedst' then
			add(entry.url, entry.url, entry.label, true)
		end
	end
	return links
end

local function archivedUrl(args, identity, entries, config)
	for _, entry in ipairs(entries) do
		if entry.kind == 'archivedst' and entry.identity then return entry.url end
	end
	if not identity or yesno(args.archivedst) == false then return nil end
	local floor = config.archivedst.minThread -- archived.st is relatively new(?), cannot format links pre start of service, layout is a bit of a mess. not a problem for the bot, who ignores pre oct 26 entries
	if floor and tonumber(identity.thread) < floor then return nil end
	local url = 'https://'
		.. config.archivedst.host
		.. '/'
		.. identity.board
		.. '/thread/'
		.. identity.thread
		.. '.html'
	if identity.post then url = url .. '#' .. identity.post end
	return url
end

local function makeLabel(args, identity, config)
	if args.title then return (args.title:gsub('%]', '&#93;')) end
	if identity then
		local label = config.text.thread:format(identity.board, identity.thread)
		if identity.post then label = label .. config.text.post:format(identity.post) end
		return label
	end
	return config.text.fallbackCaption
end

local function track(args, flags, config)
	if yesno(args.nocat) then return '' end
	if not config.trackNamespaces[mw.title.getCurrentTitle().namespace] then return '' end
	local out = {}
	for _, key in ipairs({ 'error', 'unknownHost' }) do
		if flags[key] then out[#out + 1] = '[[Category:' .. config.categories[key] .. ']]' end
	end
	return table.concat(out)
end

function p._main(args)
	local config = mw.loadData('Module:Thread archive/config')
	local entries, flags = {}, {}
	local notThread
	for _, url in ipairs(collectUrls(args)) do
		local entry = classify(url, config)
		entries[#entries + 1] = entry
		if entry.kind == 'malformed' then
			flags.error = true
			mw.addWarning(config.errors.malformed:format(mw.text.nowiki(url)))
		elseif entry.kind == 'unknown' then
			flags.unknownHost = true
		end
		if entry.notThread then
			flags.error = true
			mw.addWarning(config.errors.notThread:format(mw.text.nowiki(url)))
		end
	end

	local identity, conflict = resolveIdentity(args, entries)
	if #entries == 0 and not identity then
		flags.error = true
		mw.addWarning(config.errors.noUrl)
		return track(args, flags, config)
	end
	if conflict then
		flags.error = true
		mw.addWarning(config.errors.conflict:format(mw.text.nowiki(conflict)))
	end

	local mainUrl = archivedUrl(args, identity, entries, config)
	local label = makeLabel(args, identity, config)
	local out = {}
	if mainUrl then
		out[1] = '[' .. escapeUrl(mainUrl) .. ' ' .. label .. ']'
	else
		out[1] = label
	end
	for _, link in ipairs(buildLinks(entries, config)) do
		out[#out + 1] = ' ([' .. escapeUrl(link.url) .. ' ' .. link.label .. '])'
	end
	return table.concat(out) .. track(args, flags, config)
end

function p.main(frame)
	return p._main(getArgs(frame, { wrappers = {'Template:Thread archive', 'User:Rickert/Template:Thread archive'} }))
end

return p