From 57b2cabb0f1257548f13395f480f20511f215f25 Mon Sep 17 00:00:00 2001 From: Alicia Sykes Date: Sun, 10 May 2026 07:16:04 +0100 Subject: [PATCH 01/11] fix: False positive PhishTank hit for unverified urls --- src/client/analysis/rules/threats.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/client/analysis/rules/threats.ts b/src/client/analysis/rules/threats.ts index 99a5c9d..6e74526 100644 --- a/src/client/analysis/rules/threats.ts +++ b/src/client/analysis/rules/threats.ts @@ -13,8 +13,10 @@ const threats: Analyzer = (d) => { if (Array.isArray(d.urlHaus?.urls) && d.urlHaus.urls.length) { out.push({ severity: 'critical', title: 'Listed on URLhaus malware feed' }); } - const phishUrl = d.phishTank?.url0?.in_database; - if (phishUrl === 'true' || phishUrl === true) { + const phish = d.phishTank?.url0; + const inDb = phish?.in_database === 'true' || phish?.in_database === true; + const valid = phish?.valid === 'true' || phish?.valid === true; + if (inDb && valid) { out.push({ severity: 'critical', title: 'Listed on PhishTank' }); } if (d.cloudmersive?.CleanResult === false) { From ae3251e47a2e87c0efa4954e46ec76d764e28023 Mon Sep 17 00:00:00 2001 From: Alicia Sykes Date: Sun, 10 May 2026 07:16:57 +0100 Subject: [PATCH 02/11] fix: Moz UA header --- api/_common/http.js | 4 +++- api/carbon.js | 4 ++-- api/redirects.js | 4 ++-- api/robots-txt.js | 3 ++- api/sitemap.js | 6 +----- api/status.js | 3 ++- api/txt-records.js | 10 +++++++++- 7 files changed, 21 insertions(+), 13 deletions(-) diff --git a/api/_common/http.js b/api/_common/http.js index 3fd1c2f..795792c 100644 --- a/api/_common/http.js +++ b/api/_common/http.js @@ -61,7 +61,9 @@ const wrapNetworkError = (error) => { return error; }; -const UA = 'web-check/1.0 (https://web-check.xyz)'; +export const UA = + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) ' + + 'Chrome/120.0.0.0 Safari/537.36 (compatible; web-check/1.0; +https://web-check.xyz)'; const send = async (method, url, body, opts = {}) => { const finalUrl = appendParams(url, opts.params); diff --git a/api/carbon.js b/api/carbon.js index 0ca5b63..a08d530 100644 --- a/api/carbon.js +++ b/api/carbon.js @@ -1,11 +1,11 @@ import middleware from './_common/middleware.js'; +import { UA } from './_common/http.js'; import { createLogger } from './_common/logger.js'; const log = createLogger('carbon'); const TIMEOUT = 8000; const MAX_BYTES = 10 * 1024 * 1024; -const USER_AGENT = 'Mozilla/5.0 (compatible; WebCheck/2.0; +https://web-check.xyz)'; // Sustainable Web Design model v3 constants, matches websitecarbon.com formula const KWH_PER_GB = 0.81; @@ -33,7 +33,7 @@ const fetchByteCount = async (url) => { const r = await fetch(url, { signal: AbortSignal.timeout(TIMEOUT), redirect: 'follow', - headers: { 'user-agent': USER_AGENT, accept: 'text/html,*/*;q=0.1' }, + headers: { 'user-agent': UA, accept: 'text/html,*/*;q=0.1' }, }); if (!r.ok) throw new Error(`status ${r.status}`); if (!r.body) return 0; diff --git a/api/redirects.js b/api/redirects.js index f6df1ef..4e3f0e6 100644 --- a/api/redirects.js +++ b/api/redirects.js @@ -1,9 +1,9 @@ import middleware from './_common/middleware.js'; +import { UA } from './_common/http.js'; import { upstreamError } from './_common/upstream.js'; const MAX_REDIRECTS = 12; const TIMEOUT_MS = 10000; -const USER_AGENT = 'Mozilla/5.0 (compatible; WebCheck/2.0; +https://web-check.xyz)'; // Walks the redirect chain manually, recording each Location header as got did const redirectsHandler = async (url) => { @@ -14,7 +14,7 @@ const redirectsHandler = async (url) => { const response = await fetch(current, { redirect: 'manual', signal: AbortSignal.timeout(TIMEOUT_MS), - headers: { 'user-agent': USER_AGENT }, + headers: { 'user-agent': UA }, }); if (response.status < 300 || response.status >= 400) { if (response.status >= 400) { diff --git a/api/robots-txt.js b/api/robots-txt.js index 57d7761..aa78220 100644 --- a/api/robots-txt.js +++ b/api/robots-txt.js @@ -22,7 +22,8 @@ const robotsHandler = async (url) => { const parsed = parseRobotsTxt(res.data || ''); return parsed.robots.length ? parsed : { skipped: 'No robots.txt rules found for this host' }; } catch (error) { - if (error.response?.status === 404) { + const status = error.response?.status; + if (status >= 400 && status < 500) { return { skipped: 'No robots.txt file present on this host' }; } return upstreamError(error, 'robots.txt fetch'); diff --git a/api/sitemap.js b/api/sitemap.js index ef7859e..330690b 100644 --- a/api/sitemap.js +++ b/api/sitemap.js @@ -10,11 +10,7 @@ const MAX_DEPTH = 3; const MAX_CHILD_SITEMAPS = 25; const MAX_URLS = 5000; -// Browser-ish headers so picky CDNs do not return 406/403 to the default Node UA -const HEADERS = { - 'user-agent': 'Mozilla/5.0 (compatible; web-check-bot/1.0; +https://web-check.xyz)', - accept: 'application/xml, text/xml, application/rss+xml, */*;q=0.1', -}; +const HEADERS = { accept: 'application/xml, text/xml, application/rss+xml, */*;q=0.1' }; // Reduce a target URL to its origin so child paths resolve cleanly const toOrigin = (url) => { diff --git a/api/status.js b/api/status.js index d3e37c9..4244224 100644 --- a/api/status.js +++ b/api/status.js @@ -1,6 +1,7 @@ import https from 'https'; import { performance, PerformanceObserver } from 'perf_hooks'; import middleware from './_common/middleware.js'; +import { UA } from './_common/http.js'; const statusHandler = async (url) => { if (!url) { @@ -23,7 +24,7 @@ const statusHandler = async (url) => { try { startTime = performance.now(); const response = await new Promise((resolve, reject) => { - const req = https.get(url, (res) => { + const req = https.get(url, { headers: { 'user-agent': UA } }, (res) => { let data = ''; responseCode = res.statusCode; res.on('data', (chunk) => { diff --git a/api/txt-records.js b/api/txt-records.js index 7630360..e52fb36 100644 --- a/api/txt-records.js +++ b/api/txt-records.js @@ -2,9 +2,17 @@ import dns from 'dns/promises'; import middleware from './_common/middleware.js'; import { parseTarget } from './_common/parse-target.js'; +const NO_RECORDS = new Set(['ENODATA', 'ENOTFOUND', 'NXDOMAIN']); + const txtRecordHandler = async (url) => { const { hostname } = parseTarget(url); - const txtRecords = await dns.resolveTxt(hostname); + let txtRecords; + try { + txtRecords = await dns.resolveTxt(hostname); + } catch (error) { + if (NO_RECORDS.has(error.code)) return { skipped: 'No TXT records for this host' }; + throw error; + } // Join chunks (DNS splits long records at 255 bytes), then key=value const result = {}; for (const chunks of txtRecords) { From 60d19bfd167220d12ae12076b351c0d6263fee4f Mon Sep 17 00:00:00 2001 From: Alicia Sykes Date: Sun, 10 May 2026 08:21:42 +0100 Subject: [PATCH 03/11] feat: TLS security audit can poll for retries --- api/tls-labs.js | 18 +++++----- src/client/components/misc/AdvisoryPanel.tsx | 1 + src/client/hooks/useJobs.ts | 6 +++- src/client/jobs/registry.ts | 35 +++++++++++++++++++- src/client/jobs/types.ts | 1 + 5 files changed, 50 insertions(+), 11 deletions(-) diff --git a/api/tls-labs.js b/api/tls-labs.js index 900a3a7..09ad72d 100644 --- a/api/tls-labs.js +++ b/api/tls-labs.js @@ -5,24 +5,24 @@ import { upstreamError } from './_common/upstream.js'; const SSL_LABS = 'https://api.ssllabs.com/api/v3/analyze'; -// Pull a cached SSL Labs report; skip if no fresh cache available +// Return cached report if ready, pending status while a scan is running, else skip const tlsLabsHandler = async (url) => { const { hostname } = parseTarget(url); try { const res = await httpGet(SSL_LABS, { - params: { host: hostname, fromCache: 'on', maxAge: 24, all: 'done' }, + params: { host: hostname, fromCache: 'on', maxAge: 168, all: 'done' }, timeout: 8000, headers: { 'User-Agent': 'web-check (https://web-check.xyz)' }, }); const data = res.data; - if (!data || data.status !== 'READY' || !data.endpoints?.length) { - return { - skipped: - 'No cached SSL Labs report for this host. ' + - 'Run a fresh scan at https://www.ssllabs.com/ssltest/', - }; + if (data?.status === 'READY' && data.endpoints?.length) return data; + if (data?.status === 'DNS' || data?.status === 'IN_PROGRESS') { + return { pending: true }; } - return data; + if (data?.status === 'ERROR') { + return { error: `SSL Labs: ${data.statusMessage || 'Assessment failed'}` }; + } + return { skipped: 'No SSL Labs report available for this host' }; } catch (error) { return upstreamError(error, 'SSL Labs lookup'); } diff --git a/src/client/components/misc/AdvisoryPanel.tsx b/src/client/components/misc/AdvisoryPanel.tsx index 6e869c7..b0e8e64 100644 --- a/src/client/components/misc/AdvisoryPanel.tsx +++ b/src/client/components/misc/AdvisoryPanel.tsx @@ -25,6 +25,7 @@ const META: Record = { const Wrapper = styled(Card)` margin: 0 auto; width: 95vw; + max-height: 100%; h2 { margin: 0 0 0.75rem 0; } diff --git a/src/client/hooks/useJobs.ts b/src/client/hooks/useJobs.ts index 3168f1d..1926780 100644 --- a/src/client/hooks/useJobs.ts +++ b/src/client/hooks/useJobs.ts @@ -187,7 +187,11 @@ const useJobs = (address: string, addressType: AddressType, jobs: JobSpec[]) => const budget = parseInt((import.meta.env.PUBLIC_API_TIMEOUT_LIMIT as string) || '45000', 10); const timer = setTimeout(() => { const stuck = Object.entries(stateRef.current) - .filter(([_, e]) => e?.state === 'loading') + .filter(([id, e]) => { + if (e?.state !== 'loading') return false; + const owner = jobs.find((j) => j.cards.some((c) => c.id === id)); + return !owner?.noClientTimeout; + }) .map(([id]) => id); if (!stuck.length) return; dispatch({ diff --git a/src/client/jobs/registry.ts b/src/client/jobs/registry.ts index fc463e6..3ae4091 100644 --- a/src/client/jobs/registry.ts +++ b/src/client/jobs/registry.ts @@ -53,6 +53,38 @@ const fetchAndProcess = return raw?.error ? raw : process(raw); }; +// Sleep ms, reject AbortError if signal fires +const sleep = (ms: number, signal: AbortSignal) => + new Promise((resolve, reject) => { + if (signal.aborted) return reject(new DOMException('aborted', 'AbortError')); + const onAbort = () => { + clearTimeout(timer); + reject(new DOMException('aborted', 'AbortError')); + }; + const remove = () => { + signal.removeEventListener('abort', onAbort); + resolve(); + }; + const timer = setTimeout(remove, ms); + signal.addEventListener('abort', onAbort, { once: true }); + }); + +// Build a fetcher that re-runs while the body has { pending: true } +const fetchAndPoll = (path: string) => { + const fetchOnce = fetchAndProcess(path); + const maxAttempts = 6; + const maxDuration = 30000; + return async (ctx: JobContext) => { + for (let i = 0; i < maxAttempts; i++) { + const raw = await fetchOnce(ctx); + if (!raw?.pending) return raw; + if (i === maxAttempts - 1) break; + await sleep(maxDuration, ctx.signal); + } + return { error: 'Timed-out waiting for assessment' }; + }; +}; + const card = ( id: string, title: string, @@ -148,11 +180,12 @@ export const jobs: JobSpec[] = [ { id: 'tls-labs', expectedAddressTypes: [...URL_ONLY], + noClientTimeout: true, cards: [ card('tls-security-audit', 'TLS Security Audit', ['security'], TlsSecurityAuditCard), card('tls-client-compat', 'TLS Client Compatibility', ['security'], TlsClientCompatCard), ], - fetcher: fetchAndProcess('tls-labs?url=${url}'), + fetcher: fetchAndPoll('tls-labs?url=${url}'), }, { id: 'trace-route', diff --git a/src/client/jobs/types.ts b/src/client/jobs/types.ts index 11819fe..244536e 100644 --- a/src/client/jobs/types.ts +++ b/src/client/jobs/types.ts @@ -24,6 +24,7 @@ export interface JobSpec { fetcher: (ctx: JobContext) => Promise; expectedAddressTypes?: AddressType[]; needsIp?: boolean; + noClientTimeout?: boolean; } export interface JobEntry { From 33941f7e42d16900bccc6488d511b669cb081681 Mon Sep 17 00:00:00 2001 From: Alicia Sykes Date: Sun, 10 May 2026 09:08:06 +0100 Subject: [PATCH 04/11] ref: Cleaner loading bar --- src/client/components/misc/ProgressBar.tsx | 202 +++++++++------------ 1 file changed, 90 insertions(+), 112 deletions(-) diff --git a/src/client/components/misc/ProgressBar.tsx b/src/client/components/misc/ProgressBar.tsx index 6e36997..8b13065 100644 --- a/src/client/components/misc/ProgressBar.tsx +++ b/src/client/components/misc/ProgressBar.tsx @@ -15,20 +15,12 @@ export interface LoadingJob { retry?: () => void; } -const STATUS_EMOJI: Record = { - success: 'βœ…', - loading: 'πŸ”„', - error: '❌', - 'timed-out': '⏸️', - skipped: '⏭️', -}; - -const STATE_COLOR: Record = { - success: colors.success, - loading: colors.info, - error: colors.danger, - 'timed-out': colors.warning, - skipped: colors.neutral, +const STATE_META: Record = { + success: { emoji: 'βœ…', color: colors.success }, + loading: { emoji: 'πŸ”„', color: colors.info }, + error: { emoji: '❌', color: colors.danger }, + 'timed-out': { emoji: '⏸️', color: colors.warning }, + skipped: { emoji: '⏭️', color: colors.neutral }, }; // Tally jobs by their loading state in a single pass @@ -56,6 +48,7 @@ const stateToPercent = (jobs: LoadingJob[]): Record => { const LoadCard = styled(Card)` margin: 0 auto; width: 95vw; + max-height: 100%; position: relative; `; @@ -103,6 +96,10 @@ const ProgressBarSegment = styled.div<{ color: string; width: number }>` transition: width 0.5s ease-in-out; `; +const StateLabel = styled.span<{ color: string }>` + color: ${(p) => p.color}; +`; + const Details = styled.details` summary { margin: 0.5rem 0; @@ -150,21 +147,13 @@ const Details = styled.details` } `; -const StatusInfoWrapper = styled.div` - display: flex; - justify-content: space-between; - align-items: center; - .run-status { - color: ${colors.textColorSecondary}; - margin: 0; - } -`; - const AboutPageLink = styled.a` color: ${colors.primary}; `; const SummaryContainer = styled.div` + display: flex; + align-items: center; margin: 0.5rem 0; &.error-info { color: ${colors.danger}; @@ -193,6 +182,10 @@ const SummaryContainer = styled.div` .timed-out { color: ${colors.error}; } + .elapsed { + color: ${colors.textColorSecondary}; + margin-left: auto; + } `; const ReShowRow = styled.div` @@ -295,129 +288,114 @@ interface JobListItemProps { showErrorModal: (job: LoadingJob, isInfo?: boolean) => void; } +const REASON_LABEL: Partial> = { + error: 'β–  Show Error', + 'timed-out': 'β–  Show Timeout Reason', + skipped: 'β–  Show Skip Reason', +}; + // One row in the details list, showing job state, time and any actions const JobListItem = ({ job, showJobDocs, showErrorModal }: JobListItemProps): ReactNode => { const { name, state, timeTaken, retry, error } = job; const canRetry = retry && state !== 'success' && state !== 'loading'; - const canShowError = error && (state === 'error' || state === 'timed-out' || state === 'skipped'); - + const reasonLabel = error ? REASON_LABEL[state] : undefined; return (
  • - ({state}). + ({state}) {timeTaken && state !== 'loading' ? ` Took ${timeTaken} ms` : ''} {canRetry && ( ↻ Retry )} - {canShowError && ( + {reasonLabel && ( showErrorModal(job, state === 'skipped')} > - {state === 'timed-out' ? 'β–  Show Timeout Reason' : 'β–  Show Error'} + {reasonLabel} )}
    • ); }; -// Single-line "Running X of Y / Finished in Z" status with shared elapsed time -const RunningText = ({ jobs, elapsedMs }: { jobs: LoadingJob[]; elapsedMs: number }): ReactNode => { - const total = allCardIds.length; - const done = total - jobs.filter((j) => j.state === 'loading').length; - const isDone = done >= total; - return ( -

    - {isDone ? 'Finished in ' : `Running ${done} of ${total} jobs - `} - {elapsedMs >= 10_000 ? `${(elapsedMs / 1000).toFixed(1)} s` : `${elapsedMs} ms`} -

    - ); -}; - -// Compact one-liner shown alongside the "Show Load State" button when collapsed -const LoadSummary = ({ - jobs, - elapsedMs, - onOpen, -}: { +interface LoadSummaryProps { jobs: LoadingJob[]; elapsedMs: number; onOpen: () => void; -}): ReactNode => { +} + +// Compact one-liner shown alongside the "Show Load State" button when collapsed +const LoadSummary = ({ jobs, elapsedMs, onOpen }: LoadSummaryProps): ReactNode => { const total = allCardIds.length; - const counts = countByState(jobs); - const extras: string[] = []; - if (counts.error) extras.push(`${counts.error} failed`); - if (counts['timed-out']) extras.push(`${counts['timed-out']} timed out`); - if (counts.skipped) extras.push(`${counts.skipped} skipped`); + const c = countByState(jobs); + const issues = c.error + c['timed-out'] + c.skipped; + const sec = (elapsedMs / 1000).toFixed(1); + const text = c.loading + ? `Loading ${total - c.loading} of ${total}` + (elapsedMs < 15000 ? ` (${sec}s)` : '') + : `Finished ${total} lookups in ${sec}s`; return ( - {counts.success}/{total} lookups complete - {extras.length > 0 && ( + {text} + {issues > 0 && ( <> - {' '} + {' Β· '} )} - {elapsedMs ? `, took ${(elapsedMs / 1000).toFixed(1)}s` : ''} ); }; -const pluralJobs = (n: number) => `${n} ${n === 1 ? 'job' : 'jobs'}`; +type ChipKey = Exclude; -type ChipKey = 'success' | 'skipped' | 'timed-out' | 'error'; - -const CHIPS: Record = { - success: { cls: 'success', label: 'successful' }, - skipped: { cls: 'skipped', label: 'skipped' }, - 'timed-out': { cls: 'timed-out', label: 'timed out' }, - error: { cls: 'error', label: 'failed' }, +const CHIP_LABEL: Record = { + success: 'successful', + skipped: 'skipped', + 'timed-out': 'timed out', + error: 'failed', }; -// Inline tally chip; renders nothing for zero so callers can always include it -const Chip = ({ count, cls, label }: { count: number; cls: string; label: string }) => - count > 0 ? ( - - {pluralJobs(count)} {label}{' '} - - ) : null; +interface SummaryTextProps { + jobs: LoadingJob[]; + elapsedMs: number; +} // Heading-style summary that adapts to loading, all-success and partial-failure -const SummaryText = ({ jobs }: { jobs: LoadingJob[] }): ReactNode => { +const SummaryText = ({ jobs, elapsedMs }: SummaryTextProps): ReactNode => { const total = allCardIds.length; - const counts = countByState(jobs); - const chips = (keys: ChipKey[]) => - keys.map((k) => ); - - if (counts.loading > 0) { - return ( - - - Loading {total - counts.loading} / {total} Jobs - - {chips(['skipped', 'timed-out', 'error'])} - + const c = countByState(jobs); + const isDone = c.loading === 0; + const hasIssues = c.error > 0 || c['timed-out'] > 0; + const elapsed = elapsedMs >= 10_000 ? `${(elapsedMs / 1000).toFixed(1)} s` : `${elapsedMs} ms`; + const chip = (k: ChipKey) => + c[k] > 0 && ( + + {c[k]} {c[k] === 1 ? 'job' : 'jobs'} {CHIP_LABEL[k]}{' '} + ); - } - const hasIssues = counts.error > 0 || counts['timed-out'] > 0; - if (!hasIssues) { - return ( - - {counts.success} Jobs Completed Successfully - {chips(['skipped'])} - - ); - } + const cls = !isDone ? 'loading-info' : hasIssues ? 'error-info' : 'success-info'; + const heading = !isDone + ? `Loading ${total - c.loading} / ${total} Jobs` + : !hasIssues + ? `${c.success} Jobs Completed Successfully` + : null; + const keys: ChipKey[] = !isDone + ? ['skipped', 'timed-out', 'error'] + : hasIssues + ? ['success', 'skipped', 'timed-out', 'error'] + : ['skipped']; return ( - - {chips(['success', 'skipped', 'timed-out', 'error'])} + + {heading && {heading}} + {keys.map(chip)} + {isDone ? `Done in ${elapsed}` : elapsed} ); }; @@ -442,24 +420,27 @@ const ProgressLoader = ({ loadStatus, showModal, showJobDocs }: ProgressLoaderPr return () => clearInterval(id); }, [isDone]); - // Auto-collapse once all jobs finish, leaving the "Finished in" line briefly visible + // Auto-collapse the full loader after a fixed window so it does not hog the page useEffect(() => { - if (!isDone) return; - const t = setTimeout(() => setHideLoader(true), 1500); + const t = setTimeout(() => setHideLoader(true), 15000); return () => clearTimeout(t); + }, []); + + // Also collapse as soon as every job has reached a terminal state + useEffect(() => { + if (isDone) setHideLoader(true); }, [isDone]); const colorFor = (state: LoadingState) => - state === 'success' && isDone ? colors.primary : STATE_COLOR[state]; + state === 'success' && isDone ? colors.primary : STATE_META[state].color; const showErrorModal = (job: LoadingJob, isInfo?: boolean) => { showModal( - Error Details for {job.name} + Details for {job.name}

    - The {job.name} job failed with an {job.state} state - {job.timeTaken !== undefined ? ` after ${job.timeTaken} ms` : ''}. The server responded - with the following error: + The {job.name} job ended with state '{job.state}' + {job.timeTaken !== undefined ? ` after ${job.timeTaken} ms` : ''}. Server response: 

    {job.error}
    , @@ -495,10 +476,7 @@ const ProgressLoader = ({ loadStatus, showModal, showJobDocs }: ProgressLoaderPr /> ))} - - - - +
    Show Details
      From 52b3e19d1c4e211d803c41e16385938c971496cf Mon Sep 17 00:00:00 2001 From: Alicia Sykes Date: Sun, 10 May 2026 18:26:19 +0100 Subject: [PATCH 05/11] feat: Shows more informative error message for big fails --- src/client/components/misc/NoResults.tsx | 112 +++++++++++++++++++++ src/client/components/misc/ProgressBar.tsx | 19 ++-- src/client/hooks/useJobs.ts | 12 ++- src/client/views/Results.tsx | 26 ++++- 4 files changed, 156 insertions(+), 13 deletions(-) create mode 100644 src/client/components/misc/NoResults.tsx diff --git a/src/client/components/misc/NoResults.tsx b/src/client/components/misc/NoResults.tsx new file mode 100644 index 0000000..c04d36a --- /dev/null +++ b/src/client/components/misc/NoResults.tsx @@ -0,0 +1,112 @@ +import styled from '@emotion/styled'; +import colors from 'client/styles/colors'; +import { StyledCard } from 'client/components/Form/Card'; +import Heading from 'client/components/Form/Heading'; + +const Wrapper = styled(StyledCard)` + margin: 0 auto; + width: 95vw; + display: flex; + flex-direction: column; + gap: 0.75rem; + h2 { + margin: 0; + } + p { + margin: 0; + } + .target { + font-family: var(--font-mono); + background: ${colors.background}; + padding: 0.4rem 0.6rem; + border-radius: 4px; + word-break: break-all; + align-self: flex-start; + max-width: 100%; + color: ${colors.textColor}; + } + .reasons { + margin: 0; + padding-left: 1.25rem; + color: ${colors.textColorSecondary}; + li { + padding: 0.15rem 0; + } + } + .detail { + color: ${colors.textColorSecondary}; + font-size: 0.85rem; + word-break: break-word; + } +`; + +type Kind = 'unreachable' | 'invalid' | 'api-down' | 'disabled'; + +const VARIANT: Record = { + unreachable: { + title: 'Cannot Reach This Site', + description: 'We could not resolve an IP address for this host, so checks cannot run', + reasons: [ + 'The domain might be misspelled or no longer registered', + 'The website may be offline or temporarily unreachable', + 'A DNS resolution issue may be affecting the lookup', + 'A firewall or geo-block may be preventing access', + ], + }, + invalid: { + title: 'Invalid Input', + description: 'That does not look like a valid URL or IP address, so checks cannot run', + reasons: [ + 'Enter a domain (example.com) or an IPv4 / IPv6 address', + 'Check for typos or stray characters in the input', + 'Avoid spaces and unsupported symbols in the address', + ], + }, + 'api-down': { + title: 'Service Unavailable', + description: 'Most checks failed because the Web-Check API could not be reached', + reasons: [ + 'The API may be down, restarting or rate-limited', + 'A self-hosted instance might be misconfigured or offline', + 'A network or firewall issue could be blocking the API', + ], + }, + disabled: { + title: 'Web-Check is Paused', + description: 'This instance has been temporarily disabled, so checks cannot run', + reasons: [ + 'The public instance may be paused to manage running costs', + 'A self-hosted instance may be in maintenance mode', + 'You can run your own copy from the open-source repo on GitHub', + ], + }, +}; + +interface Props { + address: string; + error?: string; + kind?: Kind; +} + +// Surface a friendly explanation when input is invalid or the host is unreachable +const NoResults = ({ address, error, kind = 'unreachable' }: Props): JSX.Element => { + const { title, description, reasons } = VARIANT[kind]; + return ( + + + {title} + +

      {description}

      + {address} +

      Possible reasons:

      +
        + {reasons.map((r) => ( +
      • {r}
        • + ))} +
      + {error && Lookup error: {error}} +       + ); +}; + +export default NoResults; diff --git a/src/client/components/misc/ProgressBar.tsx b/src/client/components/misc/ProgressBar.tsx index 8b13065..6d95d19 100644 --- a/src/client/components/misc/ProgressBar.tsx +++ b/src/client/components/misc/ProgressBar.tsx @@ -1,4 +1,4 @@ -import { useState, useEffect, type ReactNode } from 'react'; +import { useState, useEffect, useRef, useCallback, type ReactNode } from 'react'; import styled from '@emotion/styled'; import colors from 'client/styles/colors'; import Card from 'client/components/Form/Card'; @@ -420,16 +420,19 @@ const ProgressLoader = ({ loadStatus, showModal, showJobDocs }: ProgressLoaderPr return () => clearInterval(id); }, [isDone]); - // Auto-collapse the full loader after a fixed window so it does not hog the page - useEffect(() => { - const t = setTimeout(() => setHideLoader(true), 15000); - return () => clearTimeout(t); + // Auto-collapse once when 75% of jobs have settled + const autoCollapsedRef = useRef(false); + const autoCollapse = useCallback(() => { + if (autoCollapsedRef.current) return; + autoCollapsedRef.current = true; + setHideLoader(true); }, []); - // Also collapse as soon as every job has reached a terminal state useEffect(() => { - if (isDone) setHideLoader(true); - }, [isDone]); + const total = loadStatus.length || 1; + const settled = loadStatus.filter((j) => j.state !== 'loading').length; + if (settled / total >= 0.75) autoCollapse(); + }, [loadStatus, autoCollapse]); const colorFor = (state: LoadingState) => state === 'success' && isDone ? colors.primary : STATE_META[state].color; diff --git a/src/client/hooks/useJobs.ts b/src/client/hooks/useJobs.ts index 1926780..855ad5a 100644 --- a/src/client/hooks/useJobs.ts +++ b/src/client/hooks/useJobs.ts @@ -58,6 +58,7 @@ const apiBase = (import.meta.env.PUBLIC_API_ENDPOINT || '/api') as string; const useJobs = (address: string, addressType: AddressType, jobs: JobSpec[]) => { const [state, dispatch] = useReducer(reducer, initialState); const [ipAddress, setIpAddress] = useState(); + const [ipLookupError, setIpLookupError] = useState(); const startTime = useRef(Date.now()).current; const controllers = useRef>({}); const fired = useRef>(new Set()); @@ -82,8 +83,9 @@ const useJobs = (address: string, addressType: AddressType, jobs: JobSpec[]) => .then((raw: any) => { if (controller.signal.aborted) return; const timeTaken = Date.now() - startTime; - if (job.id === 'get-ip' && typeof raw === 'string') { - setIpAddress(raw); + if (job.id === 'get-ip') { + if (typeof raw === 'string') setIpAddress(raw); + else if (raw?.error) setIpLookupError(raw.error); return; } if (raw?.skipped) { @@ -104,6 +106,7 @@ const useJobs = (address: string, addressType: AddressType, jobs: JobSpec[]) => if (controller.signal.aborted || err?.name === 'AbortError') return; const timeTaken = Date.now() - startTime; const message = err?.message || 'Unknown error'; + if (job.id === 'get-ip') return; const outcome = isTimeout(message) ? 'timed-out' : 'error'; dispatch({ type: 'error', cardIds, outcome, error: message, timeTaken }); cardIds.forEach((id) => logJobOutcome(outcome, id, timeTaken, message)); @@ -138,6 +141,7 @@ const useJobs = (address: string, addressType: AddressType, jobs: JobSpec[]) => if (!address || addressType === 'empt' || addressType === 'err') return; fired.current.clear(); + setIpLookupError(undefined); if (addressType === 'ipV4' || addressType === 'ipV6') setIpAddress(address); else setIpAddress(undefined); @@ -183,7 +187,7 @@ const useJobs = (address: string, addressType: AddressType, jobs: JobSpec[]) => // Single client-side budget for stuck jobs, resets on new input useEffect(() => { - if (!address) return; + if (!address || addressType === 'empt' || addressType === 'err') return; const budget = parseInt((import.meta.env.PUBLIC_API_TIMEOUT_LIMIT as string) || '45000', 10); const timer = setTimeout(() => { const stuck = Object.entries(stateRef.current) @@ -219,7 +223,7 @@ const useJobs = (address: string, addressType: AddressType, jobs: JobSpec[]) => [jobs, runJob, state, ipAddress], ); - return { state, retry }; + return { state, retry, ipLookupError }; }; export default useJobs; diff --git a/src/client/views/Results.tsx b/src/client/views/Results.tsx index 7db23cf..e971028 100644 --- a/src/client/views/Results.tsx +++ b/src/client/views/Results.tsx @@ -18,11 +18,13 @@ import ProgressBar, { import ActionButtons from 'client/components/misc/ActionButtons'; import AdditionalResources from 'client/components/misc/AdditionalResources'; import AdvisoryPanel from 'client/components/misc/AdvisoryPanel'; +import NoResults from 'client/components/misc/NoResults'; import ResultsMasonryGrid from 'client/components/misc/ResultsMasonryGrid'; import ViewRaw from 'client/components/misc/ViewRaw'; import { determineAddressType, type AddressType } from 'client/utils/address-type-checker'; import { hasData } from 'client/utils/result-processor'; +import keys from 'client/utils/get-keys'; import useJobs from 'client/hooks/useJobs'; import { jobs, allCards, allCardIds } from 'client/jobs/registry'; import { runAnalysis } from 'client/analysis/registry'; @@ -81,7 +83,7 @@ const Results = (props: { address?: string }): JSX.Element => { if (addressType === 'empt') setAddressType(determineAddressType(address)); }, [address, addressType]); - const { state: jobsState, retry } = useJobs(address, addressType, jobs); + const { state: jobsState, retry, ipLookupError } = useJobs(address, addressType, jobs); // Shape useJobs state for the existing ProgressBar contract const loadingJobs: LoadingJob[] = useMemo( @@ -137,6 +139,26 @@ const Results = (props: { address?: string }): JSX.Element => { const findings = useMemo(() => runAnalysis(jobsState), [jobsState]); + // Detect a catastrophic API outage when the bulk of settled jobs error or time out + const apiUnreachable = useMemo(() => { + const entries = Object.values(jobsState); + const settled = entries.filter((e) => e?.state !== 'loading'); + const dead = settled.filter((e) => e?.state === 'error' || e?.state === 'timed-out'); + return settled.length >= entries.length / 2 && dead.length / settled.length >= 0.9; + }, [jobsState]); + + // Pick the highest-priority error state, if any + let errorKind: 'invalid' | 'unreachable' | 'api-down' | 'disabled' | null = null; + if (keys.disableEverything) { + errorKind = 'disabled'; + } else if (addressType === 'err') { + errorKind = 'invalid'; + } else if (ipLookupError) { + errorKind = 'unreachable'; + } else if (apiUnreachable) { + errorKind = 'api-down'; + } + const jumpToCard = (id: string) => { const el = document.getElementById(`card-${id}`); if (!el) return; @@ -161,6 +183,7 @@ const Results = (props: { address?: string }): JSX.Element => { )} + {errorKind && } j.state !== 'loading').length < 5} /> @@ -192,6 +215,7 @@ const Results = (props: { address?: string }): JSX.Element => { }))} /> + setModalOpen(false)}> {modalContent} From bf431dcf449bf42eee28b2ebd8106f95145bcbcb Mon Sep 17 00:00:00 2001 From: Alicia Sykes Date: Sun, 10 May 2026 18:59:20 +0100 Subject: [PATCH 06/11] feat: Protocol no longer required when searching --- src/client/utils/address-type-checker.ts | 9 +++++++ src/client/views/Home.tsx | 17 +++++-------- src/client/views/Results.tsx | 9 +++++-- src/components/homepage/AnimatedInput.astro | 13 +++++++++- src/components/homepage/HeroForm.astro | 28 +++------------------ src/pages/check/[...target].astro | 17 +++++++------ 6 files changed, 46 insertions(+), 47 deletions(-) diff --git a/src/client/utils/address-type-checker.ts b/src/client/utils/address-type-checker.ts index 9856014..94be8b7 100644 --- a/src/client/utils/address-type-checker.ts +++ b/src/client/utils/address-type-checker.ts @@ -42,3 +42,12 @@ export const determineAddressType = (address: string | undefined): AddressType = if (isUrl(address)) return 'url'; return 'err'; }; + +// Strip protocol and path/query/hash so the route param stays a bare host +export const normalizeAddress = (input: string | undefined): string => { + if (!input) return ''; + let s = input.trim().replace(/^https?:\/\//i, ''); + const stop = s.search(/[/?#]/); + if (stop !== -1) s = s.slice(0, stop); + return s; +}; diff --git a/src/client/views/Home.tsx b/src/client/views/Home.tsx index 7ac2de6..37c295b 100644 --- a/src/client/views/Home.tsx +++ b/src/client/views/Home.tsx @@ -11,7 +11,7 @@ import FancyBackground from 'client/components/misc/FancyBackground'; import docs from 'client/utils/docs'; import colors from 'client/styles/colors'; -import { determineAddressType } from 'client/utils/address-type-checker'; +import { determineAddressType, normalizeAddress } from 'client/utils/address-type-checker'; const HomeContainer = styled.section` display: flex; @@ -140,7 +140,7 @@ const SiteFeaturesWrapper = styled(StyledCard)` `; const Home = (): JSX.Element => { - const defaultPlaceholder = 'e.g. https://duck.com/'; + const defaultPlaceholder = 'e.g. duck.com'; const [userInput, setUserInput] = useState(''); const [errorMsg, setErrMsg] = useState(''); const [placeholder] = useState(defaultPlaceholder); @@ -149,18 +149,17 @@ const Home = (): JSX.Element => { const location = useLocation(); - /* Redirect strait to results, if somehow we land on /check?url=[] */ useEffect(() => { const query = new URLSearchParams(location.search); const urlFromQuery = query.get('url'); if (urlFromQuery) { - navigate(`/check/${encodeURIComponent(urlFromQuery)}`, { replace: true }); + const target = normalizeAddress(urlFromQuery); + if (target) navigate(`/check/${target}`, { replace: true }); } }, [navigate, location.search]); - /* Check is valid address, either show err or redirect to results page */ const submit = () => { - let address = userInput.endsWith('/') ? userInput.slice(0, -1) : userInput; + const address = normalizeAddress(userInput); const addressType = determineAddressType(address); if (addressType === 'empt') { @@ -168,12 +167,8 @@ const Home = (): JSX.Element => { } else if (addressType === 'err') { setErrMsg('Must be a valid URL, IPv4 or IPv6 Address'); } else { - // if the addressType is 'url' and address doesn't start with 'http://' or 'https://', prepend 'https://' - if (addressType === 'url' && !/^https?:\/\//i.test(address)) { - address = 'https://' + address; - } const resultRouteParams: NavigateOptions = { state: { address, addressType } }; - navigate(`/check/${encodeURIComponent(address)}`, resultRouteParams); + navigate(`/check/${address}`, resultRouteParams); } }; diff --git a/src/client/views/Results.tsx b/src/client/views/Results.tsx index e971028..03c857c 100644 --- a/src/client/views/Results.tsx +++ b/src/client/views/Results.tsx @@ -58,7 +58,8 @@ const ResultsContent = styled.section` const makeSiteName = (address: string): string => { try { - return new URL(address).hostname.replace('www.', ''); + const withScheme = /^https?:\/\//i.test(address) ? address : `https://${address}`; + return new URL(withScheme).hostname.replace(/^www\./, ''); } catch { return address; } @@ -175,7 +176,11 @@ const Results = (props: { address?: string }): JSX.Element => { {address && ( {addressType === 'url' && ( - + )} diff --git a/src/components/homepage/AnimatedInput.astro b/src/components/homepage/AnimatedInput.astro index 2f3c528..5a65bd5 100644 --- a/src/components/homepage/AnimatedInput.astro +++ b/src/components/homepage/AnimatedInput.astro @@ -11,7 +11,18 @@ const placeholders = [ ---
      - +
      { diff --git a/src/components/homepage/HeroForm.astro b/src/components/homepage/HeroForm.astro index 7010651..08e24f5 100644 --- a/src/components/homepage/HeroForm.astro +++ b/src/components/homepage/HeroForm.astro @@ -28,37 +28,15 @@ import Screenshots from './Screenshots.astro';
      diff --git a/src/pages/check/[...target].astro b/src/pages/check/[...target].astro index 1747203..b4767fb 100644 --- a/src/pages/check/[...target].astro +++ b/src/pages/check/[...target].astro @@ -2,6 +2,7 @@ import BaseLayout from '@layouts/Base.astro'; import Main from '../../client/main.tsx'; import '../../client/styles/index.css'; +import { normalizeAddress } from '../../client/utils/address-type-checker'; export const prerender = false; @@ -10,7 +11,8 @@ const { search } = new URL(Astro.request.url); const searchUrl = new URLSearchParams(search).get('url'); if (searchUrl) { - Astro.redirect(`/check/${encodeURIComponent(searchUrl)}`); + const target = normalizeAddress(searchUrl); + if (target) Astro.redirect(`/check/${target}`); } --- @@ -27,22 +29,21 @@ if (searchUrl) { From 3ae307e7df323fca07bd3b5bbc55ff26e74bc61e Mon Sep 17 00:00:00 2001 From: Alicia Sykes Date: Sun, 10 May 2026 19:14:44 +0100 Subject: [PATCH 07/11] feat: Adds skip reasons for IP looksups on non-IP jobs --- src/client/components/misc/ProgressBar.tsx | 3 ++- src/client/hooks/useJobs.ts | 29 ++++++++++++++++------ 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/client/components/misc/ProgressBar.tsx b/src/client/components/misc/ProgressBar.tsx index 6d95d19..e4c2659 100644 --- a/src/client/components/misc/ProgressBar.tsx +++ b/src/client/components/misc/ProgressBar.tsx @@ -438,12 +438,13 @@ const ProgressLoader = ({ loadStatus, showModal, showJobDocs }: ProgressLoaderPr state === 'success' && isDone ? colors.primary : STATE_META[state].color; const showErrorModal = (job: LoadingJob, isInfo?: boolean) => { + const detailsLabel = job.state === 'skipped' ? 'Reason:' : 'Server response:'; showModal( Details for {job.name}

      The {job.name} job ended with state '{job.state}' - {job.timeTaken !== undefined ? ` after ${job.timeTaken} ms` : ''}. Server response: + {job.timeTaken !== undefined ? ` after ${job.timeTaken} ms` : ''}. {detailsLabel} 

      {job.error}
      , diff --git a/src/client/hooks/useJobs.ts b/src/client/hooks/useJobs.ts index 855ad5a..2bab730 100644 --- a/src/client/hooks/useJobs.ts +++ b/src/client/hooks/useJobs.ts @@ -115,9 +115,9 @@ const useJobs = (address: string, addressType: AddressType, jobs: JobSpec[]) => [address, startTime], ); - const skipJob = useCallback((job: JobSpec) => { + const skipJob = useCallback((job: JobSpec, reason?: string) => { const cardIds = job.cards.map((c) => c.id); - if (cardIds.length) dispatch({ type: 'skipped', cardIds }); + if (cardIds.length) dispatch({ type: 'skipped', cardIds, reason }); }, []); // Decide which jobs are eligible for the current input @@ -132,10 +132,25 @@ const useJobs = (address: string, addressType: AddressType, jobs: JobSpec[]) => [addressType], ); + const skipReason = useCallback( + (job: JobSpec): string => { + const allowed = job.expectedAddressTypes; + if (allowed && !allowed.includes(addressType)) { + if (addressType === 'ipV4' || addressType === 'ipV6') { + return 'This check requires a domain name and cannot be run against an IP address'; + } + return `This check is only available for ${allowed.join(', ')} input`; + } + return 'This check is not applicable for the current input'; + }, + [addressType], + ); + // Initial fan-out: fire non-IP jobs immediately, mark unsupported as skipped useEffect(() => { if (keys.disableEverything) { - jobs.forEach((j) => skipJob(j)); + const reason = 'Web-Check has been temporarily disabled on this instance'; + jobs.forEach((j) => skipJob(j, reason)); return; } if (!address || addressType === 'empt' || addressType === 'err') return; @@ -147,7 +162,7 @@ const useJobs = (address: string, addressType: AddressType, jobs: JobSpec[]) => jobs.forEach((job) => { if (!eligible(job)) { - skipJob(job); + skipJob(job, skipReason(job)); return; } if (job.needsIp) return; @@ -158,7 +173,7 @@ const useJobs = (address: string, addressType: AddressType, jobs: JobSpec[]) => Object.values(controllers.current).forEach((c) => c.abort()); controllers.current = {}; }; - }, [address, addressType, jobs, runJob, skipJob, eligible]); + }, [address, addressType, jobs, runJob, skipJob, eligible, skipReason]); // Fire IP-dependent jobs the moment we have an IP, but only once each useEffect(() => { @@ -166,12 +181,12 @@ const useJobs = (address: string, addressType: AddressType, jobs: JobSpec[]) => jobs.forEach((job) => { if (!job.needsIp || fired.current.has(job.id)) return; if (!eligible(job)) { - skipJob(job); + skipJob(job, skipReason(job)); return; } runJob(job, ipAddress); }); - }, [ipAddress, jobs, runJob, skipJob, eligible]); + }, [ipAddress, jobs, runJob, skipJob, eligible, skipReason]); // Promote any card whose fallback resolves after the primary failed useEffect(() => { From def5cf1b1a9da192760903a10ef6fa2f405b4531 Mon Sep 17 00:00:00 2001 From: Alicia Sykes Date: Sun, 10 May 2026 20:08:27 +0100 Subject: [PATCH 08/11] ref: Bump timeout --- vercel.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vercel.json b/vercel.json index 22d403e..a01c459 100644 --- a/vercel.json +++ b/vercel.json @@ -8,7 +8,7 @@ ], "functions": { "api/*.js": { - "maxDuration": 20 + "maxDuration": 45 } }, "env": { From 823d76f509d707271d12c3794456f693ce90bd52 Mon Sep 17 00:00:00 2001 From: Alicia Sykes Date: Sun, 10 May 2026 20:08:44 +0100 Subject: [PATCH 09/11] ref: Less failures for archive and quality checks --- api/archives.js | 16 ++++----- api/quality.js | 2 +- src/client/analysis/rules/quality.ts | 1 - src/client/components/Results/Archives.tsx | 8 ++--- src/client/jobs/registry.ts | 38 +++++++++++++++------- 5 files changed, 37 insertions(+), 28 deletions(-) diff --git a/api/archives.js b/api/archives.js index 4818c2a..830760e 100644 --- a/api/archives.js +++ b/api/archives.js @@ -47,32 +47,32 @@ const getScanFrequency = (firstScan, lastScan, totalScans, changeCount) => { }; const wayBackHandler = async (url) => { - const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${url}&output=json&fl=timestamp,statuscode,digest,length,offset`; + // collapse=timestamp:8 returns one row per archived day, slashing payloads + // (Wikipedia: 25MB/373k rows -> 428KB/6k rows) without losing first/last/change counts + const cdxUrl = + `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(url)}` + + `&output=json&fl=timestamp,statuscode,digest,length&collapse=timestamp:8`; try { const { data } = await httpGet(cdxUrl); - // Check there's data if (!data || !Array.isArray(data) || data.length <= 1) { return { skipped: 'Site has never before been archived via the Wayback Machine' }; } - // Remove the header row data.shift(); - // Process and return the results const firstScan = convertTimestampToDate(data[0][0]); const lastScan = convertTimestampToDate(data[data.length - 1][0]); - const totalScans = data.length; + const daysArchived = data.length; const changeCount = countPageChanges(data); return { firstScan, lastScan, - totalScans, + daysArchived, changeCount, averagePageSize: getAveragePageSize(data), - scanFrequency: getScanFrequency(firstScan, lastScan, totalScans, changeCount), - scans: data, + scanFrequency: getScanFrequency(firstScan, lastScan, daysArchived, changeCount), scanUrl: url, }; } catch (err) { diff --git a/api/quality.js b/api/quality.js index f0347f8..ee84db8 100644 --- a/api/quality.js +++ b/api/quality.js @@ -8,7 +8,7 @@ const qualityHandler = async (url) => { const endpoint = `https://www.googleapis.com/pagespeedonline/v5/runPagespeed?` + `url=${encodeURIComponent(url)}&category=PERFORMANCE&category=ACCESSIBILITY` + - `&category=BEST_PRACTICES&category=SEO&category=PWA&strategy=mobile` + + `&category=BEST_PRACTICES&category=SEO&strategy=mobile` + `&key=${auth.value}`; let data; diff --git a/src/client/analysis/rules/quality.ts b/src/client/analysis/rules/quality.ts index b88d09e..a354cef 100644 --- a/src/client/analysis/rules/quality.ts +++ b/src/client/analysis/rules/quality.ts @@ -5,7 +5,6 @@ const LABELS: Record = { accessibility: 'Accessibility', 'best-practices': 'Best Practices', seo: 'SEO', - pwa: 'PWA', }; // Convert a 0..1 lighthouse score to a severity bucket diff --git a/src/client/components/Results/Archives.tsx b/src/client/components/Results/Archives.tsx index 25a3e68..1530acb 100644 --- a/src/client/components/Results/Archives.tsx +++ b/src/client/components/Results/Archives.tsx @@ -18,14 +18,10 @@ const ArchivesCard = (props: { data: any; title: string; actionButtons: any }): - + - {data.scanFrequency?.scansPerDay > 1 ? ( - - ) : ( - - )} + View historical versions of this page{' '} diff --git a/src/client/jobs/registry.ts b/src/client/jobs/registry.ts index 3ae4091..00b53cf 100644 --- a/src/client/jobs/registry.ts +++ b/src/client/jobs/registry.ts @@ -69,22 +69,36 @@ const sleep = (ms: number, signal: AbortSignal) => signal.addEventListener('abort', onAbort, { once: true }); }); -// Build a fetcher that re-runs while the body has { pending: true } -const fetchAndPoll = (path: string) => { +// Re-run fetchOnce while shouldRetry(raw) holds, sleeping delay ms between attempts +const retrying = ( + path: string, + shouldRetry: (raw: any) => boolean, + attempts: number, + delay: number, + onExhausted: (last: any) => any, +) => { const fetchOnce = fetchAndProcess(path); - const maxAttempts = 6; - const maxDuration = 30000; return async (ctx: JobContext) => { - for (let i = 0; i < maxAttempts; i++) { - const raw = await fetchOnce(ctx); - if (!raw?.pending) return raw; - if (i === maxAttempts - 1) break; - await sleep(maxDuration, ctx.signal); + let last: any; + for (let i = 0; i < attempts; i++) { + last = await fetchOnce(ctx); + if (!shouldRetry(last)) return last; + if (i < attempts - 1) await sleep(delay, ctx.signal); } - return { error: 'Timed-out waiting for assessment' }; + return onExhausted(last); }; }; +// Re-run while the body has { pending: true } +const fetchAndPoll = (path: string) => + retrying(path, (r) => !!r?.pending, 6, 30000, () => ({ + error: 'Timed-out waiting for assessment', + })); + +// Re-run on transient errors, returning the last error if all attempts fail +const fetchAndRetry = (path: string) => + retrying(path, (r) => !!r?.error, 3, 2000, (last) => last); + const card = ( id: string, title: string, @@ -130,7 +144,7 @@ export const jobs: JobSpec[] = [ id: 'quality', expectedAddressTypes: [...URL_ONLY], cards: [card('quality', 'Quality Summary', ['client'], LighthouseCard)], - fetcher: fetchAndProcess('quality?url=${url}'), + fetcher: fetchAndRetry('quality?url=${url}'), }, { id: 'tech-stack', @@ -239,7 +253,7 @@ export const jobs: JobSpec[] = [ id: 'archives', expectedAddressTypes: [...URL_ONLY], cards: [card('archives', 'Archive History', ['meta'], ArchivesCard)], - fetcher: fetchAndProcess('archives?url=${url}'), + fetcher: fetchAndRetry('archives?url=${url}'), }, { id: 'rank', From cd112f88fee4f382d5d00c299610fcd24d078b1c Mon Sep 17 00:00:00 2001 From: Alicia Sykes Date: Sun, 10 May 2026 20:09:46 +0100 Subject: [PATCH 10/11] fix: formatting --- src/client/jobs/registry.ts | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/client/jobs/registry.ts b/src/client/jobs/registry.ts index 00b53cf..63ff2e8 100644 --- a/src/client/jobs/registry.ts +++ b/src/client/jobs/registry.ts @@ -91,13 +91,25 @@ const retrying = ( // Re-run while the body has { pending: true } const fetchAndPoll = (path: string) => - retrying(path, (r) => !!r?.pending, 6, 30000, () => ({ - error: 'Timed-out waiting for assessment', - })); + retrying( + path, + (r) => !!r?.pending, + 6, + 30000, + () => ({ + error: 'Timed-out waiting for assessment', + }), + ); // Re-run on transient errors, returning the last error if all attempts fail const fetchAndRetry = (path: string) => - retrying(path, (r) => !!r?.error, 3, 2000, (last) => last); + retrying( + path, + (r) => !!r?.error, + 3, + 2000, + (last) => last, + ); const card = ( id: string, From c4f65e4b44c8f55291ef6889dd3194e71c033ce2 Mon Sep 17 00:00:00 2001 From: Alicia Sykes Date: Sun, 10 May 2026 20:27:40 +0100 Subject: [PATCH 11/11] ref: Remove per-job timeouts --- api/dnssec.js | 1 - api/hsts.js | 6 ------ api/rank.js | 5 +---- api/shodan.js | 4 +--- api/tls-labs.js | 1 - 5 files changed, 2 insertions(+), 15 deletions(-) diff --git a/api/dnssec.js b/api/dnssec.js index 6ef983b..0664799 100644 --- a/api/dnssec.js +++ b/api/dnssec.js @@ -7,7 +7,6 @@ const queryDns = async (domain, type) => { const res = await httpGet('https://dns.google/resolve', { params: { name: domain, type }, headers: { Accept: 'application/dns-json' }, - timeout: 5000, }); return res.data; }; diff --git a/api/hsts.js b/api/hsts.js index 63c6f07..a0c307b 100644 --- a/api/hsts.js +++ b/api/hsts.js @@ -22,18 +22,12 @@ const evaluate = (header) => { return verdict('Site is compatible with the HSTS preload list!', true, header); }; -const REQUEST_TIMEOUT = 5000; - const hstsHandler = async (url) => new Promise((resolve) => { const req = https.request(url, (res) => { resolve(evaluate(res.headers['strict-transport-security'])); res.resume(); }); - req.setTimeout(REQUEST_TIMEOUT, () => { - req.destroy(); - resolve({ error: 'HSTS check timed out' }); - }); req.on('error', (e) => resolve({ error: `HSTS check failed: ${e.message}` })); req.end(); }); diff --git a/api/rank.js b/api/rank.js index b5d45d5..b44c874 100644 --- a/api/rank.js +++ b/api/rank.js @@ -10,10 +10,7 @@ const rankHandler = async (url) => { ? { auth: { username: TRANCO_USERNAME, password: TRANCO_API_KEY } } : {}; try { - const response = await httpGet(`https://tranco-list.eu/api/ranks/domain/${domain}`, { - timeout: 5000, - ...auth, - }); + const response = await httpGet(`https://tranco-list.eu/api/ranks/domain/${domain}`, auth); if (!response.data?.ranks?.length) { return { skipped: `${domain} isn't ranked in the top 1 million sites yet`, diff --git a/api/shodan.js b/api/shodan.js index 0615681..51a2eea 100644 --- a/api/shodan.js +++ b/api/shodan.js @@ -9,9 +9,7 @@ const shodanHandler = async (url) => { if (auth.skipped) return auth; const { hostname } = parseTarget(url); try { - const res = await httpGet(`https://api.shodan.io/shodan/host/${hostname}?key=${auth.value}`, { - timeout: 8000, - }); + const res = await httpGet(`https://api.shodan.io/shodan/host/${hostname}?key=${auth.value}`); return res.data; } catch (error) { return upstreamError(error, 'Shodan lookup'); diff --git a/api/tls-labs.js b/api/tls-labs.js index 09ad72d..19bd983 100644 --- a/api/tls-labs.js +++ b/api/tls-labs.js @@ -11,7 +11,6 @@ const tlsLabsHandler = async (url) => { try { const res = await httpGet(SSL_LABS, { params: { host: hostname, fromCache: 'on', maxAge: 168, all: 'done' }, - timeout: 8000, headers: { 'User-Agent': 'web-check (https://web-check.xyz)' }, }); const data = res.data;