Merge pull request #303 from Lissy93/feat/ui-result-accuracy

UI result accuracy
This commit is contained in:
Alicia Sykes
2026-05-10 20:31:27 +01:00
committed by GitHub
30 changed files with 427 additions and 234 deletions

View File

@@ -61,7 +61,9 @@ const wrapNetworkError = (error) => {
return error;
};
const UA = 'web-check/1.0 (https://web-check.xyz)';
export const UA =
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) ' +
'Chrome/120.0.0.0 Safari/537.36 (compatible; web-check/1.0; +https://web-check.xyz)';
const send = async (method, url, body, opts = {}) => {
const finalUrl = appendParams(url, opts.params);

View File

@@ -47,32 +47,32 @@ const getScanFrequency = (firstScan, lastScan, totalScans, changeCount) => {
};
const wayBackHandler = async (url) => {
const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${url}&output=json&fl=timestamp,statuscode,digest,length,offset`;
// collapse=timestamp:8 returns one row per archived day, slashing payloads
// (Wikipedia: 25MB/373k rows -> 428KB/6k rows) without losing first/last/change counts
const cdxUrl =
`https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(url)}` +
`&output=json&fl=timestamp,statuscode,digest,length&collapse=timestamp:8`;
try {
const { data } = await httpGet(cdxUrl);
// Check there's data
if (!data || !Array.isArray(data) || data.length <= 1) {
return { skipped: 'Site has never before been archived via the Wayback Machine' };
}
// Remove the header row
data.shift();
// Process and return the results
const firstScan = convertTimestampToDate(data[0][0]);
const lastScan = convertTimestampToDate(data[data.length - 1][0]);
const totalScans = data.length;
const daysArchived = data.length;
const changeCount = countPageChanges(data);
return {
firstScan,
lastScan,
totalScans,
daysArchived,
changeCount,
averagePageSize: getAveragePageSize(data),
scanFrequency: getScanFrequency(firstScan, lastScan, totalScans, changeCount),
scans: data,
scanFrequency: getScanFrequency(firstScan, lastScan, daysArchived, changeCount),
scanUrl: url,
};
} catch (err) {

View File

@@ -1,11 +1,11 @@
import middleware from './_common/middleware.js';
import { UA } from './_common/http.js';
import { createLogger } from './_common/logger.js';
const log = createLogger('carbon');
const TIMEOUT = 8000;
const MAX_BYTES = 10 * 1024 * 1024;
const USER_AGENT = 'Mozilla/5.0 (compatible; WebCheck/2.0; +https://web-check.xyz)';
// Sustainable Web Design model v3 constants, matches websitecarbon.com formula
const KWH_PER_GB = 0.81;
@@ -33,7 +33,7 @@ const fetchByteCount = async (url) => {
const r = await fetch(url, {
signal: AbortSignal.timeout(TIMEOUT),
redirect: 'follow',
headers: { 'user-agent': USER_AGENT, accept: 'text/html,*/*;q=0.1' },
headers: { 'user-agent': UA, accept: 'text/html,*/*;q=0.1' },
});
if (!r.ok) throw new Error(`status ${r.status}`);
if (!r.body) return 0;

View File

@@ -7,7 +7,6 @@ const queryDns = async (domain, type) => {
const res = await httpGet('https://dns.google/resolve', {
params: { name: domain, type },
headers: { Accept: 'application/dns-json' },
timeout: 5000,
});
return res.data;
};

View File

@@ -22,18 +22,12 @@ const evaluate = (header) => {
return verdict('Site is compatible with the HSTS preload list!', true, header);
};
const REQUEST_TIMEOUT = 5000;
const hstsHandler = async (url) =>
new Promise((resolve) => {
const req = https.request(url, (res) => {
resolve(evaluate(res.headers['strict-transport-security']));
res.resume();
});
req.setTimeout(REQUEST_TIMEOUT, () => {
req.destroy();
resolve({ error: 'HSTS check timed out' });
});
req.on('error', (e) => resolve({ error: `HSTS check failed: ${e.message}` }));
req.end();
});

View File

@@ -8,7 +8,7 @@ const qualityHandler = async (url) => {
const endpoint =
`https://www.googleapis.com/pagespeedonline/v5/runPagespeed?` +
`url=${encodeURIComponent(url)}&category=PERFORMANCE&category=ACCESSIBILITY` +
`&category=BEST_PRACTICES&category=SEO&category=PWA&strategy=mobile` +
`&category=BEST_PRACTICES&category=SEO&strategy=mobile` +
`&key=${auth.value}`;
let data;

View File

@@ -10,10 +10,7 @@ const rankHandler = async (url) => {
? { auth: { username: TRANCO_USERNAME, password: TRANCO_API_KEY } }
: {};
try {
const response = await httpGet(`https://tranco-list.eu/api/ranks/domain/${domain}`, {
timeout: 5000,
...auth,
});
const response = await httpGet(`https://tranco-list.eu/api/ranks/domain/${domain}`, auth);
if (!response.data?.ranks?.length) {
return {
skipped: `${domain} isn't ranked in the top 1 million sites yet`,

View File

@@ -1,9 +1,9 @@
import middleware from './_common/middleware.js';
import { UA } from './_common/http.js';
import { upstreamError } from './_common/upstream.js';
const MAX_REDIRECTS = 12;
const TIMEOUT_MS = 10000;
const USER_AGENT = 'Mozilla/5.0 (compatible; WebCheck/2.0; +https://web-check.xyz)';
// Walks the redirect chain manually, recording each Location header as got did
const redirectsHandler = async (url) => {
@@ -14,7 +14,7 @@ const redirectsHandler = async (url) => {
const response = await fetch(current, {
redirect: 'manual',
signal: AbortSignal.timeout(TIMEOUT_MS),
headers: { 'user-agent': USER_AGENT },
headers: { 'user-agent': UA },
});
if (response.status < 300 || response.status >= 400) {
if (response.status >= 400) {

View File

@@ -22,7 +22,8 @@ const robotsHandler = async (url) => {
const parsed = parseRobotsTxt(res.data || '');
return parsed.robots.length ? parsed : { skipped: 'No robots.txt rules found for this host' };
} catch (error) {
if (error.response?.status === 404) {
const status = error.response?.status;
if (status >= 400 && status < 500) {
return { skipped: 'No robots.txt file present on this host' };
}
return upstreamError(error, 'robots.txt fetch');

View File

@@ -9,9 +9,7 @@ const shodanHandler = async (url) => {
if (auth.skipped) return auth;
const { hostname } = parseTarget(url);
try {
const res = await httpGet(`https://api.shodan.io/shodan/host/${hostname}?key=${auth.value}`, {
timeout: 8000,
});
const res = await httpGet(`https://api.shodan.io/shodan/host/${hostname}?key=${auth.value}`);
return res.data;
} catch (error) {
return upstreamError(error, 'Shodan lookup');

View File

@@ -10,11 +10,7 @@ const MAX_DEPTH = 3;
const MAX_CHILD_SITEMAPS = 25;
const MAX_URLS = 5000;
// Browser-ish headers so picky CDNs do not return 406/403 to the default Node UA
const HEADERS = {
'user-agent': 'Mozilla/5.0 (compatible; web-check-bot/1.0; +https://web-check.xyz)',
accept: 'application/xml, text/xml, application/rss+xml, */*;q=0.1',
};
const HEADERS = { accept: 'application/xml, text/xml, application/rss+xml, */*;q=0.1' };
// Reduce a target URL to its origin so child paths resolve cleanly
const toOrigin = (url) => {

View File

@@ -1,6 +1,7 @@
import https from 'https';
import { performance, PerformanceObserver } from 'perf_hooks';
import middleware from './_common/middleware.js';
import { UA } from './_common/http.js';
const statusHandler = async (url) => {
if (!url) {
@@ -23,7 +24,7 @@ const statusHandler = async (url) => {
try {
startTime = performance.now();
const response = await new Promise((resolve, reject) => {
const req = https.get(url, (res) => {
const req = https.get(url, { headers: { 'user-agent': UA } }, (res) => {
let data = '';
responseCode = res.statusCode;
res.on('data', (chunk) => {

View File

@@ -5,24 +5,23 @@ import { upstreamError } from './_common/upstream.js';
const SSL_LABS = 'https://api.ssllabs.com/api/v3/analyze';
// Pull a cached SSL Labs report; skip if no fresh cache available
// Return cached report if ready, pending status while a scan is running, else skip
const tlsLabsHandler = async (url) => {
const { hostname } = parseTarget(url);
try {
const res = await httpGet(SSL_LABS, {
params: { host: hostname, fromCache: 'on', maxAge: 24, all: 'done' },
timeout: 8000,
params: { host: hostname, fromCache: 'on', maxAge: 168, all: 'done' },
headers: { 'User-Agent': 'web-check (https://web-check.xyz)' },
});
const data = res.data;
if (!data || data.status !== 'READY' || !data.endpoints?.length) {
return {
skipped:
'No cached SSL Labs report for this host. ' +
'Run a fresh scan at https://www.ssllabs.com/ssltest/',
};
if (data?.status === 'READY' && data.endpoints?.length) return data;
if (data?.status === 'DNS' || data?.status === 'IN_PROGRESS') {
return { pending: true };
}
return data;
if (data?.status === 'ERROR') {
return { error: `SSL Labs: ${data.statusMessage || 'Assessment failed'}` };
}
return { skipped: 'No SSL Labs report available for this host' };
} catch (error) {
return upstreamError(error, 'SSL Labs lookup');
}

View File

@@ -2,9 +2,17 @@ import dns from 'dns/promises';
import middleware from './_common/middleware.js';
import { parseTarget } from './_common/parse-target.js';
const NO_RECORDS = new Set(['ENODATA', 'ENOTFOUND', 'NXDOMAIN']);
const txtRecordHandler = async (url) => {
const { hostname } = parseTarget(url);
const txtRecords = await dns.resolveTxt(hostname);
let txtRecords;
try {
txtRecords = await dns.resolveTxt(hostname);
} catch (error) {
if (NO_RECORDS.has(error.code)) return { skipped: 'No TXT records for this host' };
throw error;
}
// Join chunks (DNS splits long records at 255 bytes), then key=value
const result = {};
for (const chunks of txtRecords) {

View File

@@ -5,7 +5,6 @@ const LABELS: Record<string, string> = {
accessibility: 'Accessibility',
'best-practices': 'Best Practices',
seo: 'SEO',
pwa: 'PWA',
};
// Convert a 0..1 lighthouse score to a severity bucket

View File

@@ -13,8 +13,10 @@ const threats: Analyzer = (d) => {
if (Array.isArray(d.urlHaus?.urls) && d.urlHaus.urls.length) {
out.push({ severity: 'critical', title: 'Listed on URLhaus malware feed' });
}
const phishUrl = d.phishTank?.url0?.in_database;
if (phishUrl === 'true' || phishUrl === true) {
const phish = d.phishTank?.url0;
const inDb = phish?.in_database === 'true' || phish?.in_database === true;
const valid = phish?.valid === 'true' || phish?.valid === true;
if (inDb && valid) {
out.push({ severity: 'critical', title: 'Listed on PhishTank' });
}
if (d.cloudmersive?.CleanResult === false) {

View File

@@ -18,14 +18,10 @@ const ArchivesCard = (props: { data: any; title: string; actionButtons: any }):
<Card heading={props.title} actionButtons={props.actionButtons}>
<Row lbl="First Scan" val={data.firstScan} />
<Row lbl="Last Scan" val={data.lastScan} />
<Row lbl="Total Scans" val={data.totalScans} />
<Row lbl="Days Archived" val={data.daysArchived} />
<Row lbl="Change Count" val={data.changeCount} />
<Row lbl="Avg Size" val={`${data.averagePageSize} bytes`} />
{data.scanFrequency?.scansPerDay > 1 ? (
<Row lbl="Avg Scans Per Day" val={data.scanFrequency.scansPerDay} />
) : (
<Row lbl="Avg Days between Scans" val={data.scanFrequency.daysBetweenScans} />
)}
<Row lbl="Avg Days between Archives" val={data.scanFrequency.daysBetweenScans} />
<Note>
View historical versions of this page{' '}

View File

@@ -25,6 +25,7 @@ const META: Record<Severity, SevMeta> = {
const Wrapper = styled(Card)`
margin: 0 auto;
width: 95vw;
max-height: 100%;
h2 {
margin: 0 0 0.75rem 0;
}

View File

@@ -0,0 +1,112 @@
import styled from '@emotion/styled';
import colors from 'client/styles/colors';
import { StyledCard } from 'client/components/Form/Card';
import Heading from 'client/components/Form/Heading';
const Wrapper = styled(StyledCard)`
margin: 0 auto;
width: 95vw;
display: flex;
flex-direction: column;
gap: 0.75rem;
h2 {
margin: 0;
}
p {
margin: 0;
}
.target {
font-family: var(--font-mono);
background: ${colors.background};
padding: 0.4rem 0.6rem;
border-radius: 4px;
word-break: break-all;
align-self: flex-start;
max-width: 100%;
color: ${colors.textColor};
}
.reasons {
margin: 0;
padding-left: 1.25rem;
color: ${colors.textColorSecondary};
li {
padding: 0.15rem 0;
}
}
.detail {
color: ${colors.textColorSecondary};
font-size: 0.85rem;
word-break: break-word;
}
`;
type Kind = 'unreachable' | 'invalid' | 'api-down' | 'disabled';
const VARIANT: Record<Kind, { title: string; description: string; reasons: string[] }> = {
unreachable: {
title: 'Cannot Reach This Site',
description: 'We could not resolve an IP address for this host, so checks cannot run',
reasons: [
'The domain might be misspelled or no longer registered',
'The website may be offline or temporarily unreachable',
'A DNS resolution issue may be affecting the lookup',
'A firewall or geo-block may be preventing access',
],
},
invalid: {
title: 'Invalid Input',
description: 'That does not look like a valid URL or IP address, so checks cannot run',
reasons: [
'Enter a domain (example.com) or an IPv4 / IPv6 address',
'Check for typos or stray characters in the input',
'Avoid spaces and unsupported symbols in the address',
],
},
'api-down': {
title: 'Service Unavailable',
description: 'Most checks failed because the Web-Check API could not be reached',
reasons: [
'The API may be down, restarting or rate-limited',
'A self-hosted instance might be misconfigured or offline',
'A network or firewall issue could be blocking the API',
],
},
disabled: {
title: 'Web-Check is Paused',
description: 'This instance has been temporarily disabled, so checks cannot run',
reasons: [
'The public instance may be paused to manage running costs',
'A self-hosted instance may be in maintenance mode',
'You can run your own copy from the open-source repo on GitHub',
],
},
};
interface Props {
address: string;
error?: string;
kind?: Kind;
}
// Surface a friendly explanation when input is invalid or the host is unreachable
const NoResults = ({ address, error, kind = 'unreachable' }: Props): JSX.Element => {
const { title, description, reasons } = VARIANT[kind];
return (
<Wrapper role="alert">
<Heading as="h2" align="left" color={colors.danger}>
{title}
</Heading>
<p>{description}</p>
<code className="target">{address}</code>
<p>Possible reasons:</p>
<ul className="reasons">
{reasons.map((r) => (
<li key={r}>{r}</li>
))}
</ul>
{error && <span className="detail">Lookup error: {error}</span>}
</Wrapper>
);
};
export default NoResults;

View File

@@ -1,4 +1,4 @@
import { useState, useEffect, type ReactNode } from 'react';
import { useState, useEffect, useRef, useCallback, type ReactNode } from 'react';
import styled from '@emotion/styled';
import colors from 'client/styles/colors';
import Card from 'client/components/Form/Card';
@@ -15,20 +15,12 @@ export interface LoadingJob {
retry?: () => void;
}
const STATUS_EMOJI: Record<LoadingState, string> = {
success: '✅',
loading: '🔄',
error: '❌',
'timed-out': '⏸️',
skipped: '⏭️',
};
const STATE_COLOR: Record<LoadingState, string> = {
success: colors.success,
loading: colors.info,
error: colors.danger,
'timed-out': colors.warning,
skipped: colors.neutral,
const STATE_META: Record<LoadingState, { emoji: string; color: string }> = {
success: { emoji: '✅', color: colors.success },
loading: { emoji: '🔄', color: colors.info },
error: { emoji: '❌', color: colors.danger },
'timed-out': { emoji: '⏸️', color: colors.warning },
skipped: { emoji: '⏭️', color: colors.neutral },
};
// Tally jobs by their loading state in a single pass
@@ -56,6 +48,7 @@ const stateToPercent = (jobs: LoadingJob[]): Record<LoadingState, number> => {
const LoadCard = styled(Card)`
margin: 0 auto;
width: 95vw;
max-height: 100%;
position: relative;
`;
@@ -103,6 +96,10 @@ const ProgressBarSegment = styled.div<{ color: string; width: number }>`
transition: width 0.5s ease-in-out;
`;
const StateLabel = styled.span<{ color: string }>`
color: ${(p) => p.color};
`;
const Details = styled.details`
summary {
margin: 0.5rem 0;
@@ -150,21 +147,13 @@ const Details = styled.details`
}
`;
const StatusInfoWrapper = styled.div`
display: flex;
justify-content: space-between;
align-items: center;
.run-status {
color: ${colors.textColorSecondary};
margin: 0;
}
`;
const AboutPageLink = styled.a`
color: ${colors.primary};
`;
const SummaryContainer = styled.div`
display: flex;
align-items: center;
margin: 0.5rem 0;
&.error-info {
color: ${colors.danger};
@@ -193,6 +182,10 @@ const SummaryContainer = styled.div`
.timed-out {
color: ${colors.error};
}
.elapsed {
color: ${colors.textColorSecondary};
margin-left: auto;
}
`;
const ReShowRow = styled.div`
@@ -295,129 +288,114 @@ interface JobListItemProps {
showErrorModal: (job: LoadingJob, isInfo?: boolean) => void;
}
const REASON_LABEL: Partial<Record<LoadingState, string>> = {
error: '■ Show Error',
'timed-out': '■ Show Timeout Reason',
skipped: '■ Show Skip Reason',
};
// One row in the details list, showing job state, time and any actions
const JobListItem = ({ job, showJobDocs, showErrorModal }: JobListItemProps): ReactNode => {
const { name, state, timeTaken, retry, error } = job;
const canRetry = retry && state !== 'success' && state !== 'loading';
const canShowError = error && (state === 'error' || state === 'timed-out' || state === 'skipped');
const reasonLabel = error ? REASON_LABEL[state] : undefined;
return (
<li>
<button type="button" className="docs" onClick={() => showJobDocs(name)}>
{STATUS_EMOJI[state]} {name}
{STATE_META[state].emoji} {name}
</button>
<span style={{ color: STATE_COLOR[state] }}> ({state})</span>.
<StateLabel color={STATE_META[state].color}> ({state})</StateLabel>
<i>{timeTaken && state !== 'loading' ? ` Took ${timeTaken} ms` : ''}</i>
{canRetry && (
<FailedJobActionButton type="button" onClick={retry}>
Retry
</FailedJobActionButton>
)}
{canShowError && (
{reasonLabel && (
<FailedJobActionButton
type="button"
onClick={() => showErrorModal(job, state === 'skipped')}
>
{state === 'timed-out' ? '■ Show Timeout Reason' : '■ Show Error'}
{reasonLabel}
</FailedJobActionButton>
)}
</li>
);
};
// Single-line "Running X of Y / Finished in Z" status with shared elapsed time
const RunningText = ({ jobs, elapsedMs }: { jobs: LoadingJob[]; elapsedMs: number }): ReactNode => {
const total = allCardIds.length;
const done = total - jobs.filter((j) => j.state === 'loading').length;
const isDone = done >= total;
return (
<p className="run-status">
{isDone ? 'Finished in ' : `Running ${done} of ${total} jobs - `}
{elapsedMs >= 10_000 ? `${(elapsedMs / 1000).toFixed(1)} s` : `${elapsedMs} ms`}
</p>
);
};
// Compact one-liner shown alongside the "Show Load State" button when collapsed
const LoadSummary = ({
jobs,
elapsedMs,
onOpen,
}: {
interface LoadSummaryProps {
jobs: LoadingJob[];
elapsedMs: number;
onOpen: () => void;
}): ReactNode => {
}
// Compact one-liner shown alongside the "Show Load State" button when collapsed
const LoadSummary = ({ jobs, elapsedMs, onOpen }: LoadSummaryProps): ReactNode => {
const total = allCardIds.length;
const counts = countByState(jobs);
const extras: string[] = [];
if (counts.error) extras.push(`${counts.error} failed`);
if (counts['timed-out']) extras.push(`${counts['timed-out']} timed out`);
if (counts.skipped) extras.push(`${counts.skipped} skipped`);
const c = countByState(jobs);
const issues = c.error + c['timed-out'] + c.skipped;
const sec = (elapsedMs / 1000).toFixed(1);
const text = c.loading
? `Loading ${total - c.loading} of ${total}` + (elapsedMs < 15000 ? ` (${sec}s)` : '')
: `Finished ${total} lookups in ${sec}s`;
return (
<span className="summary">
{counts.success}/{total} lookups complete
{extras.length > 0 && (
{text}
{issues > 0 && (
<>
{' '}
{' · '}
<button type="button" className="extras" onClick={onOpen}>
({extras.join(', ')})
{issues} {issues === 1 ? 'issue' : 'issues'}
</button>
</>
)}
{elapsedMs ? `, took ${(elapsedMs / 1000).toFixed(1)}s` : ''}
</span>
);
};
const pluralJobs = (n: number) => `${n} ${n === 1 ? 'job' : 'jobs'}`;
type ChipKey = Exclude<LoadingState, 'loading'>;
type ChipKey = 'success' | 'skipped' | 'timed-out' | 'error';
const CHIPS: Record<ChipKey, { cls: string; label: string }> = {
success: { cls: 'success', label: 'successful' },
skipped: { cls: 'skipped', label: 'skipped' },
'timed-out': { cls: 'timed-out', label: 'timed out' },
error: { cls: 'error', label: 'failed' },
const CHIP_LABEL: Record<ChipKey, string> = {
success: 'successful',
skipped: 'skipped',
'timed-out': 'timed out',
error: 'failed',
};
// Inline tally chip; renders nothing for zero so callers can always include it
const Chip = ({ count, cls, label }: { count: number; cls: string; label: string }) =>
count > 0 ? (
<span className={cls}>
{pluralJobs(count)} {label}{' '}
</span>
) : null;
interface SummaryTextProps {
jobs: LoadingJob[];
elapsedMs: number;
}
// Heading-style summary that adapts to loading, all-success and partial-failure
const SummaryText = ({ jobs }: { jobs: LoadingJob[] }): ReactNode => {
const SummaryText = ({ jobs, elapsedMs }: SummaryTextProps): ReactNode => {
const total = allCardIds.length;
const counts = countByState(jobs);
const chips = (keys: ChipKey[]) =>
keys.map((k) => <Chip key={k} count={counts[k]} {...CHIPS[k]} />);
if (counts.loading > 0) {
return (
<SummaryContainer className="loading-info">
<b>
Loading {total - counts.loading} / {total} Jobs
</b>
{chips(['skipped', 'timed-out', 'error'])}
</SummaryContainer>
const c = countByState(jobs);
const isDone = c.loading === 0;
const hasIssues = c.error > 0 || c['timed-out'] > 0;
const elapsed = elapsedMs >= 10_000 ? `${(elapsedMs / 1000).toFixed(1)} s` : `${elapsedMs} ms`;
const chip = (k: ChipKey) =>
c[k] > 0 && (
<span key={k} className={k}>
{c[k]} {c[k] === 1 ? 'job' : 'jobs'} {CHIP_LABEL[k]}{' '}
</span>
);
}
const hasIssues = counts.error > 0 || counts['timed-out'] > 0;
if (!hasIssues) {
return (
<SummaryContainer className="success-info">
<b>{counts.success} Jobs Completed Successfully</b>
{chips(['skipped'])}
</SummaryContainer>
);
}
const cls = !isDone ? 'loading-info' : hasIssues ? 'error-info' : 'success-info';
const heading = !isDone
? `Loading ${total - c.loading} / ${total} Jobs`
: !hasIssues
? `${c.success} Jobs Completed Successfully`
: null;
const keys: ChipKey[] = !isDone
? ['skipped', 'timed-out', 'error']
: hasIssues
? ['success', 'skipped', 'timed-out', 'error']
: ['skipped'];
return (
<SummaryContainer className="error-info">
{chips(['success', 'skipped', 'timed-out', 'error'])}
<SummaryContainer className={cls}>
{heading && <b>{heading}</b>}
{keys.map(chip)}
<span className="elapsed">{isDone ? `Done in ${elapsed}` : elapsed}</span>
</SummaryContainer>
);
};
@@ -442,24 +420,31 @@ const ProgressLoader = ({ loadStatus, showModal, showJobDocs }: ProgressLoaderPr
return () => clearInterval(id);
}, [isDone]);
// Auto-collapse once all jobs finish, leaving the "Finished in" line briefly visible
// Auto-collapse once when 75% of jobs have settled
const autoCollapsedRef = useRef(false);
const autoCollapse = useCallback(() => {
if (autoCollapsedRef.current) return;
autoCollapsedRef.current = true;
setHideLoader(true);
}, []);
useEffect(() => {
if (!isDone) return;
const t = setTimeout(() => setHideLoader(true), 1500);
return () => clearTimeout(t);
}, [isDone]);
const total = loadStatus.length || 1;
const settled = loadStatus.filter((j) => j.state !== 'loading').length;
if (settled / total >= 0.75) autoCollapse();
}, [loadStatus, autoCollapse]);
const colorFor = (state: LoadingState) =>
state === 'success' && isDone ? colors.primary : STATE_COLOR[state];
state === 'success' && isDone ? colors.primary : STATE_META[state].color;
const showErrorModal = (job: LoadingJob, isInfo?: boolean) => {
const detailsLabel = job.state === 'skipped' ? 'Reason:' : 'Server response:';
showModal(
<ErrorModalContent>
<Heading as="h3">Error Details for {job.name}</Heading>
<Heading as="h3">Details for {job.name}</Heading>
<p>
The {job.name} job failed with an {job.state} state
{job.timeTaken !== undefined ? ` after ${job.timeTaken} ms` : ''}. The server responded
with the following error:
The {job.name} job ended with state '{job.state}'
{job.timeTaken !== undefined ? ` after ${job.timeTaken} ms` : ''}. {detailsLabel}
</p>
<pre className={isInfo ? 'info' : 'error'}>{job.error}</pre>
</ErrorModalContent>,
@@ -495,10 +480,7 @@ const ProgressLoader = ({ loadStatus, showModal, showJobDocs }: ProgressLoaderPr
/>
))}
</ProgressBarContainer>
<StatusInfoWrapper>
<SummaryText jobs={loadStatus} />
<RunningText jobs={loadStatus} elapsedMs={elapsedMs} />
</StatusInfoWrapper>
<SummaryText jobs={loadStatus} elapsedMs={elapsedMs} />
<Details>
<summary>Show Details</summary>
<ul>

View File

@@ -58,6 +58,7 @@ const apiBase = (import.meta.env.PUBLIC_API_ENDPOINT || '/api') as string;
const useJobs = (address: string, addressType: AddressType, jobs: JobSpec[]) => {
const [state, dispatch] = useReducer(reducer, initialState);
const [ipAddress, setIpAddress] = useState<string | undefined>();
const [ipLookupError, setIpLookupError] = useState<string | undefined>();
const startTime = useRef(Date.now()).current;
const controllers = useRef<Record<string, AbortController>>({});
const fired = useRef<Set<string>>(new Set());
@@ -82,8 +83,9 @@ const useJobs = (address: string, addressType: AddressType, jobs: JobSpec[]) =>
.then((raw: any) => {
if (controller.signal.aborted) return;
const timeTaken = Date.now() - startTime;
if (job.id === 'get-ip' && typeof raw === 'string') {
setIpAddress(raw);
if (job.id === 'get-ip') {
if (typeof raw === 'string') setIpAddress(raw);
else if (raw?.error) setIpLookupError(raw.error);
return;
}
if (raw?.skipped) {
@@ -104,6 +106,7 @@ const useJobs = (address: string, addressType: AddressType, jobs: JobSpec[]) =>
if (controller.signal.aborted || err?.name === 'AbortError') return;
const timeTaken = Date.now() - startTime;
const message = err?.message || 'Unknown error';
if (job.id === 'get-ip') return;
const outcome = isTimeout(message) ? 'timed-out' : 'error';
dispatch({ type: 'error', cardIds, outcome, error: message, timeTaken });
cardIds.forEach((id) => logJobOutcome(outcome, id, timeTaken, message));
@@ -112,9 +115,9 @@ const useJobs = (address: string, addressType: AddressType, jobs: JobSpec[]) =>
[address, startTime],
);
const skipJob = useCallback((job: JobSpec) => {
const skipJob = useCallback((job: JobSpec, reason?: string) => {
const cardIds = job.cards.map((c) => c.id);
if (cardIds.length) dispatch({ type: 'skipped', cardIds });
if (cardIds.length) dispatch({ type: 'skipped', cardIds, reason });
}, []);
// Decide which jobs are eligible for the current input
@@ -129,21 +132,37 @@ const useJobs = (address: string, addressType: AddressType, jobs: JobSpec[]) =>
[addressType],
);
const skipReason = useCallback(
(job: JobSpec): string => {
const allowed = job.expectedAddressTypes;
if (allowed && !allowed.includes(addressType)) {
if (addressType === 'ipV4' || addressType === 'ipV6') {
return 'This check requires a domain name and cannot be run against an IP address';
}
return `This check is only available for ${allowed.join(', ')} input`;
}
return 'This check is not applicable for the current input';
},
[addressType],
);
// Initial fan-out: fire non-IP jobs immediately, mark unsupported as skipped
useEffect(() => {
if (keys.disableEverything) {
jobs.forEach((j) => skipJob(j));
const reason = 'Web-Check has been temporarily disabled on this instance';
jobs.forEach((j) => skipJob(j, reason));
return;
}
if (!address || addressType === 'empt' || addressType === 'err') return;
fired.current.clear();
setIpLookupError(undefined);
if (addressType === 'ipV4' || addressType === 'ipV6') setIpAddress(address);
else setIpAddress(undefined);
jobs.forEach((job) => {
if (!eligible(job)) {
skipJob(job);
skipJob(job, skipReason(job));
return;
}
if (job.needsIp) return;
@@ -154,7 +173,7 @@ const useJobs = (address: string, addressType: AddressType, jobs: JobSpec[]) =>
Object.values(controllers.current).forEach((c) => c.abort());
controllers.current = {};
};
}, [address, addressType, jobs, runJob, skipJob, eligible]);
}, [address, addressType, jobs, runJob, skipJob, eligible, skipReason]);
// Fire IP-dependent jobs the moment we have an IP, but only once each
useEffect(() => {
@@ -162,12 +181,12 @@ const useJobs = (address: string, addressType: AddressType, jobs: JobSpec[]) =>
jobs.forEach((job) => {
if (!job.needsIp || fired.current.has(job.id)) return;
if (!eligible(job)) {
skipJob(job);
skipJob(job, skipReason(job));
return;
}
runJob(job, ipAddress);
});
}, [ipAddress, jobs, runJob, skipJob, eligible]);
}, [ipAddress, jobs, runJob, skipJob, eligible, skipReason]);
// Promote any card whose fallback resolves after the primary failed
useEffect(() => {
@@ -183,11 +202,15 @@ const useJobs = (address: string, addressType: AddressType, jobs: JobSpec[]) =>
// Single client-side budget for stuck jobs, resets on new input
useEffect(() => {
if (!address) return;
if (!address || addressType === 'empt' || addressType === 'err') return;
const budget = parseInt((import.meta.env.PUBLIC_API_TIMEOUT_LIMIT as string) || '45000', 10);
const timer = setTimeout(() => {
const stuck = Object.entries(stateRef.current)
.filter(([_, e]) => e?.state === 'loading')
.filter(([id, e]) => {
if (e?.state !== 'loading') return false;
const owner = jobs.find((j) => j.cards.some((c) => c.id === id));
return !owner?.noClientTimeout;
})
.map(([id]) => id);
if (!stuck.length) return;
dispatch({
@@ -215,7 +238,7 @@ const useJobs = (address: string, addressType: AddressType, jobs: JobSpec[]) =>
[jobs, runJob, state, ipAddress],
);
return { state, retry };
return { state, retry, ipLookupError };
};
export default useJobs;

View File

@@ -53,6 +53,64 @@ const fetchAndProcess =
return raw?.error ? raw : process(raw);
};
// Sleep ms, reject AbortError if signal fires
const sleep = (ms: number, signal: AbortSignal) =>
new Promise<void>((resolve, reject) => {
if (signal.aborted) return reject(new DOMException('aborted', 'AbortError'));
const onAbort = () => {
clearTimeout(timer);
reject(new DOMException('aborted', 'AbortError'));
};
const remove = () => {
signal.removeEventListener('abort', onAbort);
resolve();
};
const timer = setTimeout(remove, ms);
signal.addEventListener('abort', onAbort, { once: true });
});
// Re-run fetchOnce while shouldRetry(raw) holds, sleeping delay ms between attempts
const retrying = (
path: string,
shouldRetry: (raw: any) => boolean,
attempts: number,
delay: number,
onExhausted: (last: any) => any,
) => {
const fetchOnce = fetchAndProcess(path);
return async (ctx: JobContext) => {
let last: any;
for (let i = 0; i < attempts; i++) {
last = await fetchOnce(ctx);
if (!shouldRetry(last)) return last;
if (i < attempts - 1) await sleep(delay, ctx.signal);
}
return onExhausted(last);
};
};
// Re-run while the body has { pending: true }
const fetchAndPoll = (path: string) =>
retrying(
path,
(r) => !!r?.pending,
6,
30000,
() => ({
error: 'Timed-out waiting for assessment',
}),
);
// Re-run on transient errors, returning the last error if all attempts fail
const fetchAndRetry = (path: string) =>
retrying(
path,
(r) => !!r?.error,
3,
2000,
(last) => last,
);
const card = (
id: string,
title: string,
@@ -98,7 +156,7 @@ export const jobs: JobSpec[] = [
id: 'quality',
expectedAddressTypes: [...URL_ONLY],
cards: [card('quality', 'Quality Summary', ['client'], LighthouseCard)],
fetcher: fetchAndProcess('quality?url=${url}'),
fetcher: fetchAndRetry('quality?url=${url}'),
},
{
id: 'tech-stack',
@@ -148,11 +206,12 @@ export const jobs: JobSpec[] = [
{
id: 'tls-labs',
expectedAddressTypes: [...URL_ONLY],
noClientTimeout: true,
cards: [
card('tls-security-audit', 'TLS Security Audit', ['security'], TlsSecurityAuditCard),
card('tls-client-compat', 'TLS Client Compatibility', ['security'], TlsClientCompatCard),
],
fetcher: fetchAndProcess('tls-labs?url=${url}'),
fetcher: fetchAndPoll('tls-labs?url=${url}'),
},
{
id: 'trace-route',
@@ -206,7 +265,7 @@ export const jobs: JobSpec[] = [
id: 'archives',
expectedAddressTypes: [...URL_ONLY],
cards: [card('archives', 'Archive History', ['meta'], ArchivesCard)],
fetcher: fetchAndProcess('archives?url=${url}'),
fetcher: fetchAndRetry('archives?url=${url}'),
},
{
id: 'rank',

View File

@@ -24,6 +24,7 @@ export interface JobSpec {
fetcher: (ctx: JobContext) => Promise<any>;
expectedAddressTypes?: AddressType[];
needsIp?: boolean;
noClientTimeout?: boolean;
}
export interface JobEntry {

View File

@@ -42,3 +42,12 @@ export const determineAddressType = (address: string | undefined): AddressType =
if (isUrl(address)) return 'url';
return 'err';
};
// Strip protocol and path/query/hash so the route param stays a bare host
export const normalizeAddress = (input: string | undefined): string => {
if (!input) return '';
let s = input.trim().replace(/^https?:\/\//i, '');
const stop = s.search(/[/?#]/);
if (stop !== -1) s = s.slice(0, stop);
return s;
};

View File

@@ -11,7 +11,7 @@ import FancyBackground from 'client/components/misc/FancyBackground';
import docs from 'client/utils/docs';
import colors from 'client/styles/colors';
import { determineAddressType } from 'client/utils/address-type-checker';
import { determineAddressType, normalizeAddress } from 'client/utils/address-type-checker';
const HomeContainer = styled.section`
display: flex;
@@ -140,7 +140,7 @@ const SiteFeaturesWrapper = styled(StyledCard)`
`;
const Home = (): JSX.Element => {
const defaultPlaceholder = 'e.g. https://duck.com/';
const defaultPlaceholder = 'e.g. duck.com';
const [userInput, setUserInput] = useState('');
const [errorMsg, setErrMsg] = useState('');
const [placeholder] = useState(defaultPlaceholder);
@@ -149,18 +149,17 @@ const Home = (): JSX.Element => {
const location = useLocation();
/* Redirect strait to results, if somehow we land on /check?url=[] */
useEffect(() => {
const query = new URLSearchParams(location.search);
const urlFromQuery = query.get('url');
if (urlFromQuery) {
navigate(`/check/${encodeURIComponent(urlFromQuery)}`, { replace: true });
const target = normalizeAddress(urlFromQuery);
if (target) navigate(`/check/${target}`, { replace: true });
}
}, [navigate, location.search]);
/* Check is valid address, either show err or redirect to results page */
const submit = () => {
let address = userInput.endsWith('/') ? userInput.slice(0, -1) : userInput;
const address = normalizeAddress(userInput);
const addressType = determineAddressType(address);
if (addressType === 'empt') {
@@ -168,12 +167,8 @@ const Home = (): JSX.Element => {
} else if (addressType === 'err') {
setErrMsg('Must be a valid URL, IPv4 or IPv6 Address');
} else {
// if the addressType is 'url' and address doesn't start with 'http://' or 'https://', prepend 'https://'
if (addressType === 'url' && !/^https?:\/\//i.test(address)) {
address = 'https://' + address;
}
const resultRouteParams: NavigateOptions = { state: { address, addressType } };
navigate(`/check/${encodeURIComponent(address)}`, resultRouteParams);
navigate(`/check/${address}`, resultRouteParams);
}
};

View File

@@ -18,11 +18,13 @@ import ProgressBar, {
import ActionButtons from 'client/components/misc/ActionButtons';
import AdditionalResources from 'client/components/misc/AdditionalResources';
import AdvisoryPanel from 'client/components/misc/AdvisoryPanel';
import NoResults from 'client/components/misc/NoResults';
import ResultsMasonryGrid from 'client/components/misc/ResultsMasonryGrid';
import ViewRaw from 'client/components/misc/ViewRaw';
import { determineAddressType, type AddressType } from 'client/utils/address-type-checker';
import { hasData } from 'client/utils/result-processor';
import keys from 'client/utils/get-keys';
import useJobs from 'client/hooks/useJobs';
import { jobs, allCards, allCardIds } from 'client/jobs/registry';
import { runAnalysis } from 'client/analysis/registry';
@@ -56,7 +58,8 @@ const ResultsContent = styled.section`
const makeSiteName = (address: string): string => {
try {
return new URL(address).hostname.replace('www.', '');
const withScheme = /^https?:\/\//i.test(address) ? address : `https://${address}`;
return new URL(withScheme).hostname.replace(/^www\./, '');
} catch {
return address;
}
@@ -81,7 +84,7 @@ const Results = (props: { address?: string }): JSX.Element => {
if (addressType === 'empt') setAddressType(determineAddressType(address));
}, [address, addressType]);
const { state: jobsState, retry } = useJobs(address, addressType, jobs);
const { state: jobsState, retry, ipLookupError } = useJobs(address, addressType, jobs);
// Shape useJobs state for the existing ProgressBar contract
const loadingJobs: LoadingJob[] = useMemo(
@@ -137,6 +140,26 @@ const Results = (props: { address?: string }): JSX.Element => {
const findings = useMemo(() => runAnalysis(jobsState), [jobsState]);
// Detect a catastrophic API outage when the bulk of settled jobs error or time out
const apiUnreachable = useMemo(() => {
const entries = Object.values(jobsState);
const settled = entries.filter((e) => e?.state !== 'loading');
const dead = settled.filter((e) => e?.state === 'error' || e?.state === 'timed-out');
return settled.length >= entries.length / 2 && dead.length / settled.length >= 0.9;
}, [jobsState]);
// Pick the highest-priority error state, if any
let errorKind: 'invalid' | 'unreachable' | 'api-down' | 'disabled' | null = null;
if (keys.disableEverything) {
errorKind = 'disabled';
} else if (addressType === 'err') {
errorKind = 'invalid';
} else if (ipLookupError) {
errorKind = 'unreachable';
} else if (apiUnreachable) {
errorKind = 'api-down';
}
const jumpToCard = (id: string) => {
const el = document.getElementById(`card-${id}`);
if (!el) return;
@@ -153,7 +176,11 @@ const Results = (props: { address?: string }): JSX.Element => {
{address && (
<Heading color={colors.textColor} size="medium">
{addressType === 'url' && (
<a target="_blank" rel="noreferrer" href={address}>
<a
target="_blank"
rel="noreferrer"
href={/^https?:\/\//i.test(address) ? address : `https://${address}`}
>
<img width="32px" alt="" src={`https://icon.horse/icon/${makeSiteName(address)}`} />
</a>
)}
@@ -161,6 +188,7 @@ const Results = (props: { address?: string }): JSX.Element => {
</Heading>
)}
</Nav>
{errorKind && <NoResults kind={errorKind} address={address} error={ipLookupError} />}
<ProgressBar loadStatus={loadingJobs} showModal={showErrorModal} showJobDocs={showInfo} />
<Loader show={loadingJobs.filter((j) => j.state !== 'loading').length < 5} />
<AdvisoryPanel findings={findings} onJumpTo={jumpToCard} />
@@ -192,6 +220,7 @@ const Results = (props: { address?: string }): JSX.Element => {
}))}
/>
<AdditionalResources url={address} />
<Modal isOpen={modalOpen} closeModal={() => setModalOpen(false)}>
{modalContent}
</Modal>

View File

@@ -11,7 +11,18 @@ const placeholders = [
---
<div class="input-container">
<input required id="url-input" type="url" name="url" placeholder="E.g. duck.com" />
<input
required
id="url-input"
type="text"
name="url"
inputmode="url"
autocapitalize="off"
autocomplete="off"
autocorrect="off"
spellcheck="false"
placeholder="E.g. duck.com"
/>
<div class="placeholder-container">
<span class="starter" aria-hidden="true">E.g.</span>
{

View File

@@ -28,37 +28,15 @@ import Screenshots from './Screenshots.astro';
</div>
<script>
/**
* Form management actions (validation, submission, etc.)
* We just use normal, old school JavaScript for this
*/
import { normalizeAddress } from '../../client/utils/address-type-checker';
// Select the form and input elements from the DOM
const form = document.getElementById('live-start');
const urlInput = document.getElementById('url-input') as HTMLInputElement;
// Submit Event - called when user submits form with a valid URL
// Gets and checks the URL, then redirects user to /check/:url
form?.addEventListener('submit', (event) => {
event.preventDefault();
const url = urlInput.value.trim();
if (url) {
const encodedUrl = encodeURIComponent(url);
window.location.href = `/check/${encodedUrl}`;
}
});
// User presses enter, forgets to add protocol
// Will add https:// to the URL, and retry form submit
urlInput?.addEventListener('keydown', (event) => {
if (event.key === 'Enter') {
const url = urlInput.value.trim();
const urlWithoutProtocolRegex = /^[a-zA-Z0-9]+[a-zA-Z0-9.-]*\.[a-zA-Z]{2,}$/;
if (url && !/^https?:\/\//i.test(url) && urlWithoutProtocolRegex.test(url)) {
urlInput.value = 'https://' + url;
form?.dispatchEvent(new Event('submit'));
}
}
const target = normalizeAddress(urlInput.value);
if (target) window.location.href = `/check/${target}`;
});
</script>

View File

@@ -2,6 +2,7 @@
import BaseLayout from '@layouts/Base.astro';
import Main from '../../client/main.tsx';
import '../../client/styles/index.css';
import { normalizeAddress } from '../../client/utils/address-type-checker';
export const prerender = false;
@@ -10,7 +11,8 @@ const { search } = new URL(Astro.request.url);
const searchUrl = new URLSearchParams(search).get('url');
if (searchUrl) {
Astro.redirect(`/check/${encodeURIComponent(searchUrl)}`);
const target = normalizeAddress(searchUrl);
if (target) Astro.redirect(`/check/${target}`);
}
---
@@ -27,22 +29,21 @@ if (searchUrl) {
</BaseLayout>
<script>
// Fallback, if Astro hasn't initialized the RC comp yet, then check the url
// And if form has been submitted with ?url=, redirect to the results page
import { normalizeAddress } from '../../client/utils/address-type-checker';
const searchParams = new URL(window.location.href).searchParams;
if (searchParams.has('url')) {
window.location.href = `/check/${encodeURIComponent(searchParams.get('url') || '')}`;
const target = normalizeAddress(searchParams.get('url') || '');
if (target) window.location.href = `/check/${target}`;
}
// And add a manual no-react form submit handler
const form = document.querySelector<HTMLFormElement>('form');
if (form) {
form.addEventListener('submit', function (event: Event) {
event.preventDefault();
const input = (this as HTMLFormElement).querySelector<HTMLInputElement>('input[name="url"]');
if (input && input.value) {
window.location.href = `/check/${encodeURIComponent(input.value)}`;
}
const target = normalizeAddress(input?.value);
if (target) window.location.href = `/check/${target}`;
});
}
</script>

View File

@@ -8,7 +8,7 @@
],
"functions": {
"api/*.js": {
"maxDuration": 20
"maxDuration": 45
}
},
"env": {