diff --git a/.env.sample b/.env.sample index 2930ea4..5b37299 100644 --- a/.env.sample +++ b/.env.sample @@ -21,7 +21,7 @@ WHO_API_KEY='' # CHROME_PATH='/usr/bin/chromium' # The path the the Chromium executable # PORT='3000' # Port to serve the API, when running server.js # DISABLE_GUI='false' # Disable the GUI, and only serve the API -# API_TIMEOUT_LIMIT='10000' # The timeout limit for API requests, in milliseconds +# PUBLIC_API_TIMEOUT_LIMIT='25000'# Timeout for API requests, in milliseconds # API_CORS_ORIGIN='*' # Enable CORS, by setting your allowed hostname(s) here # API_ENABLE_RATE_LIMIT='true' # Enable rate limiting for the API # REACT_APP_API_ENDPOINT='/api' # The endpoint for the API (can be local or remote) diff --git a/.github/README.md b/.github/README.md index 757247f..20b1f50 100644 --- a/.github/README.md +++ b/.github/README.md @@ -847,7 +847,7 @@ Key | Value ---|--- `PORT` | Port to serve the API, when running server.js (e.g. `3000`) `API_ENABLE_RATE_LIMIT` | Enable rate-limiting for the /api endpoints (e.g. `true`) -`API_TIMEOUT_LIMIT` | The timeout limit for API requests, in milliseconds (e.g. `10000`) +`PUBLIC_API_TIMEOUT_LIMIT` | The timeout limit for API requests, in milliseconds (e.g. `25000`) `API_CORS_ORIGIN` | Enable CORS, by setting your allowed hostname(s) here (e.g. `example.com`) `CHROME_PATH` | The path the Chromium executable (e.g. `/usr/bin/chromium`) `DISABLE_GUI` | Disable the GUI, and only serve the API (e.g. `false`) diff --git a/.gitignore b/.gitignore index d818df7..6e01eac 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,4 @@ Thumbs.db *.swp *.swo +.vercel diff --git a/.yarnrc b/.yarnrc new file mode 100644 index 0000000..93303a2 --- /dev/null +++ b/.yarnrc @@ -0,0 +1,4 @@ +# engines.node is pinned to 20.x so @astrojs/vercel@7.x emits a supported +# Vercel runtime for the SSR _render function. Local devs on a different +# Node version can still run yarn install via this flag. +--install.ignore-engines true diff --git a/Dockerfile b/Dockerfile index a01ddd8..a8507b0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Specify the Node.js version to use -ARG NODE_VERSION=21 +ARG NODE_VERSION=20 # Specify the Debian version to use, the default is "bullseye" ARG DEBIAN_VERSION=bullseye diff --git a/api/_common/logger.js b/api/_common/logger.js index 84fe4fc..2e96a72 100644 --- a/api/_common/logger.js +++ b/api/_common/logger.js @@ -1,4 +1,3 @@ -// Lightweight structured logger. Honours LOG_LEVEL env (debug, info, warn, error, silent). const LEVELS = { debug: 10, info: 20, warn: 30, error: 40, silent: 99 }; const THRESHOLD = LEVELS[(process.env.LOG_LEVEL || 'info').toLowerCase()] ?? LEVELS.info; @@ -17,7 +16,7 @@ const write = (level, stream, scope, msg, extra) => { stream.write(fmt(level, scope, msg, extra) + '\n'); }; -// Returns a logger pinned to a scope (e.g. an API route name). +// Logger scoped to a route name; honours LOG_LEVEL env. export const createLogger = (scope) => ({ debug: (msg, extra) => write('debug', process.stdout, scope, msg, extra), info: (msg, extra) => write('info', process.stdout, scope, msg, extra), diff --git a/api/_common/middleware.js b/api/_common/middleware.js index 2c5d801..0f2ca92 100644 --- a/api/_common/middleware.js +++ b/api/_common/middleware.js @@ -2,8 +2,7 @@ const normalizeUrl = (url) => { return url.startsWith('http') ? url : `https://${url}`; }; -// If present, set a shorter timeout for API requests -const TIMEOUT = process.env.API_TIMEOUT_LIMIT ? parseInt(process.env.API_TIMEOUT_LIMIT, 10) : 60000; +const TIMEOUT = parseInt(process.env.PUBLIC_API_TIMEOUT_LIMIT || '60000', 10); // If present, set CORS allowed origins for responses const ALLOWED_ORIGINS = process.env.API_CORS_ORIGIN || '*'; @@ -27,7 +26,7 @@ const headers = { const timeoutErrorMsg = 'You can re-trigger this request, by clicking "Retry"\n' + 'If you\'re running your own instance of Web Check, then you can ' + 'resolve this issue, by increasing the timeout limit in the ' -+ '`API_TIMEOUT_LIMIT` environmental variable to a higher value (in milliseconds), ' ++ '`PUBLIC_API_TIMEOUT_LIMIT` environmental variable to a higher value (in milliseconds), ' + 'or if you\'re hosting on Vercel increase the maxDuration in vercel.json.\n\n' + `The public instance currently has a lower timeout of ${TIMEOUT}ms ` + 'in order to keep running costs affordable, so that Web Check can ' diff --git a/api/_common/parse-target.js b/api/_common/parse-target.js index 3cc1470..ee39306 100644 --- a/api/_common/parse-target.js +++ b/api/_common/parse-target.js @@ -1,11 +1,10 @@ -// Parse a user-supplied target into a normalised form. -// Strips protocol/port/path so DNS-touching endpoints get a bare hostname. +// Normalise a user-supplied target, stripping :port for DNS lookups. export const parseTarget = (input) => { if (!input) throw new Error('No target provided'); const normalised = /^https?:\/\//i.test(input) ? input : `https://${input}`; let u; try { u = new URL(normalised); } - catch (err) { throw new Error(`Invalid URL: ${input}`); } + catch { throw new Error(`Invalid URL: ${input}`); } return { hostname: u.hostname, port: u.port || null, diff --git a/api/get-ip.js b/api/get-ip.js index 71c6938..dc46cf2 100644 --- a/api/get-ip.js +++ b/api/get-ip.js @@ -2,7 +2,6 @@ import dns from 'dns'; import middleware from './_common/middleware.js'; import { parseTarget } from './_common/parse-target.js'; -// Resolve the IP address for the target hostname. const lookupAsync = (address) => new Promise((resolve, reject) => { dns.lookup(address, (err, ip, family) => { if (err) reject(err); else resolve({ ip, family }); diff --git a/api/screenshot.js b/api/screenshot.js index 8c267cc..eb18a9c 100644 --- a/api/screenshot.js +++ b/api/screenshot.js @@ -9,7 +9,7 @@ import { createLogger } from './_common/logger.js'; const log = createLogger('screenshot'); -// Capture a screenshot via the system Chromium binary; faster cold-start than puppeteer. +// Screenshot via the system Chromium binary. const directChromiumScreenshot = async (url) => { const tmpDir = '/tmp'; const screenshotPath = path.join(tmpDir, `screenshot-${randomUUID()}.png`); @@ -39,7 +39,7 @@ const directChromiumScreenshot = async (url) => { }); }; -// Fallback path that uses puppeteer with the bundled chrome-aws-lambda binary. +// Fallback to puppeteer when the direct Chromium binary call fails. const puppeteerScreenshot = async (targetUrl) => { let browser = null; try { diff --git a/api/sitemap.js b/api/sitemap.js index b85420b..28c6a7c 100644 --- a/api/sitemap.js +++ b/api/sitemap.js @@ -6,13 +6,13 @@ const HARD_TIMEOUT = 5000; const MAX_DEPTH = 3; const MAX_CHILD_SITEMAPS = 25; -// Fetch a single XML sitemap and parse it. +// Fetch and parse a sitemap XML. const fetchSitemap = async (sitemapUrl) => { const res = await axios.get(sitemapUrl, { timeout: HARD_TIMEOUT }); return new xml2js.Parser().parseStringPromise(res.data); }; -// Find a sitemap URL listed in robots.txt as a fallback when /sitemap.xml is missing. +// Pull a Sitemap: line out of robots.txt. const findSitemapInRobots = async (baseUrl) => { const robots = await axios.get(`${baseUrl}/robots.txt`, { timeout: HARD_TIMEOUT }); for (const line of robots.data.split('\n')) { @@ -23,7 +23,7 @@ const findSitemapInRobots = async (baseUrl) => { return null; }; -// Recursively expand a sitemap-index into its child url sets. +// Recursively flatten a sitemap-index into its child url sets. const expandSitemap = async (parsed, depth) => { if (!parsed?.sitemapindex?.sitemap || depth >= MAX_DEPTH) return parsed; const children = parsed.sitemapindex.sitemap diff --git a/package.json b/package.json index 080fb34..174541b 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "version": "2.0.2", "homepage": "https://web-check.xyz", "engines": { - "node": ">=20" + "node": "20.x" }, "scripts": { "start": "node server", diff --git a/server.js b/server.js index 82ce2a4..2ba1cad 100644 --- a/server.js +++ b/server.js @@ -13,10 +13,8 @@ dotenv.config(); // Create the Express app const app = express(); -// Trust X-Forwarded-* headers when running behind a reverse proxy -// (e.g. Traefik, nginx). Configurable via TRUST_PROXY env var. const trustProxy = process.env.TRUST_PROXY; -if (trustProxy !== undefined && trustProxy !== '') { +if (trustProxy) { const parsed = /^\d+$/.test(trustProxy) ? parseInt(trustProxy, 10) : trustProxy === 'true' ? true @@ -110,7 +108,7 @@ const renderPlaceholderPage = async (res, msgId, logs) => { app.get(API_DIR, async (req, res) => { const results = {}; const { url } = req.query; - const maxExecutionTime = process.env.API_TIMEOUT_LIMIT || 20000; + const maxExecutionTime = process.env.PUBLIC_API_TIMEOUT_LIMIT || 20000; const executeHandler = async (handler, req) => { return new Promise(async (resolve, reject) => { diff --git a/src/web-check-live/utils/get-keys.ts b/src/web-check-live/utils/get-keys.ts index d698873..c05baf6 100644 --- a/src/web-check-live/utils/get-keys.ts +++ b/src/web-check-live/utils/get-keys.ts @@ -1,7 +1,7 @@ const keys = { - shodan: import.meta.env.REACT_APP_SHODAN_API_KEY || "default_value_if_not_set", - whoApi: import.meta.env.REACT_APP_WHO_API_KEY || "default_value_if_not_set", + shodan: import.meta.env.PUBLIC_SHODAN_API_KEY || "default_value_if_not_set", + whoApi: import.meta.env.PUBLIC_WHO_API_KEY || "default_value_if_not_set", disableEverything: import.meta.env.VITE_DISABLE_EVERYTHING === 'true', }; // const keys = process && process.env ? { diff --git a/src/web-check-live/utils/logger.ts b/src/web-check-live/utils/logger.ts new file mode 100644 index 0000000..22b4bbc --- /dev/null +++ b/src/web-check-live/utils/logger.ts @@ -0,0 +1,55 @@ +import colors from 'web-check-live/styles/colors'; + +type Outcome = 'success' | 'error' | 'timed-out'; + +const HEADING: Record = { + success: 'Fetch Success', + error: 'Fetch Error', + 'timed-out': 'Fetch Timeout', +}; + +const VERB: Record = { + success: 'succeeded in', + error: 'failed after', + 'timed-out': 'timed out after', +}; + +const ACCENT: Record = { + success: colors.success, + error: colors.danger, + 'timed-out': colors.info, +}; + +// HH:MM:SS clock prefix. +const stamp = (d: Date) => + `[${`${d.getHours()}`.padStart(2, '0')}:` + + `${`${d.getMinutes()}`.padStart(2, '0')}:` + + `${`${d.getSeconds()}`.padStart(2, '0')}]`; + +// Fancy console banner showing the result of a job, plus the trailing detail (error or hint). +export const logJobOutcome = ( + outcome: Outcome, + job: string, + timeTaken: number, + detail?: string, +) => { + const accent = ACCENT[outcome]; + const trail = outcome === 'success' + ? `\n%cRun %cwindow.webCheck['${job}']%c to inspect the raw results` + : `, with the following error:%c\n${detail || 'Unknown error'}`; + const styles = [ + `background:${accent};color:${colors.background};padding:4px 8px;font-size:16px;border-radius:2px;`, + `font-weight:bold;color:${accent};`, + `color:${accent};`, + ]; + const trailStyles = outcome === 'success' + ? [`color:#1d8242;`, `color:#1d8242;text-decoration:underline;`, `color:#1d8242;`] + : [`color:${colors.warning};`]; + console.log( + `%c${HEADING[outcome]} - ${job}%c\n\n${stamp(new Date())}%c The ${job} job ${VERB[outcome]} ${timeTaken}ms${trail}`, + ...styles, + ...trailStyles, + ); +}; + +export default logJobOutcome; diff --git a/src/web-check-live/utils/result-processor.ts b/src/web-check-live/utils/result-processor.ts index 2e1369f..2f67d25 100644 --- a/src/web-check-live/utils/result-processor.ts +++ b/src/web-check-live/utils/result-processor.ts @@ -63,8 +63,18 @@ export interface ServerInfo { type?: string, }; -export const getServerInfo = (response: any): ServerInfo => { - return { +// Whether a result has any meaningful value worth rendering a card for. +export const hasData = (r: any): boolean => { + if (r === null || r === undefined) return false; + if (typeof r === 'boolean' || typeof r === 'number') return true; + if (typeof r === 'string') return r.trim().length > 0; + if (Array.isArray(r)) return r.some(hasData); + if (typeof r === 'object') return Object.values(r).some(hasData); + return true; +}; + +export const getServerInfo = (response: any): ServerInfo | null => { + const info: ServerInfo = { org: response.org, asn: response.asn, isp: response.isp, @@ -74,6 +84,7 @@ export const getServerInfo = (response: any): ServerInfo => { loc: response.city ? `${response.city}, ${response.country_name}` : '', type: response.tags ? response.tags.toString() : '', }; + return Object.values(info).some(Boolean) ? info : null; }; export interface HostNames { @@ -95,7 +106,7 @@ export const getHostNames = (response: any): HostNames | null => { export interface ShodanResults { hostnames: HostNames | null, - serverInfo: ServerInfo, + serverInfo: ServerInfo | null, } export const parseShodanResults = (response: any): ShodanResults => { diff --git a/src/web-check-live/views/Results.tsx b/src/web-check-live/views/Results.tsx index 5ca6197..b9dd529 100644 --- a/src/web-check-live/views/Results.tsx +++ b/src/web-check-live/views/Results.tsx @@ -59,13 +59,15 @@ import TlsIssueAnalysisCard from 'web-check-live/components/Results/TlsIssueAnal import TlsClientSupportCard from 'web-check-live/components/Results/TlsClientSupport'; import keys from 'web-check-live/utils/get-keys'; +import { logJobOutcome } from 'web-check-live/utils/logger'; import { determineAddressType, type AddressType } from 'web-check-live/utils/address-type-checker'; import useMotherHook from 'web-check-live/hooks/motherOfAllHooks'; import { getLocation, type ServerLocation, type Cookie, applyWhoIsResults, type Whois, - parseShodanResults, type ShodanResults + parseShodanResults, type ShodanResults, + hasData, } from 'web-check-live/utils/result-processor'; const ResultsOuter = styled.div` @@ -171,65 +173,25 @@ const Results = (props: { address?: string } ): JSX.Element => { }; const updateTags = (tag: string) => { // Remove current tag if it exists, otherwise add it - // setTags(tags.includes(tag) ? tags.filter(t => t !== tag) : [...tags, tag]); setTags(tags.includes(tag) ? tags.filter(t => t !== tag) : [tag]); }; const updateLoadingJobs = useCallback((jobs: string | string[], newState: LoadingState, error?: string, retry?: () => void, data?: any) => { (typeof jobs === 'string' ? [jobs] : jobs).forEach((job: string) => { - const now = new Date(); - const timeTaken = now.getTime() - startTime; - setLoadingJobs((prevJobs) => { - const newJobs = prevJobs.map((loadingJob: LoadingJob) => { - if (job.includes(loadingJob.name)) { - return { ...loadingJob, error, state: newState, timeTaken, retry }; - } - return loadingJob; - }); - - const timeString = `[${now.getHours().toString().padStart(2, '0')}:` - +`${now.getMinutes().toString().padStart(2, '0')}:` - + `${now.getSeconds().toString().padStart(2, '0')}]`; - - + const timeTaken = Date.now() - startTime; + setLoadingJobs((prevJobs) => prevJobs.map((loadingJob: LoadingJob) => + job.includes(loadingJob.name) + ? { ...loadingJob, error, state: newState, timeTaken, retry } + : loadingJob + )); if (newState === 'success') { - console.log( - `%cFetch Success - ${job}%c\n\n${timeString}%c The ${job} job succeeded in ${timeTaken}ms` - + `\n%cRun %cwindow.webCheck['${job}']%c to inspect the raw the results`, - `background:${colors.success};color:${colors.background};padding: 4px 8px;font-size:16px;`, - `font-weight: bold; color: ${colors.success};`, - `color: ${colors.success};`, - `color: #1d8242;`,`color: #1d8242;text-decoration:underline;`,`color: #1d8242;`, - ); + logJobOutcome('success', job, timeTaken); if (!(window as any).webCheck) (window as any).webCheck = {}; if (data) (window as any).webCheck[job] = data; + } else if (newState === 'error' || newState === 'timed-out') { + logJobOutcome(newState, job, timeTaken, error); } - - if (newState === 'error') { - console.log( - `%cFetch Error - ${job}%c\n\n${timeString}%c The ${job} job failed ` - +`after ${timeTaken}ms, with the following error:%c\n${error}`, - `background: ${colors.danger}; color:${colors.background}; padding: 4px 8px; font-size: 16px;`, - `font-weight: bold; color: ${colors.danger};`, - `color: ${colors.danger};`, - `color: ${colors.warning};`, - ); - } - - if (newState === 'timed-out') { - console.log( - `%cFetch Timeout - ${job}%c\n\n${timeString}%c The ${job} job timed out ` - +`after ${timeTaken}ms, with the following error:%c\n${error}`, - `background: ${colors.info}; color:${colors.background}; padding: 4px 8px; font-size: 16px;`, - `font-weight: bold; color: ${colors.info};`, - `color: ${colors.info};`, - `color: ${colors.warning};`, - ); - } - - return newJobs; }); - }); }, [startTime]); const parseJson = (response: Response): Promise => { @@ -249,7 +211,7 @@ const Results = (props: { address?: string } ): JSX.Element => { const urlTypeOnly = ['url'] as AddressType[]; // Many jobs only run with these address types const api = import.meta.env.PUBLIC_API_ENDPOINT || '/api'; // Where is the API hosted? - + // Fetch and parse IP address for given URL const [ipAddress, setIpAddress] = useMotherHook({ jobId: 'get-ip', @@ -267,7 +229,7 @@ const Results = (props: { address?: string } ): JSX.Element => { if (addressType === 'ipV4' && address) { setIpAddress(address); } - }, [address, addressType, setIpAddress]); + }, [address, addressType, setIpAddress]); // Get IP address location info const [locationResults, updateLocationResults] = useMotherHook({ @@ -559,19 +521,27 @@ const Results = (props: { address?: string } ): JSX.Element => { }), }); - /* Cancel remaining jobs after 10 second timeout */ + // Promote screenshot job to success if the lighthouse fallback resolves with data useEffect(() => { - const checkJobs = () => { + const job = loadingJobs.find(j => j.name === 'screenshot'); + if (job?.state === 'success') return; + const fallback = lighthouseResults?.fullPageScreenshot?.screenshot; + if (!hasData(screenshotResult) && hasData(fallback)) { + updateLoadingJobs('screenshot', 'success'); + } + }, [screenshotResult, lighthouseResults, loadingJobs, updateLoadingJobs]); + + // Cancel remaining jobs after timeout window has passed + useEffect(() => { + const budget = parseInt(import.meta.env.PUBLIC_API_TIMEOUT_LIMIT || '25000', 10); + const timeoutId = setTimeout(() => { loadingJobs.forEach(job => { if (job.state === 'loading') { updateLoadingJobs(job.name, 'timed-out'); } }); - }; - const timeoutId = setTimeout(checkJobs, 10000); - return () => { - clearTimeout(timeoutId); - }; + }, budget); + return () => clearTimeout(timeoutId); }, [loadingJobs, updateLoadingJobs]); const makeSiteName = (address: string): string => { @@ -865,11 +835,11 @@ const Results = (props: { address?: string } ): JSX.Element => { setModalContent(content); setModalOpen(true); }; - + return (