diff --git a/Dockerfile b/Dockerfile index a8507b0..d9b3e20 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Specify the Node.js version to use -ARG NODE_VERSION=20 +ARG NODE_VERSION=22 # Specify the Debian version to use, the default is "bullseye" ARG DEBIAN_VERSION=bullseye diff --git a/api/_common/http.js b/api/_common/http.js new file mode 100644 index 0000000..738e2c0 --- /dev/null +++ b/api/_common/http.js @@ -0,0 +1,108 @@ +// Thin fetch wrapper matching the axios shape used across the api: opts.params, +// opts.headers, opts.auth, opts.timeout, opts.validateStatus; returns +// { data, status, statusText, headers }; throws errors with response/code + +const DEFAULT_TIMEOUT = 60000; + +const buildAuth = (auth) => { + if (!auth?.username) return null; + return 'Basic ' + Buffer.from(`${auth.username}:${auth.password}`).toString('base64'); +}; + +const appendParams = (url, params) => { + if (!params) return url; + const u = new URL(url); + for (const [k, v] of Object.entries(params)) u.searchParams.set(k, v); + return u.href; +}; + +const headersToObject = (headers) => { + const out = {}; + for (const [k, v] of headers.entries()) { + const key = k.toLowerCase(); + if (key === 'set-cookie') { + out[key] = headers.getSetCookie ? headers.getSetCookie() : v.split(/, (?=[^;]+=)/); + } else { + out[key] = v; + } + } + return out; +}; + +// Auto-parse JSON when the response advertises it, fall back to raw text +const parseBody = async (response) => { + const ct = (response.headers.get('content-type') || '').toLowerCase(); + const text = await response.text(); + if (!text) return ct.includes('json') ? null : ''; + if (ct.includes('json')) { + try { return JSON.parse(text); } catch { return text; } + } + return text; +}; + +const isOk = (status, validate) => + validate ? validate(status) : (status >= 200 && status < 300); + +const wrapNetworkError = (error) => { + if (error.name === 'TimeoutError' || error.name === 'AbortError') { + const e = new Error(error.message || 'Request timed out'); + e.code = 'ECONNABORTED'; + return e; + } + const code = error.cause?.code; + if (code) { + const e = new Error(error.message); + e.code = code; + return e; + } + return error; +}; + +const send = async (method, url, body, opts = {}) => { + const finalUrl = appendParams(url, opts.params); + const headers = { ...opts.headers }; + const authHeader = buildAuth(opts.auth); + if (authHeader) headers.authorization = authHeader; + + const init = { + method, + headers, + signal: AbortSignal.timeout(opts.timeout || DEFAULT_TIMEOUT), + }; + + if (body !== undefined && body !== null) { + if (typeof body === 'object') { + init.body = JSON.stringify(body); + const hasCt = Object.keys(headers).some(k => k.toLowerCase() === 'content-type'); + if (!hasCt) init.headers['content-type'] = 'application/json'; + } else { + init.body = body; + } + } + + let response; + try { + response = await fetch(finalUrl, init); + } catch (error) { + throw wrapNetworkError(error); + } + + const data = await parseBody(response); + const result = { + data, + status: response.status, + statusText: response.statusText, + headers: headersToObject(response.headers), + }; + + if (!isOk(response.status, opts.validateStatus)) { + const err = new Error(`Request failed with status code ${response.status}`); + err.response = result; + throw err; + } + + return result; +}; + +export const httpGet = (url, opts) => send('GET', url, null, opts); +export const httpPost = (url, body, opts) => send('POST', url, body, opts); diff --git a/api/archives.js b/api/archives.js index 7c6ae99..febb161 100644 --- a/api/archives.js +++ b/api/archives.js @@ -1,5 +1,5 @@ -import axios from 'axios'; import middleware from './_common/middleware.js'; +import { httpGet } from './_common/http.js'; const convertTimestampToDate = (timestamp) => { const [year, month, day, hour, minute, second] = [ @@ -50,7 +50,7 @@ const wayBackHandler = async (url) => { const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${url}&output=json&fl=timestamp,statuscode,digest,length,offset`; try { - const { data } = await axios.get(cdxUrl); + const { data } = await httpGet(cdxUrl); // Check there's data if (!data || !Array.isArray(data) || data.length <= 1) { diff --git a/api/cookies.js b/api/cookies.js index a86ccb6..ace2f81 100644 --- a/api/cookies.js +++ b/api/cookies.js @@ -1,21 +1,21 @@ -import axios from 'axios'; import puppeteer from 'puppeteer'; import middleware from './_common/middleware.js'; +import { httpGet } from './_common/http.js'; const getPuppeteerCookies = async (url) => { const browser = await puppeteer.launch({ - headless: 'new', + headless: true, args: ['--no-sandbox', '--disable-setuid-sandbox'], }); try { const page = await browser.newPage(); const navigationPromise = page.goto(url, { waitUntil: 'networkidle2' }); - const timeoutPromise = new Promise((_, reject) => + const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('Puppeteer took too long!')), 3000) ); await Promise.race([navigationPromise, timeoutPromise]); - return await page.cookies(); + return await browser.cookies(); } finally { await browser.close(); } @@ -26,19 +26,13 @@ const cookieHandler = async (url) => { let clientCookies = null; try { - const response = await axios.get(url, { - withCredentials: true, - maxRedirects: 5, - }); + const response = await httpGet(url); headerCookies = response.headers['set-cookie']; } catch (error) { if (error.response) { return { error: `Request failed with status ${error.response.status}: ${error.message}` }; - } else if (error.request) { - return { error: `No response received: ${error.message}` }; - } else { - return { error: `Error setting up request: ${error.message}` }; } + return { error: `No response received: ${error.message}` }; } try { diff --git a/api/dns-server.js b/api/dns-server.js index 0ff17f1..46b00ca 100644 --- a/api/dns-server.js +++ b/api/dns-server.js @@ -1,6 +1,6 @@ import { promises as dnsPromises } from 'dns'; -import axios from 'axios'; import middleware from './_common/middleware.js'; +import { httpGet } from './_common/http.js'; import { parseTarget } from './_common/parse-target.js'; import { upstreamError } from './_common/upstream.js'; @@ -14,8 +14,7 @@ const dnsHandler = async (url) => { } const results = await Promise.all(addresses.map(async (address) => { const hostname = await dnsPromises.reverse(address).catch(() => null); - const dohDirectSupports = await axios - .get(`https://${address}/dns-query`) + const dohDirectSupports = await httpGet(`https://${address}/dns-query`) .then(() => true) .catch(() => false); return { address, hostname, dohDirectSupports }; diff --git a/api/firewall.js b/api/firewall.js index 42f4566..9c48709 100644 --- a/api/firewall.js +++ b/api/firewall.js @@ -1,5 +1,5 @@ -import axios from 'axios'; import middleware from './_common/middleware.js'; +import { httpGet } from './_common/http.js'; import { parseTarget } from './_common/parse-target.js'; import { upstreamError } from './_common/upstream.js'; @@ -8,7 +8,7 @@ const hasWaf = (waf) => ({ hasWaf: true, waf }); const firewallHandler = async (url) => { const { href } = parseTarget(url); try { - const response = await axios.get(href); + const response = await httpGet(href); const headers = response.headers; if (headers['server'] && headers['server'].includes('cloudflare')) { diff --git a/api/headers.js b/api/headers.js index cfa1427..c84be19 100644 --- a/api/headers.js +++ b/api/headers.js @@ -1,10 +1,10 @@ -import axios from 'axios'; import middleware from './_common/middleware.js'; +import { httpGet } from './_common/http.js'; import { upstreamError } from './_common/upstream.js'; const headersHandler = async (url) => { try { - const response = await axios.get(url, { + const response = await httpGet(url, { validateStatus: (status) => status >= 200 && status < 600, }); return response.headers; diff --git a/api/http-security.js b/api/http-security.js index ca70234..bc29994 100644 --- a/api/http-security.js +++ b/api/http-security.js @@ -1,10 +1,10 @@ -import axios from 'axios'; import middleware from './_common/middleware.js'; +import { httpGet } from './_common/http.js'; import { upstreamError } from './_common/upstream.js'; const httpsSecHandler = async (url) => { try { - const { headers } = await axios.get(url); + const { headers } = await httpGet(url); return { strictTransportPolicy: !!headers['strict-transport-security'], xFrameOptions: !!headers['x-frame-options'], diff --git a/api/legacy-rank.js b/api/legacy-rank.js deleted file mode 100644 index 301cc2c..0000000 --- a/api/legacy-rank.js +++ /dev/null @@ -1,70 +0,0 @@ -import axios from 'axios'; -import unzipper from 'unzipper'; -import csv from 'csv-parser'; -import fs from 'fs'; -import middleware from './_common/middleware.js'; - -// Should also work with the following sources: -// https://www.domcop.com/files/top/top10milliondomains.csv.zip -// https://tranco-list.eu/top-1m.csv.zip -// https://www.domcop.com/files/top/top10milliondomains.csv.zip -// https://radar.cloudflare.com/charts/LargerTopDomainsTable/attachment?id=525&top=1000000 -// https://statvoo.com/dl/top-1million-sites.csv.zip - -const FILE_URL = 'https://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip'; -const TEMP_FILE_PATH = '/tmp/top-1m.csv'; - -const rankHandler = async (url) => { - let domain = null; - - try { - domain = new URL(url).hostname; - } catch (e) { - throw new Error('Invalid URL'); - } - -// Download and unzip the file if not in cache -if (!fs.existsSync(TEMP_FILE_PATH)) { - const response = await axios({ - method: 'GET', - url: FILE_URL, - responseType: 'stream' - }); - - await new Promise((resolve, reject) => { - response.data - .pipe(unzipper.Extract({ path: '/tmp' })) - .on('close', resolve) - .on('error', reject); - }); -} - -// Parse the CSV and find the rank -return new Promise((resolve, reject) => { - const csvStream = fs.createReadStream(TEMP_FILE_PATH) - .pipe(csv({ - headers: ['rank', 'domain'], - })) - .on('data', (row) => { - if (row.domain === domain) { - csvStream.destroy(); - resolve({ - domain: domain, - rank: row.rank, - isFound: true, - }); - } - }) - .on('end', () => { - resolve({ - skipped: `Skipping, as ${domain} is not present in the Umbrella top 1M list.`, - domain: domain, - isFound: false, - }); - }) - .on('error', reject); -}); -}; - -export const handler = middleware(rankHandler); -export default handler; diff --git a/api/linked-pages.js b/api/linked-pages.js index 096da4e..9516e5f 100644 --- a/api/linked-pages.js +++ b/api/linked-pages.js @@ -1,13 +1,13 @@ -import axios from 'axios'; import * as cheerio from 'cheerio'; import urlLib from 'url'; import middleware from './_common/middleware.js'; +import { httpGet } from './_common/http.js'; import { upstreamError } from './_common/upstream.js'; const linkedPagesHandler = async (url) => { let response; try { - response = await axios.get(url); + response = await httpGet(url); } catch (error) { return upstreamError(error, 'Linked pages fetch'); } diff --git a/api/location.js b/api/location.js index d4c0051..a029ffa 100644 --- a/api/location.js +++ b/api/location.js @@ -1,5 +1,5 @@ -import axios from 'axios'; import middleware from './_common/middleware.js'; +import { httpGet } from './_common/http.js'; import { parseTarget } from './_common/parse-target.js'; import { upstreamError } from './_common/upstream.js'; @@ -7,7 +7,7 @@ import { upstreamError } from './_common/upstream.js'; const locationHandler = async (url) => { const { hostname } = parseTarget(url); try { - const res = await axios.get(`https://ipapi.co/${hostname}/json/`, { timeout: 5000 }); + const res = await httpGet(`https://ipapi.co/${hostname}/json/`, { timeout: 5000 }); if (res.data?.error) return { skipped: res.data.reason || 'Lookup unavailable' }; return res.data; } catch (error) { diff --git a/api/quality.js b/api/quality.js index bbff8fc..71fab0f 100644 --- a/api/quality.js +++ b/api/quality.js @@ -1,5 +1,5 @@ -import axios from 'axios'; import middleware from './_common/middleware.js'; +import { httpGet } from './_common/http.js'; import { requireEnv, upstreamError } from './_common/upstream.js'; const qualityHandler = async (url) => { @@ -12,7 +12,7 @@ const qualityHandler = async (url) => { let data; try { - data = (await axios.get(endpoint)).data; + data = (await httpGet(endpoint)).data; } catch (error) { return upstreamError(error, 'Quality check'); } diff --git a/api/rank.js b/api/rank.js index e80f82e..265bec2 100644 --- a/api/rank.js +++ b/api/rank.js @@ -1,5 +1,5 @@ -import axios from 'axios'; import middleware from './_common/middleware.js'; +import { httpGet } from './_common/http.js'; import { parseTarget } from './_common/parse-target.js'; import { upstreamError } from './_common/upstream.js'; @@ -10,7 +10,7 @@ const rankHandler = async (url) => { ? { auth: { username: TRANCO_USERNAME, password: TRANCO_API_KEY } } : {}; try { - const response = await axios.get( + const response = await httpGet( `https://tranco-list.eu/api/ranks/domain/${domain}`, { timeout: 5000, ...auth }, ); diff --git a/api/redirects.js b/api/redirects.js index 1cea0f8..f6df1ef 100644 --- a/api/redirects.js +++ b/api/redirects.js @@ -1,21 +1,40 @@ -import got from 'got'; import middleware from './_common/middleware.js'; import { upstreamError } from './_common/upstream.js'; +const MAX_REDIRECTS = 12; +const TIMEOUT_MS = 10000; +const USER_AGENT = 'Mozilla/5.0 (compatible; WebCheck/2.0; +https://web-check.xyz)'; + +// Walks the redirect chain manually, recording each Location header as got did const redirectsHandler = async (url) => { const redirects = [url]; + let current = url; try { - await got(url, { - followRedirect: true, - maxRedirects: 12, - hooks: { - beforeRedirect: [ - (_options, response) => { redirects.push(response.headers.location); }, - ], - }, - }); + for (let i = 0; i < MAX_REDIRECTS; i++) { + const response = await fetch(current, { + redirect: 'manual', + signal: AbortSignal.timeout(TIMEOUT_MS), + headers: { 'user-agent': USER_AGENT }, + }); + if (response.status < 300 || response.status >= 400) { + if (response.status >= 400) { + const err = new Error(`HTTP ${response.status}`); + err.response = { status: response.status }; + throw err; + } + break; + } + const location = response.headers.get('location'); + if (!location) break; + redirects.push(location); + current = new URL(location, current).href; + } return { redirects }; } catch (error) { + if (error.cause?.code) error.code = error.cause.code; + if (error.name === 'TimeoutError' || error.name === 'AbortError') { + error.code = 'ECONNABORTED'; + } return upstreamError(error, 'Redirect lookup'); } }; diff --git a/api/robots-txt.js b/api/robots-txt.js index e65e40d..adc58d8 100644 --- a/api/robots-txt.js +++ b/api/robots-txt.js @@ -1,5 +1,5 @@ -import axios from 'axios'; import middleware from './_common/middleware.js'; +import { httpGet } from './_common/http.js'; import { parseTarget } from './_common/parse-target.js'; import { upstreamError } from './_common/upstream.js'; @@ -17,7 +17,7 @@ const parseRobotsTxt = (content) => { const robotsHandler = async (url) => { const { protocol, hostname } = parseTarget(url); try { - const res = await axios.get(`${protocol}//${hostname}/robots.txt`); + const res = await httpGet(`${protocol}//${hostname}/robots.txt`); const parsed = parseRobotsTxt(res.data || ''); return parsed.robots.length ? parsed diff --git a/api/screenshot.js b/api/screenshot.js index cf8c274..4e164af 100644 --- a/api/screenshot.js +++ b/api/screenshot.js @@ -1,5 +1,5 @@ import puppeteer from 'puppeteer-core'; -import chromium from 'chrome-aws-lambda'; +import chromium from '@sparticuz/chromium'; import { randomUUID } from 'crypto'; import { execFile } from 'child_process'; import { promises as fs } from 'fs'; @@ -46,9 +46,9 @@ const puppeteerScreenshot = async (targetUrl) => { browser = await puppeteer.launch({ args: [...chromium.args, '--no-sandbox'], defaultViewport: { width: 800, height: 600 }, - executablePath: process.env.CHROME_PATH || '/usr/bin/chromium', + executablePath: process.env.CHROME_PATH || await chromium.executablePath(), headless: true, - ignoreHTTPSErrors: true, + acceptInsecureCerts: true, ignoreDefaultArgs: ['--disable-extensions'], }); const page = await browser.newPage(); diff --git a/api/security-txt.js b/api/security-txt.js index df89b0c..bd70688 100644 --- a/api/security-txt.js +++ b/api/security-txt.js @@ -1,8 +1,6 @@ import { URL } from 'url'; -import followRedirects from 'follow-redirects'; import middleware from './_common/middleware.js'; - -const { https } = followRedirects; +import { httpGet } from './_common/http.js'; const SECURITY_TXT_PATHS = [ '/security.txt', @@ -71,26 +69,15 @@ const securityTxtHandler = async (urlParam) => { return { isPresent: false }; }; -async function fetchSecurityTxt(baseURL, path) { - return new Promise((resolve, reject) => { - const url = new URL(path, baseURL); - https.get(url.toString(), { headers: { 'User-Agent': 'curl/8.0.0' } }, (res) => { - if (res.statusCode === 200) { - let data = ''; - res.on('data', (chunk) => { - data += chunk; - }); - res.on('end', () => { - resolve(data); - }); - } else { - resolve(null); - } - }).on('error', (err) => { - reject(err); - }); +// Returns the file body when the path 200s, else null so the next path is tried +const fetchSecurityTxt = async (baseURL, path) => { + const url = new URL(path, baseURL); + const res = await httpGet(url.toString(), { + headers: { 'User-Agent': 'curl/8.0.0' }, + validateStatus: () => true, }); -} + return res.status === 200 ? res.data : null; +}; export const handler = middleware(securityTxtHandler); export default handler; diff --git a/api/shodan.js b/api/shodan.js index d07baba..5c09b5d 100644 --- a/api/shodan.js +++ b/api/shodan.js @@ -1,5 +1,5 @@ -import axios from 'axios'; import middleware from './_common/middleware.js'; +import { httpGet } from './_common/http.js'; import { parseTarget } from './_common/parse-target.js'; import { requireEnv, upstreamError } from './_common/upstream.js'; @@ -9,7 +9,7 @@ const shodanHandler = async (url) => { if (auth.skipped) return auth; const { hostname } = parseTarget(url); try { - const res = await axios.get( + const res = await httpGet( `https://api.shodan.io/shodan/host/${hostname}?key=${auth.value}`, { timeout: 8000 }, ); diff --git a/api/sitemap.js b/api/sitemap.js index 1f2971d..1aa44be 100644 --- a/api/sitemap.js +++ b/api/sitemap.js @@ -1,6 +1,6 @@ -import axios from 'axios'; import xml2js from 'xml2js'; import middleware from './_common/middleware.js'; +import { httpGet } from './_common/http.js'; import { upstreamError } from './_common/upstream.js'; const HARD_TIMEOUT = 5000; @@ -9,13 +9,13 @@ const MAX_CHILD_SITEMAPS = 25; // Fetch and parse a sitemap XML const fetchSitemap = async (sitemapUrl) => { - const res = await axios.get(sitemapUrl, { timeout: HARD_TIMEOUT }); + const res = await httpGet(sitemapUrl, { timeout: HARD_TIMEOUT }); return new xml2js.Parser().parseStringPromise(res.data); }; // Pull a Sitemap: line out of robots.txt const findSitemapInRobots = async (baseUrl) => { - const robots = await axios.get(`${baseUrl}/robots.txt`, { timeout: HARD_TIMEOUT }); + const robots = await httpGet(`${baseUrl}/robots.txt`, { timeout: HARD_TIMEOUT }); for (const line of robots.data.split('\n')) { if (line.toLowerCase().startsWith('sitemap:')) { return line.split(/\s+/)[1]?.trim() || null; diff --git a/api/social-tags.js b/api/social-tags.js index 2654907..7badb31 100644 --- a/api/social-tags.js +++ b/api/social-tags.js @@ -1,12 +1,12 @@ -import axios from 'axios'; import * as cheerio from 'cheerio'; import middleware from './_common/middleware.js'; +import { httpGet } from './_common/http.js'; import { upstreamError } from './_common/upstream.js'; const socialTagsHandler = async (url) => { let response; try { - response = await axios.get(url); + response = await httpGet(url); } catch (error) { return upstreamError(error, 'Social tags fetch'); } diff --git a/api/threats.js b/api/threats.js index 73bef96..cdd76eb 100644 --- a/api/threats.js +++ b/api/threats.js @@ -1,6 +1,6 @@ -import axios from 'axios'; import xml2js from 'xml2js'; import middleware from './_common/middleware.js'; +import { httpPost } from './_common/http.js'; import { parseTarget } from './_common/parse-target.js'; import { requireEnv, upstreamError } from './_common/upstream.js'; @@ -8,7 +8,7 @@ const safeBrowsing = async (url) => { const auth = requireEnv('GOOGLE_CLOUD_API_KEY', 'Google Safe Browsing'); if (auth.skipped) return auth; try { - const res = await axios.post( + const res = await httpPost( `https://safebrowsing.googleapis.com/v4/threatMatches:find?key=${auth.value}`, { threatInfo: { @@ -33,7 +33,7 @@ const safeBrowsing = async (url) => { const urlHaus = async (url) => { const { hostname } = parseTarget(url); try { - const res = await axios.post( + const res = await httpPost( 'https://urlhaus-api.abuse.ch/v1/host/', `host=${hostname}`, { headers: { 'Content-Type': 'application/x-www-form-urlencoded' } }, @@ -47,7 +47,7 @@ const urlHaus = async (url) => { const phishTank = async (url) => { try { const encoded = Buffer.from(url).toString('base64'); - const res = await axios.post( + const res = await httpPost( `https://checkurl.phishtank.com/checkurl/?url=${encoded}`, null, { headers: { 'User-Agent': 'phishtank/web-check' }, timeout: 3000 }, @@ -63,7 +63,7 @@ const cloudmersive = async (url) => { const auth = requireEnv('CLOUDMERSIVE_API_KEY', 'Cloudmersive'); if (auth.skipped) return auth; try { - const res = await axios.post( + const res = await httpPost( 'https://api.cloudmersive.com/virus/scan/website', `Url=${encodeURIComponent(url)}`, { diff --git a/api/tls-labs.js b/api/tls-labs.js index 69798da..e69d87d 100644 --- a/api/tls-labs.js +++ b/api/tls-labs.js @@ -1,5 +1,5 @@ -import axios from 'axios'; import middleware from './_common/middleware.js'; +import { httpGet } from './_common/http.js'; import { parseTarget } from './_common/parse-target.js'; import { upstreamError } from './_common/upstream.js'; @@ -9,7 +9,7 @@ const SSL_LABS = 'https://api.ssllabs.com/api/v3/analyze'; const tlsLabsHandler = async (url) => { const { hostname } = parseTarget(url); try { - const res = await axios.get(SSL_LABS, { + const res = await httpGet(SSL_LABS, { params: { host: hostname, fromCache: 'on', maxAge: 24, all: 'done' }, timeout: 8000, headers: { 'User-Agent': 'web-check (https://web-check.xyz)' }, diff --git a/api/whois-pro.js b/api/whois-pro.js index 8179719..6aa2085 100644 --- a/api/whois-pro.js +++ b/api/whois-pro.js @@ -1,5 +1,5 @@ -import axios from 'axios'; import middleware from './_common/middleware.js'; +import { httpGet } from './_common/http.js'; import { parseTarget } from './_common/parse-target.js'; import { requireEnv, upstreamError } from './_common/upstream.js'; @@ -9,7 +9,7 @@ const whoisProHandler = async (url) => { const { hostname } = parseTarget(url); let data; try { - const res = await axios.get( + const res = await httpGet( `https://api.whoapi.com/?domain=${hostname}&r=whois&apikey=${auth.value}`, { timeout: 8000 }, ); diff --git a/api/whois.js b/api/whois.js index 084e3e9..0c85d4e 100644 --- a/api/whois.js +++ b/api/whois.js @@ -1,7 +1,7 @@ import net from 'net'; import psl from 'psl'; -import axios from 'axios'; import middleware from './_common/middleware.js'; +import { httpPost } from './_common/http.js'; import { createLogger } from './_common/logger.js'; const log = createLogger('whois'); @@ -71,8 +71,8 @@ const fetchFromInternic = async (hostname) => new Promise((resolve, reject) => { const fetchFromMyAPI = async (hostname) => { try { - const response = await axios.post('https://whois-api-zeta.vercel.app/', { - domain: hostname + const response = await httpPost('https://whois-api-zeta.vercel.app/', { + domain: hostname, }); return response.data; } catch (error) { diff --git a/astro.config.mjs b/astro.config.mjs index 1790e63..8ed2959 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -7,7 +7,7 @@ import partytown from '@astrojs/partytown'; import sitemap from '@astrojs/sitemap'; // Adapters -import vercelAdapter from '@astrojs/vercel/serverless'; +import vercelAdapter from '@astrojs/vercel'; import netlifyAdapter from '@astrojs/netlify'; import nodeAdapter from '@astrojs/node'; import cloudflareAdapter from '@astrojs/cloudflare'; @@ -22,8 +22,8 @@ const unwrapEnvVar = (varName, fallbackValue) => { // Determine the deploy target (vercel, netlify, cloudflare, node) const deployTarget = unwrapEnvVar('PLATFORM', 'node').toLowerCase(); -// Determine the output mode (server, hybrid or static) -const output = unwrapEnvVar('OUTPUT', 'hybrid'); +// Determine the output mode (static or server). Mixed prerender supported in static mode +const output = unwrapEnvVar('OUTPUT', 'static'); // The FQDN of where the site is hosted (used for sitemaps & canonical URLs) const site = unwrapEnvVar('SITE_URL', 'https://web-check.xyz'); diff --git a/package.json b/package.json index 2f8a26d..2ef1312 100644 --- a/package.json +++ b/package.json @@ -4,11 +4,10 @@ "version": "2.0.2", "homepage": "https://web-check.xyz", "engines": { - "node": "20.x" + "node": ">=22" }, "scripts": { "start": "node server", - "start-pm": "pm2 start server.js -i max", "build": "astro check && astro build", "dev:vercel": "PLATFORM='vercel' npx vercel dev", "dev:netlify": "PLATFORM='netlify' npx netlify dev", @@ -17,58 +16,63 @@ "dev": "concurrently -c magenta,cyan -n backend,frontend 'yarn dev:api' 'yarn dev:astro'" }, "dependencies": { - "@astrojs/check": "^0.5.10", - "@astrojs/react": "^3.6.3", + "@astrojs/check": "^0.9.9", + "@astrojs/react": "^5.0.4", "@emotion/react": "^11.14.0", "@emotion/styled": "^11.14.1", - "@fortawesome/fontawesome-svg-core": "^6.7.2", - "@fortawesome/free-brands-svg-icons": "^6.7.2", - "@fortawesome/free-regular-svg-icons": "^6.7.2", - "@fortawesome/free-solid-svg-icons": "^6.7.2", + "@fortawesome/fontawesome-svg-core": "^7.2.0", + "@fortawesome/free-brands-svg-icons": "^7.2.0", + "@fortawesome/free-regular-svg-icons": "^7.2.0", + "@fortawesome/free-solid-svg-icons": "^7.2.0", "@fortawesome/svelte-fontawesome": "^0.2.4", - "@types/react": "^18.3.28", - "@types/react-dom": "^18.3.7", - "astro": "^4.16.19", - "axios": "^1.16.0", + "@sparticuz/chromium": "^148.0.0", + "@types/react": "^19.2.14", + "@types/react-dom": "^19.2.3", + "astro": "^6.2.2", "cheerio": "^1.2.0", - "chrome-aws-lambda": "^10.1.0", - "chromium": "^3.0.3", - "connect-history-api-fallback": "^2.0.0", - "cors": "^2.8.5", - "csv-parser": "^3.2.0", - "dotenv": "^16.6.1", - "express": "^4.21.2", - "express-rate-limit": "^7.5.1", - "framer-motion": "^11.18.2", - "got": "^14.6.6", - "pm2": "^5.4.3", + "cors": "^2.8.6", + "dotenv": "^17.4.2", + "express": "^5.2.1", + "express-rate-limit": "^8.5.0", + "framer-motion": "^12.38.0", + "prop-types": "^15.8.1", "psl": "^1.15.0", - "puppeteer": "^22.15.0", - "puppeteer-core": "^22.15.0", - "react": "^18.3.1", - "react-dom": "^18.3.1", + "puppeteer": "^24.42.0", + "puppeteer-core": "^24.42.0", + "react": "^19.2.5", + "react-dom": "^19.2.5", "react-masonry-css": "^1.0.16", - "react-router-dom": "^6.30.3", + "react-router-dom": "^7.14.2", "react-simple-maps": "^3.0.0", - "react-toastify": "^10.0.6", - "recharts": "^2.15.4", - "svelte": "^4.2.20", - "typescript": "^5.9.3", - "unzipper": "^0.11.6", - "url-parse": "^1.5.10", + "react-toastify": "^11.1.0", + "recharts": "^3.8.1", + "svelte": "^5.55.5", + "typescript": "^6.0.3", "wappalyzer": "^6.10.66", "xml2js": "^0.6.2" }, + "resolutions": { + "tar-fs": "^2.1.4", + "basic-ftp": "^5.2.0", + "picomatch": "^2.3.2", + "systeminformation": "^5.31.0", + "d3-color": "^3.1.0", + "braces": "^3.0.3", + "wappalyzer/**/ws": "^8.17.1", + "@vercel/routing-utils/path-to-regexp": "^6.3.0", + "uuid": "^14.0.0", + "yaml": "^2.8.3" + }, "devDependencies": { - "@astrojs/cloudflare": "^10.4.2", - "@astrojs/netlify": "^5.5.4", - "@astrojs/node": "^8.3.4", + "@astrojs/cloudflare": "^13.3.1", + "@astrojs/netlify": "^7.0.8", + "@astrojs/node": "^10.0.6", "@astrojs/partytown": "^2.1.7", - "@astrojs/sitemap": "~3.4.1", - "@astrojs/svelte": "^5.7.3", + "@astrojs/sitemap": "^3.7.2", + "@astrojs/svelte": "^8.1.0", "@astrojs/ts-plugin": "^1.10.7", - "@astrojs/vercel": "^7.8.2", - "concurrently": "^8.2.2", + "@astrojs/vercel": "^10.0.6", + "concurrently": "^9.2.1", "nodemon": "^3.1.14", "sass": "^1.99.0" }, diff --git a/public/index.html b/public/index.html deleted file mode 100644 index af45561..0000000 --- a/public/index.html +++ /dev/null @@ -1,38 +0,0 @@ - - - - - - - - - - - Web Check - - - - - - - - - - - - - - -
- - diff --git a/server.js b/server.js index bb6d367..26c475d 100644 --- a/server.js +++ b/server.js @@ -5,7 +5,6 @@ import cors from 'cors'; import dotenv from 'dotenv'; import express from 'express'; import rateLimit from 'express-rate-limit'; -import historyApiFallback from 'connect-history-api-fallback'; // Load environment variables from .env file dotenv.config(); @@ -56,7 +55,7 @@ const makeLimiterResponseMsg = (retryAfter) => { // Create rate limiters for each time frame const limiters = limits.map(limit => rateLimit({ windowMs: limit.timeFrame * 1000, - max: limit.max, + limit: limit.max, standardHeaders: true, legacyHeaders: false, message: { error: makeLimiterResponseMsg(limit.messageTime) } @@ -181,14 +180,6 @@ if (process.env.DISABLE_GUI && process.env.DISABLE_GUI !== 'false') { }); } -// Handle SPA routing -app.use(historyApiFallback({ - rewrites: [ - { from: new RegExp(`^${API_DIR}/.*$`), to: (context) => context.parsedUrl.path }, - { from: /^.*$/, to: '/index.html' } - ] -})); - // Anything left unhandled (which isn't an API endpoint), return a 404 app.use((req, res, next) => { if (!req.path.startsWith(`${API_DIR}/`)) { diff --git a/src/components/homepage/AnimatedInput.astro b/src/components/homepage/AnimatedInput.astro index d74202f..0b2f098 100644 --- a/src/components/homepage/AnimatedInput.astro +++ b/src/components/homepage/AnimatedInput.astro @@ -107,8 +107,8 @@ document.addEventListener('DOMContentLoaded', () => {