diff --git a/server/src/adsb.mjs b/server/src/adsb.mjs index 288dc19..c184666 100644 --- a/server/src/adsb.mjs +++ b/server/src/adsb.mjs @@ -19,70 +19,78 @@ const HISTORY_TTL = 1000 * 60 * 60; // 1 hour const ADSB_TTL = 1000; const CSV_URL = 'https://s3.opensky-network.org/data-samples/metadata/aircraft-database-complete-2024-06.csv'; -const AIRLINES_URL = 'https://raw.githubusercontent.com/jpatokal/openflights/master/data/airlines.dat' - -const MILITARY_OPERATORS = ['air force', 'airforce', 'army', 'navy', 'marine', 'coast guard', 'military', 'defence', 'defense', 'law', 'luftwaffe', 'RAF', 'USAF', 'USN', 'USMC']; +const AIRLINES_URL = 'https://raw.githubusercontent.com/jpatokal/openflights/master/data/airlines.dat'; const CARGO_OPERATORS = ['fedex', 'ups', 'dhl', 'cargo', 'freight', 'logistic', 'atlas air', 'kalitta', 'air freight']; const PASSENGER_OPERATORS = ['airlines', 'airways', 'jetblue', 'easyjet', 'ryanair', 'delta', 'united', 'american', 'avianca', 'southwest', 'lufthansa', 'emirates', 'british']; -// const MILITARY_CLASSES = ['H1M', 'H2M', 'L1M', 'L2M', 'L4M', 'A0']; +const MILITARY_CLASSES = ['H1M', 'H2M', 'L1M', 'L2M', 'L4M', 'A0']; +let db; let passengerOperators = PASSENGER_OPERATORS; - let adsbCache = null; let adsbCacheTs = 0; const noRecord = []; const history = new Map(); const milRanges = []; -let db; -function normalizeAirlineName(value) { - return String(value || '') - .normalize('NFKD') - .replace(/[\u0300-\u036f]/g, '') - .toLowerCase() - .replace(/&/g, ' and ') - .replace(/[^a-z0-9]+/g, ' ') - .trim() - .replace(/\s+/g, ' '); +function backfillModel(aircraft) { + if (!aircraft) return {}; + const similar = db.prepare(` + SELECT + manufacturer, + model, + engines, + categoryDescription, + class + FROM aircraft + WHERE aircraft = ? + AND ( + manufacturer IS NOT NULL + OR model IS NOT NULL + OR engines IS NOT NULL + ) + LIMIT 1 + `).get(aircraft); + if (!similar) return {}; + return { + manufacturer: similar.manufacturer || null, + model: similar.model || null, + engines: similar.engines || null, + categoryDescription: similar.categoryDescription || null, + class: similar.class || null + }; } -async function syncAirlines() { - try { - if (!fs.existsSync(AIRLINES_CACHE)) { - console.log('✈️ Downloading airline database'); - const res = await fetchWithTimeout(AIRLINES_URL, 10000); - if (!res.ok) throw new Error(`HTTP ${res.status}`); - fs.writeFileSync(AIRLINES_CACHE, await res.text()); - } - - const text = fs.readFileSync(AIRLINES_CACHE, 'utf8'); - const airlines = fromCsv(text, true); - const names = new Set(); - for (const row of airlines) { - if (row.active !== 'Y') continue; - for (const value of [row.name, row.alias, row.callsign]) { - const normalized = normalizeAirlineName(value); - if (normalized.length >= 3) names.add(normalized); - } - } - for (const value of PASSENGER_OPERATORS) names.add(normalizeAirlineName(value)); - passengerOperators = [...names].sort((a, b) => b.length - a.length); - console.log(`✈️ Airline operators loaded: ${passengerOperators.length}`); - } catch (e) { - console.warn(`⚠️ Could not load airline database: ${e.message}`); - passengerOperators = PASSENGER_OPERATORS.map(normalizeAirlineName); - } +async function fetchRegistration1(icao) { + const url = `https://hexdatabase.com/h/${icao}`; + const resp = await fetch(url); + if (!resp.ok) return null; + const html = await resp.text(); + const tableMatch = html.match(//i); + if (!tableMatch) return null; + const $ = cheerio.load(tableMatch[0]); + const row = $('tr').filter((_, el) => $(el).text()?.toLowerCase()?.includes(icao.toLowerCase())).first(); + if (!row.length) return null; + const cells = row.find('td'); + return { + registration: $(cells[1]).text().trim() || null, + aircraft: $(cells[2]).text().trim() || null, + operator: $(cells[3]).text().trim() || null, + serialNumber: $(cells[5]).text().trim() || null + }; } -setInterval(() => { - const cutoff = Date.now() - HISTORY_TTL; - - for (const [key, trail] of history) { - const last = trail.at(-1); - - if (!last || last.ts < cutoff) history.delete(key); - } -}, 1000 * 60 * 5); +async function fetchRegistration2(icao) { + const resp = await fetchWithTimeout(`https://hexdb.io/api/v1/aircraft/${icao}`); + if (!resp.ok) return null; + const found = await resp.json(); + return { + registration: found.Registration || null, + manufacturer: found.Manufacturer || null, + aircraft: found.ICAOTypeCode || null, + model: found.Type || null, + operator: found.RegisteredOwners || null + }; +} function fetchWithTimeout(url, ms = 5000) { const controller = new AbortController(); @@ -90,29 +98,6 @@ function fetchWithTimeout(url, ms = 5000) { return fetch(url, { signal: controller.signal }).finally(() => clearTimeout(id)); } -async function syncMilitaryRanges() { - const { ADSB_URL } = cfg(); - try { - const page = await fetch(ADSB_URL + ':8080').then(resp => resp.text()); - const db = /databaseFolder = "(db-.+?)"/.exec(page); - const res = await fetch(ADSB_URL + `:8080/${db[1]}/ranges.js`); - if (!res.ok) throw new Error(`HTTP ${res.status}`); - const { military } = await res.json(); - db.exec('DELETE FROM military_ranges'); - const insert = db.prepare('INSERT INTO military_ranges (start, end) VALUES (?, ?)'); - const insertMany = db.transaction(ranges => { - for (const [s, e] of ranges) insert.run(s.toUpperCase(), e.toUpperCase()); - }); - insertMany(military); - milRanges.length = 0; - milRanges.push(...military.map(([s, e]) => [s.toUpperCase(), e.toUpperCase()])); - } catch (e) { - const rows = db.prepare('SELECT start, end FROM military_ranges').all(); - milRanges.length = 0; - milRanges.push(...rows.map(r => [r.start, r.end])); - } -} - export async function initAircraftDb() { if (!fs.existsSync(DATA)) fs.mkdirSync(DATA, { recursive: true }); if (!fs.existsSync(IMAGES_DIR)) fs.mkdirSync(IMAGES_DIR, { recursive: true }); @@ -142,7 +127,7 @@ export async function initAircraftDb() { `); await syncAirlines(); - await syncMilitaryRanges(); + await syncMilitaryIcao(); if(missing) { const res = await fetch(CSV_URL); @@ -200,111 +185,104 @@ export async function initAircraftDb() { } } -function isIcaoInMilitaryRange(icao) { +async function syncAirlines() { + try { + if (!fs.existsSync(AIRLINES_CACHE)) { + console.log('✈️ Downloading airline database'); + const res = await fetchWithTimeout(AIRLINES_URL, 10000); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + fs.writeFileSync(AIRLINES_CACHE, await res.text()); + } + + const text = fs.readFileSync(AIRLINES_CACHE, 'utf8'); + const airlines = fromCsv(text, true); + const names = new Set(); + for (const row of airlines) { + if (row.active !== 'Y') continue; + for (const value of [row.name, row.alias, row.callsign]) { + const normalized = normalizeName(value); + if (normalized.length >= 3) names.add(normalized); + } + } + for (const value of PASSENGER_OPERATORS) names.add(normalizeName(value)); + passengerOperators = [...names].sort((a, b) => b.length - a.length); + console.log(`✈️ Airline operators loaded: ${passengerOperators.length}`); + } catch (e) { + console.warn(`⚠️ Could not load airline database: ${e.message}`); + passengerOperators = PASSENGER_OPERATORS.map(normalizeName); + } +} + +async function syncMilitaryIcao() { + const { ADSB_URL } = cfg(); + try { + const page = await fetch(ADSB_URL + ':8080').then(resp => resp.text()); + const db = /databaseFolder = "(db-.+?)"/.exec(page); + const res = await fetch(ADSB_URL + `:8080/${db[1]}/ranges.js`); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const { military } = await res.json(); + db.exec('DELETE FROM military_ranges'); + const insert = db.prepare('INSERT INTO military_ranges (start, end) VALUES (?, ?)'); + const insertMany = db.transaction(ranges => { + for (const [s, e] of ranges) insert.run(s.toUpperCase(), e.toUpperCase()); + }); + insertMany(military); + milRanges.length = 0; + milRanges.push(...military.map(([s, e]) => [s.toUpperCase(), e.toUpperCase()])); + } catch (e) { + const rows = db.prepare('SELECT start, end FROM military_ranges').all(); + milRanges.length = 0; + milRanges.push(...rows.map(r => [r.start, r.end])); + } +} + +function isMilitaryIcao(icao) { if (!icao) return false; const hex = icao.toUpperCase(); return milRanges.some(([s, e]) => hex >= s && hex <= e); } -function wordMatch(text, keyword) { - if (!text) return false; - const escaped = keyword.replace(/[.*+?^${}()|[\]\\]/gi, '\\$&'); - return new RegExp(`(? fields.some(f => { - const value = normalizeAirlineName(f); - return value && list.some(k => value.includes(normalizeAirlineName(k))); + const value = normalizeName(f); + return value && list.some(k => value.includes(normalizeName(k))); }); - const operatorFields = [row.operator, row.operatorCallsign]; - const ownerFields = [row.owner]; - const allFields = [...operatorFields, ...ownerFields, row.categoryDescription]; - - if (isIcaoInMilitaryRange(row.icao) || matchesAny(allFields, MILITARY_OPERATORS)) { + const allFields = [row.operator, row.operatorCallsign, row.owner, row.categoryDescription]; + if (isMilitaryIcao(row.icao) || MILITARY_CLASSES.includes(row.class)) { return 'military'; } - if (row.categoryDescription?.toLowerCase().includes('cargo') || matchesAny(operatorFields, CARGO_OPERATORS)) { + if (row.categoryDescription?.toLowerCase().includes('cargo') || matchesAny(allFields, CARGO_OPERATORS)) { return 'cargo'; } - if (matchesAny(operatorFields, passengerOperators)) return 'passenger'; + if (matchesAny(allFields, passengerOperators)) return 'passenger'; if (row.owner && !row.operator) return 'private'; return 'unknown'; } -async function fetchHexDb(icao) { - const resp = await fetchWithTimeout(`https://hexdb.io/api/v1/aircraft/${icao}`); - if (!resp.ok) return null; - const found = await resp.json(); - return { - registration: found.Registration || null, - manufacturer: found.Manufacturer || null, - aircraft: found.ICAOTypeCode || null, - model: found.Type || null, - operator: found.RegisteredOwners || null - }; +function normalizeName(value) { + return String(value || '') + .normalize('NFKD') + .replace(/[\u0300-\u036f]/g, '') + .toLowerCase() + .replace(/&/g, ' and ') + .replace(/[^a-z0-9]+/g, ' ') + .trim() + .replace(/\s+/g, ' '); } -async function scrapeHexDatabase(icao) { - const url = `https://hexdatabase.com/h/${icao}`; - const resp = await fetch(url); - if (!resp.ok) return null; - const html = await resp.text(); - const tableMatch = html.match(//i); - if (!tableMatch) return null; - const $ = cheerio.load(tableMatch[0]); - const row = $('tr').filter((_, el) => $(el).text()?.toLowerCase()?.includes(icao.toLowerCase())).first(); - if (!row.length) return null; - const cells = row.find('td'); - return { - registration: $(cells[1]).text().trim() || null, - aircraft: $(cells[2]).text().trim() || null, - operator: $(cells[3]).text().trim() || null, - serialNumber: $(cells[5]).text().trim() || null - }; -} - -function backfillFromSimilar(aircraft) { - if (!aircraft) return {}; - const similar = db.prepare(` - SELECT - manufacturer, - model, - engines, - categoryDescription, - class - FROM aircraft - WHERE aircraft = ? - AND ( - manufacturer IS NOT NULL - OR model IS NOT NULL - OR engines IS NOT NULL - ) - LIMIT 1 - `).get(aircraft); - if (!similar) return {}; - return { - manufacturer: similar.manufacturer || null, - model: similar.model || null, - engines: similar.engines || null, - categoryDescription: similar.categoryDescription || null, - class: similar.class || null - }; -} - -export async function enrichAircraft(a) { +export async function enrich(a) { if (!a.hex) return a; const icao = a.hex.toUpperCase(); const row = db.prepare('SELECT * FROM aircraft WHERE icao = ?').get(icao); if (row?.aircraft) return { ...a, ...row, type: classifyAircraft(row) }; if (noRecord.includes(icao)) return { icao, ...a, type: 'unknown' }; - const hexDbPromise = fetchHexDb(icao); - const scrapePromise = scrapeHexDatabase(icao); + const hexDbPromise = fetchRegistration2(icao); + const scrapePromise = fetchRegistration1(icao); let found = await hexDbPromise.catch(() => {}); if (!found) found = await scrapePromise.catch(() => {}); if (!found) { @@ -313,7 +291,7 @@ export async function enrichAircraft(a) { } const merged = { ...row, ...found }; if (merged.aircraft && (!merged.manufacturer || !merged.model)) { - const similar = backfillFromSimilar(merged.aircraft); + const similar = backfillModel(merged.aircraft); Object.assign(merged, similar); } if(found) { @@ -349,7 +327,7 @@ export async function enrichAircraft(a) { return { icao, ...a, ...merged, type: classifyAircraft(merged) }; } -export async function getADSB() { +export async function get() { if(adsbCache && Date.now() - adsbCacheTs < ADSB_TTL) return adsbCache; const { ADSB_URL } = cfg(); @@ -375,7 +353,7 @@ export async function getADSB() { } adsbCache = await Promise.all(aircraft.map(async a => { - a = await enrichAircraft(a); + a = await enrich(a); a.icon = getIcon(a); return a; })); @@ -383,11 +361,7 @@ export async function getADSB() { return adsbCache; } -export async function getADSBHistory(icao) { - return { history: history.get(icao?.toLowerCase()) || [] }; -} - -export async function getADSBRange() { +export async function getAttenuation() { const { ADSB_URL } = cfg(); if (!ADSB_URL) return []; const r = await fetch(`${ADSB_URL}:8080/data/outline.json`); @@ -395,7 +369,11 @@ export async function getADSBRange() { return j?.actualRange?.last24h?.points || []; } -export async function getADSBImage(icao) { +export async function getHistory(icao) { + return { history: history.get(icao?.toLowerCase()) || [] }; +} + +export async function getIcaoImage(icao) { function randomUA() { const v = 36 + Math.floor(Math.random() * 40); const builds = [ @@ -487,9 +465,18 @@ export async function getADSBImage(icao) { icao = icao.toLowerCase(); if(!fs.existsSync(IMAGES_DIR)) fs.mkdirSync(IMAGES_DIR, { recursive: true }); - const aircraft = await enrichAircraft({ hex: icao }); + const aircraft = await enrich({ hex: icao }); const modelType = aircraft.aircraft || aircraft.model || aircraft.type; const generic = genericImage(modelType); const specific = await icaoImage(icao); return specific || await generic; } + +// Purge old history +setInterval(() => { + const cutoff = Date.now() - HISTORY_TTL; + for (const [key, trail] of history) { + const last = trail.at(-1); + if (!last || last.ts < cutoff) history.delete(key); + } +}, 1000 * 60 * 5); diff --git a/server/src/server.mjs b/server/src/server.mjs index 980f782..870b850 100644 --- a/server/src/server.mjs +++ b/server/src/server.mjs @@ -8,7 +8,7 @@ import {apiReference} from '@scalar/express-api-reference'; import {spec} from './spec.mjs'; import {existsSync} from 'fs'; import {getAIS, getAISImage} from './ais.mjs'; -import {getADSBImage, getADSB, getADSBHistory, getADSBRange, initAircraftDb} from './adsb.mjs'; +import {getIcaoImage, get, getHistory, getAttenuation, initAircraftDb} from './adsb.mjs'; import {fetchIcon} from './openweather.mjs'; import {getForecast, getWeatherCondition, forecastTTL} from './forecast.mjs'; import {dailyWeather, hourlyWeather} from './openmeteo.mjs'; @@ -129,7 +129,7 @@ app.get('/api/daily', asyncHandler(async (req, res) => { // ── Position ────────────────────────────────────────────────────────────────── -app.get('/api/range', asyncHandler(async (req, res) => res.json(await getADSBRange()))); +app.get('/api/range', asyncHandler(async (req, res) => res.json(await getAttenuation()))); app.get('/api/position', asyncHandler(async (req, res) => { const {fields} = req.query; const data = await getCoords(); @@ -155,12 +155,12 @@ app.get('/api/space', async (req, res) => { // ── ADSB/AIS/SAT/RadioSonde Proxy ──────────────────────────────────────────────────────────── -app.get('/api/adsb', asyncHandler(async (req, res) => res.json(await getADSB()))); +app.get('/api/adsb', asyncHandler(async (req, res) => res.json(await get()))); app.get('/api/adsb/:icao/image', asyncHandler(async (req, res) => { - const buffer = await getADSBImage(req.params.icao); + const buffer = await getIcaoImage(req.params.icao); res.contentType('image/jpeg').send(buffer) })); -app.get('/api/adsb/:icao', asyncHandler(async (req, res) => res.json(await getADSBHistory(req.params.icao)))); +app.get('/api/adsb/:icao', asyncHandler(async (req, res) => res.json(await getHistory(req.params.icao)))); app.get('/api/ais', asyncHandler(async (req, res) => res.json(await getAIS()))); app.get('/api/ais/:mmsi/image', asyncHandler(async (req, res) => {