import { cfg } from './config.mjs'; import { resolve, dirname } from 'path'; import { fileURLToPath } from 'url'; import * as fs from 'node:fs'; import Database from 'better-sqlite3'; import { fromCsv } from '@ztimson/utils'; import {getIcon} from './adsb-shapes.mjs'; import * as cheerio from 'cheerio'; const DIR = dirname(fileURLToPath(import.meta.url)); const DATA = resolve(DIR, '../data'); const DB_PATH = resolve(DATA, 'aircraft.db'); const CSV_CACHE = resolve(DATA, 'aircraft_db.csv'); const MAX_HISTORY = 500; const HISTORY_TTL = 1000 * 60 * 60; // 1 hour const CSV_URL = 'https://s3.opensky-network.org/data-samples/metadata/aircraft-database-complete-2024-06.csv'; const MIL_RANGES_URL = ':8080/db-3.14.1708/ranges.js'; const MILITARY_OPERATORS = ['air force', 'army', 'navy', 'marine', 'coast guard', 'military', 'defence', 'defense', 'luftwaffe', 'RAF', 'USAF', 'USN', 'USMC']; const CARGO_OPERATORS = ['fedex', 'ups', 'dhl', 'cargo', 'freight', 'logistic', 'atlas air', 'kalitta', 'air freight']; const PASSENGER_OPERATORS = ['airlines', 'airways', 'air ', 'jet', 'easyjet', 'ryanair', 'delta', 'united', 'american', 'southwest', 'lufthansa', 'emirates', 'british']; const MILITARY_CLASSES = ['H1M', 'H2M', 'L1M', 'L2M', 'L4M', 'A0']; const noRecord = []; const history = new Map(); const milRanges = []; let db; setInterval(() => { const cutoff = Date.now() - HISTORY_TTL; for (const [key, trail] of history) { const last = trail.at(-1); if (!last || last.ts < cutoff) history.delete(key); } }, 1000 * 60 * 5); function fetchWithTimeout(url, ms = 5000) { const controller = new AbortController(); const id = setTimeout(() => controller.abort(), ms); return fetch(url, { signal: controller.signal }).finally(() => clearTimeout(id)); } async function syncMilitaryRanges() { const { ADSB_URL } = cfg(); try { const res = await fetch(ADSB_URL + MIL_RANGES_URL); if (!res.ok) throw new Error(`HTTP ${res.status}`); const { military } = await res.json(); db.exec('DELETE FROM military_ranges'); const insert = db.prepare('INSERT INTO military_ranges (start, end) VALUES (?, ?)'); const insertMany = db.transaction(ranges => { for (const [s, e] of ranges) insert.run(s.toUpperCase(), e.toUpperCase()); }); insertMany(military); milRanges.length = 0; milRanges.push(...military.map(([s, e]) => [s.toUpperCase(), e.toUpperCase()])); } catch (e) { const rows = db.prepare('SELECT start, end FROM military_ranges').all(); milRanges.length = 0; milRanges.push(...rows.map(r => [r.start, r.end])); } } export async function initAircraftDb() { if (!fs.existsSync(DATA)) fs.mkdirSync(DATA, { recursive: true }); const missing = !fs.existsSync(DB_PATH); if (missing) console.log(`✈️ Building database`); db = new Database(DB_PATH); db.exec(` CREATE TABLE IF NOT EXISTS aircraft ( icao TEXT PRIMARY KEY, built TEXT, categoryDescription TEXT, country TEXT, engines TEXT, class TEXT, manufacturer TEXT, model TEXT, modes TEXT, operator TEXT, operatorCallsign TEXT, owner TEXT, registration TEXT, serialNumber TEXT, aircraft TEXT ); CREATE TABLE IF NOT EXISTS military_ranges ( start TEXT NOT NULL, end TEXT NOT NULL ); `); await syncMilitaryRanges(); if (missing) { const res = await fetch(CSV_URL); if (!res.ok) throw new Error(`HTTP ${res.status}`); const text = await res.text(); fs.writeFileSync(CSV_CACHE, text); const csv = fromCsv(text, true); const insert = db.prepare(` INSERT OR REPLACE INTO aircraft VALUES ( @icao, @built, @categoryDescription, @country, @engines, @class, @manufacturer, @model, @modes, @operator, @operatorCallsign, @owner, @registration, @serialNumber, @aircraft ) `); const insertMany = db.transaction(rows => { for (const row of rows) insert.run(row); }); insertMany(csv.map(row => ({ icao: (row.icao24 || '').toUpperCase(), built: row.built || null, categoryDescription: row.categoryDescription || null, country: row.country || null, engines: row.engines || null, class: row.icaoAircraftClass || null, manufacturer: row.manufacturerName || row.manufacturerIcao || null, model: row.model || null, modes: row.modes || null, operator: row.operator || null, operatorCallsign: row.operatorCallsign || null, owner: row.owner || null, registration: row.registration || null, serialNumber: row.serialNumber || null, aircraft: row.typecode || null, }))); fs.unlinkSync(CSV_CACHE); console.log(`✈️ Aircraft imported: ${csv.length}`); } else { const count = db.prepare('SELECT COUNT(*) as c FROM aircraft').get(); console.log(`✈️ Aircraft loaded: ${count.c}`); } } function isIcaoInMilitaryRange(icao) { if (!icao) return false; const hex = icao.toUpperCase(); return milRanges.some(([s, e]) => hex >= s && hex <= e); } function wordMatch(text, keyword) { if (!text) return false; const escaped = keyword.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); return new RegExp(`(? fields.some(f => list.some(k => wordMatch(f, k))); const operatorFields = [row.operator, row.operatorCallsign]; const ownerFields = [row.owner]; const allFields = [...operatorFields, ...ownerFields, row.categoryDescription]; if (isIcaoInMilitaryRange(row.icao) || MILITARY_CLASSES.includes(row.class) || matchesAny(allFields, MILITARY_OPERATORS)) return 'military'; if (row.categoryDescription?.toLowerCase().includes('cargo') || matchesAny(operatorFields, CARGO_OPERATORS)) return 'cargo'; if (matchesAny(operatorFields, PASSENGER_OPERATORS)) return 'passenger'; if (row.owner && !row.operator) return 'private'; return 'unknown'; } async function fetchHexDb(icao) { const resp = await fetchWithTimeout(`https://hexdb.io/api/v1/aircraft/${icao}`); if (!resp.ok) return null; const found = await resp.json(); return { registration: found.Registration || null, manufacturer: found.Manufacturer || null, aircraft: found.ICAOTypeCode || null, model: found.Type || null, operator: found.RegisteredOwners || null, }; } async function scrapeHexDatabase(icao) { const url = `https://hexdatabase.com/h/${icao}`; const resp = await fetch(url); if (!resp.ok) return null; const $ = cheerio.load(await resp.text()); const row = $('tr').filter((_, el) => $(el).text().toLowerCase().includes(icao.toLowerCase())).first(); if (!row.length) return null; const cells = row.find('td'); return { country: $(cells[1]).text().trim() || null, registration: $(cells[2]).text().trim() || null, aircraft: $(cells[3]).text().trim() || null, operator: $(cells[4]).text().trim() || null, serialNumber: $(cells[6]).text().trim() || null, }; } function backfillFromSimilar(aircraft) { if (!aircraft) return {}; const similar = db.prepare(` SELECT manufacturer, model, engines, categoryDescription, class FROM aircraft WHERE aircraft = ? AND (manufacturer IS NOT NULL OR model IS NOT NULL OR engines IS NOT NULL) LIMIT 1 `).get(aircraft); if (!similar) return {}; return { manufacturer: similar.manufacturer || null, model: similar.model || null, engines: similar.engines || null, categoryDescription: similar.categoryDescription || null, class: similar.class || null, }; } export async function enrichAircraft(a) { if (!a.hex) return a; const icao = a.hex.toUpperCase(); // 1. Check own DB first const row = db.prepare('SELECT * FROM aircraft WHERE icao = ?').get(icao); if(row?.manufacturer && row?.model) return { ...a, ...row, type: classifyAircraft(row) }; // 2. Race the two external sources if(noRecord.includes(icao)) return { icao, ...a, type: 'unknown' }; let found = scrapeHexDatabase(icao); if(!found) { noRecord.push(icao); return { icao, ...a, type: 'unknown' }; } // 3. Backfill manufacturer/model/engines from similar aircraft type in DB const merged = { ...row, ...found }; if (merged.aircraft && (!merged.manufacturer || !merged.model)) { const similar = backfillFromSimilar(merged.aircraft); Object.assign(merged, similar); } // 4. Save back to DB if (found) { db.prepare(` INSERT INTO aircraft (icao, registration, manufacturer, aircraft, model, operator, country, serialNumber, engines, categoryDescription, class) VALUES (@icao, @registration, @manufacturer, @aircraft, @model, @operator, @country, @serialNumber, @engines, @categoryDescription, @class) ON CONFLICT(icao) DO UPDATE SET registration = COALESCE(excluded.registration, registration), manufacturer = COALESCE(excluded.manufacturer, manufacturer), aircraft = COALESCE(excluded.aircraft, aircraft), model = COALESCE(excluded.model, model), operator = COALESCE(excluded.operator, operator), country = COALESCE(excluded.country, country), serialNumber = COALESCE(excluded.serialNumber, serialNumber), engines = COALESCE(excluded.engines, engines), categoryDescription = COALESCE(excluded.categoryDescription, categoryDescription), class = COALESCE(excluded.class, class) `).run({ icao, registration: merged.registration || null, manufacturer: merged.manufacturer || null, aircraft: merged.aircraft || null, model: merged.model || null, operator: merged.operator || null, country: merged.country || null, serialNumber: merged.serialNumber || null, engines: merged.engines || null, categoryDescription: merged.categoryDescription || null, class: merged.class || null, }); } return {icao, ...a, ...merged, type: classifyAircraft(merged) } } export async function getADSB() { const { ADSB_URL } = cfg(); if (!ADSB_URL) return []; const r = await fetchWithTimeout(`${ADSB_URL}:8080/data/aircraft.json`); const j = await r.json(); const aircraft = j.aircraft || []; for (const a of aircraft) { a.type = 'unknown'; if (!a.hex || !a.lat || !a.lon) continue; const key = a.hex.toLowerCase(); if (!history.has(key)) history.set(key, []); const trail = history.get(key); trail.push({ latitude: a.lat, longitude: a.lon, altitude: a.alt_baro || 0, ts: Date.now() }); if (trail.length > MAX_HISTORY) trail.shift(); } return Promise.all(aircraft.map(async a => { a = await enrichAircraft(a); a.icon = getIcon(a); return a; })); } export async function getADSBHistory(icao) { return { history: history.get(icao?.toLowerCase()) || [] }; } export async function getADSBRange() { const { ADSB_URL } = cfg(); if (!ADSB_URL) return []; const r = await fetch(`${ADSB_URL}:8080/data/outline.json`); const j = await r.json(); return j?.actualRange?.last24h?.points || []; }