import { cfg } from './config.mjs';
import { resolve, dirname } from 'path';
import { fileURLToPath } from 'url';
import * as fs from 'node:fs';
import Database from 'better-sqlite3';
import { fromCsv } from '@ztimson/utils';
import { getIcon } from './adsb-shapes.mjs';
import * as cheerio from 'cheerio';
import sharp from 'sharp';
const DIR = dirname(fileURLToPath(import.meta.url));
const DATA = resolve(DIR, '../data');
const IMAGES_DIR = resolve(DATA, 'aircraft');
const DB_PATH = resolve(DATA, 'aircraft.db');
const CSV_CACHE = resolve(DATA, 'aircraft_db.csv');
const AIRLINES_CACHE = resolve(DATA, 'airlines.csv');
const MAX_HISTORY = 500;
const HISTORY_TTL = 1000 * 60 * 60; // 1 hour
const ADSB_TTL = 1000;
const CSV_URL = 'https://s3.opensky-network.org/data-samples/metadata/aircraft-database-complete-2024-06.csv';
const AIRLINES_URL = 'https://raw.githubusercontent.com/jpatokal/openflights/master/data/airlines.dat';
const CARGO_OPERATORS = ['fedex', 'ups', 'dhl', 'cargo', 'freight', 'logistic', 'atlas air', 'kalitta', 'air freight'];
const PASSENGER_OPERATORS = ['airlines', 'airways', 'jetblue', 'easyjet', 'ryanair', 'delta', 'united', 'american', 'avianca', 'southwest', 'lufthansa', 'emirates', 'british'];
const MILITARY_CLASSES = ['H1M', 'H2M', 'L1M', 'L2M', 'L4M', 'A0'];
let db;
let passengerOperators = PASSENGER_OPERATORS;
let adsbCache = null;
let adsbCacheTs = 0;
const noRecord = [];
const history = new Map();
const milRanges = [];
function backfillModel(aircraft) {
if (!aircraft) return {};
const similar = db.prepare(`
SELECT
manufacturer,
model,
engines,
categoryDescription,
class
FROM aircraft
WHERE aircraft = ?
AND (
manufacturer IS NOT NULL
OR model IS NOT NULL
OR engines IS NOT NULL
)
LIMIT 1
`).get(aircraft);
if (!similar) return {};
return {
manufacturer: similar.manufacturer || null,
model: similar.model || null,
engines: similar.engines || null,
categoryDescription: similar.categoryDescription || null,
class: similar.class || null
};
}
async function fetchRegistration1(icao) {
const url = `https://hexdatabase.com/h/${icao}`;
const resp = await fetch(url);
if (!resp.ok) return null;
const html = await resp.text();
const tableMatch = html.match(/
/i);
if (!tableMatch) return null;
const $ = cheerio.load(tableMatch[0]);
const row = $('tr').filter((_, el) => $(el).text()?.toLowerCase()?.includes(icao.toLowerCase())).first();
if (!row.length) return null;
const cells = row.find('td');
return {
registration: $(cells[1]).text().trim() || null,
aircraft: $(cells[2]).text().trim() || null,
operator: $(cells[3]).text().trim() || null,
serialNumber: $(cells[5]).text().trim() || null
};
}
async function fetchRegistration2(icao) {
const resp = await fetchWithTimeout(`https://hexdb.io/api/v1/aircraft/${icao}`);
if (!resp.ok) return null;
const found = await resp.json();
return {
registration: found.Registration || null,
manufacturer: found.Manufacturer || null,
aircraft: found.ICAOTypeCode || null,
model: found.Type || null,
operator: found.RegisteredOwners || null
};
}
function fetchWithTimeout(url, ms = 5000) {
const controller = new AbortController();
const id = setTimeout(() => controller.abort(), ms);
return fetch(url, { signal: controller.signal }).finally(() => clearTimeout(id));
}
export async function initAircraftDb() {
if (!fs.existsSync(DATA)) fs.mkdirSync(DATA, { recursive: true });
if (!fs.existsSync(IMAGES_DIR)) fs.mkdirSync(IMAGES_DIR, { recursive: true });
const missing = !fs.existsSync(DB_PATH);
if (missing) console.log(`✈️ Building database`);
db = new Database(DB_PATH);
db.exec(`
CREATE TABLE IF NOT EXISTS aircraft (
icao TEXT PRIMARY KEY,
built TEXT,
categoryDescription TEXT,
country TEXT,
engines TEXT,
class TEXT,
manufacturer TEXT,
model TEXT,
modes TEXT,
operator TEXT,
operatorCallsign TEXT,
owner TEXT,
registration TEXT,
serialNumber TEXT,
aircraft TEXT
);
CREATE TABLE IF NOT EXISTS military_ranges (start TEXT NOT NULL, end TEXT NOT NULL);
`);
await syncAirlines();
await syncMilitaryIcao();
if(missing) {
const res = await fetch(CSV_URL);
if(!res.ok) throw new Error(`HTTP ${res.status}`);
const text = await res.text();
fs.writeFileSync(CSV_CACHE, text);
const csv = fromCsv(text, true);
const insert = db.prepare(`
INSERT OR REPLACE INTO aircraft VALUES (
@icao,
@built,
@categoryDescription,
@country,
@engines,
@class,
@manufacturer,
@model,
@modes,
@operator,
@operatorCallsign,
@owner,
@registration,
@serialNumber,
@aircraft
)
`);
const insertMany = db.transaction(rows => {
for (const row of rows) insert.run(row);
});
insertMany(csv.map(row => ({
icao: (row.icao24 || '').toUpperCase(),
built: row.built || null,
categoryDescription: row.categoryDescription || null,
country: row.country || null,
engines: row.engines || null,
class: row.icaoAircraftClass || null,
manufacturer: row.manufacturerName || row.manufacturerIcao || null,
model: row.model || null,
modes: row.modes || null,
operator: row.operator || null,
operatorCallsign: row.operatorCallsign || null,
owner: row.owner || null,
registration: row.registration || null,
serialNumber: row.serialNumber || null,
aircraft: row.typecode || null
})));
fs.unlinkSync(CSV_CACHE);
console.log(`✈️ Registrations imported: ${csv.length}`);
} else {
const count = db.prepare('SELECT COUNT(*) as c FROM aircraft').get();
console.log(`✈️ Registrations loaded: ${count.c}`);
}
}
function isMilitaryIcao(icao) {
if (!icao) return false;
const hex = icao.toUpperCase();
return milRanges.some(([s, e]) => hex >= s && hex <= e);
}
async function syncAirlines() {
try {
if (!fs.existsSync(AIRLINES_CACHE)) {
console.log('✈️ Downloading airline database');
const res = await fetchWithTimeout(AIRLINES_URL, 10000);
if(!res.ok) throw new Error(`HTTP ${res.status}`);
fs.writeFileSync(AIRLINES_CACHE, await res.text());
}
const text = fs.readFileSync(AIRLINES_CACHE, 'utf8');
const airlines = text.split('\n').slice(1).map(row =>
row.split(',')[1]?.slice(1, -1).toLowerCase()).filter(Boolean).flat();
passengerOperators = airlines.sort((a, b) => b.length - a.length);
console.log(`✈️ Airlines loaded: ${airlines.length}`);
} catch (e) {
console.warn(`⚠️ Could not load airline database: ${e.message}`);
}
}
async function syncMilitaryIcao() {
const { ADSB_URL } = cfg();
try {
const page = await fetch(ADSB_URL + ':8080').then(resp => resp.text());
const db = /databaseFolder = "(db-.+?)"/.exec(page);
const res = await fetch(ADSB_URL + `:8080/${db[1]}/ranges.js`);
if (!res.ok) throw new Error(`HTTP ${res.status}`);
const { military } = await res.json();
db.exec('DELETE FROM military_ranges');
const insert = db.prepare('INSERT INTO military_ranges (start, end) VALUES (?, ?)');
const insertMany = db.transaction(ranges => {
for (const [s, e] of ranges) insert.run(s.toUpperCase(), e.toUpperCase());
});
insertMany(military);
milRanges.length = 0;
milRanges.push(...military.map(([s, e]) => [s.toUpperCase(), e.toUpperCase()]));
} catch (e) {
const rows = db.prepare('SELECT start, end FROM military_ranges').all();
milRanges.length = 0;
milRanges.push(...rows.map(r => [r.start, r.end]));
}
}
export function classifyAircraft(row) {
if (!row) return 'unknown';
const matchesAny = (fields, list) => fields.some(f => {
const value = normalizeName(f);
return value && list.some(k => value.includes(normalizeName(k)));
});
if (isMilitaryIcao(row.icao) || MILITARY_CLASSES.includes(row.class)) {
return 'military';
}
const operators = [row.operator, row.operatorCallsign];
if (row.categoryDescription?.toLowerCase().includes('cargo') || matchesAny(operators, CARGO_OPERATORS)) return 'cargo';
if (matchesAny(operators, passengerOperators)) return 'passenger';
if (row.owner && !row.operator) return 'private';
return 'unknown';
}
function normalizeName(value) {
return String(value || '')
.normalize('NFKD')
.replace(/[\u0300-\u036f]/g, '')
.toLowerCase()
.replace(/&/g, ' and ')
.replace(/[^a-z0-9]+/g, ' ')
.trim()
.replace(/\s+/g, ' ');
}
export async function enrich(a) {
if (!a.hex) return a;
const icao = a.hex.toUpperCase();
const row = db.prepare('SELECT * FROM aircraft WHERE icao = ?').get(icao);
if (row?.aircraft) return { ...a, ...row, type: classifyAircraft(row) };
if (noRecord.includes(icao)) return { icao, ...a, type: 'unknown' };
const hexDbPromise = fetchRegistration2(icao);
const scrapePromise = fetchRegistration1(icao);
let found = await hexDbPromise.catch(() => {});
if (!found) found = await scrapePromise.catch(() => {});
if (!found) {
noRecord.push(icao);
return { icao, ...a, type: 'unknown' };
}
const merged = { ...row, ...found };
if (merged.aircraft && (!merged.manufacturer || !merged.model)) {
const similar = backfillModel(merged.aircraft);
Object.assign(merged, similar);
}
if(found) {
db.prepare(`
INSERT INTO aircraft (icao, registration, manufacturer, aircraft, model, operator, country, serialNumber, engines, categoryDescription, class)
VALUES (@icao, @registration, @manufacturer, @aircraft, @model, @operator, @country, @serialNumber, @engines, @categoryDescription, @class)
ON CONFLICT(icao) DO UPDATE SET
registration = COALESCE(excluded.registration, registration),
manufacturer = COALESCE(excluded.manufacturer, manufacturer),
aircraft = COALESCE(excluded.aircraft, aircraft),
model = COALESCE(excluded.model, model),
operator = COALESCE(excluded.operator, operator),
country = COALESCE(excluded.country, country),
serialNumber = COALESCE(excluded.serialNumber, serialNumber),
engines = COALESCE(excluded.engines, engines),
categoryDescription = COALESCE(excluded.categoryDescription, categoryDescription),
class = COALESCE(excluded.class, class)
`).run({
icao,
registration: merged.registration || null,
manufacturer: merged.manufacturer || null,
aircraft: merged.aircraft || null,
model: merged.model || null,
operator: merged.operator || null,
country: merged.country || null,
serialNumber: merged.serialNumber || null,
engines: merged.engines || null,
categoryDescription: merged.categoryDescription || null,
class: merged.class || null
});
}
return { icao, ...a, ...merged, type: classifyAircraft(merged) };
}
export async function get() {
if(adsbCache && Date.now() - adsbCacheTs < ADSB_TTL) return adsbCache;
const { ADSB_URL } = cfg();
if(!ADSB_URL) return [];
const r = await fetchWithTimeout(`${ADSB_URL}:8080/data/aircraft.json`);
const j = await r.json();
const aircraft = j.aircraft || [];
for (const a of aircraft) {
a.type = 'unknown';
if(!a.hex || !a.lat || !a.lon) continue;
const key = a.hex.toLowerCase();
if (!history.has(key)) history.set(key, []);
const trail = history.get(key);
trail.push({
latitude: a.lat,
longitude: a.lon,
altitude: a.alt_baro || 0,
ts: Date.now()
});
if (trail.length > MAX_HISTORY) trail.shift();
}
adsbCache = await Promise.all(aircraft.map(async a => {
a = await enrich(a);
a.icon = getIcon(a);
return a;
}));
adsbCacheTs = Date.now();
return adsbCache;
}
export async function getAttenuation() {
const { ADSB_URL } = cfg();
if (!ADSB_URL) return [];
const r = await fetch(`${ADSB_URL}:8080/data/outline.json`);
const j = await r.json();
return j?.actualRange?.last24h?.points || [];
}
export async function getHistory(icao) {
return { history: history.get(icao?.toLowerCase()) || [] };
}
export async function getIcaoImage(icao) {
const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.0.0 Safari/537.36';
async function duckduckgo(query) {
async function getVqd(query) {
const url = `https://duckduckgo.com/?q=${encodeURIComponent(query)}&iax=images&ia=images`;
const res = await fetch(url, {
headers: {
'User-Agent': UA,
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.9'
}
});
if(!res.ok) return console.warn(`DuckDuckGo Search failed: ${res.status}`);
const html = await res.text();
const match = html.match(/vqd=["']([^"']+)["']/);
return match?.[1] || null;
}
const vqd = await getVqd(query);
if (!vqd) throw new Error('Could not get vqd token');
await new Promise(resolve => setTimeout(resolve, 800 + Math.random() * 500));
const searchUrl = `https://duckduckgo.com/i.js?q=${encodeURIComponent(query)}&o=json&vqd=${encodeURIComponent(vqd)}&f=,,,,,&p=1`;
const data = await fetch(searchUrl, {
headers: {
'User-Agent': UA,
'Accept': 'application/json, text/javascript, */*; q=0.01',
'Accept-Language': 'en-US,en;q=0.9',
'Referer': `https://duckduckgo.com/?q=${encodeURIComponent(query)}&iax=images&ia=images`
}
}).then(async resp => {
if (!resp.ok) return console.warn(`DuckDuckGo image failed: ${resp.status}`);
const text = await resp.text();
try {
return JSON.parse(text);
} catch {
console.warn(`DuckDuckGo returned invalid JSON: ${text.slice(0, 200)}`);
return null;
}
});
if (data?.results?.length) {
for (const result of data.results) {
if (!result.image) continue;
try {
const imgRes = await fetch(result.image, {
headers: {
'Accept': 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8',
'User-Agent': UA,
'Referer': result.url || 'https://duckduckgo.com/'
}
});
if (imgRes.ok) return imgRes.blob();
console.warn(`Image fetch failed: ${imgRes.status} ${result.image}`);
} catch (e) {
console.warn(`Image fetch failed: ${result.image} - ${e.message}`);
}
}
}
return null;
}
async function genericImage(modelType) {
if (!modelType) return null;
const filePath = resolve(IMAGES_DIR, 'generic', `${modelType}.jpg`);
if (fs.existsSync(filePath)) return fs.readFileSync(filePath);
const blob = await duckduckgo(`${modelType} Aircraft`);
if (!blob) return null;
fs.mkdirSync(dirname(filePath), { recursive: true });
const buffer = Buffer.from(await blob.arrayBuffer());
const metadata = await sharp(buffer).metadata();
const width = metadata.width || 1200;
const height = metadata.height || 800;
const fontSize = Math.max(48, Math.round(Math.min(width, height) * 0.16));
const watermark = Buffer.from(`
`);
const watermarked = await sharp(buffer).composite([{ input: watermark }]).jpeg().toBuffer();
fs.writeFileSync(filePath, watermarked);
return watermarked;
}
async function icaoImage(icao) {
const filePath = resolve(IMAGES_DIR, `${icao}.jpg`);
if (fs.existsSync(filePath)) return fs.readFileSync(filePath);
const planeSpotters = await fetch(`https://api.planespotters.net/pub/photos/hex/${icao}`, {
headers: { 'User-Agent': 'open-sight.net (zaktimson@gmail.com)' }
}).then(resp => resp.ok ? resp.json() : null).catch(() => null);
const src = planeSpotters?.photos?.[0]?.thumbnail_large?.src || planeSpotters?.photos?.[0]?.thumbnail?.src;
if (!src) return null;
const resp = await fetch(src, {
headers: {
'Accept': 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8',
'User-Agent': UA,
'Referer': 'https://www.planespotters.net/'
}
});
if (!resp.ok) return null;
fs.mkdirSync(IMAGES_DIR, { recursive: true });
const buffer = Buffer.from(await resp.arrayBuffer());
fs.writeFileSync(filePath, buffer);
return buffer;
}
icao = icao.toLowerCase();
if (!fs.existsSync(IMAGES_DIR)) fs.mkdirSync(IMAGES_DIR, { recursive: true });
const aircraft = await enrich({ hex: icao });
const modelType = aircraft.aircraft || aircraft.model || aircraft.type;
const specific = await icaoImage(icao);
if (specific) return specific;
return genericImage(modelType);
}
// Purge old history
setInterval(() => {
const cutoff = Date.now() - HISTORY_TTL;
for (const [key, trail] of history) {
const last = trail.at(-1);
if (!last || last.ts < cutoff) history.delete(key);
}
}, 1000 * 60 * 5);