330 lines
12 KiB
JavaScript
330 lines
12 KiB
JavaScript
import { cfg } from './config.mjs';
|
|
import { resolve, dirname } from 'path';
|
|
import { fileURLToPath } from 'url';
|
|
import * as fs from 'node:fs';
|
|
import Database from 'better-sqlite3';
|
|
import { fromCsv } from '@ztimson/utils';
|
|
import {getIcon} from './adsb-shapes.mjs';
|
|
import * as cheerio from 'cheerio';
|
|
|
|
const DIR = dirname(fileURLToPath(import.meta.url));
|
|
const DATA = resolve(DIR, '../data');
|
|
const DB_PATH = resolve(DATA, 'aircraft.db');
|
|
const CSV_CACHE = resolve(DATA, 'aircraft_db.csv');
|
|
const MAX_HISTORY = 500;
|
|
const HISTORY_TTL = 1000 * 60 * 60; // 1 hour
|
|
|
|
const ADSB_TTL = 1000;
|
|
const CSV_URL = 'https://s3.opensky-network.org/data-samples/metadata/aircraft-database-complete-2024-06.csv';
|
|
const MIL_RANGES_URL = ':8080/db-3.14.1708/ranges.js';
|
|
|
|
const MILITARY_OPERATORS = ['air force', 'army', 'navy', 'marine', 'coast guard', 'military', 'defence', 'defense', 'luftwaffe', 'RAF', 'USAF', 'USN', 'USMC'];
|
|
const CARGO_OPERATORS = ['fedex', 'ups', 'dhl', 'cargo', 'freight', 'logistic', 'atlas air', 'kalitta', 'air freight'];
|
|
const PASSENGER_OPERATORS = ['airlines', 'airways', 'air ', 'jet', 'jetblue', 'easyjet', 'ryanair', 'delta', 'united', 'american', 'avianca', 'southwest', 'lufthansa', 'emirates', 'british'];
|
|
const MILITARY_CLASSES = ['H1M', 'H2M', 'L1M', 'L2M', 'L4M', 'A0'];
|
|
|
|
let adsbCache = null;
|
|
let adsbCacheTs = 0;
|
|
const noRecord = [];
|
|
const history = new Map();
|
|
const milRanges = [];
|
|
let db;
|
|
|
|
setInterval(() => {
|
|
const cutoff = Date.now() - HISTORY_TTL;
|
|
for (const [key, trail] of history) {
|
|
const last = trail.at(-1);
|
|
if (!last || last.ts < cutoff) history.delete(key);
|
|
}
|
|
}, 1000 * 60 * 5);
|
|
|
|
function fetchWithTimeout(url, ms = 5000) {
|
|
const controller = new AbortController();
|
|
const id = setTimeout(() => controller.abort(), ms);
|
|
return fetch(url, { signal: controller.signal }).finally(() => clearTimeout(id));
|
|
}
|
|
|
|
async function syncMilitaryRanges() {
|
|
const { ADSB_URL } = cfg();
|
|
try {
|
|
const res = await fetch(ADSB_URL + MIL_RANGES_URL);
|
|
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
|
const { military } = await res.json();
|
|
|
|
db.exec('DELETE FROM military_ranges');
|
|
const insert = db.prepare('INSERT INTO military_ranges (start, end) VALUES (?, ?)');
|
|
const insertMany = db.transaction(ranges => {
|
|
for (const [s, e] of ranges) insert.run(s.toUpperCase(), e.toUpperCase());
|
|
});
|
|
|
|
insertMany(military);
|
|
milRanges.length = 0;
|
|
milRanges.push(...military.map(([s, e]) => [s.toUpperCase(), e.toUpperCase()]));
|
|
} catch (e) {
|
|
const rows = db.prepare('SELECT start, end FROM military_ranges').all();
|
|
milRanges.length = 0;
|
|
milRanges.push(...rows.map(r => [r.start, r.end]));
|
|
}
|
|
}
|
|
|
|
export async function initAircraftDb() {
|
|
if (!fs.existsSync(DATA)) fs.mkdirSync(DATA, { recursive: true });
|
|
|
|
const missing = !fs.existsSync(DB_PATH);
|
|
if (missing) console.log(`✈️ Building database`);
|
|
|
|
db = new Database(DB_PATH);
|
|
db.exec(`
|
|
CREATE TABLE IF NOT EXISTS aircraft (
|
|
icao TEXT PRIMARY KEY,
|
|
built TEXT,
|
|
categoryDescription TEXT,
|
|
country TEXT,
|
|
engines TEXT,
|
|
class TEXT,
|
|
manufacturer TEXT,
|
|
model TEXT,
|
|
modes TEXT,
|
|
operator TEXT,
|
|
operatorCallsign TEXT,
|
|
owner TEXT,
|
|
registration TEXT,
|
|
serialNumber TEXT,
|
|
aircraft TEXT
|
|
);
|
|
CREATE TABLE IF NOT EXISTS military_ranges (
|
|
start TEXT NOT NULL,
|
|
end TEXT NOT NULL
|
|
);
|
|
`);
|
|
|
|
await syncMilitaryRanges();
|
|
|
|
if (missing) {
|
|
const res = await fetch(CSV_URL);
|
|
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
|
const text = await res.text();
|
|
fs.writeFileSync(CSV_CACHE, text);
|
|
|
|
const csv = fromCsv(text, true);
|
|
const insert = db.prepare(`
|
|
INSERT OR REPLACE INTO aircraft VALUES (
|
|
@icao, @built, @categoryDescription, @country, @engines,
|
|
@class, @manufacturer, @model, @modes, @operator,
|
|
@operatorCallsign, @owner, @registration, @serialNumber, @aircraft
|
|
)
|
|
`);
|
|
|
|
const insertMany = db.transaction(rows => {
|
|
for (const row of rows) insert.run(row);
|
|
});
|
|
|
|
insertMany(csv.map(row => ({
|
|
icao: (row.icao24 || '').toUpperCase(),
|
|
built: row.built || null,
|
|
categoryDescription: row.categoryDescription || null,
|
|
country: row.country || null,
|
|
engines: row.engines || null,
|
|
class: row.icaoAircraftClass || null,
|
|
manufacturer: row.manufacturerName || row.manufacturerIcao || null,
|
|
model: row.model || null,
|
|
modes: row.modes || null,
|
|
operator: row.operator || null,
|
|
operatorCallsign: row.operatorCallsign || null,
|
|
owner: row.owner || null,
|
|
registration: row.registration || null,
|
|
serialNumber: row.serialNumber || null,
|
|
aircraft: row.typecode || null,
|
|
})));
|
|
|
|
fs.unlinkSync(CSV_CACHE);
|
|
console.log(`✈️ Aircraft imported: ${csv.length}`);
|
|
} else {
|
|
const count = db.prepare('SELECT COUNT(*) as c FROM aircraft').get();
|
|
console.log(`✈️ Aircraft loaded: ${count.c}`);
|
|
}
|
|
}
|
|
|
|
function isIcaoInMilitaryRange(icao) {
|
|
if (!icao) return false;
|
|
const hex = icao.toUpperCase();
|
|
return milRanges.some(([s, e]) => hex >= s && hex <= e);
|
|
}
|
|
|
|
function wordMatch(text, keyword) {
|
|
if (!text) return false;
|
|
const escaped = keyword.replace(/[.*+?^${}()|[\]\\]/gi, '\\$&');
|
|
return new RegExp(`(?<![\\w])${escaped}(?![\\w])`, 'i').test(text);
|
|
}
|
|
|
|
export function classifyAircraft(row) {
|
|
if (!row) return 'unknown';
|
|
|
|
const matchesAny = (fields, list) =>
|
|
fields.some(f => list.some(k => wordMatch(f, k)));
|
|
|
|
const operatorFields = [row.operator, row.operatorCallsign];
|
|
const ownerFields = [row.owner];
|
|
const allFields = [...operatorFields, ...ownerFields, row.categoryDescription];
|
|
|
|
if (isIcaoInMilitaryRange(row.icao) || MILITARY_CLASSES.includes(row.class) || matchesAny(allFields, MILITARY_OPERATORS)) return 'military';
|
|
if (row.categoryDescription?.toLowerCase().includes('cargo') || matchesAny(operatorFields, CARGO_OPERATORS)) return 'cargo';
|
|
if (matchesAny(operatorFields, PASSENGER_OPERATORS)) return 'passenger';
|
|
if (row.owner && !row.operator) return 'private';
|
|
|
|
return 'unknown';
|
|
}
|
|
|
|
async function fetchHexDb(icao) {
|
|
const resp = await fetchWithTimeout(`https://hexdb.io/api/v1/aircraft/${icao}`);
|
|
if (!resp.ok) return null;
|
|
const found = await resp.json();
|
|
return {
|
|
registration: found.Registration || null,
|
|
manufacturer: found.Manufacturer || null,
|
|
aircraft: found.ICAOTypeCode || null,
|
|
model: found.Type || null,
|
|
operator: found.RegisteredOwners || null,
|
|
};
|
|
}
|
|
|
|
async function scrapeHexDatabase(icao) {
|
|
const url = `https://hexdatabase.com/h/${icao}`;
|
|
const resp = await fetch(url);
|
|
if (!resp.ok) return null;
|
|
|
|
const html = await resp.text();
|
|
const tableMatch = html.match(/<table[\s\S]*?<\/table>/i);
|
|
if (!tableMatch) return null;
|
|
|
|
const $ = cheerio.load(tableMatch[0]);
|
|
const row = $('tr').filter((_, el) => $(el).text().toLowerCase().includes(icao.toLowerCase())).first();
|
|
if (!row.length) return null;
|
|
|
|
const cells = row.find('td');
|
|
return {
|
|
registration: $(cells[1]).text().trim() || null,
|
|
aircraft: $(cells[2]).text().trim() || null,
|
|
operator: $(cells[3]).text().trim() || null,
|
|
serialNumber: $(cells[5]).text().trim() || null,
|
|
};
|
|
}
|
|
|
|
function backfillFromSimilar(aircraft) {
|
|
if (!aircraft) return {};
|
|
const similar = db.prepare(`
|
|
SELECT manufacturer, model, engines, categoryDescription, class
|
|
FROM aircraft
|
|
WHERE aircraft = ?
|
|
AND (manufacturer IS NOT NULL OR model IS NOT NULL OR engines IS NOT NULL)
|
|
LIMIT 1
|
|
`).get(aircraft);
|
|
if (!similar) return {};
|
|
return {
|
|
manufacturer: similar.manufacturer || null,
|
|
model: similar.model || null,
|
|
engines: similar.engines || null,
|
|
categoryDescription: similar.categoryDescription || null,
|
|
class: similar.class || null,
|
|
};
|
|
}
|
|
|
|
export async function enrichAircraft(a) {
|
|
if (!a.hex) return a;
|
|
const icao = a.hex.toUpperCase();
|
|
|
|
// 1. Check own DB first
|
|
const row = db.prepare('SELECT * FROM aircraft WHERE icao = ?').get(icao);
|
|
if(row?.aircraft) return { ...a, ...row, type: classifyAircraft(row) };
|
|
|
|
// 2. Race the two external sources
|
|
if(noRecord.includes(icao)) return { icao, ...a, type: 'unknown' };
|
|
const hexDbPromise = fetchHexDb(icao);
|
|
const scrapePromise = scrapeHexDatabase(icao);
|
|
let found = await hexDbPromise;
|
|
if(!found) found = await scrapePromise;
|
|
if(!found) {
|
|
noRecord.push(icao);
|
|
return { icao, ...a, type: 'unknown' };
|
|
}
|
|
|
|
// 3. Backfill manufacturer/model/engines from similar aircraft type in DB
|
|
const merged = { ...row, ...found };
|
|
if (merged.aircraft && (!merged.manufacturer || !merged.model)) {
|
|
const similar = backfillFromSimilar(merged.aircraft);
|
|
Object.assign(merged, similar);
|
|
}
|
|
|
|
// 4. Save back to DB
|
|
if (found) {
|
|
db.prepare(`
|
|
INSERT INTO aircraft (icao, registration, manufacturer, aircraft, model, operator, country, serialNumber, engines, categoryDescription, class)
|
|
VALUES (@icao, @registration, @manufacturer, @aircraft, @model, @operator, @country, @serialNumber, @engines, @categoryDescription, @class)
|
|
ON CONFLICT(icao) DO UPDATE SET
|
|
registration = COALESCE(excluded.registration, registration),
|
|
manufacturer = COALESCE(excluded.manufacturer, manufacturer),
|
|
aircraft = COALESCE(excluded.aircraft, aircraft),
|
|
model = COALESCE(excluded.model, model),
|
|
operator = COALESCE(excluded.operator, operator),
|
|
country = COALESCE(excluded.country, country),
|
|
serialNumber = COALESCE(excluded.serialNumber, serialNumber),
|
|
engines = COALESCE(excluded.engines, engines),
|
|
categoryDescription = COALESCE(excluded.categoryDescription, categoryDescription),
|
|
class = COALESCE(excluded.class, class)
|
|
`).run({
|
|
icao,
|
|
registration: merged.registration || null,
|
|
manufacturer: merged.manufacturer || null,
|
|
aircraft: merged.aircraft || null,
|
|
model: merged.model || null,
|
|
operator: merged.operator || null,
|
|
country: merged.country || null,
|
|
serialNumber: merged.serialNumber || null,
|
|
engines: merged.engines || null,
|
|
categoryDescription: merged.categoryDescription || null,
|
|
class: merged.class || null,
|
|
});
|
|
}
|
|
|
|
return {icao, ...a, ...merged, type: classifyAircraft(merged) }
|
|
}
|
|
|
|
export async function getADSB() {
|
|
if (adsbCache && Date.now() - adsbCacheTs < ADSB_TTL) return adsbCache;
|
|
const { ADSB_URL } = cfg();
|
|
if (!ADSB_URL) return [];
|
|
const r = await fetchWithTimeout(`${ADSB_URL}:8080/data/aircraft.json`);
|
|
const j = await r.json();
|
|
const aircraft = j.aircraft || [];
|
|
|
|
for (const a of aircraft) {
|
|
a.type = 'unknown';
|
|
if (!a.hex || !a.lat || !a.lon) continue;
|
|
const key = a.hex.toLowerCase();
|
|
if (!history.has(key)) history.set(key, []);
|
|
const trail = history.get(key);
|
|
trail.push({ latitude: a.lat, longitude: a.lon, altitude: a.alt_baro || 0, ts: Date.now() });
|
|
if (trail.length > MAX_HISTORY) trail.shift();
|
|
}
|
|
|
|
adsbCache = await Promise.all(aircraft.map(async a => {
|
|
a = await enrichAircraft(a);
|
|
a.icon = getIcon(a);
|
|
return a;
|
|
}));
|
|
adsbCacheTs = Date.now();
|
|
return adsbCache;
|
|
}
|
|
|
|
export async function getADSBHistory(icao) {
|
|
return { history: history.get(icao?.toLowerCase()) || [] };
|
|
}
|
|
|
|
export async function getADSBRange() {
|
|
const { ADSB_URL } = cfg();
|
|
if (!ADSB_URL) return [];
|
|
const r = await fetch(`${ADSB_URL}:8080/data/outline.json`);
|
|
const j = await r.json();
|
|
return j?.actualRange?.last24h?.points || [];
|
|
}
|