More military fixes
This commit is contained in:
@@ -19,70 +19,78 @@ const HISTORY_TTL = 1000 * 60 * 60; // 1 hour
|
||||
|
||||
const ADSB_TTL = 1000;
|
||||
const CSV_URL = 'https://s3.opensky-network.org/data-samples/metadata/aircraft-database-complete-2024-06.csv';
|
||||
const AIRLINES_URL = 'https://raw.githubusercontent.com/jpatokal/openflights/master/data/airlines.dat'
|
||||
|
||||
const MILITARY_OPERATORS = ['air force', 'airforce', 'army', 'navy', 'marine', 'coast guard', 'military', 'defence', 'defense', 'law', 'luftwaffe', 'RAF', 'USAF', 'USN', 'USMC'];
|
||||
const AIRLINES_URL = 'https://raw.githubusercontent.com/jpatokal/openflights/master/data/airlines.dat';
|
||||
const CARGO_OPERATORS = ['fedex', 'ups', 'dhl', 'cargo', 'freight', 'logistic', 'atlas air', 'kalitta', 'air freight'];
|
||||
const PASSENGER_OPERATORS = ['airlines', 'airways', 'jetblue', 'easyjet', 'ryanair', 'delta', 'united', 'american', 'avianca', 'southwest', 'lufthansa', 'emirates', 'british'];
|
||||
// const MILITARY_CLASSES = ['H1M', 'H2M', 'L1M', 'L2M', 'L4M', 'A0'];
|
||||
const MILITARY_CLASSES = ['H1M', 'H2M', 'L1M', 'L2M', 'L4M', 'A0'];
|
||||
|
||||
let db;
|
||||
let passengerOperators = PASSENGER_OPERATORS;
|
||||
|
||||
let adsbCache = null;
|
||||
let adsbCacheTs = 0;
|
||||
const noRecord = [];
|
||||
const history = new Map();
|
||||
const milRanges = [];
|
||||
let db;
|
||||
|
||||
function normalizeAirlineName(value) {
|
||||
return String(value || '')
|
||||
.normalize('NFKD')
|
||||
.replace(/[\u0300-\u036f]/g, '')
|
||||
.toLowerCase()
|
||||
.replace(/&/g, ' and ')
|
||||
.replace(/[^a-z0-9]+/g, ' ')
|
||||
.trim()
|
||||
.replace(/\s+/g, ' ');
|
||||
function backfillModel(aircraft) {
|
||||
if (!aircraft) return {};
|
||||
const similar = db.prepare(`
|
||||
SELECT
|
||||
manufacturer,
|
||||
model,
|
||||
engines,
|
||||
categoryDescription,
|
||||
class
|
||||
FROM aircraft
|
||||
WHERE aircraft = ?
|
||||
AND (
|
||||
manufacturer IS NOT NULL
|
||||
OR model IS NOT NULL
|
||||
OR engines IS NOT NULL
|
||||
)
|
||||
LIMIT 1
|
||||
`).get(aircraft);
|
||||
if (!similar) return {};
|
||||
return {
|
||||
manufacturer: similar.manufacturer || null,
|
||||
model: similar.model || null,
|
||||
engines: similar.engines || null,
|
||||
categoryDescription: similar.categoryDescription || null,
|
||||
class: similar.class || null
|
||||
};
|
||||
}
|
||||
|
||||
async function syncAirlines() {
|
||||
try {
|
||||
if (!fs.existsSync(AIRLINES_CACHE)) {
|
||||
console.log('✈️ Downloading airline database');
|
||||
const res = await fetchWithTimeout(AIRLINES_URL, 10000);
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
fs.writeFileSync(AIRLINES_CACHE, await res.text());
|
||||
}
|
||||
|
||||
const text = fs.readFileSync(AIRLINES_CACHE, 'utf8');
|
||||
const airlines = fromCsv(text, true);
|
||||
const names = new Set();
|
||||
for (const row of airlines) {
|
||||
if (row.active !== 'Y') continue;
|
||||
for (const value of [row.name, row.alias, row.callsign]) {
|
||||
const normalized = normalizeAirlineName(value);
|
||||
if (normalized.length >= 3) names.add(normalized);
|
||||
}
|
||||
}
|
||||
for (const value of PASSENGER_OPERATORS) names.add(normalizeAirlineName(value));
|
||||
passengerOperators = [...names].sort((a, b) => b.length - a.length);
|
||||
console.log(`✈️ Airline operators loaded: ${passengerOperators.length}`);
|
||||
} catch (e) {
|
||||
console.warn(`⚠️ Could not load airline database: ${e.message}`);
|
||||
passengerOperators = PASSENGER_OPERATORS.map(normalizeAirlineName);
|
||||
}
|
||||
async function fetchRegistration1(icao) {
|
||||
const url = `https://hexdatabase.com/h/${icao}`;
|
||||
const resp = await fetch(url);
|
||||
if (!resp.ok) return null;
|
||||
const html = await resp.text();
|
||||
const tableMatch = html.match(/<table[\s\S]*?<\/table>/i);
|
||||
if (!tableMatch) return null;
|
||||
const $ = cheerio.load(tableMatch[0]);
|
||||
const row = $('tr').filter((_, el) => $(el).text()?.toLowerCase()?.includes(icao.toLowerCase())).first();
|
||||
if (!row.length) return null;
|
||||
const cells = row.find('td');
|
||||
return {
|
||||
registration: $(cells[1]).text().trim() || null,
|
||||
aircraft: $(cells[2]).text().trim() || null,
|
||||
operator: $(cells[3]).text().trim() || null,
|
||||
serialNumber: $(cells[5]).text().trim() || null
|
||||
};
|
||||
}
|
||||
|
||||
setInterval(() => {
|
||||
const cutoff = Date.now() - HISTORY_TTL;
|
||||
|
||||
for (const [key, trail] of history) {
|
||||
const last = trail.at(-1);
|
||||
|
||||
if (!last || last.ts < cutoff) history.delete(key);
|
||||
}
|
||||
}, 1000 * 60 * 5);
|
||||
async function fetchRegistration2(icao) {
|
||||
const resp = await fetchWithTimeout(`https://hexdb.io/api/v1/aircraft/${icao}`);
|
||||
if (!resp.ok) return null;
|
||||
const found = await resp.json();
|
||||
return {
|
||||
registration: found.Registration || null,
|
||||
manufacturer: found.Manufacturer || null,
|
||||
aircraft: found.ICAOTypeCode || null,
|
||||
model: found.Type || null,
|
||||
operator: found.RegisteredOwners || null
|
||||
};
|
||||
}
|
||||
|
||||
function fetchWithTimeout(url, ms = 5000) {
|
||||
const controller = new AbortController();
|
||||
@@ -90,29 +98,6 @@ function fetchWithTimeout(url, ms = 5000) {
|
||||
return fetch(url, { signal: controller.signal }).finally(() => clearTimeout(id));
|
||||
}
|
||||
|
||||
async function syncMilitaryRanges() {
|
||||
const { ADSB_URL } = cfg();
|
||||
try {
|
||||
const page = await fetch(ADSB_URL + ':8080').then(resp => resp.text());
|
||||
const db = /databaseFolder = "(db-.+?)"/.exec(page);
|
||||
const res = await fetch(ADSB_URL + `:8080/${db[1]}/ranges.js`);
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
const { military } = await res.json();
|
||||
db.exec('DELETE FROM military_ranges');
|
||||
const insert = db.prepare('INSERT INTO military_ranges (start, end) VALUES (?, ?)');
|
||||
const insertMany = db.transaction(ranges => {
|
||||
for (const [s, e] of ranges) insert.run(s.toUpperCase(), e.toUpperCase());
|
||||
});
|
||||
insertMany(military);
|
||||
milRanges.length = 0;
|
||||
milRanges.push(...military.map(([s, e]) => [s.toUpperCase(), e.toUpperCase()]));
|
||||
} catch (e) {
|
||||
const rows = db.prepare('SELECT start, end FROM military_ranges').all();
|
||||
milRanges.length = 0;
|
||||
milRanges.push(...rows.map(r => [r.start, r.end]));
|
||||
}
|
||||
}
|
||||
|
||||
export async function initAircraftDb() {
|
||||
if (!fs.existsSync(DATA)) fs.mkdirSync(DATA, { recursive: true });
|
||||
if (!fs.existsSync(IMAGES_DIR)) fs.mkdirSync(IMAGES_DIR, { recursive: true });
|
||||
@@ -142,7 +127,7 @@ export async function initAircraftDb() {
|
||||
`);
|
||||
|
||||
await syncAirlines();
|
||||
await syncMilitaryRanges();
|
||||
await syncMilitaryIcao();
|
||||
|
||||
if(missing) {
|
||||
const res = await fetch(CSV_URL);
|
||||
@@ -200,111 +185,104 @@ export async function initAircraftDb() {
|
||||
}
|
||||
}
|
||||
|
||||
function isIcaoInMilitaryRange(icao) {
|
||||
async function syncAirlines() {
|
||||
try {
|
||||
if (!fs.existsSync(AIRLINES_CACHE)) {
|
||||
console.log('✈️ Downloading airline database');
|
||||
const res = await fetchWithTimeout(AIRLINES_URL, 10000);
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
fs.writeFileSync(AIRLINES_CACHE, await res.text());
|
||||
}
|
||||
|
||||
const text = fs.readFileSync(AIRLINES_CACHE, 'utf8');
|
||||
const airlines = fromCsv(text, true);
|
||||
const names = new Set();
|
||||
for (const row of airlines) {
|
||||
if (row.active !== 'Y') continue;
|
||||
for (const value of [row.name, row.alias, row.callsign]) {
|
||||
const normalized = normalizeName(value);
|
||||
if (normalized.length >= 3) names.add(normalized);
|
||||
}
|
||||
}
|
||||
for (const value of PASSENGER_OPERATORS) names.add(normalizeName(value));
|
||||
passengerOperators = [...names].sort((a, b) => b.length - a.length);
|
||||
console.log(`✈️ Airline operators loaded: ${passengerOperators.length}`);
|
||||
} catch (e) {
|
||||
console.warn(`⚠️ Could not load airline database: ${e.message}`);
|
||||
passengerOperators = PASSENGER_OPERATORS.map(normalizeName);
|
||||
}
|
||||
}
|
||||
|
||||
async function syncMilitaryIcao() {
|
||||
const { ADSB_URL } = cfg();
|
||||
try {
|
||||
const page = await fetch(ADSB_URL + ':8080').then(resp => resp.text());
|
||||
const db = /databaseFolder = "(db-.+?)"/.exec(page);
|
||||
const res = await fetch(ADSB_URL + `:8080/${db[1]}/ranges.js`);
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
const { military } = await res.json();
|
||||
db.exec('DELETE FROM military_ranges');
|
||||
const insert = db.prepare('INSERT INTO military_ranges (start, end) VALUES (?, ?)');
|
||||
const insertMany = db.transaction(ranges => {
|
||||
for (const [s, e] of ranges) insert.run(s.toUpperCase(), e.toUpperCase());
|
||||
});
|
||||
insertMany(military);
|
||||
milRanges.length = 0;
|
||||
milRanges.push(...military.map(([s, e]) => [s.toUpperCase(), e.toUpperCase()]));
|
||||
} catch (e) {
|
||||
const rows = db.prepare('SELECT start, end FROM military_ranges').all();
|
||||
milRanges.length = 0;
|
||||
milRanges.push(...rows.map(r => [r.start, r.end]));
|
||||
}
|
||||
}
|
||||
|
||||
function isMilitaryIcao(icao) {
|
||||
if (!icao) return false;
|
||||
const hex = icao.toUpperCase();
|
||||
return milRanges.some(([s, e]) => hex >= s && hex <= e);
|
||||
}
|
||||
|
||||
function wordMatch(text, keyword) {
|
||||
if (!text) return false;
|
||||
const escaped = keyword.replace(/[.*+?^${}()|[\]\\]/gi, '\\$&');
|
||||
return new RegExp(`(?<![\\w])${escaped}(?![\\w])`, 'i').test(text);
|
||||
}
|
||||
|
||||
export function classifyAircraft(row) {
|
||||
if (!row) return 'unknown';
|
||||
|
||||
const matchesAny = (fields, list) => fields.some(f => {
|
||||
const value = normalizeAirlineName(f);
|
||||
return value && list.some(k => value.includes(normalizeAirlineName(k)));
|
||||
const value = normalizeName(f);
|
||||
return value && list.some(k => value.includes(normalizeName(k)));
|
||||
});
|
||||
|
||||
const operatorFields = [row.operator, row.operatorCallsign];
|
||||
const ownerFields = [row.owner];
|
||||
const allFields = [...operatorFields, ...ownerFields, row.categoryDescription];
|
||||
|
||||
if (isIcaoInMilitaryRange(row.icao) || matchesAny(allFields, MILITARY_OPERATORS)) {
|
||||
const allFields = [row.operator, row.operatorCallsign, row.owner, row.categoryDescription];
|
||||
if (isMilitaryIcao(row.icao) || MILITARY_CLASSES.includes(row.class)) {
|
||||
return 'military';
|
||||
}
|
||||
|
||||
if (row.categoryDescription?.toLowerCase().includes('cargo') || matchesAny(operatorFields, CARGO_OPERATORS)) {
|
||||
if (row.categoryDescription?.toLowerCase().includes('cargo') || matchesAny(allFields, CARGO_OPERATORS)) {
|
||||
return 'cargo';
|
||||
}
|
||||
|
||||
if (matchesAny(operatorFields, passengerOperators)) return 'passenger';
|
||||
if (matchesAny(allFields, passengerOperators)) return 'passenger';
|
||||
if (row.owner && !row.operator) return 'private';
|
||||
return 'unknown';
|
||||
}
|
||||
|
||||
async function fetchHexDb(icao) {
|
||||
const resp = await fetchWithTimeout(`https://hexdb.io/api/v1/aircraft/${icao}`);
|
||||
if (!resp.ok) return null;
|
||||
const found = await resp.json();
|
||||
return {
|
||||
registration: found.Registration || null,
|
||||
manufacturer: found.Manufacturer || null,
|
||||
aircraft: found.ICAOTypeCode || null,
|
||||
model: found.Type || null,
|
||||
operator: found.RegisteredOwners || null
|
||||
};
|
||||
function normalizeName(value) {
|
||||
return String(value || '')
|
||||
.normalize('NFKD')
|
||||
.replace(/[\u0300-\u036f]/g, '')
|
||||
.toLowerCase()
|
||||
.replace(/&/g, ' and ')
|
||||
.replace(/[^a-z0-9]+/g, ' ')
|
||||
.trim()
|
||||
.replace(/\s+/g, ' ');
|
||||
}
|
||||
|
||||
async function scrapeHexDatabase(icao) {
|
||||
const url = `https://hexdatabase.com/h/${icao}`;
|
||||
const resp = await fetch(url);
|
||||
if (!resp.ok) return null;
|
||||
const html = await resp.text();
|
||||
const tableMatch = html.match(/<table[\s\S]*?<\/table>/i);
|
||||
if (!tableMatch) return null;
|
||||
const $ = cheerio.load(tableMatch[0]);
|
||||
const row = $('tr').filter((_, el) => $(el).text()?.toLowerCase()?.includes(icao.toLowerCase())).first();
|
||||
if (!row.length) return null;
|
||||
const cells = row.find('td');
|
||||
return {
|
||||
registration: $(cells[1]).text().trim() || null,
|
||||
aircraft: $(cells[2]).text().trim() || null,
|
||||
operator: $(cells[3]).text().trim() || null,
|
||||
serialNumber: $(cells[5]).text().trim() || null
|
||||
};
|
||||
}
|
||||
|
||||
function backfillFromSimilar(aircraft) {
|
||||
if (!aircraft) return {};
|
||||
const similar = db.prepare(`
|
||||
SELECT
|
||||
manufacturer,
|
||||
model,
|
||||
engines,
|
||||
categoryDescription,
|
||||
class
|
||||
FROM aircraft
|
||||
WHERE aircraft = ?
|
||||
AND (
|
||||
manufacturer IS NOT NULL
|
||||
OR model IS NOT NULL
|
||||
OR engines IS NOT NULL
|
||||
)
|
||||
LIMIT 1
|
||||
`).get(aircraft);
|
||||
if (!similar) return {};
|
||||
return {
|
||||
manufacturer: similar.manufacturer || null,
|
||||
model: similar.model || null,
|
||||
engines: similar.engines || null,
|
||||
categoryDescription: similar.categoryDescription || null,
|
||||
class: similar.class || null
|
||||
};
|
||||
}
|
||||
|
||||
export async function enrichAircraft(a) {
|
||||
export async function enrich(a) {
|
||||
if (!a.hex) return a;
|
||||
const icao = a.hex.toUpperCase();
|
||||
const row = db.prepare('SELECT * FROM aircraft WHERE icao = ?').get(icao);
|
||||
if (row?.aircraft) return { ...a, ...row, type: classifyAircraft(row) };
|
||||
if (noRecord.includes(icao)) return { icao, ...a, type: 'unknown' };
|
||||
const hexDbPromise = fetchHexDb(icao);
|
||||
const scrapePromise = scrapeHexDatabase(icao);
|
||||
const hexDbPromise = fetchRegistration2(icao);
|
||||
const scrapePromise = fetchRegistration1(icao);
|
||||
let found = await hexDbPromise.catch(() => {});
|
||||
if (!found) found = await scrapePromise.catch(() => {});
|
||||
if (!found) {
|
||||
@@ -313,7 +291,7 @@ export async function enrichAircraft(a) {
|
||||
}
|
||||
const merged = { ...row, ...found };
|
||||
if (merged.aircraft && (!merged.manufacturer || !merged.model)) {
|
||||
const similar = backfillFromSimilar(merged.aircraft);
|
||||
const similar = backfillModel(merged.aircraft);
|
||||
Object.assign(merged, similar);
|
||||
}
|
||||
if(found) {
|
||||
@@ -349,7 +327,7 @@ export async function enrichAircraft(a) {
|
||||
return { icao, ...a, ...merged, type: classifyAircraft(merged) };
|
||||
}
|
||||
|
||||
export async function getADSB() {
|
||||
export async function get() {
|
||||
if(adsbCache && Date.now() - adsbCacheTs < ADSB_TTL) return adsbCache;
|
||||
|
||||
const { ADSB_URL } = cfg();
|
||||
@@ -375,7 +353,7 @@ export async function getADSB() {
|
||||
}
|
||||
|
||||
adsbCache = await Promise.all(aircraft.map(async a => {
|
||||
a = await enrichAircraft(a);
|
||||
a = await enrich(a);
|
||||
a.icon = getIcon(a);
|
||||
return a;
|
||||
}));
|
||||
@@ -383,11 +361,7 @@ export async function getADSB() {
|
||||
return adsbCache;
|
||||
}
|
||||
|
||||
export async function getADSBHistory(icao) {
|
||||
return { history: history.get(icao?.toLowerCase()) || [] };
|
||||
}
|
||||
|
||||
export async function getADSBRange() {
|
||||
export async function getAttenuation() {
|
||||
const { ADSB_URL } = cfg();
|
||||
if (!ADSB_URL) return [];
|
||||
const r = await fetch(`${ADSB_URL}:8080/data/outline.json`);
|
||||
@@ -395,7 +369,11 @@ export async function getADSBRange() {
|
||||
return j?.actualRange?.last24h?.points || [];
|
||||
}
|
||||
|
||||
export async function getADSBImage(icao) {
|
||||
export async function getHistory(icao) {
|
||||
return { history: history.get(icao?.toLowerCase()) || [] };
|
||||
}
|
||||
|
||||
export async function getIcaoImage(icao) {
|
||||
function randomUA() {
|
||||
const v = 36 + Math.floor(Math.random() * 40);
|
||||
const builds = [
|
||||
@@ -487,9 +465,18 @@ export async function getADSBImage(icao) {
|
||||
|
||||
icao = icao.toLowerCase();
|
||||
if(!fs.existsSync(IMAGES_DIR)) fs.mkdirSync(IMAGES_DIR, { recursive: true });
|
||||
const aircraft = await enrichAircraft({ hex: icao });
|
||||
const aircraft = await enrich({ hex: icao });
|
||||
const modelType = aircraft.aircraft || aircraft.model || aircraft.type;
|
||||
const generic = genericImage(modelType);
|
||||
const specific = await icaoImage(icao);
|
||||
return specific || await generic;
|
||||
}
|
||||
|
||||
// Purge old history
|
||||
setInterval(() => {
|
||||
const cutoff = Date.now() - HISTORY_TTL;
|
||||
for (const [key, trail] of history) {
|
||||
const last = trail.at(-1);
|
||||
if (!last || last.ts < cutoff) history.delete(key);
|
||||
}
|
||||
}, 1000 * 60 * 5);
|
||||
|
||||
@@ -8,7 +8,7 @@ import {apiReference} from '@scalar/express-api-reference';
|
||||
import {spec} from './spec.mjs';
|
||||
import {existsSync} from 'fs';
|
||||
import {getAIS, getAISImage} from './ais.mjs';
|
||||
import {getADSBImage, getADSB, getADSBHistory, getADSBRange, initAircraftDb} from './adsb.mjs';
|
||||
import {getIcaoImage, get, getHistory, getAttenuation, initAircraftDb} from './adsb.mjs';
|
||||
import {fetchIcon} from './openweather.mjs';
|
||||
import {getForecast, getWeatherCondition, forecastTTL} from './forecast.mjs';
|
||||
import {dailyWeather, hourlyWeather} from './openmeteo.mjs';
|
||||
@@ -129,7 +129,7 @@ app.get('/api/daily', asyncHandler(async (req, res) => {
|
||||
|
||||
// ── Position ──────────────────────────────────────────────────────────────────
|
||||
|
||||
app.get('/api/range', asyncHandler(async (req, res) => res.json(await getADSBRange())));
|
||||
app.get('/api/range', asyncHandler(async (req, res) => res.json(await getAttenuation())));
|
||||
app.get('/api/position', asyncHandler(async (req, res) => {
|
||||
const {fields} = req.query;
|
||||
const data = await getCoords();
|
||||
@@ -155,12 +155,12 @@ app.get('/api/space', async (req, res) => {
|
||||
|
||||
// ── ADSB/AIS/SAT/RadioSonde Proxy ────────────────────────────────────────────────────────────
|
||||
|
||||
app.get('/api/adsb', asyncHandler(async (req, res) => res.json(await getADSB())));
|
||||
app.get('/api/adsb', asyncHandler(async (req, res) => res.json(await get())));
|
||||
app.get('/api/adsb/:icao/image', asyncHandler(async (req, res) => {
|
||||
const buffer = await getADSBImage(req.params.icao);
|
||||
const buffer = await getIcaoImage(req.params.icao);
|
||||
res.contentType('image/jpeg').send(buffer)
|
||||
}));
|
||||
app.get('/api/adsb/:icao', asyncHandler(async (req, res) => res.json(await getADSBHistory(req.params.icao))));
|
||||
app.get('/api/adsb/:icao', asyncHandler(async (req, res) => res.json(await getHistory(req.params.icao))));
|
||||
|
||||
app.get('/api/ais', asyncHandler(async (req, res) => res.json(await getAIS())));
|
||||
app.get('/api/ais/:mmsi/image', asyncHandler(async (req, res) => {
|
||||
|
||||
Reference in New Issue
Block a user