ADSB backfill fix?

2026-06-27 16:58:46 -04:00
parent d9fbe0568d
commit 6bc4d920f8
4 changed files with 327 additions and 24 deletions
--- a/client/src/components/Seismic.vue
+++ b/client/src/components/Seismic.vue
@@ -10,7 +10,6 @@ const canvas = ref<HTMLCanvasElement | null>(null);
 const W = 640, H = 120, MAX_PTS = 120;
 const SEG_W = W / (MAX_PTS - 1);
 const SUB = 4;
-const PAD_SEGS = 0;

 let interval: ReturnType<typeof setInterval>;
 let rafId: number;
@@ -70,7 +69,7 @@ function draw(timestamp: number) {
 	if (pointBuffer.length < 2) { rafId = requestAnimationFrame(draw); return; }

 	// Shift the whole path left by PAD_SEGS segments so tip never hits the edge
-	const xOffset = PAD_SEGS * SEG_W;
+	const xOffset = 3 * SEG_W;

 	const settledCount = pointBuffer.length - SUB;
 	const anchor = pointBuffer[settledCount - 1];
@@ -134,7 +133,7 @@ onUnmounted(() => {
 </script>

 <style scoped lang="scss">
-canvas { width: calc(100% - 16px); height: 60px; }
+canvas { width: 100%; height: 60px; }
 </style>

 <template>
--- a/server/package-lock.json
+++ b/server/package-lock.json
@@ -12,6 +12,7 @@
 				"@ztimson/utils": "^0.29.5",
 				"adm-zip": "^0.5.17",
 				"better-sqlite3": "^12.11.1",
+				"cheerio": "^1.2.0",
 				"dotenv": "^16.3.1",
 				"express": "^4.18.2",
 				"yaml": "^2.9.0"
@@ -204,6 +205,12 @@
 				"npm": "1.2.8000 || >= 1.4.16"
 			}
 		},
+		"node_modules/boolbase": {
+			"version": "1.0.0",
+			"resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
+			"integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==",
+			"license": "ISC"
+		},
 		"node_modules/buffer": {
 			"version": "5.7.1",
 			"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
@@ -266,6 +273,48 @@
 				"url": "https://github.com/sponsors/ljharb"
 			}
 		},
+		"node_modules/cheerio": {
+			"version": "1.2.0",
+			"resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.2.0.tgz",
+			"integrity": "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg==",
+			"license": "MIT",
+			"dependencies": {
+				"cheerio-select": "^2.1.0",
+				"dom-serializer": "^2.0.0",
+				"domhandler": "^5.0.3",
+				"domutils": "^3.2.2",
+				"encoding-sniffer": "^0.2.1",
+				"htmlparser2": "^10.1.0",
+				"parse5": "^7.3.0",
+				"parse5-htmlparser2-tree-adapter": "^7.1.0",
+				"parse5-parser-stream": "^7.1.2",
+				"undici": "^7.19.0",
+				"whatwg-mimetype": "^4.0.0"
+			},
+			"engines": {
+				"node": ">=20.18.1"
+			},
+			"funding": {
+				"url": "https://github.com/cheeriojs/cheerio?sponsor=1"
+			}
+		},
+		"node_modules/cheerio-select": {
+			"version": "2.1.0",
+			"resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz",
+			"integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==",
+			"license": "BSD-2-Clause",
+			"dependencies": {
+				"boolbase": "^1.0.0",
+				"css-select": "^5.1.0",
+				"css-what": "^6.1.0",
+				"domelementtype": "^2.3.0",
+				"domhandler": "^5.0.3",
+				"domutils": "^3.0.1"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/fb55"
+			}
+		},
 		"node_modules/chownr": {
 			"version": "1.1.4",
 			"resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
@@ -308,6 +357,34 @@
 			"integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==",
 			"license": "MIT"
 		},
+		"node_modules/css-select": {
+			"version": "5.2.2",
+			"resolved": "https://registry.npmjs.org/css-select/-/css-select-5.2.2.tgz",
+			"integrity": "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==",
+			"license": "BSD-2-Clause",
+			"dependencies": {
+				"boolbase": "^1.0.0",
+				"css-what": "^6.1.0",
+				"domhandler": "^5.0.2",
+				"domutils": "^3.0.1",
+				"nth-check": "^2.0.1"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/fb55"
+			}
+		},
+		"node_modules/css-what": {
+			"version": "6.2.2",
+			"resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz",
+			"integrity": "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==",
+			"license": "BSD-2-Clause",
+			"engines": {
+				"node": ">= 6"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/fb55"
+			}
+		},
 		"node_modules/debug": {
 			"version": "2.6.9",
 			"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
@@ -369,6 +446,61 @@
 				"node": ">=8"
 			}
 		},
+		"node_modules/dom-serializer": {
+			"version": "2.0.0",
+			"resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz",
+			"integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==",
+			"license": "MIT",
+			"dependencies": {
+				"domelementtype": "^2.3.0",
+				"domhandler": "^5.0.2",
+				"entities": "^4.2.0"
+			},
+			"funding": {
+				"url": "https://github.com/cheeriojs/dom-serializer?sponsor=1"
+			}
+		},
+		"node_modules/domelementtype": {
+			"version": "2.3.0",
+			"resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz",
+			"integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==",
+			"funding": [
+				{
+					"type": "github",
+					"url": "https://github.com/sponsors/fb55"
+				}
+			],
+			"license": "BSD-2-Clause"
+		},
+		"node_modules/domhandler": {
+			"version": "5.0.3",
+			"resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz",
+			"integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==",
+			"license": "BSD-2-Clause",
+			"dependencies": {
+				"domelementtype": "^2.3.0"
+			},
+			"engines": {
+				"node": ">= 4"
+			},
+			"funding": {
+				"url": "https://github.com/fb55/domhandler?sponsor=1"
+			}
+		},
+		"node_modules/domutils": {
+			"version": "3.2.2",
+			"resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz",
+			"integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==",
+			"license": "BSD-2-Clause",
+			"dependencies": {
+				"dom-serializer": "^2.0.0",
+				"domelementtype": "^2.3.0",
+				"domhandler": "^5.0.3"
+			},
+			"funding": {
+				"url": "https://github.com/fb55/domutils?sponsor=1"
+			}
+		},
 		"node_modules/dotenv": {
 			"version": "16.6.1",
 			"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.6.1.tgz",
@@ -410,6 +542,31 @@
 				"node": ">= 0.8"
 			}
 		},
+		"node_modules/encoding-sniffer": {
+			"version": "0.2.1",
+			"resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.1.tgz",
+			"integrity": "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw==",
+			"license": "MIT",
+			"dependencies": {
+				"iconv-lite": "^0.6.3",
+				"whatwg-encoding": "^3.1.1"
+			},
+			"funding": {
+				"url": "https://github.com/fb55/encoding-sniffer?sponsor=1"
+			}
+		},
+		"node_modules/encoding-sniffer/node_modules/iconv-lite": {
+			"version": "0.6.3",
+			"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
+			"integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
+			"license": "MIT",
+			"dependencies": {
+				"safer-buffer": ">= 2.1.2 < 3.0.0"
+			},
+			"engines": {
+				"node": ">=0.10.0"
+			}
+		},
 		"node_modules/end-of-stream": {
 			"version": "1.4.5",
 			"resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz",
@@ -419,6 +576,18 @@
 				"once": "^1.4.0"
 			}
 		},
+		"node_modules/entities": {
+			"version": "4.5.0",
+			"resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
+			"integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==",
+			"license": "BSD-2-Clause",
+			"engines": {
+				"node": ">=0.12"
+			},
+			"funding": {
+				"url": "https://github.com/fb55/entities?sponsor=1"
+			}
+		},
 		"node_modules/es-define-property": {
 			"version": "1.0.1",
 			"resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
@@ -655,6 +824,37 @@
 				"node": ">= 0.4"
 			}
 		},
+		"node_modules/htmlparser2": {
+			"version": "10.1.0",
+			"resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz",
+			"integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==",
+			"funding": [
+				"https://github.com/fb55/htmlparser2?sponsor=1",
+				{
+					"type": "github",
+					"url": "https://github.com/sponsors/fb55"
+				}
+			],
+			"license": "MIT",
+			"dependencies": {
+				"domelementtype": "^2.3.0",
+				"domhandler": "^5.0.3",
+				"domutils": "^3.2.2",
+				"entities": "^7.0.1"
+			}
+		},
+		"node_modules/htmlparser2/node_modules/entities": {
+			"version": "7.0.1",
+			"resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz",
+			"integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==",
+			"license": "BSD-2-Clause",
+			"engines": {
+				"node": ">=0.12"
+			},
+			"funding": {
+				"url": "https://github.com/fb55/entities?sponsor=1"
+			}
+		},
 		"node_modules/http-errors": {
 			"version": "2.0.1",
 			"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz",
@@ -875,6 +1075,18 @@
 				"node": ">=10"
 			}
 		},
+		"node_modules/nth-check": {
+			"version": "2.1.1",
+			"resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz",
+			"integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==",
+			"license": "BSD-2-Clause",
+			"dependencies": {
+				"boolbase": "^1.0.0"
+			},
+			"funding": {
+				"url": "https://github.com/fb55/nth-check?sponsor=1"
+			}
+		},
 		"node_modules/object-inspect": {
 			"version": "1.13.4",
 			"resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz",
@@ -908,6 +1120,55 @@
 				"wrappy": "1"
 			}
 		},
+		"node_modules/parse5": {
+			"version": "7.3.0",
+			"resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz",
+			"integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==",
+			"license": "MIT",
+			"dependencies": {
+				"entities": "^6.0.0"
+			},
+			"funding": {
+				"url": "https://github.com/inikulin/parse5?sponsor=1"
+			}
+		},
+		"node_modules/parse5-htmlparser2-tree-adapter": {
+			"version": "7.1.0",
+			"resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz",
+			"integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==",
+			"license": "MIT",
+			"dependencies": {
+				"domhandler": "^5.0.3",
+				"parse5": "^7.0.0"
+			},
+			"funding": {
+				"url": "https://github.com/inikulin/parse5?sponsor=1"
+			}
+		},
+		"node_modules/parse5-parser-stream": {
+			"version": "7.1.2",
+			"resolved": "https://registry.npmjs.org/parse5-parser-stream/-/parse5-parser-stream-7.1.2.tgz",
+			"integrity": "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==",
+			"license": "MIT",
+			"dependencies": {
+				"parse5": "^7.0.0"
+			},
+			"funding": {
+				"url": "https://github.com/inikulin/parse5?sponsor=1"
+			}
+		},
+		"node_modules/parse5/node_modules/entities": {
+			"version": "6.0.1",
+			"resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz",
+			"integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==",
+			"license": "BSD-2-Clause",
+			"engines": {
+				"node": ">=0.12"
+			},
+			"funding": {
+				"url": "https://github.com/fb55/entities?sponsor=1"
+			}
+		},
 		"node_modules/parseurl": {
 			"version": "1.3.3",
 			"resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
@@ -1364,6 +1625,15 @@
 				"node": ">= 0.6"
 			}
 		},
+		"node_modules/undici": {
+			"version": "7.28.0",
+			"resolved": "https://registry.npmjs.org/undici/-/undici-7.28.0.tgz",
+			"integrity": "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==",
+			"license": "MIT",
+			"engines": {
+				"node": ">=20.18.1"
+			}
+		},
 		"node_modules/unpipe": {
 			"version": "1.0.0",
 			"resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
@@ -1403,6 +1673,40 @@
 				"node": ">= 0.8"
 			}
 		},
+		"node_modules/whatwg-encoding": {
+			"version": "3.1.1",
+			"resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz",
+			"integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==",
+			"deprecated": "Use @exodus/bytes instead for a more spec-conformant and faster implementation",
+			"license": "MIT",
+			"dependencies": {
+				"iconv-lite": "0.6.3"
+			},
+			"engines": {
+				"node": ">=18"
+			}
+		},
+		"node_modules/whatwg-encoding/node_modules/iconv-lite": {
+			"version": "0.6.3",
+			"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
+			"integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
+			"license": "MIT",
+			"dependencies": {
+				"safer-buffer": ">= 2.1.2 < 3.0.0"
+			},
+			"engines": {
+				"node": ">=0.10.0"
+			}
+		},
+		"node_modules/whatwg-mimetype": {
+			"version": "4.0.0",
+			"resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz",
+			"integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==",
+			"license": "MIT",
+			"engines": {
+				"node": ">=18"
+			}
+		},
 		"node_modules/wrappy": {
 			"version": "1.0.2",
 			"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
--- a/server/package.json
+++ b/server/package.json
@@ -11,6 +11,7 @@
 		"@ztimson/utils": "^0.29.5",
 		"adm-zip": "^0.5.17",
 		"better-sqlite3": "^12.11.1",
+		"cheerio": "^1.2.0",
 		"dotenv": "^16.3.1",
 		"express": "^4.18.2",
 		"yaml": "^2.9.0"
--- a/server/src/adsb.mjs
+++ b/server/src/adsb.mjs
@@ -5,6 +5,7 @@ import * as fs from 'node:fs';
 import Database from 'better-sqlite3';
 import { fromCsv } from '@ztimson/utils';
 import {getIcon} from './adsb-shapes.mjs';
+import * as cheerio from 'cheerio';

 const DIR            = dirname(fileURLToPath(import.meta.url));
 const DATA           = resolve(DIR, '../data');
@@ -171,27 +172,6 @@ export function classifyAircraft(row) {
 	return 'unknown';
 }

-async function scrapeHexDatabase(icao) {
-	const url = `https://hexdatabase.com/h/${icao}`;
-	try {
-		const resp = await fetch(url);
-		if (!resp.ok) return null;
-		const html = await resp.text();
-		const rowMatch = html.match(new RegExp(`<tr[^>]*>.*?<span[^>]*>.*?${icao}.*?<\/span>.*?<\/tr>`, 's'));
-		if (!rowMatch) return null;
-		const cells = rowMatch.match(/<td[^>]*>.*?<\/td>/g) || [];
-		return {
-			country:      cells[1]?.replace(/<[^>]*>/g, '').trim() || null,
-			registration: cells[2]?.replace(/<[^>]*>/g, '').trim() || null,
-			aircraft:     cells[3]?.replace(/<[^>]*>/g, '').trim() || null,
-			operator:     cells[4]?.replace(/<[^>]*>/g, '').trim() || null,
-			serialNumber: cells[6]?.replace(/<[^>]*>/g, '').trim() || null,
-		};
-	} catch (e) {
-		return null;
-	}
-}
-
 async function fetchHexDb(icao) {
 	const resp = await fetchWithTimeout(`https://hexdb.io/api/v1/aircraft/${icao}`);
 	if (!resp.ok) return null;
@@ -205,6 +185,25 @@ async function fetchHexDb(icao) {
 	};
 }

+async function scrapeHexDatabase(icao) {
+	const url = `https://hexdatabase.com/h/${icao}`;
+	const resp = await fetch(url);
+	if (!resp.ok) return null;
+
+	const $ = cheerio.load(await resp.text());
+	const row = $('tr').filter((_, el) => $(el).text().toLowerCase().includes(icao.toLowerCase())).first();
+	if (!row.length) return null;
+
+	const cells = row.find('td');
+	return {
+		country:      $(cells[1]).text().trim() || null,
+		registration: $(cells[2]).text().trim() || null,
+		aircraft:     $(cells[3]).text().trim() || null,
+		operator:     $(cells[4]).text().trim() || null,
+		serialNumber: $(cells[6]).text().trim() || null,
+	};
+}
+
 function backfillFromSimilar(aircraft) {
 	if (!aircraft) return {};
 	const similar = db.prepare(`