Skip to content

Cache adsb.lol responses and stop writing them back to the readsb file #19

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Purple10101 merged 2 commits into from
Aug 24, 2026
Merged

Cache adsb.lol responses and stop writing them back to the readsb file #19

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9837379
Cache adsb.lol responses and stop writing them back to the readsb file
Purple10101 Aug 24, 2026
8bbf79a
Throttle upstream retries while adsb.lol is failing
Purple10101 Aug 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions .env.example
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,3 +18,23 @@ ADSBLOL_ENABLED=true
# Radius in nautical miles for adsb.lol queries
# Aircraft within this radius of your receiver location will be fetched
ADSBLOL_RADIUS=40

# User-Agent sent to adsb.lol. They reject missing or generic agents with
# 403 "User-Agent too generic; include valid contact info." Set this to
# something that identifies you and carries a real contact address.
ADSBLOL_USER_AGENT=retina-node/1.0 (+https://github.com/offworldlabs/tar1090-node)

# How long an adsb.lol response may be reused before refetching (ms).
# Also throttles retries while adsb.lol is failing. Consumers project
# positions forward from the payload's own `now`, and blah2-api refuses to
# extrapolate more than 5 s, so keep this comfortably inside that budget.
ADSBLOL_CACHE_TTL_MS=3000

# Upstream request timeout (ms). Must stay below the consumer's HTTP timeout
# (blah2-api uses 5000) so a slow upstream degrades into stale-but-served.
ADSBLOL_TIMEOUT_MS=3000

# How long the last good response keeps being served while adsb.lol is
# failing (ms). Past this the proxy reports source 'none' rather than
# passing off arbitrarily old data as current.
ADSBLOL_MAX_STALE_MS=60000
160 changes: 106 additions & 54 deletions proxy/server.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,16 @@ const RECEIVER_LON = parseFloat(process.env.RECEIVER_LON || '0');
const ADSBLOL_RADIUS = parseInt(process.env.ADSBLOL_RADIUS || '40');
const PORT = parseInt(process.env.PROXY_PORT || '3005');

// How long an adsb.lol response may be reused before we refetch. Consumers
// correct for staleness by projecting positions forward from the payload's own
// `now`, but blah2-api's lib/extrapolation.js refuses to project more than 5 s,
// so this must stay comfortably inside that budget.
const CACHE_TTL_MS = parseInt(process.env.ADSBLOL_CACHE_TTL_MS || '3000');
// Must stay below the consumer's HTTP timeout (blah2-api uses 5000 ms) so a
// slow upstream degrades into stale-but-served rather than a client timeout.
const UPSTREAM_TIMEOUT_MS = parseInt(process.env.ADSBLOL_TIMEOUT_MS || '3000');
// How long we keep serving the last good response while adsb.lol is failing.
const MAX_STALE_MS = parseInt(process.env.ADSBLOL_MAX_STALE_MS || '60000');
// adsb.lol refuses requests whose User-Agent is missing or too generic, with
// 403 "User-Agent too generic; include valid contact info." Node's https.get
// sends no User-Agent at all unless one is set, so every request failed closed
Expand All @@ -19,16 +29,35 @@ const USER_AGENT = process.env.ADSBLOL_USER_AGENT ||

const ADSBLOL_API = `https://api.adsb.lol/v2/lat/${RECEIVER_LAT}/lon/${RECEIVER_LON}/dist/${ADSBLOL_RADIUS}`;

// Last good adsb.lol response: { payload, fetchedAt }. `payload.now` is the
// fetch time and is never restamped on serve - consumers rely on it to work out
// how stale each position is.
let cache = null;
// Shared promise for an in-progress fetch, so concurrent requests collapse into
// a single upstream call instead of one call each.
let inFlight = null;
// When the last upstream attempt started, recorded regardless of outcome.
// cache.fetchedAt only advances on success, so gating refreshes on it alone
// leaves a failing upstream unthrottled - see getAircraftData().
let lastAttemptAt = 0;

function emptyPayload() {
return { now: Date.now() / 1000, messages: 0, aircraft: [] };
}

function fetchUrl(url) {
return new Promise((resolve, reject) => {
const client = url.startsWith('https') ? https : http;
const timeout = 5000;

const req = client.get(url, { timeout, headers: { 'User-Agent': USER_AGENT } }, (res) => {
const req = client.get(url, {
timeout: UPSTREAM_TIMEOUT_MS,
headers: { 'User-Agent': USER_AGENT }
}, (res) => {
if (res.statusCode !== 200) {
// Read the body rather than discarding it: adsb.lol states the reason
// for a refusal there, and throwing it away is why a hard 403 was
// indistinguishable from "no aircraft nearby" for as long as it was.
// Consuming it also drains the socket, which res.resume() did before.
let err = '';
res.on('data', chunk => { if (err.length < 200) err += chunk; });
res.on('end', () => {
Expand Down Expand Up @@ -62,6 +91,8 @@ function convertAdsbLolToReadsb(adsbLolData) {
const aircraft = adsbLolData.ac || [];

return {
// Fetch time, not serve time. Consumers subtract each aircraft's seen_pos
// from this to recover when the position was actually observed.
now: Date.now() / 1000,
messages: 0,
aircraft: aircraft.map(ac => ({
Expand Down Expand Up @@ -97,20 +128,26 @@ function convertAdsbLolToReadsb(adsbLolData) {
};
}

async function fetchAdsbLol() {
console.log('Fetching from adsb.lol...');
const adsbLolData = await fetchUrl(ADSBLOL_API);
const convertedData = convertAdsbLolToReadsb(adsbLolData);
console.log(`adsb.lol: ${convertedData.aircraft?.length || 0} aircraft`);

// Write to local file so tar1090's backend process can read it
try {
await fs.writeFile(LOCAL_DATA_PATH, JSON.stringify(convertedData));
} catch (err) {
console.log(`Warning: Could not write to ${LOCAL_DATA_PATH}: ${err.message}`);
// Refreshes the cache, collapsing concurrent callers onto one upstream request.
// Never rejects - a failed refresh leaves the previous cache in place.
function refreshRemote() {
if (!inFlight) {
lastAttemptAt = Date.now();
console.log('Fetching from adsb.lol...');
inFlight = fetchUrl(ADSBLOL_API)
.then((raw) => {
const payload = convertAdsbLolToReadsb(raw);
cache = { payload, fetchedAt: Date.now() };
console.log(`adsb.lol: ${payload.aircraft.length} aircraft`);
})
.catch((err) => {
console.log(`adsb.lol fetch failed: ${err.message}`);
})
.finally(() => {
inFlight = null;
});
}

return { data: convertedData, source: 'adsb.lol' };
return inFlight;
}

async function readLocalFile() {
Expand All @@ -123,60 +160,74 @@ async function readLocalFile() {
}

async function getAircraftData() {
// Try local file first (readsb writes to /run/readsb/aircraft.json)
// A local receiver, when it has anything to say, always wins - and it is read
// fresh on every request rather than cached, since readsb rewrites it ~1/s.
const localData = await readLocalFile();

if (localData && localData.aircraft?.length > 0) {
console.log(`Local file: ${localData.aircraft.length} aircraft`);
return { data: localData, source: 'local' };
return { data: localData, source: 'local', ageMs: 0, stale: false };
}

// Try adsb.lol fallback if enabled
if (ADSBLOL_ENABLED) {
const reason = localData ? '0 aircraft from local' : 'local file not found';
console.log(`Falling back to adsb.lol (${reason})...`);
try {
return await fetchAdsbLol();
} catch (fallbackError) {
console.log(`adsb.lol fallback failed: ${fallbackError.message}`);
}
if (!ADSBLOL_ENABLED) {
return { data: localData || emptyPayload(), source: localData ? 'local' : 'none', ageMs: 0, stale: false };
}

// Return local data even if empty (if we got a response)
if (localData) {
return { data: localData, source: 'local' };
// Refresh only when the cached payload has aged out AND we have not already
// tried within this window. Without the second condition a failing upstream
// never advances cache.fetchedAt, so `age` stays Infinity and every request
// starts its own fetch - inFlight collapses concurrent callers but not
// sequential ones. That reverts to the full client poll rate precisely when
// adsb.lol is least able to serve it: measured at ~33 upstream requests/min
// during the 403 outage, against ~14/min once the cache was being populated.
const cacheAge = cache ? Date.now() - cache.fetchedAt : Infinity;
const sinceAttempt = Date.now() - lastAttemptAt;
if (cacheAge >= CACHE_TTL_MS && sinceAttempt >= CACHE_TTL_MS) {
await refreshRemote();
}

// No data sources available
throw new Error('No data sources available');
if (cache) {
const servedAge = Date.now() - cache.fetchedAt;
if (servedAge <= MAX_STALE_MS) {
return {
data: cache.payload,
source: 'adsb.lol',
ageMs: servedAge,
stale: servedAge >= CACHE_TTL_MS
};
}
}

// No usable remote data. Report that honestly rather than passing off an
// empty sky as a successful read.
return { data: localData || emptyPayload(), source: 'none', ageMs: 0, stale: false };
}

const server = http.createServer(async (req, res) => {
if (req.url === '/data/aircraft.json') {
try {
const { data, source } = await getAircraftData();
res.writeHead(200, {
'Content-Type': 'application/json',
'Access-Control-Allow-Origin': '*',
'X-Data-Source': source
});
res.end(JSON.stringify(data));
} catch (error) {
console.log(`Data fetch failed: ${error.message}`);
res.writeHead(200, {
'Content-Type': 'application/json',
'Access-Control-Allow-Origin': '*',
'X-Data-Source': 'none'
});
res.end(JSON.stringify({
now: Date.now() / 1000,
messages: 0,
aircraft: []
}));
}
// getAircraftData() is contracted never to throw or reject: readLocalFile()
// swallows its own errors and refreshRemote() carries an internal .catch.
// That contract is what makes it safe to call without a try/catch here - in
// an async http.createServer callback an unhandled rejection surfaces as an
// unhandledRejection rather than a clean response. Keep it if you edit them.
const { data, source, ageMs, stale } = await getAircraftData();
res.writeHead(200, {
'Content-Type': 'application/json',
'Access-Control-Allow-Origin': '*',
'Cache-Control': 'no-store',
'X-Data-Source': source,
'X-Data-Age-Ms': String(ageMs),
'X-Data-Stale': stale ? 'true' : 'false'
});
res.end(JSON.stringify(data));
} else if (req.url === '/health') {
const age = cache ? Date.now() - cache.fetchedAt : null;
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ status: 'ok' }));
res.end(JSON.stringify({
status: 'ok',
adsblol: ADSBLOL_ENABLED ? 'enabled' : 'disabled',
cacheAgeMs: age,
cachedAircraft: cache ? cache.payload.aircraft.length : 0
}));
} else {
res.writeHead(404);
res.end('Not Found');
Expand All @@ -189,5 +240,6 @@ server.listen(PORT, () => {
console.log(`adsb.lol fallback: ${ADSBLOL_ENABLED ? 'enabled' : 'disabled'}`);
if (ADSBLOL_ENABLED) {
console.log(`adsb.lol API: ${ADSBLOL_API}`);
console.log(`cache TTL: ${CACHE_TTL_MS} ms, upstream timeout: ${UPSTREAM_TIMEOUT_MS} ms`);
}
});
Loading