scrapers/osm.js

import pLimit from 'p-limit';


const overpass_endpoint = 'https://overpass-api.de/api/interpreter';
// Const overpass_endpoint = "https://maps.mail.ru/osm/tools/overpass/api/interpreter";
//const overpass_endpoint = "https://overpass.kumi.systems/api/interpreter";

async function get_road_length(insee_code) {
	console.log(`Getting data for INSEE:${insee_code}`)
	try {
		const res = await fetch(
			overpass_endpoint,
			{
				method: 'POST',
				headers: {
					Pragma: 'no-cache',
					'Cache-Control': 'no-cache',
				},
				referrer: overpass_endpoint,
				body: 'data=' + encodeURIComponent(`
		[out:json];
		area["ref:INSEE"="${insee_code}"]->.searchArea;
		way(area.searchArea);
		for(t["highway"])
		{
			 make stat "highway"=_.val, number=count(ways),length=sum(length());
			 out;
		};
		out body;
					`),
			},
		);

		if (String(res.status)[0] !== '2') {
			console.error(`Error ${res.status}`);
			return;
		}

		const text = await res.text();

		const json = JSON.parse(text);

		const data = json.elements;

		return data.map(road => ({
			type: road.tags.highway,
			count: Number(road.tags.number),
			// Len because I THINK js is making 'length' disappear in console.log
			// which drives me nuts
			len: Number(road.tags.length),
		}));
	} catch (error) {
		console.error(error);
		return undefined;
	}
}

// https://wiki.openstreetmap.org/wiki/Key:highway .
const actual_roads_types = new Set([
	'motorway',
	'trunk',
	'busway',
	'living_street',
	'residential',
	'primary',
	'secondary',
	'tertiary',
	'unclassified',
	'road',

	'motorway_link',
	'trunk_link',
	'primary_link',
	'secondary_link',
	'tertiary_link',

]);

async function get_data(node) {
	const roads = (await get_road_length(node.code));

	if (roads === undefined) {
		console.log(`[\u001B[31m${node.code}\u001B[0m] ${node.nom} \u001B[0m`);
		return undefined;
	} 
	else {
		const sum = roads.map(r => r.len).reduce((previous, curr) => previous + curr, 0);

		const filtered_sum = roads.filter(r => actual_roads_types.has(r.type)).map(r => r.len).reduce((previous, curr) => previous + curr, 0);

		// Km / point d'impot
		const ratio = (filtered_sum / 1000) / node.impots;

		console.log(`[\u001B[1;32m${node.code}\u001B[0m\u001B[1m] ${node.nom} : \u001B[1;33m${Math.round(filtered_sum / 1000)}km\u001B[0m of road for \u001B[1;33m${node.impots}%\u001B[0m impôts => \u001B[1;35m${ratio}\u001B[0m.`);

		return {
			...node,

			road_length: filtered_sum,
			ratio,

			total_highways: sum,

			roads_data: roads,
		};
	}
}

async function get_data_for_each(file) {
	const json = await Bun.file(file).json();

	const limit = pLimit(6);
	const result = await Promise.all( json.map( el => limit(() => get_data(el)) ) );

	const results_normalized = result.sort( (a,b) => Number(a.code) - Number(b.code) );

	//console.log(results_normalized);

	console.log('\n==> Saving...');
	await Bun.write(`./full-data_${file}`, JSON.stringify(results_normalized, null, 4));
	console.log('==> Done.\n');

	//for (const r of results_normalized) {
	//	console.log(`[\u001B[1;32m${r.code}\u001B[0m\u001B[1m] ${r.nom} : \u001B[1;35m${r.ratio_normalized}\u001B[0m (${r.ratio})`);
	//}

	return result;
}

(async () => {
	await get_data_for_each('departements.json');
	await get_data_for_each('arrondissements.json');
	await get_data_for_each('communes.json');

	// Get_road_length(91471).then(console.log)
})();