diff --git a/.gitignore b/.gitignore index 3d343c7..3894f7c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ node_modules npm-debug.log .DS_Store test/out/* +test/cache/* dist diff --git a/README.md b/README.md index 8fcbe52..e607c52 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,9 @@ height | Required | Height of the output image in px paddingX | 0 | (optional) Minimum distance in px between map features and map border paddingY | 0 | (optional) Minimum distance in px between map features and map border tileUrl | | (optional) Tile server URL for the map base layer or `null` for empty base layer. `{x},{y},{z}` or `{quadkey}` supported. +tileCacheFolder | | (optional) When set to an existing folder, a file cache is used +tileCacheLifetime | 86400 | (optional) Time before tile in cache expire and will be reloaded +tileCacheAutoPurge | true | (optional) Should the Filebased TileCache automatically purged tileSubdomains | [] | (optional) Subdomains of tile server, usage `['a', 'b', 'c']` tileSize | 256 | (optional) Tile size in pixel tileRequestTimeout | | (optional) Timeout for the tiles request @@ -65,6 +68,7 @@ Method | Description [addMultiPolygon](#addmultipolygon-options) | Adds a multipolygon to the map [addCircle](#addcircle-options) | Adds a circle to the map [addText](#addtext-options) | Adds text to the map +[clearCache](#clearcache) | Manually clear the base layer tile cache [render](#render-center-zoom) | Renders the map and added features [image.save](#imagesave-filename-outputoptions) | Saves the map image to a file [image.buffer](#imagebuffer-mime-outputoptions) | Saves the map image to a buffer @@ -267,6 +271,12 @@ center | | (optional) Set center of map to a specific coo zoom | | (optional) Set a specific zoom level. *** +#### clearCache () +clear the file based Tile cache. +Can be used, if tileCacheAutoPurge is set to false to clear the cache +``` +map.clearCache(); +``` #### image.save (fileName, [outputOptions]) Saves the image to a file in `fileName`. diff --git a/package-lock.json b/package-lock.json index 742078b..5f472d0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1739,6 +1739,17 @@ "type-detect": "^4.0.5" } }, + "chai-image": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chai-image/-/chai-image-3.0.0.tgz", + "integrity": "sha512-qbGaZvqRJ4bSdqhwrGhsl9+4KIYwqjC/Be56IFyrQLhncqWTjP0sXoEFe4CjLnXX2DabUKSJRZBfSvIUIYhF8Q==", + "dev": true, + "requires": { + "mkdirp": "^1.0.0", + "pixelmatch": "^5.0.2", + "pngjs": "^6.0.0" + } + }, "chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -3236,6 +3247,12 @@ "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" }, + "mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true + }, "mkdirp-classic": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", @@ -3636,6 +3653,23 @@ "integrity": "sha512-8V9+HQPupnaXMA23c5hvl69zXvTwTzyAYasnkb0Tts4XvO4CliqONMOnvlq26rkhLC3nWDFBJf73LU1e1VZLaQ==", "dev": true }, + "pixelmatch": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-5.2.1.tgz", + "integrity": "sha512-WjcAdYSnKrrdDdqTcVEY7aB7UhhwjYQKYhHiBXdJef0MOaQeYpUdQ+iVyBLa5YBKS8MPVPPMX7rpOByISLpeEQ==", + "dev": true, + "requires": { + "pngjs": "^4.0.1" + }, + "dependencies": { + "pngjs": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-4.0.1.tgz", + "integrity": "sha512-rf5+2/ioHeQxR6IxuYNYGFytUyG3lma/WW1nsmjeHlWwtb2aByla6dkVc8pmJ9nplzkTA0q2xx7mMWrOTqT4Gg==", + "dev": true + } + } + }, "pkg-dir": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-3.0.0.tgz", @@ -3645,6 +3679,12 @@ "find-up": "^3.0.0" } }, + "pngjs": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-6.0.0.tgz", + "integrity": "sha512-TRzzuFRRmEoSW/p1KVAmiOgPco2Irlah+bGFCeNfJXxxYGwSw7YwAOAcd7X28K/m5bjBWKsC29KyoMfHbypayg==", + "dev": true + }, "prebuild-install": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", diff --git a/package.json b/package.json index 4650d67..30786ff 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "@babel/preset-stage-2": "^7.8.3", "@babel/register": "^7.13.16", "chai": "^4.3.4", + "chai-image": "^3.0.0", "eslint": "^7.27.0", "eslint-config-airbnb-base": "^14.2.1", "eslint-plugin-import": "^2.23.3", diff --git a/src/staticmaps.js b/src/staticmaps.js index 249b3da..717672b 100644 --- a/src/staticmaps.js +++ b/src/staticmaps.js @@ -4,7 +4,9 @@ import find from 'lodash.find'; import uniqBy from 'lodash.uniqby'; import url from 'url'; import chunk from 'lodash.chunk'; - +import { createHash } from 'crypto'; +import fs from 'fs'; +import path from 'path'; import Image from './image'; import IconMarker from './marker'; import Polyline from './polyline'; @@ -29,6 +31,13 @@ class StaticMaps { this.padding = [this.paddingX, this.paddingY]; this.tileUrl = 'tileUrl' in this.options ? this.options.tileUrl : 'https://tile.openstreetmap.org/{z}/{x}/{y}.png'; this.tileSize = this.options.tileSize || 256; + + this.tileCacheFolder = this.options.tileCacheFolder || null; + this.tileCacheAutoPurge = typeof this.options.tileCacheAutoPurge !== 'undefined' ? + this.options.tileCacheAutoPurge : true; + this.tileCacheLifetime = this.options.tileCacheLifetime || 86400; + this.tileCacheHits = 0; + this.tileSubdomains = this.options.tileSubdomains || this.options.subdomains || []; this.tileRequestTimeout = this.options.tileRequestTimeout; this.tileRequestHeader = this.options.tileRequestHeader; @@ -120,9 +129,48 @@ class StaticMaps { this.drawBaselayer(), this.loadMarker(), ]); + + // when a cache folder is configured and auto purge is enable + // clear cache in 10% of all executions + if (this.tileCacheFolder !== null + && this.tileCacheAutoPurge === true + && Math.random() * 10 <= 1) { + this.clearCache(); + } + return this.drawFeatures(); } + getTileCacheHits() { + return this.tileCacheHits; + } + + async clearCache() { + if (this.tileCacheFolder !== null) { + const now = new Date().getTime(); + + fs.readdir(this.tileCacheFolder, (err, files) => { + files.forEach((file, index) => { + fs.stat(path.join(this.tileCacheFolder, file), (err, stat) => { + if (err) { + return console.error(err); + } + + const fileMTime = new Date(stat.mtime).getTime() + this.tileCacheLifetime * 1000; + + if (now > fileMTime) { + return fs.unlink(path.join(this.tileCacheFolder, file), (err) => { + if (err) { + return console.error(err); + } + }); + } + }); + }); + }); + } + } + /** * calculate common extent of all current map features */ @@ -487,6 +535,7 @@ class StaticMaps { * Fetching tile from endpoint */ async getTile(data) { + const options = { url: data.url, responseType: 'buffer', @@ -496,14 +545,61 @@ class StaticMaps { }; try { - const res = await got.get(options); - const { body, headers } = res; + let cacheFile = null; + + if (this.tileCacheFolder !== null) { + const cacheKey = createHash('sha256').update(data.url).digest('hex'); + cacheFile = path.join(this.tileCacheFolder, cacheKey); + + if (fs.existsSync(cacheFile)) { + const stats = fs.statSync(cacheFile); + + const seconds = (new Date().getTime() - stats.mtime) / 1000; + + // If TTL expire, delete file + if (seconds < this.tileCacheLifetime) { + let cacheData; + try { + cacheData = JSON.parse(fs.readFileSync(cacheFile)); + } catch(e) { + try { + cacheData = JSON.parse(fs.readFileSync(cacheFile)); + } catch(e) {} + } + + if(cacheData && cacheData.length > 0) { + try { + cacheData = Buffer.from(cacheData, 'base64'); + + if(cacheData && cacheData.length > 0) { + const responseContent = { + success: true, + tile: { + url: data.url, + box: data.box, + body: cacheData, + }, + }; + + this.tileCacheHits++; + + return responseContent; + } + } catch (e) {} + } + } + + fs.rmSync(cacheFile); + } + } + + let res = await got.get(options); + const { body, headers } = res; const contentType = headers['content-type']; if (!contentType.startsWith('image/')) throw new Error('Tiles server response with wrong data'); - // console.log(headers); - return { + const responseContent = { success: true, tile: { url: data.url, @@ -511,6 +607,22 @@ class StaticMaps { body, }, }; + + if (this.tileCacheFolder !== null) { + fs.writeFile(cacheFile, JSON.stringify(responseContent.tile.body.toString('base64')), (err) => { + if (err) { + console.error(err); + } + // file written successfully + }); + + if (typeof responseContent.tile.body === 'string') { + responseContent.tile.body = Buffer.from(responseContent.tile.body, 'base64'); + } + } + + return responseContent; + } catch (error) { return { success: false, diff --git a/test/staticmaps.js b/test/staticmaps.js index b21682d..9dd5c3c 100644 --- a/test/staticmaps.js +++ b/test/staticmaps.js @@ -5,11 +5,18 @@ import GeoJSON from './static/geojson'; import MultiPolygonGeometry from './static/multipolygonGeometry'; import Route from './static/routeLong'; -const { expect } = require('chai'); +import * as chai from "chai"; +import { chaiImage } from "chai-image"; + +chai.use(chaiImage); +const expect = chai.expect; + +const fs = require('fs'); const markerPath = path.join(__dirname, 'marker.png'); describe('StaticMap', () => { + describe('Initializing ...', () => { it('without any arguments', () => { expect(() => { @@ -96,7 +103,7 @@ describe('StaticMap', () => { await map.render([13.437524, 52.4945528], 12); await map.image.save('test/out/04-marker.png'); }).timeout(0); - + /* it('render w/ remote url icon', async () => { const options = { width: 500, @@ -198,6 +205,7 @@ describe('StaticMap', () => { await map.render(); await map.image.save('test/out/05-annotations-nobaselayer.png'); }).timeout(0); + */ }); describe('Rendering w/ polylines ...', () => { @@ -415,5 +423,123 @@ describe('StaticMap', () => { await map.render([13.437524, 52.4945528], 13); await map.image.save('test/out/10-subdomains.png'); }).timeout(0); + }); + + describe('Tile cache', () => { + var cacheFolder = path.resolve(__dirname, 'cache'); + + if (!fs.existsSync(cacheFolder)){ + fs.mkdirSync(cacheFolder); + } + + function sleep(ms) { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); + } + + function clearCache() { + return new Promise((resolve) => { + let files = fs.readdirSync(cacheFolder); + + files.forEach(file => { + fs.unlinkSync(path.join(cacheFolder, file)); + }); + + resolve(); + }); + } + + it('call clearCache manually', async () => { + await clearCache(); + let files = fs.readdirSync(cacheFolder); + expect(files.length).to.be.equal(0, 'cache must start empty'); + const testFilepath = path.join(cacheFolder, 'testfile'); + fs.writeFileSync(testFilepath, 'nothing'); + files = fs.readdirSync(cacheFolder); + expect(files.length).to.be.equal(1, 'we created a single testfile'); + + const options = { + width: 500, + height: 500, + tileCacheFolder: cacheFolder, + tileCacheAutoPurge: false, + tileCacheLifetime: 86400 + }; + + const map = new StaticMaps(options); + map.clearCache(); + + // we must wait, until cache is cleared + await sleep(1000); + + files = fs.readdirSync(cacheFolder); + expect(files.length).to.be.equal(1, 'cache folder must be contain testfile, because within Lifetime'); + + const time = new Date('2000-01-01 18:00:00'); + fs.utimesSync(testFilepath, time, time); + + map.clearCache(); + + // we must wait, until cache is cleared + await sleep(1000); + + files = fs.readdirSync(cacheFolder); + expect(files.length).to.be.equal(0, 'cache folder must be empty after clearCache'); + }).timeout(0); + + it('generate map with cache', async () => { + await clearCache(); + + const options = { + width: 500, + height: 500, + tileCacheFolder: cacheFolder, + tileCacheAutoPurge: false, + tileCacheLifetime: 86400 + }; + + let map = new StaticMaps(options); + + const marker = { + img: markerPath, + offsetX: 24, + offsetY: 48, + width: 48, + height: 48, + }; + + marker.coord = [13.437524, 52.4945528]; + map.addMarker(marker); + + marker.coord = [13.430524, 52.4995528]; + map.addMarker(marker); + + await map.render([13.437524, 52.4945528], 12); + await map.image.save('test/out/11-marker.png'); + + expect(map.getTileCacheHits()).to.be.equal(0); + + let files = fs.readdirSync(cacheFolder); + + map = new StaticMaps(options); + + marker.coord = [13.437524, 52.4945528]; + map.addMarker(marker); + + marker.coord = [13.430524, 52.4995528]; + map.addMarker(marker); + + await map.render([13.437524, 52.4945528], 12); + const bugCompare = fs.readFileSync('test/out/11-marker.png'); + + const bufActual = await map.image.buffer('image/png'); + + // must use all existing cache files + expect(map.getTileCacheHits()).to.be.equal(files.length); + // must match image without cache + expect(bufActual).to.matchImage(bugCompare); + }).timeout(0); + }); });