From 834b2f780741a65ebb372f8b34f874fab85aaa1d Mon Sep 17 00:00:00 2001 From: Bryan Housel Date: Mon, 24 Feb 2020 12:40:18 -0500 Subject: [PATCH] Initial commit Mostly a clone of osmlab/osm-community-index --- .editorconfig | 13 ++ .eslintrc | 13 ++ .gitignore | 7 + .npmignore | 11 + .travis.yml | 4 + .tx/config | 9 + CHANGELOG.md | 18 ++ CONTRIBUTING.md | 153 ++++++++++++++ LICENSE.md | 15 ++ README.md | 63 ++++++ RELEASE.md | 18 ++ build.js | 242 ++++++++++++++++++++++ build_dist.js | 100 ++++++++++ dist/.gitkeep | 0 features/.gitkeep | 0 i18n/.gitkeep | 0 package.json | 36 ++++ schema/feature.json | 15 ++ schema/geojson.json | 353 +++++++++++++++++++++++++++++++++ schema/resource.json | 103 ++++++++++ sources/africa/.gitkeep | 0 sources/asia/.gitkeep | 0 sources/europe/.gitkeep | 0 sources/north-america/.gitkeep | 0 sources/oceania/.gitkeep | 0 sources/south-america/.gitkeep | 0 sources/world/.gitkeep | 0 stats.js | 67 +++++++ 28 files changed, 1240 insertions(+) create mode 100644 .editorconfig create mode 100644 .eslintrc create mode 100644 .gitignore create mode 100644 .npmignore create mode 100644 .travis.yml create mode 100644 .tx/config create mode 100644 CHANGELOG.md create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE.md create mode 100644 README.md create mode 100644 RELEASE.md create mode 100644 build.js create mode 100644 build_dist.js create mode 100644 dist/.gitkeep create mode 100644 features/.gitkeep create mode 100644 i18n/.gitkeep create mode 100644 package.json create mode 100644 schema/feature.json create mode 100644 schema/geojson.json create mode 100644 schema/resource.json create mode 100644 sources/africa/.gitkeep create mode 100644 sources/asia/.gitkeep create mode 100644 sources/europe/.gitkeep create mode 100644 sources/north-america/.gitkeep create mode 100644 sources/oceania/.gitkeep create mode 100644 sources/south-america/.gitkeep create mode 100644 sources/world/.gitkeep create mode 100644 stats.js diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..05f9e08 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,13 @@ +root = true + +[*] +trim_trailing_whitespace = true +insert_final_newline = true + +# for ESLint +[*.js] +end_of_line = lf + +[*.{js,json,css,html}] +indent_style = space +indent_size = 2 \ No newline at end of file diff --git a/.eslintrc b/.eslintrc new file mode 100644 index 0000000..67596fb --- /dev/null +++ b/.eslintrc @@ -0,0 +1,13 @@ +{ + "env": { + "node": true, + "es6": true + }, + "parserOptions": { + "ecmaVersion": 6, + "sourceType": "script" + }, + "extends": [ + "eslint:recommended" + ] +} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b34ea3a --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +/.nyc_output/ +/coverage/ +/node_modules/ + +.DS_Store +npm-debug.log +package-lock.json diff --git a/.npmignore b/.npmignore new file mode 100644 index 0000000..9180ca3 --- /dev/null +++ b/.npmignore @@ -0,0 +1,11 @@ +/.nyc_output/ +/.tx/ +/coverage/ +/node_modules/ +/test/ + +.DS_Store +.travis.yml +npm-debug.log +package-lock.json +RELEASE.md diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..e2d26a9 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,4 @@ +language: node_js +node_js: + - "10" + - "12" diff --git a/.tx/config b/.tx/config new file mode 100644 index 0000000..a1ffb5a --- /dev/null +++ b/.tx/config @@ -0,0 +1,9 @@ +[main] +host = https://www.transifex.com +minimum_perc = 1 + +[id-editor.community] +file_filter = i18n/.yaml +source_file = i18n/en.yaml +source_lang = en +type = YAML diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..e014989 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,18 @@ +# What's New + +**location-conflation** is an open source project. You can submit bug reports, help out, +or learn more by visiting our project page on GitHub: :octocat: https://github.com/ideditor/imagery-index + +Please star our project on GitHub to show your support! :star: + +_Breaking changes, which may affect downstream projects, are marked with a_ :warning: + + + + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..8a39dcd --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,153 @@ +## Contributing + +*If you don't understand the explanation below, feel free to [post an Issue](https://github.com/ideditor/imagery-index/issues) to describe your imagery sources. That page contains some pointers to help you fill in all the info we need. You do need [a Github account](https://github.com/join) to be able to post an Issue.* + +There are 2 kinds of files in this project: + +* Under `sources/` there are `.json` files to describe the imagery sources +* Under `features/` there are custom `.geojson` files + +### tl;dr + +To add your imagery source to the index: + +* Add source `.json` files under the `sources/` folder + * Each file contains info about what the source is (slack, forum, mailinglist, facebook, etc.) + * Each file also contains info about which locations the source is active. The locations can be country or region codes, points, or custom `.geojson` files in the `features/*` folder. + * You can copy and change an existing file to get started. +* run `npm run test` + * This will check the files for errors and make them pretty. + * If you don't have Node installed, you can skip this step and we will do it for you. +* If there are no errors, submit a pull request. + + +### Installing + +* Clone this project, for example: + `git clone git@github.com:ideditor/imagery-index.git` +* `cd` into the project folder, +* Run `npm install` to install libraries + + +### Sources + +These are `*.json` files found under the `sources/` folder. +Each source file contains a single JSON object with information about the imagery source. + +Source files look like this: + +```js +{ + "id": + "type": + "locationSet": { "include": ["us"] } + "name": + "description": + "extendedDescription": + "url": +} +``` + +Here are the properties that a source file can contain: + +* __`id`__ - (required) A unique identifier for the source. +* __`type`__ - (required) Type of imagery source, one of `tms` or `wms`. +* __`locationSet`__ - (required) Where the imagery source is active (see below for details). +* __`name`__ - (required) Display name for this imagery source +(in English, will be sent to Transifex for translation to other languages) +* __`description`__ - (required) One line description of the imagery source +(in English, will be sent to Transifex for translation to other languages) +* __`extendedDescription`__ - (optional) Longer description of the imagery source +(in English, will be sent to Transifex for translation to other languages) +* __`url`__ - (required) A url template for the imagery source + + +#### locationSet + +Each source must have a `locationSet` to define where the source is active. + +```js +"locationSet": { + "include": [ Array of locations ], // required + "exclude": [ Array of locations ] // optional +} +``` + +The "locations" can be any of the following: +* Codes recognized by the [country-coder library](https://github.com/ideditor/country-coder#readme). These should be [ISO 3166-1 2 or 3 letter country codes or UN M.49 numeric codes](https://en.wikipedia.org/wiki/List_of_countries_by_United_Nations_geoscheme).
_Example: `"de"`_ +* Points as `[longitude, latitude]` coordinate pairs. A 25km radius circle will be computed around the point.
_Example: `[8.67039, 49.41882]`_ +* Filenames for `.geojson` features. If you want to use your own features, you'll need to add these under the `features/` folder. Each `Feature` must have an `id` that ends in `.geojson`.
_Example: `"de-hamburg.geojson"`_
Tip: You can use [geojson.io](http://geojson.io) or other tools to create these. + +See [location-conflation](https://github.com/ideditor/location-conflation#readme) project for details and examples. + + +### Features + +These are optional `*.geojson` files found under the `features/` folder. Each feature file contains a single GeoJSON `Feature` for a region where a imagery source is active. + +Feature files look like this: + +```js +{ + "type": "Feature", + "id": "boston_metro.geojson", + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [...] + } +} +``` + +Note: A `FeatureCollection` containing a single `Feature` is ok too - the build script can handle this. + +There are many online tools to create or modify these `.geojson` files. +Drawing a simple shape with [geojson.io](http://geojson.io) works great. + + +### Building + +* Just `npm run test` + * This will check the files for errors and make them pretty. + + +### Translations + +All imagery sources automatically support localization of the +`name`, `description`, and `extendedDescription` text. These fields +should be written in US English. + +Translations are managed using the +[Transifex](https://www.transifex.com/projects/p/id-editor/) platform. +After signing up, you can go to [iD's project page](https://www.transifex.com/projects/p/id-editor/), +select a language and click **Translate** to start translating. + +The translation strings for this project are located in a resource called +[**imagery**](https://www.transifex.com/openstreetmap/id-editor/imagery/). + + +#### For maintainers + +Transifex will automatically fetch the source file from this repository daily. +We need to manually pull down and check in the translation files whenever we +make a new release (see [RELEASE.md](RELEASE.md)). + +To work with translation files, +[install the Transifex Client](https://docs.transifex.com/client/introduction) software. + +The Transifex Client uses a file +[`~/.transifex.rc`](https://docs.transifex.com/client/client-configuration#-transifexrc) +to store your username and password. + +Note that you can also use a +[Transifex API Token](https://docs.transifex.com/api/introduction#authentication) +in place of your username and password. In this usage, the username is `api` +and the password is the generated API token. + +Once you have installed the client and setup the `~/.transifex.rc` file, you can +use the following commands: + +* `tx push -s` - upload latest source `/i18n/en.yaml` file to Transifex +* `tx pull -a` - download latest translation files to `/i18n/.yaml` + +For convenience you can also run these commands as `npm run txpush` or `npm run txpull`. diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..080e0b5 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,15 @@ +ISC License + +Copyright (c) 2020, imagery-index contributors + +Permission to use, copy, modify, and/or distribute this software for any purpose +with or without fee is hereby granted, provided that the above copyright notice +and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF +THIS SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..8dee278 --- /dev/null +++ b/README.md @@ -0,0 +1,63 @@ + +# imagery-index + +🛰 An index of aerial and satellite imagery useful for mapping. + + +### About the index + +#### tl;dr + +To add an imagery source to the index: + +* Add source `.json` files under the `sources/` folder + * Each file contains info about the imagery source: name, url template, license requirements + * Each file also contains info about which locations the imagery covers. The locations can be country or region codes, points, or custom `.geojson` files in the `features/*` folder. + * You can copy and change an existing file to get started. +* Run `npm run test` + * This will check the files for errors and make them pretty. + * If you don't have Node installed, you can skip this step and we will do it for you. +* If there are no errors, submit a pull request. + +:point_right: See [CONTRIBUTING.md](CONTRIBUTING.md) for full details about how to add an imagery source to this index. + + +#### Source files + +The source files for this index are stored in two kinds of files: + +* Under `sources/` there are `.json` files to describe the imagery sources +* Under `features/` there are custom `.geojson` files + + +#### Distributed Files + +Several files are published under `dist/`. These are generated - do not edit them. + +* todo + + +#### Prerequisites + +* [Node.js](https://nodejs.org/) version 10 or newer +* [`git`](https://www.atlassian.com/git/tutorials/install-git/) for your platform + + +#### Installing + +* Clone this project, for example: + `git clone git@github.com:ideditor/imagery-index.git` +* `cd` into the project folder, +* Run `npm install` to install libraries + + +#### Building + +* Just `npm run test` + * This will check the files for errors and make them pretty. + + +### License + +imagery-index is available under the [ISC License](https://opensource.org/licenses/ISC). +See the [LICENSE.md](LICENSE.md) file for more details. diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 0000000..29f51b6 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,18 @@ +## Release Checklist + +### Update master branch, tag and publish +- [ ] git checkout master +- [ ] npm install +- [ ] npm run test +- [ ] npm run dist +- [ ] npm run txpull +- [ ] git add . && git commit -m 'npm run txpull' +- [ ] Update `CHANGELOG.md` +- [ ] Update version number in `package.json` +- [ ] git add . && git commit -m 'vA.B.C' +- [ ] git tag vA.B.C +- [ ] git push origin master vA.B.C +- [ ] npm publish + +Open https://github.com/ideditor/imagery-index/tags +Click "Add Release Notes" and link to the CHANGELOG diff --git a/build.js b/build.js new file mode 100644 index 0000000..86fbd20 --- /dev/null +++ b/build.js @@ -0,0 +1,242 @@ +const calcArea = require('@mapbox/geojson-area'); +const colors = require('colors/safe'); +const fs = require('fs'); +const glob = require('glob'); +const LocationConflation = require('@ideditor/location-conflation'); +const path = require('path'); +const precision = require('geojson-precision'); +const prettyStringify = require('json-stringify-pretty-compact'); +const rewind = require('geojson-rewind'); +const shell = require('shelljs'); +const Validator = require('jsonschema').Validator; +const YAML = require('js-yaml'); + +const geojsonSchema = require('./schema/geojson.json'); +const featureSchema = require('./schema/feature.json'); +const sourceSchema = require('./schema/source.json'); + +let v = new Validator(); +v.addSchema(geojsonSchema, 'http://json.schemastore.org/geojson.json'); + +buildAll(); + + +function buildAll() { + const START = '🏗 ' + colors.yellow('Building data...'); + const END = '👍 ' + colors.green('data built'); + + console.log(''); + console.log(START); + console.time(END); + + // Start clean + shell.rm('-f', [ + 'dist/featureCollection.json', + 'dist/sources.json', + 'i18n/en.yaml' + ]); + + // Features + let tstrings = {}; // translation strings + const features = collectFeatures(); + const featureCollection = { type: 'FeatureCollection', features: features }; + fs.writeFileSync('dist/featureCollection.json', prettyStringify(featureCollection, { maxLength: 9999 })); + + // Sources + const sources = collectSources(tstrings, featureCollection); + fs.writeFileSync('dist/sources.json', prettyStringify({ sources: sort(sources) }, { maxLength: 9999 })); + fs.writeFileSync('i18n/en.yaml', YAML.safeDump({ en: sort(tstrings) }, { lineWidth: -1 }) ); + + console.timeEnd(END); +} + + +function collectFeatures() { + let features = []; + let files = {}; + process.stdout.write('📦 Features: '); + + glob.sync('features/**/*.geojson').forEach(file => { + const contents = fs.readFileSync(file, 'utf8'); + let parsed; + try { + parsed = JSON.parse(contents); + } catch (jsonParseError) { + console.error(colors.red(`Error - ${jsonParseError.message} in:`)); + console.error(' ' + colors.yellow(file)); + process.exit(1); + } + + let feature = precision(rewind(parsed, true), 5); + let fc = feature.features; + + // A FeatureCollection with a single feature inside (geojson.io likes to make these). + if (feature.type === 'FeatureCollection' && Array.isArray(fc) && fc.length === 1) { + feature = fc[0]; + } + + // warn if this feature is so small it would better be represented as a point. + let area = calcArea.geometry(feature.geometry) / 1e6; // m² to km² + area = Number(area.toFixed(2)); + if (area < 2000) { + console.warn(colors.yellow(`Warning - small area (${area} km²). Use a point 'includeLocation' instead.`)); + console.warn(' ' + colors.yellow(file)); + } + + // use the filename as the feature.id + const id = path.basename(file).toLowerCase(); + feature.id = id; + + // sort properties + let obj = {}; + if (feature.type) { obj.type = feature.type; } + if (feature.id) { obj.id = feature.id; } + if (feature.properties) { obj.properties = feature.properties; } + if (feature.geometry) { obj.geometry = feature.geometry; } + feature = obj; + + validateFile(file, feature, featureSchema); + prettifyFile(file, feature, contents); + + if (files[id]) { + console.error(colors.red('Error - Duplicate filenames: ') + colors.yellow(id)); + console.error(' ' + colors.yellow(files[id])); + console.error(' ' + colors.yellow(file)); + process.exit(1); + } + features.push(feature); + files[id] = file; + + process.stdout.write(colors.green('✓')); + }); + + process.stdout.write(' ' + Object.keys(files).length + '\n'); + + return features; +} + + +function collectSources(tstrings, featureCollection) { + let sources = {}; + let files = {}; + const loco = new LocationConflation(featureCollection); + process.stdout.write('📦 Sources: '); + + glob.sync('sources/**/*.json').forEach(file => { + let contents = fs.readFileSync(file, 'utf8'); + + let source; + try { + source = JSON.parse(contents); + } catch (jsonParseError) { + console.error(colors.red(`Error - ${jsonParseError.message} in:`)); + console.error(' ' + colors.yellow(file)); + process.exit(1); + } + + // // sort properties and array values + // let obj = {}; + // if (source.id) { obj.id = source.id; } + // if (source.type) { obj.type = source.type; } + + // if (source.locationSet) { + // obj.locationSet = {}; + // if (source.locationSet.include) { obj.locationSet.include = source.locationSet.include; } + // if (source.locationSet.exclude) { obj.locationSet.exclude = source.locationSet.exclude; } + // } + + // if (source.languageCodes) { obj.languageCodes = source.languageCodes.sort(); } + // if (source.name) { obj.name = source.name; } + // if (source.description) { obj.description = source.description; } + // if (source.extendedDescription) { obj.extendedDescription = source.extendedDescription; } + // if (source.url) { obj.url = source.url; } + // if (source.signupUrl) { obj.signupUrl = source.signupUrl; } + // if (source.contacts) { obj.contacts = source.contacts; } + // if (source.order) { obj.order = source.order; } + // if (source.events) { obj.events = source.events; } + // source = obj; + + validateFile(file, source, sourceSchema); + + (source.locationSet.include || []).forEach(location => { + if (!loco.validateLocation(location)) { + console.error(colors.red('Error - Invalid include location: ') + colors.yellow(location)); + console.error(' ' + colors.yellow(file)); + process.exit(1); + } + }); + + (source.locationSet.exclude || []).forEach(location => { + if (!loco.validateLocation(location)) { + console.error(colors.red('Error - Invalid exclude location: ') + colors.yellow(location)); + console.error(' ' + colors.yellow(file)); + process.exit(1); + } + }); + + prettifyFile(file, source, contents); + + let sourceId = source.id; + if (files[sourceId]) { + console.error(colors.red('Error - Duplicate source id: ') + colors.yellow(sourceId)); + console.error(' ' + colors.yellow(files[sourceId])); + console.error(' ' + colors.yellow(file)); + process.exit(1); + } + + sources[sourceId] = source; + files[sourceId] = file; + + // collect translation strings for this source + tstrings[sourceId] = { + name: source.name, + description: source.description + }; + + if (source.extendedDescription) { + tstrings[sourceId].extendedDescription = source.extendedDescription; + } + + process.stdout.write(colors.green('✓')); + }); + + process.stdout.write(' ' + Object.keys(files).length + '\n'); + + return sources; +} + + +function validateFile(file, source, schema) { + const validationErrors = v.validate(source, schema).errors; + if (validationErrors.length) { + console.error(colors.red('Error - Schema validation:')); + console.error(' ' + colors.yellow(file + ': ')); + validationErrors.forEach(error => { + if (error.property) { + console.error(' ' + colors.yellow(error.property + ' ' + error.message)); + } else { + console.error(' ' + colors.yellow(error)); + } + }); + process.exit(1); + } +} + + +function prettifyFile(file, object, contents) { + const pretty = prettyStringify(object, { maxLength: 100 }); + if (pretty !== contents) { + fs.writeFileSync(file, pretty); + } +} + + +// Returns an object with sorted keys and sorted values. +// (This is useful for file diffing) +function sort(obj) { + let sorted = {}; + Object.keys(obj).sort().forEach(k => { + sorted[k] = Array.isArray(obj[k]) ? obj[k].sort() : obj[k]; + }); + return sorted; +} diff --git a/build_dist.js b/build_dist.js new file mode 100644 index 0000000..eb24b34 --- /dev/null +++ b/build_dist.js @@ -0,0 +1,100 @@ +const colors = require('colors/safe'); +const fs = require('fs'); +const LocationConflation = require('@ideditor/location-conflation'); +const prettyStringify = require('json-stringify-pretty-compact'); +const shell = require('shelljs'); + +const featureCollection = require('./dist/featureCollection.json'); +const sources = require('./dist/sources.json').sources; + +buildAll(); + + +function buildAll() { + const START = '🏗 ' + colors.yellow('Building dist...'); + const END = '👍 ' + colors.green('dist built'); + + console.log(''); + console.log(START); + console.time(END); + + // Start clean + shell.rm('-f', [ + 'dist/combined.geojson', + 'dist/combined.min.geojson', + 'dist/featureCollection.min.json', + 'dist/sources.min.json' + ]); + + const combined = generateCombined(sources, featureCollection); + + // Save individual data files + fs.writeFileSync('dist/combined.geojson', prettyStringify(combined) ); + fs.writeFileSync('dist/combined.min.geojson', JSON.stringify(combined) ); + fs.writeFileSync('dist/featureCollection.min.json', JSON.stringify(featureCollection) ); + fs.writeFileSync('dist/sources.min.json', JSON.stringify({ sources: sources }) ); + + console.timeEnd(END); +} + + +function deepClone(obj) { + return JSON.parse(JSON.stringify(obj)); +} + + +// Generate a combined GeoJSON FeatureCollection +// containing all the features w/ sources stored in properties +// +// { +// type: 'FeatureCollection', +// features: [ +// { +// type: 'Feature', +// id: 'Q117', +// geometry: { ... }, +// properties: { +// 'area': 297118.3, +// 'sources': { +// 'osm-gh-facebook': { ... }, +// 'osm-gh-twitter': { ... }, +// 'talk-gh': { ... } +// } +// } +// }, { +// type: 'Feature', +// id: 'Q1019', +// geometry: { ... }, +// properties: { +// 'area': 964945.85, +// 'sources': { +// 'osm-mg-facebook': { ... }, +// 'osm-mg-twitter': { ... }, +// 'talk-mg': { ... } +// } +// } +// }, +// ... +// ] +// } +// +function generateCombined(sources, featureCollection) { + let keepFeatures = {}; + const loco = new LocationConflation(featureCollection); + + Object.keys(sources).forEach(resourceId => { + const resource = sources[resourceId]; + const feature = loco.resolveLocationSet(resource.locationSet); + + let keepFeature = keepFeatures[feature.id]; + if (!keepFeature) { + keepFeature = deepClone(feature); + keepFeature.properties.sources = {}; + keepFeatures[feature.id] = keepFeature; + } + + keepFeature.properties.sources[resourceId] = deepClone(resource); + }); + + return { type: 'FeatureCollection', features: Object.values(keepFeatures) }; +} diff --git a/dist/.gitkeep b/dist/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/features/.gitkeep b/features/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/i18n/.gitkeep b/i18n/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/package.json b/package.json new file mode 100644 index 0000000..26acc47 --- /dev/null +++ b/package.json @@ -0,0 +1,36 @@ +{ + "name": "@ideditor/imagery-index", + "version": "0.0.0", + "license": "ISC", + "repository": "ideditor/imagery-index", + "author": "Bryan Housel ", + "description": "An index of background imagery useful for mapping", + "keywords": ["imagery"], + "scripts": { + "build": "node build.js", + "dist": "node build_dist.js", + "lint": "eslint *.js", + "test": "npm run build && npm run lint", + "stats": "node stats.js", + "txpull": "tx pull -a", + "txpush": "tx push -s" + }, + "devDependencies": { + "@ideditor/location-conflation": "0.3.0", + "@mapbox/geojson-area": "^0.2.2", + "bytes": "^3.1.0", + "colors": "^1.4.0", + "easy-table": "^1.1.1", + "eslint": "^6.8.0", + "geojson-precision": "^0.4.0", + "geojson-rewind": "^0.3.1", + "glob": "^7.1.6", + "js-yaml": "^3.13.1", + "json-stringify-pretty-compact": "^2.0.0", + "jsonschema": "^1.2.5", + "shelljs": "^0.8.3" + }, + "engines": { + "node": ">=10" + } +} diff --git a/schema/feature.json b/schema/feature.json new file mode 100644 index 0000000..6366b80 --- /dev/null +++ b/schema/feature.json @@ -0,0 +1,15 @@ +{ + "title": "Feature", + "description": "A GeoJSON Feature", + "allOf": [ + { "$ref": "http://json.schemastore.org/geojson.json#/definitions/feature" } + ], + "type": "object", + "required": ["id"], + "properties": { + "id": { + "type": "string", + "pattern": "^[-_.A-Za-z0-9]+$" + } + } +} diff --git a/schema/geojson.json b/schema/geojson.json new file mode 100644 index 0000000..79b591b --- /dev/null +++ b/schema/geojson.json @@ -0,0 +1,353 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "GeoJSON Object", + "type": "object", + "description": "This object represents a geometry, feature, or collection of features.", + "additionalProperties": true, + "required": ["type"], + + "properties": { + + "type": { + "title": "Type", + "type": "string", + "description": "The type of GeoJSON object.", + "enum": [ + "Point", + "MultiPoint", + "LineString", + "MultiLineString", + "Polygon", + "MultiPolygon", + "GeometryCollection", + "Feature", + "FeatureCollection" + ] + }, + + "crs": { + "title": "Coordinate Reference System (CRS)", + "description": "The coordinate reference system (CRS) of a GeoJSON object is determined by its `crs` member (referred to as the CRS object below). If an object has no crs member, then its parent or grandparent object's crs member may be acquired. If no crs member can be so acquired, the default CRS shall apply to the GeoJSON object.\n\n* The default CRS is a geographic coordinate reference system, using the WGS84 datum, and with longitude and latitude units of decimal degrees.\n\n* The value of a member named `crs` must be a JSON object (referred to as the CRS object below) or JSON null. If the value of CRS is null, no CRS can be assumed.\n\n* The crs member should be on the top-level GeoJSON object in a hierarchy (in feature collection, feature, geometry order) and should not be repeated or overridden on children or grandchildren of the object.\n\n* A non-null CRS object has two mandatory members: `type` and `properties`.\n\n* The value of the type member must be a string, indicating the type of CRS object.\n\n* The value of the properties member must be an object.\n\n* CRS shall not change coordinate ordering.", + + "oneOf": [ + { "type": "null" }, + { + "type": "object", + "required": ["type", "properties"], + "properties": { + "type": { + "title": "CRS Type", + "type": "string", + "description": "The value of the type member must be a string, indicating the type of CRS object.", + "minLength": 1 + }, + "properties": { + "title": "CRS Properties", + "type": "object" + } + } + } + ], + + "not": { + "anyOf": [ + + { + "properties": { + "type": { "enum": ["name"] }, + "properties": { + "not": { + "required": ["name"], + "properties": { + "name": { + "type": "string", + "minLength": 1 + } + } + } + } + } + }, + + { + "properties": { + "type": { "enum": ["link"] }, + "properties": { + "not": { + "title": "Link Object", + "type": "object", + "required": ["href"], + "properties": { + + "href": { + "title": "href", + "type": "string", + "description": "The value of the required `href` member must be a dereferenceable URI.", + "format": "uri" + }, + + "type": { + "title": "Link Object Type", + "type": "string", + "description": "The value of the optional `type` member must be a string that hints at the format used to represent CRS parameters at the provided URI. Suggested values are: `proj4`, `ogcwkt`, `esriwkt`, but others can be used." + } + + } + } + } + } + } + + ] + } + + }, + + "bbox": { + "title": "Bounding Box", + "type": "array", + "description": "To include information on the coordinate range for geometries, features, or feature collections, a GeoJSON object may have a member named `bbox`. The value of the bbox member must be a 2*n array where n is the number of dimensions represented in the contained geometries, with the lowest values for all axes followed by the highest values. The axes order of a bbox follows the axes order of geometries. In addition, the coordinate reference system for the bbox is assumed to match the coordinate reference system of the GeoJSON object of which it is a member.", + "minItems": 4, + "items": { + "type": "number" + } + } + + }, + + "oneOf": [ + + { + "title": "Point", + "description": "For type `Point`, the `coordinates` member must be a single position.", + "required": ["coordinates"], + "properties": { + "type": { "enum": ["Point"] }, + "coordinates": { + "allOf": [ + { "$ref": "#/definitions/coordinates" }, + { "$ref": "#/definitions/position" } + ] + } + }, + "allOf": [{ "$ref": "#/definitions/geometry" }] + }, + + { + "title": "Multi Point Geometry", + "description": "For type `MultiPoint`, the `coordinates` member must be an array of positions.", + "required": ["coordinates"], + "properties": { + "type": { "enum": ["MultiPoint"] }, + "coordinates": { + "allOf": [ + { "$ref": "#/definitions/coordinates" }, + { + "items": { "$ref": "#/definitions/position" } + } + ] + } + }, + "allOf": [{ "$ref": "#/definitions/geometry" }] + }, + + { + "title": "Line String", + "description": "For type `LineString`, the `coordinates` member must be an array of two or more positions.\n\nA LinearRing is closed LineString with 4 or more positions. The first and last positions are equivalent (they represent equivalent points). Though a LinearRing is not explicitly represented as a GeoJSON geometry type, it is referred to in the Polygon geometry type definition.", + "required": ["coordinates"], + "properties": { + "type": { "enum": ["LineString"] }, + "coordinates": { "$ref": "#/definitions/lineStringCoordinates" } + }, + "allOf": [{ "$ref": "#/definitions/geometry" }] + }, + + { + "title": "MultiLineString", + "description": "For type `MultiLineString`, the `coordinates` member must be an array of LineString coordinate arrays.", + "required": ["coordinates"], + "properties": { + "type": { "enum": ["MultiLineString"] }, + "coordinates": { + "allOf": [ + { "$ref": "#/definitions/coordinates" }, + { + "items": { "$ref": "#/definitions/lineStringCoordinates" } + } + ] + } + }, + "allOf": [{ "$ref": "#/definitions/geometry" }] + }, + + { + "title": "Polygon", + "description": "For type `Polygon`, the `coordinates` member must be an array of LinearRing coordinate arrays. For Polygons with multiple rings, the first must be the exterior ring and any others must be interior rings or holes.", + "required": ["coordinates"], + "properties": { + "type": { "enum": ["Polygon"] }, + "coordinates": { "$ref": "#/definitions/polygonCoordinates" } + }, + "allOf": [{ "$ref": "#/definitions/geometry" }] + }, + + { + "title": "Multi-Polygon Geometry", + "description": "For type `MultiPolygon`, the `coordinates` member must be an array of Polygon coordinate arrays.", + "required": ["coordinates"], + "properties": { + "type": { "enum": ["MultiPolygon"] }, + "coordinates": { + "allOf": [ + { "$ref": "#/definitions/coordinates" }, + { + "items": { "$ref": "#/definitions/polygonCoordinates" } + } + ] + } + }, + "allOf": [{ "$ref": "#/definitions/geometry" }] + }, + + { + "title": "Geometry Collection", + "description": "A GeoJSON object with type `GeometryCollection` is a geometry object which represents a collection of geometry objects.\n\nA geometry collection must have a member with the name `geometries`. The value corresponding to `geometries` is an array. Each element in this array is a GeoJSON geometry object.", + "required": ["geometries"], + "properties": { + "type": { "enum": ["GeometryCollection"] }, + "geometries": { + "title": "Geometries", + "type": "array", + "items": { "$ref": "#/definitions/geometry" } + } + }, + "allOf": [{ "$ref": "#/definitions/geometry" }] + }, + + { "$ref": "#/definitions/feature" }, + + { + "title": "Feature Collection", + "description": "A GeoJSON object with the type `FeatureCollection` is a feature collection object.\n\nAn object of type `FeatureCollection` must have a member with the name `features`. The value corresponding to `features` is an array. Each element in the array is a feature object as defined above.", + "required": ["features"], + "properties": { + "type": { "enum": ["FeatureCollection"] }, + "features": { + "title": "Features", + "type": "array", + "items": { "$ref": "#/definitions/feature" } + } + } + } + + ], + + "definitions": { + + "coordinates": { + "title": "Coordinates", + "type": "array", + "items": { + "oneOf": [ + { "type": "array" }, + { "type": "number" } + ] + } + }, + + "geometry": { + "title": "Geometry", + "description": "A geometry is a GeoJSON object where the type member's value is one of the following strings: `Point`, `MultiPoint`, `LineString`, `MultiLineString`, `Polygon`, `MultiPolygon`, or `GeometryCollection`.", + "properties": { + "type": { + "enum": [ + "Point", + "MultiPoint", + "LineString", + "MultiLineString", + "Polygon", + "MultiPolygon", + "GeometryCollection" + ] + } + } + }, + + "feature": { + "title": "Feature", + "description": "A GeoJSON object with the type `Feature` is a feature object.\n\n* A feature object must have a member with the name `geometry`. The value of the geometry member is a geometry object as defined above or a JSON null value.\n\n* A feature object must have a member with the name `properties`. The value of the properties member is an object (any JSON object or a JSON null value).\n\n* If a feature has a commonly used identifier, that identifier should be included as a member of the feature object with the name `id`.", + "required": ["geometry", "properties"], + + "properties": { + + "type": { "enum": ["Feature"] }, + + "geometry": { + "title": "Geometry", + "oneOf": [ + { "$ref": "#/definitions/geometry" }, + { "type": "null" } + ] + }, + + "properties": { + "title": "Properties", + "oneOf": [ + { "type": "object" }, + { "type": "null" } + ] + }, + + "id": {} + + } + }, + + "linearRingCoordinates": { + "title": "Linear Ring Coordinates", + "description": "A LinearRing is closed LineString with 4 or more positions. The first and last positions are equivalent (they represent equivalent points). Though a LinearRing is not explicitly represented as a GeoJSON geometry type, it is referred to in the Polygon geometry type definition.", + "allOf": [ + { "$ref": "#/definitions/lineStringCoordinates" }, + { + "minItems": 4 + } + ] + }, + + "lineStringCoordinates": { + "title": "Line String Coordinates", + "description": "For type `LineString`, the `coordinates` member must be an array of two or more positions.", + "allOf": [ + { "$ref": "#/definitions/coordinates" }, + { + "minLength": 2, + "items": { "$ref": "#/definitions/position" } + } + ] + }, + + "polygonCoordinates": { + "title": "Polygon Coordinates", + "description": "For type `Polygon`, the `coordinates` member must be an array of LinearRing coordinate arrays. For Polygons with multiple rings, the first must be the exterior ring and any others must be interior rings or holes.", + "allOf": [ + { "$ref": "#/definitions/coordinates" }, + { + "items": { "$ref": "#/definitions/linearRingCoordinates" } + } + ] + }, + + "position": { + "title": "Position", + "type": "array", + "description": "A position is the fundamental geometry construct. The `coordinates` member of a geometry object is composed of one position (in the case of a Point geometry), an array of positions (LineString or MultiPoint geometries), an array of arrays of positions (Polygons, MultiLineStrings), or a multidimensional array of positions (MultiPolygon).\n\nA position is represented by an array of numbers. There must be at least two elements, and may be more. The order of elements must follow x, y, z order (easting, northing, altitude for coordinates in a projected coordinate reference system, or longitude, latitude, altitude for coordinates in a geographic coordinate reference system). Any number of additional elements are allowed -- interpretation and meaning of additional elements is beyond the scope of this specification.", + "minItems": 2, + "additionalItems": true, + "items": { + "type": "number" + } + } + + } + +} diff --git a/schema/resource.json b/schema/resource.json new file mode 100644 index 0000000..2e8d4a7 --- /dev/null +++ b/schema/resource.json @@ -0,0 +1,103 @@ +{ + "title": "Resource", + "description": "An imagery source", + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "type", + "locationSet", + "name", + "description", + "url" + ], + + "properties": { + "id": { + "description": "(required) A unique identifier for the resource", + "type": "string", + "pattern": "^[-_.A-Za-z0-9]+$" + }, + "type": { + "description": "(required) Type of imagery source", + "type": "string", + "enum": ["tms", "wms"] + }, + "locationSet": { + "$comment": "See location-conflation documentation for compatible values: https://github.com/ideditor/location-conflation#readme", + "description": "(required) included and excluded locations for this resource", + "type": "object", + "additionalProperties": false, + "required": [ + "include" + ], + "properties": { + "include": { + "description": "(required) locations included", + "type": "array", + "uniqueItems": true, + "items": { + "anyOf": [ + { "$ref": "#/definitions/countryCoder" }, + { "$ref": "#/definitions/coordinatePair" }, + { "$ref": "#/definitions/geojsonFilename" } + ] + } + }, + "exclude": { + "description": "(optional) locations excluded", + "type": "array", + "uniqueItems": true, + "items": { + "anyOf": [ + { "$ref": "#/definitions/countryCoder" }, + { "$ref": "#/definitions/coordinatePair" }, + { "$ref": "#/definitions/geojsonFilename" } + ] + } + } + } + }, + "name": { + "$comment": "Assumed to be in English, this value will be sent to Transifex for translation", + "description": "(required) Display name for this imagery source", + "type": "string" + }, + "description": { + "$comment": "Assumed to be in English, this value will be sent to Transifex for translation", + "description": "(required) One line description of the imagery source", + "type": "string" + }, + "extendedDescription": { + "$comment": "Assumed to be in English, this value will be sent to Transifex for translation", + "description": "(optional) Longer description of the imagery source", + "type": "string" + }, + "url": { + "description": "(required) A url template for the imagery source", + "type": "string" + } + }, + + "definitions": { + "countryCoder": { + "$comment": "See country-coder documentation for compatible values: https://github.com/ideditor/country-coder#readme", + "description": "A country code (ISO 3166-1, United Nations M49, or anything recognized by country-coder)", + "type": "string" + }, + "coordinatePair": { + "description": "A coordinate pair formatted as [longitude, latitude]", + "type": "array", + "items": { + "type": "number", + "minItems": 2, + "maxItems": 2 + } + }, + "geojsonFilename": { + "description": "A filename for one of the geojson features in this project", + "type": "string", + "pattern": "^.*\\.geojson$" + } + } +} diff --git a/sources/africa/.gitkeep b/sources/africa/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/sources/asia/.gitkeep b/sources/asia/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/sources/europe/.gitkeep b/sources/europe/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/sources/north-america/.gitkeep b/sources/north-america/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/sources/oceania/.gitkeep b/sources/oceania/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/sources/south-america/.gitkeep b/sources/south-america/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/sources/world/.gitkeep b/sources/world/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/stats.js b/stats.js new file mode 100644 index 0000000..9ba3f2a --- /dev/null +++ b/stats.js @@ -0,0 +1,67 @@ +const bytes = require('bytes'); +const colors = require('colors/safe'); +const fs = require('fs'); +const glob = require('glob'); +const path = require('path'); +const Table = require('easy-table'); + +getStats(); + +function getStats() { + let featureSize = 0; + let sourceSize = 0; + let featureFiles = 0; + let sourceFiles = 0; + let currSize = 0; + let currFiles = 0; + + let t = new Table; + currSize = 0; + currFiles = 0; + glob.sync('features/**/*.geojson').forEach(addRow); + t.sort(['Size|des']); + console.log(t.toString()); + featureSize = bytes(currSize, { unitSeparator: ' ' }); + featureFiles = currFiles; + + t = new Table; + currSize = 0; + currFiles = 0; + glob.sync('sources/**/*.json').forEach(addRow); + t.sort(['Size|des']); + console.log(t.toString()); + sourceSize = bytes(currSize, { unitSeparator: ' ' }); + sourceFiles = currFiles; + + console.info(`\nTotals:`); + console.info(`-------`); + console.info(colors.blue.bold(`Features: ${featureSize} in ${featureFiles} files.`)); + console.info(colors.blue.bold(`Sources: ${sourceSize} in ${sourceFiles} files.`)); + console.info(''); + + + function addRow(file) { + const stats = fs.statSync(file); + const color = colorBytes(stats.size); + currSize += stats.size; + currFiles++; + + t.cell('Size', stats.size, (val, width) => { + const displaySize = bytes(stats.size, { unitSeparator: ' ' }); + return width ? Table.padLeft(displaySize, width) : color(displaySize); + }); + t.cell('File', color(path.basename(file))); + t.newRow(); + } + + function colorBytes(size) { + if (size > 1024 * 10) { // 10 KB + return colors.red; + } else if (size > 1024 * 2) { // 2 KB + return colors.yellow; + } else { + return colors.green; + } + } +} +