diff --git a/.flake8 b/.flake8 index 4bd453d25..74a039484 100644 --- a/.flake8 +++ b/.flake8 @@ -2,4 +2,4 @@ select = B,B9,BLK,C,E,F,W,S ignore = E203,W503 max-line-length = 88 -per-file-ignores = tests/*:S101, wetterdienst/__init__.py:F401 wetterdienst/cli.py:B950 +per-file-ignores = tests/*:S101, wetterdienst/__init__.py:F401, wetterdienst/cli.py:E501,B950 diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c8815a502..a7da68af4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,7 @@ Development - Add test for Jupyter notebook - Add function to discover available climate observations (time resolution, parameter, period type) +- Make the CLI work again and add software tests to prevent future havocs 0.6.0 (07.09.2020) ================== @@ -14,7 +15,6 @@ Development - enhance usage of get_nearby_stations to check for availability - output of get_nearby_stations is now a slice of meta_data DataFrame output - 0.5.0 (27.08.2020) ================== diff --git a/noxfile.py b/noxfile.py index 91fd9f29e..3bb59d71e 100644 --- a/noxfile.py +++ b/noxfile.py @@ -11,7 +11,7 @@ @nox.session(python=["3.6", "3.7", "3.8"]) def tests(session): """Run tests.""" - session.run("poetry", "install", "--no-dev", external=True) + session.run("poetry", "install", "--no-dev", "--extras=excel", external=True) install_with_constraints(session, "pytest", "pytest-notebook", "matplotlib", "mock") session.run("pytest") @@ -19,7 +19,7 @@ def tests(session): @nox.session(python=["3.7"]) def coverage(session: Session) -> None: """Run tests and upload coverage data.""" - session.run("poetry", "install", "--no-dev", external=True) + session.run("poetry", "install", "--no-dev", "--extras=excel", external=True) install_with_constraints( session, "coverage[toml]", diff --git a/poetry.lock b/poetry.lock index 4ec8a8705..bc51735a4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -335,6 +335,14 @@ optional = false python-versions = ">=2.7" version = "0.3" +[[package]] +category = "main" +description = "An implementation of lxml.xmlfile for the standard library" +name = "et-xmlfile" +optional = true +python-versions = "*" +version = "1.0.1" + [[package]] category = "dev" description = "A platform independent file lock." @@ -569,6 +577,14 @@ optional = false python-versions = "*" version = "0.2.0" +[[package]] +category = "main" +description = "Julian dates from proleptic Gregorian and Julian calendars." +name = "jdcal" +optional = true +python-versions = "*" +version = "1.4.1" + [[package]] category = "main" description = "An autocompletion tool for Python that can be used for text editors." @@ -879,6 +895,18 @@ optional = false python-versions = ">=3.5" version = "1.18.3" +[[package]] +category = "main" +description = "A Python library to read/write Excel 2010 xlsx/xlsm files" +name = "openpyxl" +optional = true +python-versions = ">=3.6," +version = "3.0.5" + +[package.dependencies] +et-xmlfile = "*" +jdcal = "*" + [[package]] category = "dev" description = "Core utilities for Python packages" @@ -1055,7 +1083,7 @@ description = "Pygments is a syntax highlighting package written in Python." name = "pygments" optional = false python-versions = ">=3.5" -version = "2.6.1" +version = "2.7.0" [[package]] category = "main" @@ -1071,7 +1099,7 @@ description = "Persistent/Functional/Immutable data structures" name = "pyrsistent" optional = false python-versions = ">=3.5" -version = "0.17.2" +version = "0.17.3" [[package]] category = "dev" @@ -1079,7 +1107,7 @@ description = "pytest: simple powerful testing with Python" name = "pytest" optional = false python-versions = ">=3.5" -version = "6.0.1" +version = "6.0.2" [package.dependencies] atomicwrites = ">=1.0" @@ -1569,10 +1597,11 @@ docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] testing = ["jaraco.itertools", "func-timeout"] [extras] +excel = ["openpyxl"] ipython = ["ipython", "ipython-genutils", "matplotlib"] [metadata] -content-hash = "b65bffb4c3a4cbe4c2d2abac74c413a7ac9a14b94ca3b55d6f4bfa8b61cdf836" +content-hash = "97bd71fcca0a610c4c6f62d647da2dda6fc7ce4950792341d83289a23365c4df" lock-version = "1.0" python-versions = "^3.6.1" @@ -1773,6 +1802,9 @@ entrypoints = [ {file = "entrypoints-0.3-py2.py3-none-any.whl", hash = "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19"}, {file = "entrypoints-0.3.tar.gz", hash = "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451"}, ] +et-xmlfile = [ + {file = "et_xmlfile-1.0.1.tar.gz", hash = "sha256:614d9722d572f6246302c4491846d2c393c199cfa4edc9af593437691683335b"}, +] filelock = [ {file = "filelock-3.0.12-py3-none-any.whl", hash = "sha256:929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836"}, {file = "filelock-3.0.12.tar.gz", hash = "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59"}, @@ -1869,6 +1901,10 @@ ipython-genutils = [ {file = "ipython_genutils-0.2.0-py2.py3-none-any.whl", hash = "sha256:72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8"}, {file = "ipython_genutils-0.2.0.tar.gz", hash = "sha256:eb2e116e75ecef9d4d228fdc66af54269afa26ab4463042e33785b887c628ba8"}, ] +jdcal = [ + {file = "jdcal-1.4.1-py2.py3-none-any.whl", hash = "sha256:1abf1305fce18b4e8aa248cf8fe0c56ce2032392bc64bbd61b5dff2a19ec8bba"}, + {file = "jdcal-1.4.1.tar.gz", hash = "sha256:472872e096eb8df219c23f2689fc336668bdb43d194094b5cc1707e1640acfc8"}, +] jedi = [ {file = "jedi-0.17.2-py2.py3-none-any.whl", hash = "sha256:98cc583fa0f2f8304968199b01b6b4b94f469a1f4a74c1560506ca2a211378b5"}, {file = "jedi-0.17.2.tar.gz", hash = "sha256:86ed7d9b750603e4ba582ea8edc678657fb4007894a12bcf6f4bb97892f31d20"}, @@ -1893,16 +1929,19 @@ kiwisolver = [ {file = "kiwisolver-1.2.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:443c2320520eda0a5b930b2725b26f6175ca4453c61f739fef7a5847bd262f74"}, {file = "kiwisolver-1.2.0-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:efcf3397ae1e3c3a4a0a0636542bcad5adad3b1dd3e8e629d0b6e201347176c8"}, {file = "kiwisolver-1.2.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:fccefc0d36a38c57b7bd233a9b485e2f1eb71903ca7ad7adacad6c28a56d62d2"}, + {file = "kiwisolver-1.2.0-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:be046da49fbc3aa9491cc7296db7e8d27bcf0c3d5d1a40259c10471b014e4e0c"}, {file = "kiwisolver-1.2.0-cp36-none-win32.whl", hash = "sha256:60a78858580761fe611d22127868f3dc9f98871e6fdf0a15cc4203ed9ba6179b"}, {file = "kiwisolver-1.2.0-cp36-none-win_amd64.whl", hash = "sha256:556da0a5f60f6486ec4969abbc1dd83cf9b5c2deadc8288508e55c0f5f87d29c"}, {file = "kiwisolver-1.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7cc095a4661bdd8a5742aaf7c10ea9fac142d76ff1770a0f84394038126d8fc7"}, {file = "kiwisolver-1.2.0-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:c955791d80e464da3b471ab41eb65cf5a40c15ce9b001fdc5bbc241170de58ec"}, {file = "kiwisolver-1.2.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:603162139684ee56bcd57acc74035fceed7dd8d732f38c0959c8bd157f913fec"}, + {file = "kiwisolver-1.2.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:63f55f490b958b6299e4e5bdac66ac988c3d11b7fafa522800359075d4fa56d1"}, {file = "kiwisolver-1.2.0-cp37-none-win32.whl", hash = "sha256:03662cbd3e6729f341a97dd2690b271e51a67a68322affab12a5b011344b973c"}, {file = "kiwisolver-1.2.0-cp37-none-win_amd64.whl", hash = "sha256:4eadb361baf3069f278b055e3bb53fa189cea2fd02cb2c353b7a99ebb4477ef1"}, {file = "kiwisolver-1.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c31bc3c8e903d60a1ea31a754c72559398d91b5929fcb329b1c3a3d3f6e72113"}, {file = "kiwisolver-1.2.0-cp38-cp38-manylinux1_i686.whl", hash = "sha256:d52b989dc23cdaa92582ceb4af8d5bcc94d74b2c3e64cd6785558ec6a879793e"}, {file = "kiwisolver-1.2.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:e586b28354d7b6584d8973656a7954b1c69c93f708c0c07b77884f91640b7657"}, + {file = "kiwisolver-1.2.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:38d05c9ecb24eee1246391820ed7137ac42a50209c203c908154782fced90e44"}, {file = "kiwisolver-1.2.0-cp38-none-win32.whl", hash = "sha256:d069ef4b20b1e6b19f790d00097a5d5d2c50871b66d10075dab78938dc2ee2cf"}, {file = "kiwisolver-1.2.0-cp38-none-win_amd64.whl", hash = "sha256:18d749f3e56c0480dccd1714230da0f328e6e4accf188dd4e6884bdd06bf02dd"}, {file = "kiwisolver-1.2.0.tar.gz", hash = "sha256:247800260cd38160c362d211dcaf4ed0f7816afb5efe56544748b21d6ad6d17f"}, @@ -1943,6 +1982,15 @@ markupsafe = [ {file = "MarkupSafe-1.1.1.tar.gz", hash = "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b"}, ] matplotlib = [ + {file = "matplotlib-3.3.1-1-cp36-cp36m-win32.whl", hash = "sha256:fab11637734eb14affb9c5e20d44d69429c18b49595d6e67c69295de24827fc4"}, + {file = "matplotlib-3.3.1-1-cp36-cp36m-win_amd64.whl", hash = "sha256:24392ac1a382ed753505286f1a1483bcfd67ed0c72d51be10c4c2013e386d0b7"}, + {file = "matplotlib-3.3.1-1-cp37-cp37m-win32.whl", hash = "sha256:c4ffb25b9855bdb6cdaf21bbd4ab2c229be539248304ac5215b94c816ea6e32e"}, + {file = "matplotlib-3.3.1-1-cp37-cp37m-win_amd64.whl", hash = "sha256:5a42c84264a1acbbf01c073a7bd05a0e80d99f94f10020d613b1b0526af9dcc2"}, + {file = "matplotlib-3.3.1-1-cp38-cp38-win32.whl", hash = "sha256:bc978374b43737f2bbc4a6ec48e52ae8c92be6278a80d0e2ce92f0eb0841f15c"}, + {file = "matplotlib-3.3.1-1-cp38-cp38-win_amd64.whl", hash = "sha256:6d0f03079f655ca0a2d2e0bf49c28e1ec43d9d544c33d8da1a88765f23018ecc"}, + {file = "matplotlib-3.3.1-1-cp39-cp39-win32.whl", hash = "sha256:2375f039b8c6ad6c1d03f01bf31f086bbbf997bf25e246f3b67f69969cde3d98"}, + {file = "matplotlib-3.3.1-1-cp39-cp39-win_amd64.whl", hash = "sha256:233bef5e3b3494f3b7057595ca814f23ba0ce67a03632ddf677be5132128b3db"}, + {file = "matplotlib-3.3.1-1-pp36-pypy36_pp73-win32.whl", hash = "sha256:f62c0b9a5d38c26673a8862cbae4d26cffcda260848e4278246b4e00f5a95eaf"}, {file = "matplotlib-3.3.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:282f8a077a1217f9f2ac178596f27c1ae94abbc6e7b785e1b8f25e83918e9199"}, {file = "matplotlib-3.3.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:83ae7261f4d5ab387be2caee29c4f499b1566f31c8ac97a0b8ab61afd9e3da92"}, {file = "matplotlib-3.3.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:1f9cf2b8500b833714a193cb24281153f5072d55b2e486009f1e81f0b7da3410"}, @@ -2058,6 +2106,10 @@ numpy = [ {file = "numpy-1.18.3-cp38-cp38-win_amd64.whl", hash = "sha256:c60175d011a2e551a2f74c84e21e7c982489b96b6a5e4b030ecdeacf2914da68"}, {file = "numpy-1.18.3.zip", hash = "sha256:e46e2384209c91996d5ec16744234d1c906ab79a701ce1a26155c9ec890b8dc8"}, ] +openpyxl = [ + {file = "openpyxl-3.0.5-py2.py3-none-any.whl", hash = "sha256:f7d666b569f729257082cf7ddc56262431878f602dcc2bc3980775c59439cdab"}, + {file = "openpyxl-3.0.5.tar.gz", hash = "sha256:18e11f9a650128a12580a58e3daba14e00a11d9e907c554a17ea016bf1a2c71b"}, +] packaging = [ {file = "packaging-20.4-py2.py3-none-any.whl", hash = "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181"}, {file = "packaging-20.4.tar.gz", hash = "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8"}, @@ -2128,6 +2180,8 @@ pillow = [ {file = "Pillow-7.2.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:5e51ee2b8114def244384eda1c82b10e307ad9778dac5c83fb0943775a653cd8"}, {file = "Pillow-7.2.0-cp38-cp38-win32.whl", hash = "sha256:725aa6cfc66ce2857d585f06e9519a1cc0ef6d13f186ff3447ab6dff0a09bc7f"}, {file = "Pillow-7.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:a060cf8aa332052df2158e5a119303965be92c3da6f2d93b6878f0ebca80b2f6"}, + {file = "Pillow-7.2.0-pp36-pypy36_pp73-macosx_10_10_x86_64.whl", hash = "sha256:9c87ef410a58dd54b92424ffd7e28fd2ec65d2f7fc02b76f5e9b2067e355ebf6"}, + {file = "Pillow-7.2.0-pp36-pypy36_pp73-manylinux2010_x86_64.whl", hash = "sha256:e901964262a56d9ea3c2693df68bc9860b8bdda2b04768821e4c44ae797de117"}, {file = "Pillow-7.2.0-pp36-pypy36_pp73-win32.whl", hash = "sha256:25930fadde8019f374400f7986e8404c8b781ce519da27792cbe46eabec00c4d"}, {file = "Pillow-7.2.0.tar.gz", hash = "sha256:97f9e7953a77d5a70f49b9a48da7776dc51e9b738151b22dacf101641594a626"}, ] @@ -2164,19 +2218,19 @@ pyflakes = [ {file = "pyflakes-2.2.0.tar.gz", hash = "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8"}, ] pygments = [ - {file = "Pygments-2.6.1-py3-none-any.whl", hash = "sha256:ff7a40b4860b727ab48fad6360eb351cc1b33cbf9b15a0f689ca5353e9463324"}, - {file = "Pygments-2.6.1.tar.gz", hash = "sha256:647344a061c249a3b74e230c739f434d7ea4d8b1d5f3721bc0f3558049b38f44"}, + {file = "Pygments-2.7.0-py3-none-any.whl", hash = "sha256:2df50d16b45b977217e02cba6c8422aaddb859f3d0570a88e09b00eafae89c6e"}, + {file = "Pygments-2.7.0.tar.gz", hash = "sha256:2594e8fdb06fef91552f86f4fd3a244d148ab24b66042036e64f29a291515048"}, ] pyparsing = [ {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"}, {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, ] pyrsistent = [ - {file = "pyrsistent-0.17.2.tar.gz", hash = "sha256:27515d2d5db0629c7dadf6fbe76973eb56f098c1b01d36de42eb69220d2c19e4"}, + {file = "pyrsistent-0.17.3.tar.gz", hash = "sha256:2e636185d9eb976a18a8a8e96efce62f2905fea90041958d8cc2a189756ebf3e"}, ] pytest = [ - {file = "pytest-6.0.1-py3-none-any.whl", hash = "sha256:8b6007800c53fdacd5a5c192203f4e531eb2a1540ad9c752e052ec0f7143dbad"}, - {file = "pytest-6.0.1.tar.gz", hash = "sha256:85228d75db9f45e06e57ef9bf4429267f81ac7c0d742cc9ed63d09886a9fe6f4"}, + {file = "pytest-6.0.2-py3-none-any.whl", hash = "sha256:0e37f61339c4578776e090c3b8f6b16ce4db333889d65d0efb305243ec544b40"}, + {file = "pytest-6.0.2.tar.gz", hash = "sha256:c8f57c2a30983f469bf03e68cdfa74dc474ce56b8f280ddcb080dfd91df01043"}, ] pytest-cov = [ {file = "pytest-cov-2.10.1.tar.gz", hash = "sha256:47bd0ce14056fdd79f93e1713f88fad7bdcc583dcd7783da86ef2f085a0bb88e"}, diff --git a/pyproject.toml b/pyproject.toml index 2f5e823b6..2c04716ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,9 +85,11 @@ importlib_metadata = {version = "1.6.1", python = "<3.8"} ipython = { version = "^7.10.1", optional = true } ipython-genutils = { version = "^0.2.0", optional = true } matplotlib = { version = "^3.0.3", optional = true } +openpyxl = { version = "^3.0.5", optional = true } [tool.poetry.extras] ipython = ["ipython", "ipython-genutils", "matplotlib"] +excel = ["openpyxl"] [tool.poetry.dev-dependencies] nox = "^2020.8.22" diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 000000000..7d4e013a1 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,168 @@ +import json +import shlex +import sys +import zipfile + +import pytest +import docopt +from wetterdienst import cli + + +def test_cli_help(): + + with pytest.raises(docopt.DocoptExit) as excinfo: + cli.run() + + response = str(excinfo.value) + assert "wetterdienst stations" in response + assert "wetterdienst readings" in response + assert "wetterdienst about" in response + + +def test_cli_about_parameters(capsys): + + sys.argv = ["wetterdienst", "about", "parameters"] + cli.run() + stdout, stderr = capsys.readouterr() + + response = stdout + assert "precipitation" in response + assert "air_temperature" in response + assert "weather_phenomena" in response + assert "radolan" in response + + +def test_cli_about_resolutions(capsys): + + sys.argv = ["wetterdienst", "about", "resolutions"] + cli.run() + stdout, stderr = capsys.readouterr() + + response = stdout + assert "1_minute" in response + assert "hourly" in response + assert "annual" in response + + +def test_cli_about_periods(capsys): + + sys.argv = ["wetterdienst", "about", "periods"] + cli.run() + stdout, stderr = capsys.readouterr() + + response = stdout + assert "historical" in response + assert "recent" in response + assert "now" in response + + +def invoke_wetterdienst_stations(format="json"): + argv = shlex.split( + f"wetterdienst stations --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --num=5 --format={format}" # noqa:E501,B950 + ) + sys.argv = argv + cli.run() + + +def invoke_wetterdienst_readings(format="json"): + argv = shlex.split( + f"wetterdienst readings --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --num=5 --date=2020-06-30 --format={format}" # noqa:E501,B950 + ) + sys.argv = argv + cli.run() + + +def test_cli_stations_json(capsys): + + invoke_wetterdienst_stations(format="json") + + stdout, stderr = capsys.readouterr() + response = json.loads(stdout) + + station_names = [station["station_name"] for station in response] + + assert "Schaafheim-Schlierbach" in station_names + assert "Offenbach-Wetterpark" in station_names + + # Please remove if too flakey. + assert station_names == [ + "Schaafheim-Schlierbach", + "Kahl/Main", + "Michelstadt-Vielbrunn", + "Offenbach-Wetterpark", + "Michelstadt", + ] + + +def test_cli_stations_geojson(capsys): + + invoke_wetterdienst_stations(format="geojson") + + stdout, stderr = capsys.readouterr() + response = json.loads(stdout) + + assert len(response["features"]) == 5 + + station_names = [station["properties"]["name"] for station in response["features"]] + + assert "Schaafheim-Schlierbach" in station_names + assert "Offenbach-Wetterpark" in station_names + + +def test_cli_stations_csv(capsys): + + invoke_wetterdienst_stations(format="csv") + + stdout, stderr = capsys.readouterr() + + assert "Schaafheim-Schlierbach" in stdout + assert "Offenbach-Wetterpark" in stdout + + +def test_cli_stations_excel(capsys): + + invoke_wetterdienst_stations(format="excel") + + # FIXME: Make --format=excel write to a designated file. + filename = "output.xlsx" + with zipfile.ZipFile(filename, "r") as zip: + payload = zip.read("xl/worksheets/sheet1.xml") + + assert b"Schaafheim-Schlierbach" in payload + assert b"Offenbach-Wetterpark" in payload + + +def test_cli_readings_json(capsys): + + invoke_wetterdienst_readings(format="json") + + stdout, stderr = capsys.readouterr() + response = json.loads(stdout) + + station_ids = list(set([reading["station_id"] for reading in response])) + + assert 2480 in station_ids + assert 4411 in station_ids + + +def test_cli_readings_csv(capsys): + + invoke_wetterdienst_readings(format="csv") + + stdout, stderr = capsys.readouterr() + + assert str(2480) in stdout + assert str(4411) in stdout + + +def test_cli_readings_excel(capsys): + + invoke_wetterdienst_stations(format="excel") + + # FIXME: Make --format=excel write to a designated file. + filename = "output.xlsx" + with zipfile.ZipFile(filename, "r") as zip: + payload = zip.read("xl/worksheets/sheet1.xml") + + assert b"2480" in payload + assert b"4411" in payload diff --git a/wetterdienst/additionals/geo_location.py b/wetterdienst/additionals/geo_location.py index 9e6ea3cc5..07bde30fe 100644 --- a/wetterdienst/additionals/geo_location.py +++ b/wetterdienst/additionals/geo_location.py @@ -58,9 +58,7 @@ def get_nearby_stations( DataFrames with valid Stations in radius per requested location """ - if (num_stations_nearby and max_distance_in_km) and ( - num_stations_nearby and max_distance_in_km - ): + if num_stations_nearby and max_distance_in_km: raise ValueError("Either set 'num_stations_nearby' or 'max_distance_in_km'.") if num_stations_nearby == 0: diff --git a/wetterdienst/additionals/util.py b/wetterdienst/additionals/util.py index 08822c3dc..17a35a195 100644 --- a/wetterdienst/additionals/util.py +++ b/wetterdienst/additionals/util.py @@ -20,7 +20,8 @@ def normalize_options(options: dict) -> Munch: for key, value in options.items(): # Add primary variant. - key = key.replace("--<>", "") + chars = "--<>" + key = key.strip(chars) normalized[key] = value # Add secondary variant. diff --git a/wetterdienst/cli.py b/wetterdienst/cli.py index e660f01fa..ef20b6fa9 100644 --- a/wetterdienst/cli.py +++ b/wetterdienst/cli.py @@ -2,7 +2,7 @@ import sys import json import logging -from typing import Tuple, List +from datetime import datetime, timedelta from docopt import docopt from munch import Munch @@ -29,26 +29,26 @@ def run(): """ Usage: - wetterdienst stations --parameter= --resolution= --period= [--station=] [--latitude=] [--longitude=] [--count=] [--distance=] [--persist] [--format=] # noqa:E501 - wetterdienst readings --parameter= --resolution= --period= --station= [--persist] [--date=] [--format=] # noqa:E501 - wetterdienst readings --parameter= --resolution= --period= --latitude= --longitude= [--count=] [--distance=] [--persist] [--date=] [--format=] # noqa:E501 + wetterdienst stations --parameter= --resolution= --period= [--station=] [--latitude=] [--longitude=] [--number=] [--distance=] [--persist] [--format=] + wetterdienst readings --parameter= --resolution= --period= --station= [--persist] [--date=] [--format=] + wetterdienst readings --parameter= --resolution= --period= --latitude= --longitude= [--number=] [--distance=] [--persist] [--date=] [--format=] wetterdienst about [parameters] [resolutions] [periods] wetterdienst about coverage [--parameter=] [--resolution=] [--period=] wetterdienst --version wetterdienst (-h | --help) Options: - --parameter= Parameter/variable, e.g. "kl", "air_temperature", "precipitation", etc. # noqa:E501 - --resolution= Dataset resolution: "annual", "monthly", "daily", "hourly", "minute_10", "minute_1" # noqa:E501 + --parameter= Parameter/variable, e.g. "kl", "air_temperature", "precipitation", etc. + --resolution= Dataset resolution: "annual", "monthly", "daily", "hourly", "minute_10", "minute_1" --period= Dataset period: "historical", "recent", "now" --station= Comma-separated list of station identifiers --latitude= Latitude for filtering by geoposition. --longitude= Longitude for filtering by geoposition. - --count= Number of nearby stations when filtering by geoposition. # noqa:E501 - --distance= Maximum distance in km when filtering by geoposition. # noqa:E501 - --persist Save and restore data to filesystem w/o going to the network # noqa:E501 - --date= Date for filtering data. Can be either a single date(time) or # noqa:E501 - an ISO-8601 time interval, see https://en.wikipedia.org/wiki/ISO_8601#Time_intervals. # noqa:E501 + --number= Number of nearby stations when filtering by geoposition. + --distance= Maximum distance in km when filtering by geoposition. + --persist Save and restore data to filesystem w/o going to the network + --date= Date for filtering data. Can be either a single date(time) or + an ISO-8601 time interval, see https://en.wikipedia.org/wiki/ISO_8601#Time_intervals. --format= Output format. [Default: json] --version Show version information --debug Enable debug messages @@ -61,55 +61,76 @@ def run(): wetterdienst stations --parameter=kl --resolution=daily --period=recent # Get list of all stations in CSV format - wetterdienst stations --parameter=kl --resolution=daily --period=recent --format=csv # noqa:E501 + wetterdienst stations --parameter=kl --resolution=daily --period=recent --format=csv # Get list of specific stations - wetterdienst stations --resolution=daily --parameter=kl --period=recent --station=1,1048,2667 # noqa:E501 + wetterdienst stations --resolution=daily --parameter=kl --period=recent --station=1,1048,4411 # Get list of specific stations in GeoJSON format - wetterdienst stations --resolution=daily --parameter=kl --period=recent --station=1,1048,2667 --format=geojson # noqa:E501 + wetterdienst stations --resolution=daily --parameter=kl --period=recent --station=1,1048,4411 --format=geojson Examples requesting readings: - # Get daily climate summary data for stations 44 and 1048 - wetterdienst readings --station=44,1048 --parameter=kl --resolution=daily --period=recent # noqa:E501 + # Get daily climate summary data for specific stations + wetterdienst readings --station=1048,4411 --parameter=kl --resolution=daily --period=recent - # Optionally save/restore to/from disk in order to avoid asking upstream servers each time # noqa:E501 - wetterdienst readings --station=44,1048 --parameter=kl --resolution=daily --period=recent --persist # noqa:E501 + # Optionally save/restore to/from disk in order to avoid asking upstream servers each time + wetterdienst readings --station=1048,4411 --parameter=kl --resolution=daily --period=recent --persist # Limit output to specific date - wetterdienst readings --station=44,1048 --parameter=kl --resolution=daily --period=recent --date=2020-05-01 # noqa:E501 + wetterdienst readings --station=1048,4411 --parameter=kl --resolution=daily --period=recent --date=2020-05-01 # Limit output to specified date range in ISO-8601 time interval format - wetterdienst readings --station=44,1048 --parameter=kl --resolution=daily --period=recent --date=2020-05-01/2020-05-05 # noqa:E501 + wetterdienst readings --station=1048,4411 --parameter=kl --resolution=daily --period=recent --date=2020-05-01/2020-05-05 # The real power horse: Acquire data across historical+recent data sets - wetterdienst readings --station=44,1048 --parameter=kl --resolution=daily --period=historical,recent --date=1969-01-01/2020-06-11 # noqa:E501 + wetterdienst readings --station=1048,4411 --parameter=kl --resolution=daily --period=historical,recent --date=1969-01-01/2020-06-11 # Acquire monthly data for 2020-05 - wetterdienst readings --station=44,1048 --parameter=kl --resolution=monthly --period=recent,historical --date=2020-05 # noqa:E501 + wetterdienst readings --station=1048,4411 --parameter=kl --resolution=monthly --period=recent,historical --date=2020-05 # Acquire monthly data from 2017-01 to 2019-12 - wetterdienst readings --station=44,1048 --parameter=kl --resolution=monthly --period=recent,historical --date=2017-01/2019-12 # noqa:E501 + wetterdienst readings --station=1048,4411 --parameter=kl --resolution=monthly --period=recent,historical --date=2017-01/2019-12 # Acquire annual data for 2019 - wetterdienst readings --station=44,1048 --parameter=kl --resolution=annual --period=recent,historical --date=2019 # noqa:E501 + wetterdienst readings --station=1048,4411 --parameter=kl --resolution=annual --period=recent,historical --date=2019 # Acquire annual data from 2010 to 2020 - wetterdienst readings --station=44,1048 --parameter=kl --resolution=annual --period=recent,historical --date=2010/2020 # noqa:E501 + wetterdienst readings --station=1048,4411 --parameter=kl --resolution=annual --period=recent,historical --date=2010/2020 # Acquire hourly data - wetterdienst readings --station=44,1048 --parameter=air_temperature --resolution=hourly --period=recent --date=2020-06-15T12 # noqa:E501 + wetterdienst readings --station=1048,4411 --parameter=air_temperature --resolution=hourly --period=recent --date=2020-06-15T12 Examples using geospatial features: - # Acquire stations and readings by geoposition, request specific number of nearby stations. # noqa:E501 - wetterdienst stations --resolution=daily --parameter=kl --period=recent --lat=50.2 --lon=10.3 --count=10 # noqa:E501 - wetterdienst readings --resolution=daily --parameter=kl --period=recent --lat=50.2 --lon=10.3 --count=10 --date=2020-06-30 # noqa:E501 + # Acquire stations and readings by geoposition, request specific number of nearby stations. + wetterdienst stations --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --num=5 + wetterdienst readings --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --num=5 --date=2020-06-30 - # Acquire stations and readings by geoposition, request stations within specific radius. # noqa:E501 - wetterdienst stations --resolution=daily --parameter=kl --period=recent --lat=50.2 --lon=10.3 --distance=20 # noqa:E501 - wetterdienst readings --resolution=daily --parameter=kl --period=recent --lat=50.2 --lon=10.3 --distance=20 --date=2020-06-30 # noqa:E501 + # Acquire stations and readings by geoposition, request stations within specific radius. + wetterdienst stations --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --distance=25 + wetterdienst readings --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --distance=25 --date=2020-06-30 + + Examples for inquring metadata: + + # Display list of available parameters (air_temperature, precipitation, pressure, ...) + wetterdienst about parameters + + # Display list of available resolutions (10_minutes, hourly, daily, ...) + wetterdienst about resolutions + + # Display list of available periods (historical, recent, now) + wetterdienst about periods + + # Display coverage/correlation between parameters, resolutions and periods. + # This can answer questions like ... + wetterdienst about coverage + + # Tell me all periods and resolutions available for 'air_temperature'. + wetterdienst about coverage --parameter=air_temperature + + # Tell me all parameters available for 'daily' resolution. + wetterdienst about coverage --resolution=daily """ @@ -141,8 +162,7 @@ def run(): df = df[df.STATION_ID.isin(station_ids)] elif options.latitude and options.longitude: - nearby_stations, distances = get_nearby(options) - df = df[df.STATION_ID.isin(nearby_stations)] + df = get_nearby(options) if df.empty: log.error("No data available for given constraints") @@ -154,8 +174,8 @@ def run(): station_ids = read_list(options.station) elif options.latitude and options.longitude: - nearby_stations, distances = get_nearby(options) - station_ids = nearby_stations + df = get_nearby(options) + station_ids = df.STATION_ID.unique() else: raise KeyError("Either --station or --lat, --lon required") @@ -220,13 +240,13 @@ def run(): # Make column names lowercase. df = df.rename(columns=str.lower) + for attribute in DWDMetaColumns.PARAMETER, DWDMetaColumns.ELEMENT: + attribute_name = attribute.value.lower() + if attribute_name in df: + df[attribute_name] = df[attribute_name].str.lower() # Output as JSON. if options.format == "json": - df[DWDMetaColumns.PARAMETER.value] = df[ - DWDMetaColumns.PARAMETER.value - ].str.lower() - df[DWDMetaColumns.ELEMENT.value] = df[DWDMetaColumns.ELEMENT.value].str.lower() output = df.to_json(orient="records", date_format="iso", indent=4) # Output as GeoJSON. @@ -240,10 +260,12 @@ def run(): output = df.to_csv(index=False, date_format="%Y-%m-%dT%H-%M-%S") # Output as XLSX. + # FIXME: Make --format=excel write to a designated file. elif options.format == "excel": # TODO: Obtain output file name from command line. - log.info('Writing "output.xlsx"') - df.to_excel("output.xlsx", index=False) + output_filename = "output.xlsx" + log.info(f"Writing {output_filename}") + df.to_excel(output_filename, index=False) return else: @@ -253,7 +275,7 @@ def run(): print(output) -def get_nearby(options: Munch) -> Tuple[List, List]: +def get_nearby(options: Munch) -> pd.DataFrame: """ Convenience utility function to dispatch command line options related to geospatial requests. @@ -265,32 +287,46 @@ def get_nearby(options: Munch) -> Tuple[List, List]: and "maximum distance in kilometers" query flavors. :param options: Normalized docopt command line options. - :return: nearby_stations, distances + :return: nearby_stations """ + # Obtain DWIM date range for station liveness. + # TODO: Obtain from user. + # However, some dates will not work and Pandas will croak with: + # KeyError: "None of [Int64Index([0, 0, 0, 0, 0], dtype='int64')] are in the [index]" + # See also https://github.com/earthobservations/wetterdienst/pull/145#discussion_r487698588. + + days500 = datetime.now() + timedelta(days=-500) + now = datetime.now() + timedelta(days=-2) + + minimal_date = datetime(days500.year, days500.month, days500.day) + maximal_date = datetime(now.year, now.month, now.day) + nearby_baseline_args = dict( - latitudes=[float(options.latitude)], - longitudes=[float(options.longitude)], + latitude=float(options.latitude), + longitude=float(options.longitude), + minimal_available_date=minimal_date, + maximal_available_date=maximal_date, parameter=options.parameter, time_resolution=options.resolution, period_type=options.period, ) if options.latitude and options.longitude: - if options.count: - nearby_stations, distances = get_nearby_stations( + if options.number: + nearby_stations = get_nearby_stations( **nearby_baseline_args, - num_stations_nearby=int(options.count), + num_stations_nearby=int(options.number), ) elif options.distance: - nearby_stations, distances = get_nearby_stations( + nearby_stations = get_nearby_stations( **nearby_baseline_args, max_distance_in_km=int(options.distance), ) - return nearby_stations, distances + return nearby_stations - return [], [] + return pd.DataFrame() def about(options: Munch): @@ -304,7 +340,11 @@ def about(options: Munch): def output(thing): for item in thing: if item: - print("-", item.value) + if hasattr(item, "value"): + value = item.value + else: + value = item + print("-", value) if options.parameters: output(Parameter) @@ -325,8 +365,6 @@ def output(thing): ) else: - log.error( - 'Invoke "wetterdienst about" with one of "parameters", "resolutions" or ' - '"periods"' - ) + log.error('Please invoke "wetterdienst about" with one of these subcommands:') + output(["parameters", "resolutions", "periods", "coverage"]) sys.exit(1)