From 13181a06dad62a7b9fdb20731f21dbfe83035b54 Mon Sep 17 00:00:00 2001 From: Neal Chambers Date: Fri, 6 Oct 2023 09:25:01 +0900 Subject: [PATCH 1/9] Add vcrpy to Dali-RP2 --- mypy.ini | 4 ++++ setup.cfg | 3 +++ tests/conftest.py | 27 +++++++++++++++++++++++++++ 3 files changed, 34 insertions(+) create mode 100755 tests/conftest.py diff --git a/mypy.ini b/mypy.ini index 1f5d5bb5..96205f99 100644 --- a/mypy.ini +++ b/mypy.ini @@ -41,6 +41,10 @@ warn_unused_ignores = True ignore_errors = False mypy_path = $MYPY_CONFIG_FILE_DIR/src/stubs/ +[mypy-conftest] +disallow_any_explicit = False +disallow_any_expr = False + [mypy-dali.abstract_input_filler] disallow_any_decorated = False disallow_any_explicit = False diff --git a/setup.cfg b/setup.cfg index 098b888f..a370ad2b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,6 +26,7 @@ project_urls = User Documentation = https://github.com/eprbell/dali-rp2/blob/main/README.md Contact = https://eprbell.github.io/eprbell/about.html +# vcpy > 4.4.0 is only compatible with Python 3.10+ [options] package_dir = = src @@ -39,10 +40,12 @@ install_requires = prezzemolo>=0.0.4 progressbar2>=4.2.0 pyexcel-ezodf>=0.3.4 + pytest-recording==0.13.0 python-dateutil>=2.8.2 pytz>=2021.3 requests>=2.26.0 rp2>=1.5.0 + vcrpy==4.4.0 [options.extras_require] dev = diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100755 index 00000000..ace8e504 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,27 @@ +# Copyright 2023 Neal Chambers +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, List, Union + +import pytest + + +@pytest.fixture(scope="session") +def vcr_config() -> Dict[str, Union[bool, List[str], str]]: + return { + "filter_headers": ["authorization"], + "filter_query_parameters": ["access_key"], + "ignore_localhost": True, + "record_mode": "none", + } From 9ddc06e1a03367e14008f2b0f9b233ca847f363a Mon Sep 17 00:00:00 2001 From: Neal Chambers Date: Fri, 6 Oct 2023 08:38:45 +0900 Subject: [PATCH 2/9] Add Access Key to Abstract Pair Converter Plugin --- src/dali/abstract_pair_converter_plugin.py | 109 +++++++++++++-------- src/dali/plugin/pair_converter/ccxt.py | 3 +- 2 files changed, 68 insertions(+), 44 deletions(-) diff --git a/src/dali/abstract_pair_converter_plugin.py b/src/dali/abstract_pair_converter_plugin.py index 9216f310..6dcd93dc 100644 --- a/src/dali/abstract_pair_converter_plugin.py +++ b/src/dali/abstract_pair_converter_plugin.py @@ -21,7 +21,7 @@ from requests.models import Response from requests.sessions import Session from rp2.rp2_decimal import ZERO, RP2Decimal -from rp2.rp2_error import RP2RuntimeError, RP2TypeError +from rp2.rp2_error import RP2RuntimeError, RP2TypeError, RP2ValueError from dali.cache import load_from_cache, save_to_cache from dali.configuration import HISTORICAL_PRICE_KEYWORD_SET @@ -31,13 +31,15 @@ from dali.transaction_manifest import TransactionManifest # exchangerates.host keywords +_ACCESS_KEY: str = "access_key" +_CURRENCIES: str = "currencies" +_DATE: str = "date" +_QUOTES: str = "quotes" _SUCCESS: str = "success" -_SYMBOLS: str = "symbols" -_RATES: str = "rates" # exchangerates.host urls -_EXCHANGE_BASE_URL: str = "https://api.exchangerate.host/" -_EXCHANGE_SYMBOLS_URL: str = "https://api.exchangerate.host/symbols" +_EXCHANGE_BASE_URL: str = "http://api.exchangerate.host/" +_EXCHANGE_SYMBOLS_URL: str = "http://api.exchangerate.host/list" _DAYS_IN_SECONDS: int = 86400 _FIAT_EXCHANGE: str = "exchangerate.host" @@ -58,6 +60,8 @@ _STANDARD_WEIGHT: float = 1 _STANDARD_INCREMENT: float = 1 +_CONFIG_FILE_URL: str = "https://github.com/eprbell/dali-rp2/blob/main/docs/configuration_file.md" + class AssetPairAndTimestamp(NamedTuple): timestamp: datetime @@ -70,7 +74,7 @@ class AbstractPairConverterPlugin: __ISSUES_URL: str = "https://github.com/eprbell/dali-rp2/issues" __TIMEOUT: int = 30 - def __init__(self, historical_price_type: str, fiat_priority: Optional[str] = None) -> None: + def __init__(self, historical_price_type: str, fiat_access_key: Optional[str] = None, fiat_priority: Optional[str] = None) -> None: if not isinstance(historical_price_type, str): raise RP2TypeError(f"historical_price_type is not a string: {historical_price_type}") if historical_price_type not in HISTORICAL_PRICE_KEYWORD_SET: @@ -94,6 +98,13 @@ def __init__(self, historical_price_type: str, fiat_priority: Optional[str] = No weight += _STANDARD_INCREMENT else: self.__fiat_priority = _FIAT_PRIORITY + self.__fiat_access_key: Optional[str] = None + if fiat_access_key: + self.__fiat_access_key = fiat_access_key + else: + LOGGER.warning( + "No Fiat Access Key. Fiat pricing will NOT be available. To enable fiat pricing, an access key from exchangerate.host is required." + ) def name(self) -> str: raise NotImplementedError("Abstract method: it must be implemented in the plugin class") @@ -181,28 +192,25 @@ def get_conversion_rate(self, timestamp: datetime, from_asset: str, to_asset: st return result def _build_fiat_list(self) -> None: + self._check_fiat_access_key() try: - response: Response = self.__session.get(_EXCHANGE_SYMBOLS_URL, timeout=self.__TIMEOUT) - # { - # 'motd': - # { - # 'msg': 'If you or your company ...', - # 'url': 'https://exchangerate.host/#/donate' - # }, - # 'success': True, - # 'symbols': - # { - # 'AED': - # { - # 'description': 'United Arab Emirates Dirham', - # 'code': 'AED' - # }, - # ... - # } + response: Response = self.__session.get(_EXCHANGE_SYMBOLS_URL, params={_ACCESS_KEY: self.__fiat_access_key}, timeout=self.__TIMEOUT) + # { + # "success": true, + # "terms": "https://exchangerate.host/terms", + # "privacy": "https://exchangerate.host/privacy", + # "currencies": { + # "AED": "United Arab Emirates Dirham", + # "AFN": "Afghan Afghani", + # "ALL": "Albanian Lek", + # "AMD": "Armenian Dram", + # "ANG": "Netherlands Antillean Guilder", + # [...] + # } # } data: Any = response.json() if data[_SUCCESS]: - self.__fiat_list = [fiat_iso for fiat_iso in data[_SYMBOLS] if fiat_iso != "BTC"] + self.__fiat_list = [fiat_iso for fiat_iso in data[_CURRENCIES] if fiat_iso != "BTC"] else: if "message" in data: LOGGER.error("Error %d: %s: %s", response.status_code, _EXCHANGE_SYMBOLS_URL, data["message"]) @@ -245,37 +253,44 @@ def _is_fiat(self, asset: str) -> bool: return asset in self.__fiat_list def _get_fiat_exchange_rate(self, timestamp: datetime, from_asset: str, to_asset: str) -> Optional[HistoricalBar]: + self._check_fiat_access_key() + if from_asset != "USD": + raise RP2ValueError("Fiat conversion is only available from USD at this time.") result: Optional[HistoricalBar] = None - params: Dict[str, Any] = {"base": from_asset, "symbols": to_asset} + params: Dict[str, Any] = {_ACCESS_KEY: self.__fiat_access_key, _DATE: timestamp.strftime("%Y-%m-%d"), _CURRENCIES: to_asset} request_count: int = 0 # exchangerate.host only gives us daily accuracy, which should be suitable for tax reporting while request_count < 5: try: - response: Response = self.__session.get(f"{_EXCHANGE_BASE_URL}{timestamp.strftime('%Y-%m-%d')}", params=params, timeout=self.__TIMEOUT) + response: Response = self.__session.get(f"{_EXCHANGE_BASE_URL}", params=params, timeout=self.__TIMEOUT) # { - # 'motd': - # { - # 'msg': 'If you or your company ...', - # 'url': 'https://exchangerate.host/#/donate' - # }, - # 'success': True, - # 'historical': True, - # 'base': 'EUR', - # 'date': '2020-04-04', - # 'rates': - # { - # 'USD': 1.0847, ... // float, Lists all supported currencies unless you specify - # } + # "success": true, + # "terms": "https://exchangerate.host/terms", + # "privacy": "https://exchangerate.host/privacy", + # "historical": true, + # "date": "2005-02-01", + # "timestamp": 1107302399, + # "source": "USD", + # "quotes": { + # "USDAED": 3.67266, + # "USDALL": 96.848753, + # "USDAMD": 475.798297, + # "USDANG": 1.790403, + # "USDARS": 2.918969, + # "USDAUD": 1.293878, + # [...] + # } # } data: Any = response.json() if data[_SUCCESS]: + market: str = f"USD{to_asset}" result = HistoricalBar( duration=timedelta(seconds=_DAYS_IN_SECONDS), timestamp=timestamp, - open=RP2Decimal(str(data[_RATES][to_asset])), - high=RP2Decimal(str(data[_RATES][to_asset])), - low=RP2Decimal(str(data[_RATES][to_asset])), - close=RP2Decimal(str(data[_RATES][to_asset])), + open=RP2Decimal(str(data[_QUOTES][market])), + high=RP2Decimal(str(data[_QUOTES][market])), + low=RP2Decimal(str(data[_QUOTES][market])), + close=RP2Decimal(str(data[_QUOTES][market])), volume=ZERO, ) break @@ -289,3 +304,11 @@ def _get_fiat_exchange_rate(self, timestamp: datetime, from_asset: str, to_asset raise RP2RuntimeError("JSON decode error") from exc return result + + def _check_fiat_access_key(self) -> None: + if self.__fiat_access_key is None: + raise RP2ValueError( + f"No fiat access key. To convert fiat assets, please acquire an access key from exchangerate.host." + f"The access key will then need to be added to the configuration file. For more details visit " + f"{_CONFIG_FILE_URL}" + ) diff --git a/src/dali/plugin/pair_converter/ccxt.py b/src/dali/plugin/pair_converter/ccxt.py index b0896c2a..80bf35c8 100755 --- a/src/dali/plugin/pair_converter/ccxt.py +++ b/src/dali/plugin/pair_converter/ccxt.py @@ -208,6 +208,7 @@ def __init__( self, historical_price_type: str, default_exchange: Optional[str] = None, + fiat_access_key: Optional[str] = None, fiat_priority: Optional[str] = None, google_api_key: Optional[str] = None, exchange_locked: Optional[bool] = None, @@ -218,7 +219,7 @@ def __init__( fiat_priority_cache_modifier = fiat_priority if fiat_priority else "" self.__cache_modifier = "_".join(x for x in [exchange_cache_modifier, fiat_priority_cache_modifier] if x) - super().__init__(historical_price_type=historical_price_type, fiat_priority=fiat_priority) + super().__init__(historical_price_type=historical_price_type, fiat_access_key=fiat_access_key, fiat_priority=fiat_priority) self.__logger: logging.Logger = create_logger(f"{self.name()}/{historical_price_type}") self.__exchanges: Dict[str, Exchange] = {} From f0a4aa286c4faeff54c00122e0272aab1f246486 Mon Sep 17 00:00:00 2001 From: Neal Chambers Date: Fri, 6 Oct 2023 09:16:19 +0900 Subject: [PATCH 3/9] Implement Vcrpy in CCXT tests --- mypy.ini | 1 + tests/cassettes/test_plugin_ccxt/unified.yaml | 98 +++++++++++++++++++ tests/test_plugin_ccxt.py | 49 +++++++--- 3 files changed, 136 insertions(+), 12 deletions(-) create mode 100644 tests/cassettes/test_plugin_ccxt/unified.yaml diff --git a/mypy.ini b/mypy.ini index 96205f99..94723759 100644 --- a/mypy.ini +++ b/mypy.ini @@ -175,6 +175,7 @@ disallow_any_expr = False [mypy-test_plugin_ccxt] disallow_any_explicit = False disallow_any_expr = False +disallow_any_decorated = False [mypy-test_mapped_graph] disallow_any_explicit = False diff --git a/tests/cassettes/test_plugin_ccxt/unified.yaml b/tests/cassettes/test_plugin_ccxt/unified.yaml new file mode 100644 index 00000000..124eb71e --- /dev/null +++ b/tests/cassettes/test_plugin_ccxt/unified.yaml @@ -0,0 +1,98 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.31.0 + method: GET + uri: http://api.exchangerate.host/list?access_key=BOGUS_KEY + response: + body: + string: '{"success":true,"terms":"https:\/\/currencylayer.com\/terms","privacy":"https:\/\/currencylayer.com\/privacy","currencies":{"AED":"United + Arab Emirates Dirham","AFN":"Afghan Afghani","ALL":"Albanian Lek","AMD":"Armenian + Dram","ANG":"Netherlands Antillean Guilder","AOA":"Angolan Kwanza","ARS":"Argentine + Peso","AUD":"Australian Dollar","AWG":"Aruban Florin","AZN":"Azerbaijani Manat","BAM":"Bosnia-Herzegovina + Convertible Mark","BBD":"Barbadian Dollar","BDT":"Bangladeshi Taka","BGN":"Bulgarian + Lev","BHD":"Bahraini Dinar","BIF":"Burundian Franc","BMD":"Bermudan Dollar","BND":"Brunei + Dollar","BOB":"Bolivian Boliviano","BRL":"Brazilian Real","BSD":"Bahamian + Dollar","BTC":"Bitcoin","BTN":"Bhutanese Ngultrum","BWP":"Botswanan Pula","BYN":"New + Belarusian Ruble","BYR":"Belarusian Ruble","BZD":"Belize Dollar","CAD":"Canadian + Dollar","CDF":"Congolese Franc","CHF":"Swiss Franc","CLF":"Chilean Unit of + Account (UF)","CLP":"Chilean Peso","CNY":"Chinese Yuan","COP":"Colombian Peso","CRC":"Costa + Rican Col\u00f3n","CUC":"Cuban Convertible Peso","CUP":"Cuban Peso","CVE":"Cape + Verdean Escudo","CZK":"Czech Republic Koruna","DJF":"Djiboutian Franc","DKK":"Danish + Krone","DOP":"Dominican Peso","DZD":"Algerian Dinar","EGP":"Egyptian Pound","ERN":"Eritrean + Nakfa","ETB":"Ethiopian Birr","EUR":"Euro","FJD":"Fijian Dollar","FKP":"Falkland + Islands Pound","GBP":"British Pound Sterling","GEL":"Georgian Lari","GGP":"Guernsey + Pound","GHS":"Ghanaian Cedi","GIP":"Gibraltar Pound","GMD":"Gambian Dalasi","GNF":"Guinean + Franc","GTQ":"Guatemalan Quetzal","GYD":"Guyanaese Dollar","HKD":"Hong Kong + Dollar","HNL":"Honduran Lempira","HRK":"Croatian Kuna","HTG":"Haitian Gourde","HUF":"Hungarian + Forint","IDR":"Indonesian Rupiah","ILS":"Israeli New Sheqel","IMP":"Manx pound","INR":"Indian + Rupee","IQD":"Iraqi Dinar","IRR":"Iranian Rial","ISK":"Icelandic Kr\u00f3na","JEP":"Jersey + Pound","JMD":"Jamaican Dollar","JOD":"Jordanian Dinar","JPY":"Japanese Yen","KES":"Kenyan + Shilling","KGS":"Kyrgystani Som","KHR":"Cambodian Riel","KMF":"Comorian Franc","KPW":"North + Korean Won","KRW":"South Korean Won","KWD":"Kuwaiti Dinar","KYD":"Cayman Islands + Dollar","KZT":"Kazakhstani Tenge","LAK":"Laotian Kip","LBP":"Lebanese Pound","LKR":"Sri + Lankan Rupee","LRD":"Liberian Dollar","LSL":"Lesotho Loti","LTL":"Lithuanian + Litas","LVL":"Latvian Lats","LYD":"Libyan Dinar","MAD":"Moroccan Dirham","MDL":"Moldovan + Leu","MGA":"Malagasy Ariary","MKD":"Macedonian Denar","MMK":"Myanma Kyat","MNT":"Mongolian + Tugrik","MOP":"Macanese Pataca","MRO":"Mauritanian Ouguiya","MUR":"Mauritian + Rupee","MVR":"Maldivian Rufiyaa","MWK":"Malawian Kwacha","MXN":"Mexican Peso","MYR":"Malaysian + Ringgit","MZN":"Mozambican Metical","NAD":"Namibian Dollar","NGN":"Nigerian + Naira","NIO":"Nicaraguan C\u00f3rdoba","NOK":"Norwegian Krone","NPR":"Nepalese + Rupee","NZD":"New Zealand Dollar","OMR":"Omani Rial","PAB":"Panamanian Balboa","PEN":"Peruvian + Nuevo Sol","PGK":"Papua New Guinean Kina","PHP":"Philippine Peso","PKR":"Pakistani + Rupee","PLN":"Polish Zloty","PYG":"Paraguayan Guarani","QAR":"Qatari Rial","RON":"Romanian + Leu","RSD":"Serbian Dinar","RUB":"Russian Ruble","RWF":"Rwandan Franc","SAR":"Saudi + Riyal","SBD":"Solomon Islands Dollar","SCR":"Seychellois Rupee","SDG":"Sudanese + Pound","SEK":"Swedish Krona","SGD":"Singapore Dollar","SHP":"Saint Helena + Pound","SLE":"Sierra Leonean Leone","SLL":"Sierra Leonean Leone","SOS":"Somali + Shilling","SRD":"Surinamese Dollar","SSP":"South Sudanese Pound","STD":"S\u00e3o + Tom\u00e9 and Pr\u00edncipe Dobra","SVC":"Salvadoran Col\u00f3n","SYP":"Syrian + Pound","SZL":"Swazi Lilangeni","THB":"Thai Baht","TJS":"Tajikistani Somoni","TMT":"Turkmenistani + Manat","TND":"Tunisian Dinar","TOP":"Tongan Pa\u02bbanga","TRY":"Turkish Lira","TTD":"Trinidad + and Tobago Dollar","TWD":"New Taiwan Dollar","TZS":"Tanzanian Shilling","UAH":"Ukrainian + Hryvnia","UGX":"Ugandan Shilling","USD":"United States Dollar","UYU":"Uruguayan + Peso","UZS":"Uzbekistan Som","VEF":"Venezuelan Bol\u00edvar Fuerte","VES":"Sovereign + Bolivar","VND":"Vietnamese Dong","VUV":"Vanuatu Vatu","WST":"Samoan Tala","XAF":"CFA + Franc BEAC","XAG":"Silver (troy ounce)","XAU":"Gold (troy ounce)","XCD":"East + Caribbean Dollar","XDR":"Special Drawing Rights","XOF":"CFA Franc BCEAO","XPF":"CFP + Franc","YER":"Yemeni Rial","ZAR":"South African Rand","ZMK":"Zambian Kwacha + (pre-2013)","ZMW":"Zambian Kwacha","ZWL":"Zimbabwean Dollar"}}' + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 81118ef0bd388347-KIX + Connection: + - keep-alive + Content-Type: + - application/json; Charset=UTF-8 + Date: + - Thu, 05 Oct 2023 00:30:51 GMT + NEL: + - '{"success_fraction":0,"report_to":"cf-nel","max_age":604800}' + Report-To: + - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v3?s=GUKQnWrVY9SINoyRSfBqX9esLhjZY8cw2Sak9MrxDs2Ltd4rsrRNCWgMk85KWp0DbEHkgS%2FBDSe1lm3W12wm3NdNIuWbCABblFsOGKQzVwlxfYi28hrb7WUm7uIDLOBXMIhQacrQJg%3D%3D"}],"group":"cf-nel","max_age":604800}' + Server: + - cloudflare + Transfer-Encoding: + - chunked + access-control-allow-methods: + - GET, HEAD, POST, PUT, PATCH, DELETE, OPTIONS + access-control-allow-origin: + - '*' + x-apilayer-transaction-id: + - bcaa986b-3e51-4706-8af4-ee9c69e28538 + x-request-time: + - '0.007' + status: + code: 200 + message: OK +version: 1 diff --git a/tests/test_plugin_ccxt.py b/tests/test_plugin_ccxt.py index a9831155..f5cbc4e2 100755 --- a/tests/test_plugin_ccxt.py +++ b/tests/test_plugin_ccxt.py @@ -401,20 +401,24 @@ def __btcusdt_mock_unoptimized( mocker.patch.object(plugin, "_PairConverterPlugin__exchange_2_graph_tree", {TEST_EXCHANGE: simple_tree}) + @pytest.mark.default_cassette("unified.yaml") + @pytest.mark.vcr def test_unknown_exchange(self, mocker: Any, graph_optimized: MappedGraph[str], simple_tree: AVLTree[datetime, Dict[str, MappedGraph[str]]]) -> None: - plugin: PairConverterPlugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value) + plugin: PairConverterPlugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value, fiat_access_key="BOGUS_KEY") self.__btcusdt_mock_unoptimized(plugin, mocker, graph_optimized, simple_tree) assert plugin._get_pricing_exchange_for_exchange("Bogus Exchange") == TEST_EXCHANGE # pylint: disable=protected-access + @pytest.mark.default_cassette("unified.yaml") + @pytest.mark.vcr def test_historical_prices(self, mocker: Any, graph_optimized: MappedGraph[str], simple_tree: AVLTree[datetime, Dict[str, MappedGraph[str]]]) -> None: - plugin: PairConverterPlugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value) + plugin: PairConverterPlugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value, fiat_access_key="BOGUS_KEY") cache_path = os.path.join(CACHE_DIR, plugin.cache_key()) if os.path.exists(cache_path): os.remove(cache_path) # Reinstantiate plugin now that cache is gone - plugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value) + plugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value, fiat_access_key="BOGUS_KEY") self.__btcusdt_mock_unoptimized(plugin, mocker, graph_optimized, simple_tree) data = plugin.get_historic_bar_from_native_source(BTCUSDT_TIMESTAMP, "BTC", "USD", TEST_EXCHANGE) @@ -453,8 +457,10 @@ def test_historical_prices(self, mocker: Any, graph_optimized: MappedGraph[str], assert data.close == BTCUSDT_CLOSE * USDTUSD_CLOSE assert data.volume == BTCUSDT_VOLUME + USDTUSD_VOLUME + @pytest.mark.default_cassette("unified.yaml") + @pytest.mark.vcr def test_missing_historical_prices(self, mocker: Any) -> None: - plugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value) + plugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value, fiat_access_key="BOGUS_KEY") timestamp = datetime(2020, 6, 1, 0, 0) mocker.patch.object(plugin, "get_historic_bar_from_native_source").return_value = None @@ -462,8 +468,10 @@ def test_missing_historical_prices(self, mocker: Any) -> None: data = plugin.get_historic_bar_from_native_source(timestamp, "BOGUSCOIN", "JPY", TEST_EXCHANGE) assert data is None + @pytest.mark.default_cassette("unified.yaml") + @pytest.mark.vcr(record_mode="none") def test_missing_fiat_pair(self, mocker: Any, graph_optimized: MappedGraph[str], simple_tree: AVLTree[datetime, Dict[str, MappedGraph[str]]]) -> None: - plugin: PairConverterPlugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value) + plugin: PairConverterPlugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value, fiat_access_key="BOGUS_KEY") self.__btcusdt_mock_unoptimized(plugin, mocker, graph_optimized, simple_tree) mocker.patch.object(plugin, "_get_fiat_exchange_rate").return_value = HistoricalBar( @@ -488,8 +496,10 @@ def test_missing_fiat_pair(self, mocker: Any, graph_optimized: MappedGraph[str], # Some crypto assets have no fiat or stable coin pair; they are only paired with BTC or ETH (e.g. EZ or BETH) # To get an accurate fiat price, we must get the price in the base asset (e.g. BETH -> ETH) then convert that to fiat (e.g. ETH -> USD) + @pytest.mark.default_cassette("unified.yaml") + @pytest.mark.vcr def test_no_fiat_pair(self, mocker: Any, graph_optimized: MappedGraph[str], simple_tree: AVLTree[datetime, Dict[str, MappedGraph[str]]]) -> None: - plugin: PairConverterPlugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value) + plugin: PairConverterPlugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value, fiat_access_key="BOGUS_KEY") exchange = kraken( { @@ -559,8 +569,10 @@ def no_fiat_fetch_ohlcv(symbol: str, timeframe: str, timestamp: int, candles: in assert data.volume == BETHETH_VOLUME + ETHUSDT_VOLUME + USDTUSD_VOLUME # Test to make sure the default stable coin is not used with a fiat market that does exist on the exchange + @pytest.mark.default_cassette("unified.yaml") + @pytest.mark.vcr def test_nonusd_fiat_pair(self, mocker: Any, graph_optimized: MappedGraph[str], simple_tree: AVLTree[datetime, Dict[str, MappedGraph[str]]]) -> None: - plugin: PairConverterPlugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value, default_exchange="Binance.com") + plugin: PairConverterPlugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value, default_exchange="Binance.com", fiat_access_key="BOGUS_KEY") alt_exchange = binance( { "apiKey": "key", @@ -609,8 +621,10 @@ def test_nonusd_fiat_pair(self, mocker: Any, graph_optimized: MappedGraph[str], assert data.volume == BTCGBP_VOLUME # Plugin should hand off the handling of a fiat to fiat pair to the fiat converter + @pytest.mark.default_cassette("unified.yaml") + @pytest.mark.vcr def test_fiat_pair(self, mocker: Any, graph_optimized: MappedGraph[str], simple_tree: AVLTree[datetime, Dict[str, MappedGraph[str]]]) -> None: - plugin: PairConverterPlugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value) + plugin: PairConverterPlugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value, fiat_access_key="BOGUS_KEY") exchange = binance( { "apiKey": "key", @@ -644,8 +658,10 @@ def test_fiat_pair(self, mocker: Any, graph_optimized: MappedGraph[str], simple_ assert data.close == EUR_USD_RATE assert data.volume == ZERO + @pytest.mark.default_cassette("unified.yaml") + @pytest.mark.vcr def test_kraken_csv(self, mocker: Any, graph_optimized: MappedGraph[str], simple_tree: AVLTree[datetime, Dict[str, MappedGraph[str]]]) -> None: - plugin: PairConverterPlugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value, google_api_key="whatever") + plugin: PairConverterPlugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value, google_api_key="whatever", fiat_access_key="BOGUS_KEY") cache_path = os.path.join(CACHE_DIR, "Test-" + plugin.cache_key()) if os.path.exists(cache_path): @@ -710,8 +726,12 @@ def test_kraken_csv(self, mocker: Any, graph_optimized: MappedGraph[str], simple assert data.close == BTCUSDT_CLOSE * KRAKEN_CLOSE assert data.volume == BTCUSDT_VOLUME + KRAKEN_VOLUME + @pytest.mark.default_cassette("unified.yaml") + @pytest.mark.vcr def test_locked_exchange(self, mocker: Any, graph_optimized: MappedGraph[str], simple_tree: AVLTree[datetime, Dict[str, MappedGraph[str]]]) -> None: - plugin: PairConverterPlugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value, default_exchange=LOCKED_EXCHANGE, exchange_locked=True) + plugin: PairConverterPlugin = PairConverterPlugin( + Keyword.HISTORICAL_PRICE_HIGH.value, default_exchange=LOCKED_EXCHANGE, exchange_locked=True, fiat_access_key="BOGUS_KEY" + ) # Name is changed to exchange_instance to avoid conflicts with the side effect function `add_exchange_side_effect` exchange_instance = kraken( { @@ -729,6 +749,7 @@ def test_locked_exchange(self, mocker: Any, graph_optimized: MappedGraph[str], s mocker.patch.object(kraken_csv, "find_historical_bar").return_value = None mocker.patch.object(plugin, "_PairConverterPlugin__exchange_csv_reader", {LOCKED_EXCHANGE: kraken_csv}) + mocker.patch.object(plugin, "_get_request_delay").return_value = 0.0 mocker.patch.object(exchange_instance, "fetchOHLCV").return_value = [ [ BTCUSDT_TIMESTAMP.timestamp() * _MS_IN_SECOND, # UTC timestamp in milliseconds, integer @@ -765,8 +786,10 @@ def add_exchange_side_effect(exchange: str) -> MappedGraph[str]: # pylint: disa assert data plugin._cache_graph_snapshots.assert_called_once_with("not-kraken") # type: ignore # pylint: disable=protected-access, no-member + @pytest.mark.default_cassette("unified.yaml") + @pytest.mark.vcr def test_optimization_of_graph(self, mocker: Any, graph_fiat_optimized: MappedGraph[str]) -> None: - plugin: PairConverterPlugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value, google_api_key="whatever") + plugin: PairConverterPlugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value, google_api_key="whatever", fiat_access_key="BOGUS_KEY") self.__btcusdt_mock(plugin, mocker, graph_fiat_optimized) @@ -819,6 +842,8 @@ def test_optimization_of_graph(self, mocker: Any, graph_fiat_optimized: MappedGr assert new_snapshot.close == BTCUSDT_CLOSE * USDTUSD_CLOSE assert new_snapshot.volume == BTCUSDT_VOLUME + USDTUSD_VOLUME + @pytest.mark.default_cassette("unified.yaml") + @pytest.mark.vcr def test_base_universal_aliases( self, mocker: Any, @@ -826,7 +851,7 @@ def test_base_universal_aliases( simple_tree: AVLTree[datetime, Dict[str, MappedGraph[str]]], simple_pionex_tree: AVLTree[datetime, Dict[str, MappedGraph[str]]], ) -> None: - plugin: PairConverterPlugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value, google_api_key="whatever") + plugin: PairConverterPlugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value, google_api_key="whatever", fiat_access_key="BOGUS_KEY") self.__btcusdt_mock_unoptimized(plugin, mocker, graph_optimized, simple_tree) pionex_markets: Dict[str, List[str]] = TEST_MARKETS pionex_markets.update(PIONEX_MARKETS) From 6de332a0a6b4d931216747f280c4856916963eb8 Mon Sep 17 00:00:00 2001 From: Neal Chambers Date: Thu, 5 Oct 2023 23:46:54 +0900 Subject: [PATCH 4/9] Add Documentation for Access Key for Fiat Conversion --- docs/configuration_file.md | 2 ++ docs/developer_faq.md | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/configuration_file.md b/docs/configuration_file.md index 9c0edcfa..b867a62b 100644 --- a/docs/configuration_file.md +++ b/docs/configuration_file.md @@ -474,6 +474,7 @@ Initialize this plugin section as follows: [dali.plugin.pair_converter.ccxt] historical_price_type = <historical_price_type> default_exchange = <default_exchange> +fiat_access_key = <fiat_access_key> fiat_priority = <fiat_priority> google_api_key = <google_api_key> untradeable_assets = <untradeable_assets> @@ -483,6 +484,7 @@ aliases = <untradeable_assets> Where: * `` is one of `open`, `high`, `low`, `close`, `nearest`. When DaLi downloads historical market data, it captures a `bar` of data surrounding the timestamp of the transaction. Each bar has a starting timestamp, an ending timestamp, and OHLC prices. You can choose which price to select for price lookups. The open, high, low, and close prices are self-explanatory. The `nearest` price is either the open price or the close price of the bar depending on whether the transaction time is nearer the bar starting time or the bar ending time. * `default_exchange` is an optional string for the name of an exchange to use if the exchange listed in a transaction is not currently supported by the CCXT plugin. If no default is set, Kraken(US) is used. If you would like an exchange added please open an issue. The current available exchanges are "Binance.com", "Gate", "Huobi" and "Kraken". +* `fiat_access_key` is an optional access key that can be obtained from [Exchangerate.host](https://exchangerate.host/). It is required for any fiat conversions, which are typically required if the base fiat is other than USD. * `fiat_priority` is an optional list of strings in JSON format (e.g. `["_1stpriority_", "_2ndpriority_"...]`) that ranks the priority of fiat in the routing system. If no `fiat_priority` is given, the default priority is USD, JPY, KRW, EUR, GBP, AUD, which is based on the volume of the fiat market paired with BTC (ie. BTC/USD has the highest worldwide volume, then BTC/JPY, etc.). * `google_api_key` is an optional string for the Google API Key that is needed by some CSV readers, most notably the Kraken CSV reader. It is used to download the OHLCV files for a market. No data is ever sent to Google Drive. This is only used to retrieve data. To get a Google API Key, visit the [Google Console Page](https://console.developers.google.com/) and setup a new project. Be sure to enable the Google Drive API by clicking [+ ENABLE APIS AND SERVICES] and selecting the Google Drive API. * `untradeable_assets` is a comma separated list of assets that have no market, yet. These are typically assets that are farmed or given away as a part of promotion before a market is available to price them and CCXT can not automatically assign a price. If you get the error "The asset XXX or XXX is missing from graph" and the asset is untradeable, adding the untradeable asset to this list will resolve it. diff --git a/docs/developer_faq.md b/docs/developer_faq.md index 8426207f..c5240c9e 100644 --- a/docs/developer_faq.md +++ b/docs/developer_faq.md @@ -79,7 +79,7 @@ Read the [Internal Design](../README.dev.md#internal-design) section of the Deve Read about the [transaction_resolver](../src/dali/transaction_resolver.py) in the [Internal Design](../README.dev.md#the-transaction-resolver) section of the Developer Documentation. ## Why the Strange Directory Structure with Src? -Because DaLI is a [src](https://bskinn.github.io/My-How-Why-Pyproject-Src/)-[based](https://hynek.me/articles/testing-packaging/) +Because DaLI is a [src](https://bskinn.github.io/My-How-Why-Pyproject-Src/)-[based](https://hynek.me/articles/testing-packaging/) [project](https://blog.ionelmc.ro/2014/05/25/python-packaging/). From 04a5f53c948291260432da3fa1c8aba0f08f231f Mon Sep 17 00:00:00 2001 From: Neal Chambers Date: Fri, 20 Oct 2023 16:31:18 +0900 Subject: [PATCH 5/9] Rename unified Cassette to exchange_rate_host_symbol_call --- ...ml => exchange_rate_host_symbol_call.yaml} | 0 tests/test_plugin_ccxt.py | 22 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) rename tests/cassettes/test_plugin_ccxt/{unified.yaml => exchange_rate_host_symbol_call.yaml} (100%) diff --git a/tests/cassettes/test_plugin_ccxt/unified.yaml b/tests/cassettes/test_plugin_ccxt/exchange_rate_host_symbol_call.yaml similarity index 100% rename from tests/cassettes/test_plugin_ccxt/unified.yaml rename to tests/cassettes/test_plugin_ccxt/exchange_rate_host_symbol_call.yaml diff --git a/tests/test_plugin_ccxt.py b/tests/test_plugin_ccxt.py index f5cbc4e2..73238903 100755 --- a/tests/test_plugin_ccxt.py +++ b/tests/test_plugin_ccxt.py @@ -401,7 +401,7 @@ def __btcusdt_mock_unoptimized( mocker.patch.object(plugin, "_PairConverterPlugin__exchange_2_graph_tree", {TEST_EXCHANGE: simple_tree}) - @pytest.mark.default_cassette("unified.yaml") + @pytest.mark.default_cassette("exchange_rate_host_symbol_call.yaml") @pytest.mark.vcr def test_unknown_exchange(self, mocker: Any, graph_optimized: MappedGraph[str], simple_tree: AVLTree[datetime, Dict[str, MappedGraph[str]]]) -> None: plugin: PairConverterPlugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value, fiat_access_key="BOGUS_KEY") @@ -409,7 +409,7 @@ def test_unknown_exchange(self, mocker: Any, graph_optimized: MappedGraph[str], assert plugin._get_pricing_exchange_for_exchange("Bogus Exchange") == TEST_EXCHANGE # pylint: disable=protected-access - @pytest.mark.default_cassette("unified.yaml") + @pytest.mark.default_cassette("exchange_rate_host_symbol_call.yaml") @pytest.mark.vcr def test_historical_prices(self, mocker: Any, graph_optimized: MappedGraph[str], simple_tree: AVLTree[datetime, Dict[str, MappedGraph[str]]]) -> None: plugin: PairConverterPlugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value, fiat_access_key="BOGUS_KEY") @@ -457,7 +457,7 @@ def test_historical_prices(self, mocker: Any, graph_optimized: MappedGraph[str], assert data.close == BTCUSDT_CLOSE * USDTUSD_CLOSE assert data.volume == BTCUSDT_VOLUME + USDTUSD_VOLUME - @pytest.mark.default_cassette("unified.yaml") + @pytest.mark.default_cassette("exchange_rate_host_symbol_call.yaml") @pytest.mark.vcr def test_missing_historical_prices(self, mocker: Any) -> None: plugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value, fiat_access_key="BOGUS_KEY") @@ -468,7 +468,7 @@ def test_missing_historical_prices(self, mocker: Any) -> None: data = plugin.get_historic_bar_from_native_source(timestamp, "BOGUSCOIN", "JPY", TEST_EXCHANGE) assert data is None - @pytest.mark.default_cassette("unified.yaml") + @pytest.mark.default_cassette("exchange_rate_host_symbol_call.yaml") @pytest.mark.vcr(record_mode="none") def test_missing_fiat_pair(self, mocker: Any, graph_optimized: MappedGraph[str], simple_tree: AVLTree[datetime, Dict[str, MappedGraph[str]]]) -> None: plugin: PairConverterPlugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value, fiat_access_key="BOGUS_KEY") @@ -496,7 +496,7 @@ def test_missing_fiat_pair(self, mocker: Any, graph_optimized: MappedGraph[str], # Some crypto assets have no fiat or stable coin pair; they are only paired with BTC or ETH (e.g. EZ or BETH) # To get an accurate fiat price, we must get the price in the base asset (e.g. BETH -> ETH) then convert that to fiat (e.g. ETH -> USD) - @pytest.mark.default_cassette("unified.yaml") + @pytest.mark.default_cassette("exchange_rate_host_symbol_call.yaml") @pytest.mark.vcr def test_no_fiat_pair(self, mocker: Any, graph_optimized: MappedGraph[str], simple_tree: AVLTree[datetime, Dict[str, MappedGraph[str]]]) -> None: plugin: PairConverterPlugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value, fiat_access_key="BOGUS_KEY") @@ -569,7 +569,7 @@ def no_fiat_fetch_ohlcv(symbol: str, timeframe: str, timestamp: int, candles: in assert data.volume == BETHETH_VOLUME + ETHUSDT_VOLUME + USDTUSD_VOLUME # Test to make sure the default stable coin is not used with a fiat market that does exist on the exchange - @pytest.mark.default_cassette("unified.yaml") + @pytest.mark.default_cassette("exchange_rate_host_symbol_call.yaml") @pytest.mark.vcr def test_nonusd_fiat_pair(self, mocker: Any, graph_optimized: MappedGraph[str], simple_tree: AVLTree[datetime, Dict[str, MappedGraph[str]]]) -> None: plugin: PairConverterPlugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value, default_exchange="Binance.com", fiat_access_key="BOGUS_KEY") @@ -621,7 +621,7 @@ def test_nonusd_fiat_pair(self, mocker: Any, graph_optimized: MappedGraph[str], assert data.volume == BTCGBP_VOLUME # Plugin should hand off the handling of a fiat to fiat pair to the fiat converter - @pytest.mark.default_cassette("unified.yaml") + @pytest.mark.default_cassette("exchange_rate_host_symbol_call.yaml") @pytest.mark.vcr def test_fiat_pair(self, mocker: Any, graph_optimized: MappedGraph[str], simple_tree: AVLTree[datetime, Dict[str, MappedGraph[str]]]) -> None: plugin: PairConverterPlugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value, fiat_access_key="BOGUS_KEY") @@ -658,7 +658,7 @@ def test_fiat_pair(self, mocker: Any, graph_optimized: MappedGraph[str], simple_ assert data.close == EUR_USD_RATE assert data.volume == ZERO - @pytest.mark.default_cassette("unified.yaml") + @pytest.mark.default_cassette("exchange_rate_host_symbol_call.yaml") @pytest.mark.vcr def test_kraken_csv(self, mocker: Any, graph_optimized: MappedGraph[str], simple_tree: AVLTree[datetime, Dict[str, MappedGraph[str]]]) -> None: plugin: PairConverterPlugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value, google_api_key="whatever", fiat_access_key="BOGUS_KEY") @@ -726,7 +726,7 @@ def test_kraken_csv(self, mocker: Any, graph_optimized: MappedGraph[str], simple assert data.close == BTCUSDT_CLOSE * KRAKEN_CLOSE assert data.volume == BTCUSDT_VOLUME + KRAKEN_VOLUME - @pytest.mark.default_cassette("unified.yaml") + @pytest.mark.default_cassette("exchange_rate_host_symbol_call.yaml") @pytest.mark.vcr def test_locked_exchange(self, mocker: Any, graph_optimized: MappedGraph[str], simple_tree: AVLTree[datetime, Dict[str, MappedGraph[str]]]) -> None: plugin: PairConverterPlugin = PairConverterPlugin( @@ -786,7 +786,7 @@ def add_exchange_side_effect(exchange: str) -> MappedGraph[str]: # pylint: disa assert data plugin._cache_graph_snapshots.assert_called_once_with("not-kraken") # type: ignore # pylint: disable=protected-access, no-member - @pytest.mark.default_cassette("unified.yaml") + @pytest.mark.default_cassette("exchange_rate_host_symbol_call.yaml") @pytest.mark.vcr def test_optimization_of_graph(self, mocker: Any, graph_fiat_optimized: MappedGraph[str]) -> None: plugin: PairConverterPlugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value, google_api_key="whatever", fiat_access_key="BOGUS_KEY") @@ -842,7 +842,7 @@ def test_optimization_of_graph(self, mocker: Any, graph_fiat_optimized: MappedGr assert new_snapshot.close == BTCUSDT_CLOSE * USDTUSD_CLOSE assert new_snapshot.volume == BTCUSDT_VOLUME + USDTUSD_VOLUME - @pytest.mark.default_cassette("unified.yaml") + @pytest.mark.default_cassette("exchange_rate_host_symbol_call.yaml") @pytest.mark.vcr def test_base_universal_aliases( self, From 3dca2c85c493693b9b3648e3d60f30a429a09ffe Mon Sep 17 00:00:00 2001 From: Neal Chambers Date: Fri, 20 Oct 2023 16:31:44 +0900 Subject: [PATCH 6/9] Rename Config Doc File --- src/dali/abstract_pair_converter_plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dali/abstract_pair_converter_plugin.py b/src/dali/abstract_pair_converter_plugin.py index 6dcd93dc..98b1055f 100644 --- a/src/dali/abstract_pair_converter_plugin.py +++ b/src/dali/abstract_pair_converter_plugin.py @@ -60,7 +60,7 @@ _STANDARD_WEIGHT: float = 1 _STANDARD_INCREMENT: float = 1 -_CONFIG_FILE_URL: str = "https://github.com/eprbell/dali-rp2/blob/main/docs/configuration_file.md" +_CONFIG_DOC_FILE_URL: str = "https://github.com/eprbell/dali-rp2/blob/main/docs/configuration_file.md" class AssetPairAndTimestamp(NamedTuple): @@ -310,5 +310,5 @@ def _check_fiat_access_key(self) -> None: raise RP2ValueError( f"No fiat access key. To convert fiat assets, please acquire an access key from exchangerate.host." f"The access key will then need to be added to the configuration file. For more details visit " - f"{_CONFIG_FILE_URL}" + f"{_CONFIG_DOC_FILE_URL}" ) From 0a6a6c836fb05d6e5da008a2721516e620573d69 Mon Sep 17 00:00:00 2001 From: Neal Chambers Date: Fri, 20 Oct 2023 16:34:32 +0900 Subject: [PATCH 7/9] Add documentation to conftest.py --- tests/conftest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index ace8e504..7e25fc4e 100755 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,6 +16,9 @@ import pytest +# This section configures pytest-recording, which uses vcrpy under the hood. +# Documentation: https://github.com/kiwicom/pytest-recording + @pytest.fixture(scope="session") def vcr_config() -> Dict[str, Union[bool, List[str], str]]: From 4772bffdf3f1065c12037caff883959b61f6e487 Mon Sep 17 00:00:00 2001 From: Neal Chambers Date: Fri, 20 Oct 2023 16:35:48 +0900 Subject: [PATCH 8/9] Reformat Abstract Pair Converter --- src/dali/abstract_pair_converter_plugin.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/dali/abstract_pair_converter_plugin.py b/src/dali/abstract_pair_converter_plugin.py index 98b1055f..fb2f6ee0 100644 --- a/src/dali/abstract_pair_converter_plugin.py +++ b/src/dali/abstract_pair_converter_plugin.py @@ -102,9 +102,7 @@ def __init__(self, historical_price_type: str, fiat_access_key: Optional[str] = if fiat_access_key: self.__fiat_access_key = fiat_access_key else: - LOGGER.warning( - "No Fiat Access Key. Fiat pricing will NOT be available. To enable fiat pricing, an access key from exchangerate.host is required." - ) + LOGGER.warning("No Fiat Access Key. Fiat pricing will NOT be available. To enable fiat pricing, an access key from exchangerate.host is required.") def name(self) -> str: raise NotImplementedError("Abstract method: it must be implemented in the plugin class") From aaac998a257cb441f3dec005d3afacb5af7a0afe Mon Sep 17 00:00:00 2001 From: Neal Chambers Date: Fri, 20 Oct 2023 16:36:06 +0900 Subject: [PATCH 9/9] Reformat Kraken Rest Plugin --- src/dali/plugin/input/rest/kraken.py | 29 ++++++----- tests/test_plugin_kraken.py | 78 ++++++++++++++-------------- 2 files changed, 55 insertions(+), 52 deletions(-) diff --git a/src/dali/plugin/input/rest/kraken.py b/src/dali/plugin/input/rest/kraken.py index 0f051f56..675fdd92 100644 --- a/src/dali/plugin/input/rest/kraken.py +++ b/src/dali/plugin/input/rest/kraken.py @@ -128,9 +128,11 @@ def _initialize_markets(self) -> None: for markets in self._client.markets_by_id.values(): if not isinstance(markets, list): # type: ignore - exc_str = f"Expected List from Kraken CCXT Exchange, got {type(markets)} instead. " \ - f"Incompatible CCXT library - make sure to follow Dali setup instructions " \ - f"to install appropriate versions of dependencies." + exc_str = ( + f"Expected List from Kraken CCXT Exchange, got {type(markets)} instead. " + f"Incompatible CCXT library - make sure to follow Dali setup instructions " + f"to install appropriate versions of dependencies." + ) raise RP2RuntimeError(exc_str) for market in markets: # type: ignore @@ -140,12 +142,13 @@ def _initialize_markets(self) -> None: # multiple markets for a market ID. This plugin is designed for one BASE symbol to a BASE_ID. # If this condition occurs, it indicates the Exchange has changed its response to the # API call. Please report the issue to the DALI-RP2 developers. - if base_id in self.base_id_to_base and \ - market[_BASE] != self.base_id_to_base[base_id]: - exc_str = f"Unsupported BASE for BASE_ID. Please open an issue at {self.ISSUES_URL}. " \ - f"A Kraken market's BASE differs with another BASE for the same BASE_ID. " \ - f"BASE_ID={base_id}, discovered BASE={market[_BASE]}, " \ - f"previous cached base={self.base_id_to_base[base_id]}" + if base_id in self.base_id_to_base and market[_BASE] != self.base_id_to_base[base_id]: + exc_str = ( + f"Unsupported BASE for BASE_ID. Please open an issue at {self.ISSUES_URL}. " + f"A Kraken market's BASE differs with another BASE for the same BASE_ID. " + f"BASE_ID={base_id}, discovered BASE={market[_BASE]}, " + f"previous cached base={self.base_id_to_base[base_id]}" + ) raise RP2RuntimeError(exc_str) self.base_id_to_base.update({market[_BASE_ID]: market[_BASE]}) @@ -260,9 +263,11 @@ def _compute_transaction_set(self, trade_history: Dict[str, Dict[str, str]], led # the API call. Please report the issue to the DALI-RP2 developers. if len(markets) > 1: possible_quotes = [market[_QUOTE] for market in markets] - exc_str = f"Multiple quotes for pair. Please open an issue at {self.ISSUES_URL}. " \ - f"Which quote to use for {trade_history[record[_REFID]][_PAIR]} market? " \ - f"Possible quotes={possible_quotes}" + exc_str = ( + f"Multiple quotes for pair. Please open an issue at {self.ISSUES_URL}. " + f"Which quote to use for {trade_history[record[_REFID]][_PAIR]} market? " + f"Possible quotes={possible_quotes}" + ) raise RP2RuntimeError(exc_str) asset_quote: str = markets[0][_QUOTE] is_quote_asset_fiat: bool = asset_quote in _KRAKEN_FIAT_LIST diff --git a/tests/test_plugin_kraken.py b/tests/test_plugin_kraken.py index e4a04126..2849fabb 100644 --- a/tests/test_plugin_kraken.py +++ b/tests/test_plugin_kraken.py @@ -14,24 +14,17 @@ # pylint: disable=protected-access -import pytest from typing import Any, Dict +import pytest from ccxt import Exchange +from rp2.rp2_error import RP2RuntimeError from dali.configuration import Keyword from dali.in_transaction import InTransaction from dali.intra_transaction import IntraTransaction from dali.out_transaction import OutTransaction -from dali.plugin.input.rest.kraken import ( - _BASE, - _BASE_ID, - _ID, - _QUOTE, - InputPlugin -) - -from rp2.rp2_error import RP2RuntimeError +from dali.plugin.input.rest.kraken import _BASE, _BASE_ID, _ID, _QUOTE, InputPlugin @pytest.fixture(name="private_post_ledgers_return") @@ -150,12 +143,14 @@ def test_initialize_markets_exception(plugin: InputPlugin, mocker: Any) -> None: client: Exchange = plugin._client mocker.patch.object(client, "load_markets").return_value = None - mocker.patch.object(client, "markets_by_id", - { - "XLTCZUSD": {_ID: "XLTCZUSD", _BASE_ID: "XLTC", _BASE: "LTC", _QUOTE: "USD"}, - 'XLTCXXBT': {_ID: "XLTCXXBT", _BASE_ID: "XLTC", _BASE: "LTC", _QUOTE: "BTC"}, - } - ) + mocker.patch.object( + client, + "markets_by_id", + { + "XLTCZUSD": {_ID: "XLTCZUSD", _BASE_ID: "XLTC", _BASE: "LTC", _QUOTE: "USD"}, + "XLTCXXBT": {_ID: "XLTCXXBT", _BASE_ID: "XLTC", _BASE: "LTC", _QUOTE: "BTC"}, + }, + ) with pytest.raises(RP2RuntimeError) as excinfo: plugin.load(country=None) # type: ignore @@ -170,15 +165,17 @@ def test_initialize_markets_multiple_bases(plugin: InputPlugin, mocker: Any) -> client: Exchange = plugin._client mocker.patch.object(client, "load_markets").return_value = None - mocker.patch.object(client, "markets_by_id", - { - "XLTCZUSD": [{_ID: "XLTCZUSD", _BASE_ID: "XLTC", _BASE: "LTC", _QUOTE: "USD"}], - 'XLTCXXBT': [ - {_ID: "XLTCXXBT", _BASE_ID: "XLTC", _BASE: "LTC", _QUOTE: "BTC"}, - {_ID: "XLTCXXBT", _BASE_ID: "XLTC", _BASE: "XLTC", _QUOTE: "BTC"}, - ], - } - ) + mocker.patch.object( + client, + "markets_by_id", + { + "XLTCZUSD": [{_ID: "XLTCZUSD", _BASE_ID: "XLTC", _BASE: "LTC", _QUOTE: "USD"}], + "XLTCXXBT": [ + {_ID: "XLTCXXBT", _BASE_ID: "XLTC", _BASE: "LTC", _QUOTE: "BTC"}, + {_ID: "XLTCXXBT", _BASE_ID: "XLTC", _BASE: "XLTC", _QUOTE: "BTC"}, + ], + }, + ) with pytest.raises(RP2RuntimeError) as excinfo: plugin.load(country=None) # type: ignore @@ -198,15 +195,14 @@ def test_initialize_markets_multiple_quotes_to_base_pair( client: Exchange = plugin._client mocker.patch.object(client, "load_markets").return_value = None - mocker.patch.object(client, "markets_by_id", - { - "XLTCZUSD": [ - {_ID: "XLTCZUSD", _BASE_ID: "XLTC", _BASE: "LTC", _QUOTE: "USD"}, - {_ID: "XLTCZUSD", _BASE_ID: "XLTC", _BASE: "LTC", _QUOTE: "ZUSD"} - ], - 'XLTCXXBT': [{_ID: "XLTCXXBT", _BASE_ID: "XLTC", _BASE: "LTC", _QUOTE: "BTC"}], - }, - ) + mocker.patch.object( + client, + "markets_by_id", + { + "XLTCZUSD": [{_ID: "XLTCZUSD", _BASE_ID: "XLTC", _BASE: "LTC", _QUOTE: "USD"}, {_ID: "XLTCZUSD", _BASE_ID: "XLTC", _BASE: "LTC", _QUOTE: "ZUSD"}], + "XLTCXXBT": [{_ID: "XLTCXXBT", _BASE_ID: "XLTC", _BASE: "LTC", _QUOTE: "BTC"}], + }, + ) mocker.patch.object(client, "private_post_ledgers").return_value = private_post_ledgers_return mocker.patch.object(client, "private_post_tradeshistory").return_value = private_post_tradeshistory_return @@ -228,12 +224,14 @@ def test_kraken( client: Exchange = plugin._client mocker.patch.object(client, "load_markets").return_value = None - mocker.patch.object(client, "markets_by_id", - { - "XLTCZUSD": [{_ID: "XLTCZUSD", _BASE_ID: "XLTC", _BASE: "LTC", _QUOTE: "USD"}], - 'XLTCXXBT': [{_ID: "XLTCXXBT", _BASE_ID: "XLTC", _BASE: "LTC", _QUOTE: "BTC"}], - } - ) + mocker.patch.object( + client, + "markets_by_id", + { + "XLTCZUSD": [{_ID: "XLTCZUSD", _BASE_ID: "XLTC", _BASE: "LTC", _QUOTE: "USD"}], + "XLTCXXBT": [{_ID: "XLTCXXBT", _BASE_ID: "XLTC", _BASE: "LTC", _QUOTE: "BTC"}], + }, + ) mocker.patch.object(client, "private_post_ledgers").return_value = private_post_ledgers_return mocker.patch.object(client, "private_post_tradeshistory").return_value = private_post_tradeshistory_return