eprbell · eprbell · Oct 21, 2023 · Oct 6, 2023 · Oct 5, 2023 · Oct 6, 2023
diff --git a/docs/configuration_file.md b/docs/configuration_file.md
@@ -474,6 +474,7 @@ Initialize this plugin section as follows:
 [dali.plugin.pair_converter.ccxt</em>]
 historical_price_type = <em>&lt;historical_price_type&gt;</em>
 default_exchange = <em>&lt;default_exchange&gt;</em>
+fiat_access_key = <em>&lt;fiat_access_key&gt;</em>
 fiat_priority = <em>&lt;fiat_priority&gt;</em>
 google_api_key = <em>&lt;google_api_key&gt;</em>
 untradeable_assets = <em>&lt;untradeable_assets&gt;</em>
@@ -483,6 +484,7 @@ aliases = <em>&lt;untradeable_assets&gt;</em>
 Where:
 * `<historical_price_type>` is one of `open`, `high`, `low`, `close`, `nearest`. When DaLi downloads historical market data, it captures a `bar` of data surrounding the timestamp of the transaction. Each bar has a starting timestamp, an ending timestamp, and OHLC prices. You can choose which price to select for price lookups. The open, high, low, and close prices are self-explanatory. The `nearest` price is either the open price or the close price of the bar depending on whether the transaction time is nearer the bar starting time or the bar ending time.
 * `default_exchange` is an optional string for the name of an exchange to use if the exchange listed in a transaction is not currently supported by the CCXT plugin. If no default is set, Kraken(US) is used. If you would like an exchange added please open an issue. The current available exchanges are "Binance.com", "Gate", "Huobi" and "Kraken".
+* `fiat_access_key` is an optional access key that can be obtained from [Exchangerate.host](https://exchangerate.host/). It is required for any fiat conversions, which are typically required if the base fiat is other than USD.
 * `fiat_priority` is an optional list of strings in JSON format (e.g. `["_1stpriority_", "_2ndpriority_"...]`) that ranks the priority of fiat in the routing system. If no `fiat_priority` is given, the default priority is USD, JPY, KRW, EUR, GBP, AUD, which is based on the volume of the fiat market paired with BTC (ie. BTC/USD has the highest worldwide volume, then BTC/JPY, etc.).
 * `google_api_key` is an optional string for the Google API Key that is needed by some CSV readers, most notably the Kraken CSV reader. It is used to download the OHLCV files for a market. No data is ever sent to Google Drive. This is only used to retrieve data. To get a Google API Key, visit the [Google Console Page](https://console.developers.google.com/) and setup a new project. Be sure to enable the Google Drive API by clicking [+ ENABLE APIS AND SERVICES] and selecting the Google Drive API.
 * `untradeable_assets` is a comma separated list of assets that have no market, yet. These are typically assets that are farmed or given away as a part of promotion before a market is available to price them and CCXT can not automatically assign a price. If you get the error "The asset XXX or XXX is missing from graph" and the asset is untradeable, adding the untradeable asset to this list will resolve it.

diff --git a/docs/developer_faq.md b/docs/developer_faq.md
@@ -79,7 +79,7 @@ Read the [Internal Design](../README.dev.md#internal-design) section of the Deve
 Read about the [transaction_resolver](../src/dali/transaction_resolver.py) in the [Internal Design](../README.dev.md#the-transaction-resolver) section of the Developer Documentation.
 
 ## Why the Strange Directory Structure with Src?
-Because DaLI is a [src](https://bskinn.github.io/My-How-Why-Pyproject-Src/)-[based](https://hynek.me/articles/testing-packaging/) 
+Because DaLI is a [src](https://bskinn.github.io/My-How-Why-Pyproject-Src/)-[based](https://hynek.me/articles/testing-packaging/)
 <!-- markdown-link-check-disable -->
 [project](https://blog.ionelmc.ro/2014/05/25/python-packaging/).
 <!-- markdown-link-check-enable -->

diff --git a/mypy.ini b/mypy.ini
@@ -41,6 +41,10 @@ warn_unused_ignores = True
 ignore_errors = False
 mypy_path = $MYPY_CONFIG_FILE_DIR/src/stubs/
 
+[mypy-conftest]
+disallow_any_explicit = False
+disallow_any_expr = False
+
 [mypy-dali.abstract_input_filler]
 disallow_any_decorated = False
 disallow_any_explicit = False
@@ -171,6 +175,7 @@ disallow_any_expr = False
 [mypy-test_plugin_ccxt]
 disallow_any_explicit = False
 disallow_any_expr = False
+disallow_any_decorated = False
 
 [mypy-test_mapped_graph]
 disallow_any_explicit = False

diff --git a/setup.cfg b/setup.cfg
@@ -26,6 +26,7 @@ project_urls =
     User Documentation = https://github.com/eprbell/dali-rp2/blob/main/README.md
     Contact = https://eprbell.github.io/eprbell/about.html
 
+# vcpy > 4.4.0 is only compatible with Python 3.10+
 [options]
 package_dir =
     = src
@@ -39,10 +40,12 @@ install_requires =
     prezzemolo>=0.0.4
     progressbar2>=4.2.0
     pyexcel-ezodf>=0.3.4
+    pytest-recording==0.13.0
     python-dateutil>=2.8.2
     pytz>=2021.3
     requests>=2.26.0
     rp2>=1.5.0
+    vcrpy==4.4.0
 
 [options.extras_require]
 dev =

diff --git a/src/dali/abstract_pair_converter_plugin.py b/src/dali/abstract_pair_converter_plugin.py
@@ -21,7 +21,7 @@
 from requests.models import Response
 from requests.sessions import Session
 from rp2.rp2_decimal import ZERO, RP2Decimal
-from rp2.rp2_error import RP2RuntimeError, RP2TypeError
+from rp2.rp2_error import RP2RuntimeError, RP2TypeError, RP2ValueError
 
 from dali.cache import load_from_cache, save_to_cache
 from dali.configuration import HISTORICAL_PRICE_KEYWORD_SET
@@ -31,13 +31,15 @@
 from dali.transaction_manifest import TransactionManifest
 
 # exchangerates.host keywords
+_ACCESS_KEY: str = "access_key"
+_CURRENCIES: str = "currencies"
+_DATE: str = "date"
+_QUOTES: str = "quotes"
 _SUCCESS: str = "success"
-_SYMBOLS: str = "symbols"
-_RATES: str = "rates"
 
 # exchangerates.host urls
-_EXCHANGE_BASE_URL: str = "https://api.exchangerate.host/"
-_EXCHANGE_SYMBOLS_URL: str = "https://api.exchangerate.host/symbols"
+_EXCHANGE_BASE_URL: str = "http://api.exchangerate.host/"
+_EXCHANGE_SYMBOLS_URL: str = "http://api.exchangerate.host/list"
 
 _DAYS_IN_SECONDS: int = 86400
 _FIAT_EXCHANGE: str = "exchangerate.host"
@@ -58,6 +60,8 @@
 _STANDARD_WEIGHT: float = 1
 _STANDARD_INCREMENT: float = 1
 
+_CONFIG_DOC_FILE_URL: str = "https://github.com/eprbell/dali-rp2/blob/main/docs/configuration_file.md"
+
 
 class AssetPairAndTimestamp(NamedTuple):
     timestamp: datetime
@@ -70,7 +74,7 @@ class AbstractPairConverterPlugin:
     __ISSUES_URL: str = "https://github.com/eprbell/dali-rp2/issues"
     __TIMEOUT: int = 30
 
-    def __init__(self, historical_price_type: str, fiat_priority: Optional[str] = None) -> None:
+    def __init__(self, historical_price_type: str, fiat_access_key: Optional[str] = None, fiat_priority: Optional[str] = None) -> None:
         if not isinstance(historical_price_type, str):
             raise RP2TypeError(f"historical_price_type is not a string: {historical_price_type}")
         if historical_price_type not in HISTORICAL_PRICE_KEYWORD_SET:
@@ -94,6 +98,11 @@ def __init__(self, historical_price_type: str, fiat_priority: Optional[str] = No
                 weight += _STANDARD_INCREMENT
         else:
             self.__fiat_priority = _FIAT_PRIORITY
+        self.__fiat_access_key: Optional[str] = None
+        if fiat_access_key:
+            self.__fiat_access_key = fiat_access_key
+        else:
+            LOGGER.warning("No Fiat Access Key. Fiat pricing will NOT be available. To enable fiat pricing, an access key from exchangerate.host is required.")
 
     def name(self) -> str:
         raise NotImplementedError("Abstract method: it must be implemented in the plugin class")
@@ -181,28 +190,25 @@ def get_conversion_rate(self, timestamp: datetime, from_asset: str, to_asset: st
         return result
 
     def _build_fiat_list(self) -> None:
+        self._check_fiat_access_key()
         try:
-            response: Response = self.__session.get(_EXCHANGE_SYMBOLS_URL, timeout=self.__TIMEOUT)
-            # {
-            #     'motd':
-            #         {
-            #             'msg': 'If you or your company ...',
-            #             'url': 'https://exchangerate.host/#/donate'
-            #         },
-            #     'success': True,
-            #     'symbols':
-            #         {
-            #             'AED':
-            #                 {
-            #                     'description': 'United Arab Emirates Dirham',
-            #                     'code': 'AED'
-            #                 },
-            #             ...
-            #         }
+            response: Response = self.__session.get(_EXCHANGE_SYMBOLS_URL, params={_ACCESS_KEY: self.__fiat_access_key}, timeout=self.__TIMEOUT)
+            #  {
+            #     "success": true,
+            #     "terms": "https://exchangerate.host/terms",
+            #     "privacy": "https://exchangerate.host/privacy",
+            #     "currencies": {
+            #         "AED": "United Arab Emirates Dirham",
+            #         "AFN": "Afghan Afghani",
+            #         "ALL": "Albanian Lek",
+            #         "AMD": "Armenian Dram",
+            #         "ANG": "Netherlands Antillean Guilder",
+            #         [...]
+            #     }
             # }
             data: Any = response.json()
             if data[_SUCCESS]:
-                self.__fiat_list = [fiat_iso for fiat_iso in data[_SYMBOLS] if fiat_iso != "BTC"]
+                self.__fiat_list = [fiat_iso for fiat_iso in data[_CURRENCIES] if fiat_iso != "BTC"]
             else:
                 if "message" in data:
                     LOGGER.error("Error %d: %s: %s", response.status_code, _EXCHANGE_SYMBOLS_URL, data["message"])
@@ -245,37 +251,44 @@ def _is_fiat(self, asset: str) -> bool:
         return asset in self.__fiat_list
 
     def _get_fiat_exchange_rate(self, timestamp: datetime, from_asset: str, to_asset: str) -> Optional[HistoricalBar]:
+        self._check_fiat_access_key()
+        if from_asset != "USD":
+            raise RP2ValueError("Fiat conversion is only available from USD at this time.")
         result: Optional[HistoricalBar] = None
-        params: Dict[str, Any] = {"base": from_asset, "symbols": to_asset}
+        params: Dict[str, Any] = {_ACCESS_KEY: self.__fiat_access_key, _DATE: timestamp.strftime("%Y-%m-%d"), _CURRENCIES: to_asset}
         request_count: int = 0
         # exchangerate.host only gives us daily accuracy, which should be suitable for tax reporting
         while request_count < 5:
             try:
-                response: Response = self.__session.get(f"{_EXCHANGE_BASE_URL}{timestamp.strftime('%Y-%m-%d')}", params=params, timeout=self.__TIMEOUT)
+                response: Response = self.__session.get(f"{_EXCHANGE_BASE_URL}", params=params, timeout=self.__TIMEOUT)
                 # {
-                #     'motd':
-                #         {
-                #             'msg': 'If you or your company ...',
-                #             'url': 'https://exchangerate.host/#/donate'
-                #         },
-                #     'success': True,
-                #     'historical': True,
-                #     'base': 'EUR',
-                #     'date': '2020-04-04',
-                #     'rates':
-                #         {
-                #             'USD': 1.0847, ... // float, Lists all supported currencies unless you specify
-                #         }
+                #     "success": true,
+                #     "terms": "https://exchangerate.host/terms",
+                #     "privacy": "https://exchangerate.host/privacy",
+                #     "historical": true,
+                #     "date": "2005-02-01",
+                #     "timestamp": 1107302399,
+                #     "source": "USD",
+                #     "quotes": {
+                #         "USDAED": 3.67266,
+                #         "USDALL": 96.848753,
+                #         "USDAMD": 475.798297,
+                #         "USDANG": 1.790403,
+                #         "USDARS": 2.918969,
+                #         "USDAUD": 1.293878,
+                #         [...]
+                #     }
                 # }
                 data: Any = response.json()
                 if data[_SUCCESS]:
+                    market: str = f"USD{to_asset}"
                     result = HistoricalBar(
                         duration=timedelta(seconds=_DAYS_IN_SECONDS),
                         timestamp=timestamp,
-                        open=RP2Decimal(str(data[_RATES][to_asset])),
-                        high=RP2Decimal(str(data[_RATES][to_asset])),
-                        low=RP2Decimal(str(data[_RATES][to_asset])),
-                        close=RP2Decimal(str(data[_RATES][to_asset])),
+                        open=RP2Decimal(str(data[_QUOTES][market])),
+                        high=RP2Decimal(str(data[_QUOTES][market])),
+                        low=RP2Decimal(str(data[_QUOTES][market])),
+                        close=RP2Decimal(str(data[_QUOTES][market])),
                         volume=ZERO,
                     )
                 break
@@ -289,3 +302,11 @@ def _get_fiat_exchange_rate(self, timestamp: datetime, from_asset: str, to_asset
                     raise RP2RuntimeError("JSON decode error") from exc
 
         return result
+
+    def _check_fiat_access_key(self) -> None:
+        if self.__fiat_access_key is None:
+            raise RP2ValueError(
+                f"No fiat access key. To convert fiat assets, please acquire an access key from exchangerate.host."
+                f"The access key will then need to be added to the configuration file. For more details visit "
+                f"{_CONFIG_DOC_FILE_URL}"
+            )
diff --git a/src/dali/plugin/input/rest/kraken.py b/src/dali/plugin/input/rest/kraken.py
@@ -128,9 +128,11 @@ def _initialize_markets(self) -> None:
 
         for markets in self._client.markets_by_id.values():
             if not isinstance(markets, list):  # type: ignore
-                exc_str = f"Expected List from Kraken CCXT Exchange, got {type(markets)} instead. " \
-                          f"Incompatible CCXT library - make sure to follow Dali setup instructions " \
-                          f"to install appropriate versions of dependencies."
+                exc_str = (
+                    f"Expected List from Kraken CCXT Exchange, got {type(markets)} instead. "
+                    f"Incompatible CCXT library - make sure to follow Dali setup instructions "
+                    f"to install appropriate versions of dependencies."
+                )
                 raise RP2RuntimeError(exc_str)
 
             for market in markets:  # type: ignore
@@ -140,12 +142,13 @@ def _initialize_markets(self) -> None:
                 # multiple markets for a market ID. This plugin is designed for one BASE symbol to a BASE_ID.
                 # If this condition occurs, it indicates the Exchange has changed its response to the
                 # API call. Please report the issue to the DALI-RP2 developers.
-                if base_id in self.base_id_to_base and \
-                    market[_BASE] != self.base_id_to_base[base_id]:
-                    exc_str = f"Unsupported BASE for BASE_ID. Please open an issue at {self.ISSUES_URL}. " \
-                              f"A Kraken market's BASE differs with another BASE for the same BASE_ID. " \
-                              f"BASE_ID={base_id}, discovered BASE={market[_BASE]}, " \
-                              f"previous cached base={self.base_id_to_base[base_id]}"
+                if base_id in self.base_id_to_base and market[_BASE] != self.base_id_to_base[base_id]:
+                    exc_str = (
+                        f"Unsupported BASE for BASE_ID. Please open an issue at {self.ISSUES_URL}. "
+                        f"A Kraken market's BASE differs with another BASE for the same BASE_ID. "
+                        f"BASE_ID={base_id}, discovered BASE={market[_BASE]}, "
+                        f"previous cached base={self.base_id_to_base[base_id]}"
+                    )
                     raise RP2RuntimeError(exc_str)
                 self.base_id_to_base.update({market[_BASE_ID]: market[_BASE]})
 
@@ -260,9 +263,11 @@ def _compute_transaction_set(self, trade_history: Dict[str, Dict[str, str]], led
                 # the API call. Please report the issue to the DALI-RP2 developers.
                 if len(markets) > 1:
                     possible_quotes = [market[_QUOTE] for market in markets]
-                    exc_str = f"Multiple quotes for pair. Please open an issue at {self.ISSUES_URL}. " \
-                              f"Which quote to use for {trade_history[record[_REFID]][_PAIR]} market? " \
-                              f"Possible quotes={possible_quotes}"
+                    exc_str = (
+                        f"Multiple quotes for pair. Please open an issue at {self.ISSUES_URL}. "
+                        f"Which quote to use for {trade_history[record[_REFID]][_PAIR]} market? "
+                        f"Possible quotes={possible_quotes}"
+                    )
                     raise RP2RuntimeError(exc_str)
                 asset_quote: str = markets[0][_QUOTE]
                 is_quote_asset_fiat: bool = asset_quote in _KRAKEN_FIAT_LIST

diff --git a/src/dali/plugin/pair_converter/ccxt.py b/src/dali/plugin/pair_converter/ccxt.py
@@ -208,6 +208,7 @@ def __init__(
         self,
         historical_price_type: str,
         default_exchange: Optional[str] = None,
+        fiat_access_key: Optional[str] = None,
         fiat_priority: Optional[str] = None,
         google_api_key: Optional[str] = None,
         exchange_locked: Optional[bool] = None,
@@ -218,7 +219,7 @@ def __init__(
         fiat_priority_cache_modifier = fiat_priority if fiat_priority else ""
         self.__cache_modifier = "_".join(x for x in [exchange_cache_modifier, fiat_priority_cache_modifier] if x)
 
-        super().__init__(historical_price_type=historical_price_type, fiat_priority=fiat_priority)
+        super().__init__(historical_price_type=historical_price_type, fiat_access_key=fiat_access_key, fiat_priority=fiat_priority)
         self.__logger: logging.Logger = create_logger(f"{self.name()}/{historical_price_type}")
 
         self.__exchanges: Dict[str, Exchange] = {}