fix: 941 - respect rate-limits during integration tests (#943)

New files: * `too_many_requests_exception.dart`: Exception when the server returns "Too many requests". * `too_many_requests_manager.dart`: Manager dedicated to "too many requests" server response. Deleted files: * `api_get_to_be_completed_products_test.dart`: moved "searchProduct" code to `api_search_products_test` * `api_get_user_products_test.dart`: moved "searchProduct" code to `api_search_products_test` * `api_matched_product_v2_test.dart`: moved "searchProduct" code to `api_search_products_test` Impacted files: * `api_get_localized_product_test.dart`: now using the `TooManyRequestsManager` for "getProduct" queries * `api_get_product_image_ids_test.dart`: now using the `TooManyRequestsManager` for "getProduct" queries * `api_get_product_test.dart`: moved "searchProduct" code to `api_search_products_test`; now using the `TooManyRequestsManager` for "getProduct" queries * `api_json_to_from_test.dart`: now using the `TooManyRequestsManager` for "getProduct" queries * `api_matched_product_v1_test.dart`: now using the `TooManyRequestsManager` for "getProduct" queries * `api_not_food_get_product_test.dart`: now using the `TooManyRequestsManager` for "getProduct" queries * `api_ocr_ingredients_test.dart`: now using the `TooManyRequestsManager` for "getProduct" queries * `api_search_products_test.dart`: now using the `TooManyRequestsManager` for "searchProducts" queries; moved code there from other test files, gathering all "searchProducts" queries here * `get_locations_order.dart`: unrelated minor improvement * `get_proofs_order.dart`: unrelated minor improvement * `open_food_api_client.dart`: now checking for `TooManyRequestsException`s for "getProduct" and "searchProducts" queries * `openfoodfacts.dart`: added the 2 new files
openfoodfacts · Jul 7, 2024 · 881dc5b · 881dc5b
1 parent a7e195b
commit 881dc5b
Show file tree

Hide file tree

Showing 17 changed files with 664 additions and 589 deletions.
diff --git a/lib/openfoodfacts.dart b/lib/openfoodfacts.dart
@@ -145,6 +145,8 @@ export 'src/utils/server_type.dart';
 export 'src/utils/suggestion_manager.dart';
 export 'src/utils/tag_type.dart';
 export 'src/utils/tag_type_autocompleter.dart';
+export 'src/utils/too_many_requests_exception.dart';
+export 'src/utils/too_many_requests_manager.dart';
 export 'src/utils/unit_helper.dart';
 export 'src/utils/uri_helper.dart';
 export 'src/utils/uri_reader.dart';

diff --git a/lib/src/open_food_api_client.dart b/lib/src/open_food_api_client.dart
@@ -47,6 +47,7 @@ import 'utils/product_query_configurations.dart';
 import 'utils/product_search_query_configuration.dart';
 import 'utils/tag_type.dart';
 import 'utils/taxonomy_query_configuration.dart';
+import 'utils/too_many_requests_exception.dart';
 import 'utils/uri_helper.dart';
 
 /// Client calls of the Open Food Facts API
@@ -356,6 +357,7 @@ class OpenFoodAPIClient {
     final UriProductHelper uriHelper = uriHelperFoodProd,
   }) async {
     final Response response = await configuration.getResponse(user, uriHelper);
+    TooManyRequestsException.check(response);
     return response.body;
   }
 
@@ -519,6 +521,7 @@ class OpenFoodAPIClient {
     final UriProductHelper uriHelper = uriHelperFoodProd,
   }) async {
     final Response response = await configuration.getResponse(user, uriHelper);
+    TooManyRequestsException.check(response);
     final String jsonStr = _replaceQuotes(response.body);
     final SearchResult result = SearchResult.fromJson(
       HttpHelper().jsonDecode(jsonStr),

diff --git a/lib/src/prices/get_locations_order.dart b/lib/src/prices/get_locations_order.dart
@@ -2,6 +2,7 @@ import 'order_by.dart';
 
 /// Field for the "order by" clause of "get locations".
 enum GetLocationsOrderField implements OrderByField {
+  priceCount(offTag: 'price_count'),
   created(offTag: 'created'),
   updated(offTag: 'updated');
 

diff --git a/lib/src/prices/get_proofs_order.dart b/lib/src/prices/get_proofs_order.dart
@@ -2,6 +2,7 @@ import 'order_by.dart';
 
 /// Field for the "order by" clause of "get proofs".
 enum GetProofsOrderField implements OrderByField {
+  priceCount(offTag: 'price_count'),
   created(offTag: 'created');
 
   const GetProofsOrderField({required this.offTag});

diff --git a/lib/src/utils/too_many_requests_exception.dart b/lib/src/utils/too_many_requests_exception.dart
@@ -0,0 +1,19 @@
+import 'package:http/http.dart';
+
+/// Exception when the server returns "Too many requests".
+class TooManyRequestsException implements Exception {
+  const TooManyRequestsException();
+
+  /// Start of the response body when the server received too many requests.
+  static const String _tooManyRequestsError =
+      '<!DOCTYPE html><html><head><meta name="robots" content="noindex"></head><body><h1>TOO MANY REQUESTS</h1>';
+
+  static void check(final Response response) {
+    if (response.body.startsWith(_tooManyRequestsError)) {
+      throw TooManyRequestsException();
+    }
+  }
+
+  @override
+  String toString() => 'Too many requests';
+}
diff --git a/lib/src/utils/too_many_requests_manager.dart b/lib/src/utils/too_many_requests_manager.dart
@@ -0,0 +1,46 @@
+/// Manager dedicated to "too many requests" server response.
+///
+/// Typically, the server may limit the number of requests to a [maxCount]
+/// during a specific [duration].
+class TooManyRequestsManager {
+  TooManyRequestsManager({
+    required this.maxCount,
+    required this.duration,
+  });
+
+  final int maxCount;
+  final Duration duration;
+
+  final List<int> _requestTimestamps = <int>[];
+
+  /// Waits the needed duration in order to avoid "too many requests" error.
+  Future<void> waitIfNeeded() async {
+    while (_requestTimestamps.length >= maxCount) {
+      final int previousInMillis = _requestTimestamps.first;
+      final int nowInMillis = DateTime.now().millisecondsSinceEpoch;
+      final int waitingInMillis =
+          duration.inMilliseconds - nowInMillis + previousInMillis;
+      if (waitingInMillis > 0) {
+        await Future.delayed(Duration(milliseconds: waitingInMillis));
+      }
+      _requestTimestamps.removeAt(0);
+    }
+    final DateTime now = DateTime.now();
+    final int nowInMillis = now.millisecondsSinceEpoch;
+    _requestTimestamps.add(nowInMillis);
+  }
+}
+
+/// [TooManyRequestsManager] dedicated to "searchProducts" queries in PROD.
+final TooManyRequestsManager searchProductsTooManyRequestsManager =
+    TooManyRequestsManager(
+  maxCount: 10,
+  duration: Duration(minutes: 1),
+);
+
+/// [TooManyRequestsManager] dedicated to "getProduct" queries in PROD.
+final TooManyRequestsManager getProductTooManyRequestsManager =
+    TooManyRequestsManager(
+  maxCount: 100,
+  duration: Duration(minutes: 1),
+);
diff --git a/test/api_get_localized_product_test.dart b/test/api_get_localized_product_test.dart
@@ -7,6 +7,13 @@ void main() {
   OpenFoodAPIConfiguration.userAgent = TestConstants.TEST_USER_AGENT;
   OpenFoodAPIConfiguration.globalUser = TestConstants.PROD_USER;
 
+  Future<ProductResultV3> getProductV3InProd(
+    ProductQueryConfiguration configuration,
+  ) async {
+    await getProductTooManyRequestsManager.waitIfNeeded();
+    return OpenFoodAPIClient.getProductV3(configuration);
+  }
+
   group('$OpenFoodAPIClient get localized product fields', () {
     test('get packaging text in languages (Coca-Cola)', () async {
       const String barcode = '5449000000996';
@@ -22,7 +29,7 @@ void main() {
         fields: [ProductField.PACKAGING_TEXT_IN_LANGUAGES],
         version: ProductQueryVersion.v3,
       );
-      final ProductResultV3 result = await OpenFoodAPIClient.getProductV3(
+      final ProductResultV3 result = await getProductV3InProd(
         configurations,
       );
       expect(result.status, ProductResultV3.statusSuccess);
@@ -42,8 +49,7 @@ void main() {
         OpenFoodFactsLanguage.FRENCH,
       ];
 
-      final ProductResultV3 productResult =
-          await OpenFoodAPIClient.getProductV3(
+      final ProductResultV3 productResult = await getProductV3InProd(
         ProductQueryConfiguration(
           BARCODE_DANISH_BUTTER_COOKIES,
           languages: languages,
@@ -107,7 +113,7 @@ void main() {
         ],
         version: ProductQueryVersion.v3,
       );
-      final ProductResultV3 result = await OpenFoodAPIClient.getProductV3(
+      final ProductResultV3 result = await getProductV3InProd(
         configurations,
       );
       expect(result.status, ProductResultV3.statusSuccess);
@@ -265,7 +271,7 @@ void main() {
         ],
         version: ProductQueryVersion.v3,
       );
-      final ProductResultV3 result = await OpenFoodAPIClient.getProductV3(
+      final ProductResultV3 result = await getProductV3InProd(
         configurations,
       );
       expect(result.status, ProductResultV3.statusSuccess);
@@ -325,7 +331,7 @@ void main() {
         ],
         version: ProductQueryVersion.v3,
       );
-      final ProductResultV3 result = await OpenFoodAPIClient.getProductV3(
+      final ProductResultV3 result = await getProductV3InProd(
         configurations,
       );
       expect(result.status, ProductResultV3.statusSuccess);
@@ -396,7 +402,7 @@ void main() {
         ],
         version: ProductQueryVersion.v3,
       );
-      final ProductResultV3 result = await OpenFoodAPIClient.getProductV3(
+      final ProductResultV3 result = await getProductV3InProd(
         configurations,
       );
       expect(result.status, ProductResultV3.statusSuccess);
@@ -447,7 +453,7 @@ void main() {
         ],
         version: ProductQueryVersion.v3,
       );
-      final ProductResultV3 result = await OpenFoodAPIClient.getProductV3(
+      final ProductResultV3 result = await getProductV3InProd(
         configurations,
       );
       expect(result.status, ProductResultV3.statusSuccess);

diff --git a/test/api_get_product_image_ids_test.dart b/test/api_get_product_image_ids_test.dart
@@ -8,6 +8,7 @@ void main() {
 
   test('get product images (all, main and raw)', () async {
     const String barcode = '3019081238643';
+    await getProductTooManyRequestsManager.waitIfNeeded();
     final ProductResultV3 result = await OpenFoodAPIClient.getProductV3(
       ProductQueryConfiguration(
         barcode,