Merge pull request #2655 from data-for-change/dev

merge dev into master

.github/workflows/Tests.yml

@@ -99,7 +99,7 @@ jobs:
         docker logs --details db
 
     - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v2
+      uses: codecov/codecov-action@v4
       with:
         file: ./coverage.xml
         flags: unittests

anyway/flask_app.py

@@ -14,12 +14,13 @@
 from flask_babel import Babel, gettext
 from flask_compress import Compress
 from flask_cors import CORS
-from flask_restx import Resource, fields, reqparse
+from flask_restx import Resource, fields, reqparse, Model
 from sqlalchemy import and_, not_, or_
 from sqlalchemy import func
 from webassets import Environment as AssetsEnvironment, Bundle as AssetsBundle
 from webassets.ext.jinja2 import AssetsExtension
 from werkzeug.exceptions import BadRequestKeyError
+import json
 
 from anyway import utilities, secrets
 from anyway.app_and_db import api, get_cors_config
@@ -57,7 +58,8 @@
     City,
     Streets,
     Comment,
-    TelegramForwardedMessages
+    TelegramForwardedMessages,
+    NewsFlash
 )
 from anyway.request_params import get_request_params_from_request_values
 from anyway.views.news_flash.api import (
@@ -70,6 +72,7 @@
     DEFAULT_LIMIT_REQ_PARAMETER,
     DEFAULT_OFFSET_REQ_PARAMETER,
     DEFAULT_NUMBER_OF_YEARS_AGO,
+    search_newsflashes_by_resolution
 )
 from anyway.views.schools.api import (
     schools_description_api,
@@ -1128,6 +1131,17 @@ def acc_in_area_query():
     help="limit number of retrieved items to given limit",
 )
 
+newsflash_fields = tuple(column.name for column in NewsFlash.__table__.columns)
+nfbr_parser = reqparse.RequestParser()
+nfbr_parser.add_argument("resolutions", type=str, action="append", required=True,
+                         help="List of resolutions to filter by")
+nfbr_parser.add_argument("include", type=str, required=True, choices=["True", "False"],
+                         help="Flag to include or exclude the specified resolutions (True\False)")
+nfbr_parser.add_argument("limit", type=int, help="Maximum number of records to return")
+nfbr_parser.add_argument("fields", type=str, choices=(*newsflash_fields, ""), action="append",
+                          help=f"List of fields to include in the response (Empty array to fetch all).\n"
+                               f"Available values: {', '.join(newsflash_fields)}")
+
 
 def datetime_to_str(val: datetime.datetime) -> str:
     return val.strftime("%Y-%m-%d %H:%M:%S") if isinstance(val, datetime.datetime) else "None"
@@ -1184,7 +1198,32 @@ def patch(self, news_flash_id):
         return update_news_flash_qualifying(news_flash_id)
     def options(self, news_flash_id):
         return single_news_flash(news_flash_id)
-
+
+
+def filter_json_fields(json_data, fields):
+    return {field: json_data[field] for field in fields if field in json_data}
+
+
+@api.route("/api/news-flash/by-resolution", methods=["GET"])
+class RetrieveNewsFlashByResolution(Resource):
+    @api.doc("get news flash records by resolution")
+    @api.expect(nfbr_parser)
+    @api.response(404, "Parameter value not supported or missing")
+    @api.response(
+        200, "Retrieve news-flash items filtered by given parameters", news_flash_list_model
+    )
+    def get(self):
+        args = nfbr_parser.parse_args()
+        limit = args["limit"] if "limit" in args else None
+        query = search_newsflashes_by_resolution(db.session, args["resolutions"], args["include"], limit)
+        res = query.all()
+        news_flashes_jsons = [n.serialize() for n in res]
+        if not args["fields"]:
+            filtered_jsons = news_flashes_jsons
+        else:
+            filtered_jsons = [filter_json_fields(json_data, args["fields"]) for json_data in news_flashes_jsons]
+        return Response(json.dumps(filtered_jsons, default=str), mimetype="application/json")
+
 
 @api.route("/api/news-flash-new", methods=["GET"])
 class RetrieveNewsFlash(Resource):

anyway/models.py

@@ -903,7 +903,7 @@ def set_critical(
         resolution = BE_CONST.ResolutionCategories(self.resolution)
         end_time = last_accident_date.to_pydatetime().date()
         start_time = datetime.date(end_time.year + 1 - years_before, 1, 1)
-        location_info = LocationInfo()
+        location_info: LocationInfo = {}
         if resolution == BE_CONST.ResolutionCategories.SUBURBAN_ROAD:
             location_info["road1"] = self.road1
             location_info["road_segment_id"] = self.road_segment_id

anyway/views/news_flash/api.py

@@ -517,3 +517,19 @@ def get_downloaded_data(format, years_ago):
 
     headers = { 'Content-Disposition': f'attachment; filename=anyway_download_{datetime.datetime.now().strftime("%d_%m_%Y_%H_%M_%S")}.{file_type}' }
     return Response(buffer.getvalue(), mimetype=mimetype, headers=headers)
+
+
+def search_newsflashes_by_resolution(session, resolutions, include_resolutions, limit=None):
+    query = session.query(NewsFlash)
+    if include_resolutions:
+        query = query.filter(NewsFlash.resolution.in_(resolutions))
+    else:
+        query = query.filter(NewsFlash.resolution.notin_(resolutions))
+
+    query = query.filter(NewsFlash.accident == True) \
+        .order_by(NewsFlash.date.desc())
+
+    limit = DEFAULT_LIMIT_REQ_PARAMETER if not limit else limit
+    query = query.limit(limit)
+
+    return query

anyway/widgets/all_locations_widgets/killed_and_injured_count_per_age_group_widget_utils.py

@@ -4,6 +4,7 @@
 
 from flask_sqlalchemy import BaseQuery
 from sqlalchemy import func, asc
+# noinspection PyProtectedMember
 from flask_babel import _
 
 from anyway.app_and_db import db
@@ -34,6 +35,8 @@ def filter_and_group_injured_count_per_age_group(
     ) -> Dict[str, Dict[int, int]]:
         start_time = request_params.start_time
         end_time = request_params.end_time
+        cache_key = None  # prevent pylint warning
+
         if request_params.resolution == BE_CONST.ResolutionCategories.STREET:
             involve_yishuv_name = request_params.location_info["yishuv_name"]
             street1_hebrew = request_params.location_info["street1_hebrew"]

anyway/widgets/road_segment_widgets/head_on_collisions_comparison_widget.py

@@ -107,7 +107,7 @@ def is_included(self) -> bool:
                 segment_others = item["count"]
             else:
                 raise ValueError
-        segment_total = segment_h2h + segment_others
+        segment_total = segment_h2h + segment_others  # pylint: disable=E0606
         all_items = self.items[self.ALL_ROADS_SUBTITLE]
         for item in all_items:
             if item["desc"] == "frontal":
@@ -116,7 +116,7 @@ def is_included(self) -> bool:
                 all_others = item["count"]
             else:
                 raise ValueError
-        all_total = all_h2h + all_others
+        all_total = all_h2h + all_others  # pylint: disable=E0606
         return segment_h2h > 0 and (segment_h2h / segment_total) > all_h2h / all_total
 
 

anyway/widgets/segment_junctions.py

@@ -36,7 +36,14 @@ def __calc_fill_segment_junctions():
             if t.road not in rkj:
                 rkj[t.road] = {}
                 road_last_junction_km[t.road] = -1
-            rkj[t.road][t.km] = t.non_urban_intersection
+            if t.km not in rkj[t.road]:
+                rkj[t.road][t.km] = []
+            else:
+                logging.debug(
+                    f"Two junctions in same location:road:{t.road},km:{t.km},1:"
+                    f"{rkj[t.road][t.km]},2:{t.non_urban_intersection}."
+                )
+            rkj[t.road][t.km].append(t.non_urban_intersection)
             if road_last_junction_km[t.road] < t.km:
                 road_last_junction_km[t.road] = t.km
         tmp: List[RoadSegments] = db.session.query(RoadSegments).all()
@@ -45,11 +52,10 @@ def __calc_fill_segment_junctions():
             if seg.road not in rkj:
                 logging.warning(f"No junctions in road {seg.road}.")
                 continue
-            junctions = [
-                rkj[seg.road][km]
-                for km in rkj[seg.road].keys()
-                if is_junction_km_in_segment(km, seg, road_last_junction_km.get(seg.road))
-            ]
+            junctions = []
+            for km in rkj[seg.road].keys():
+                if is_junction_km_in_segment(km, seg, road_last_junction_km.get(seg.road)):
+                    junctions.extend(rkj[seg.road][km])
             res[seg_id] = junctions
         return res
 

tests/test_infographics_utils.py

@@ -54,9 +54,11 @@ class TestInfographicsUtilsCase(unittest.TestCase):
     rjks = [
         RoadJunctionKM(road=1, non_urban_intersection=1, km=1.0),
         RoadJunctionKM(road=1, non_urban_intersection=2, km=2.0),
+        RoadJunctionKM(road=1, non_urban_intersection=22, km=2.0),
         RoadJunctionKM(road=1, non_urban_intersection=3, km=3.0),
         RoadJunctionKM(road=10, non_urban_intersection=1, km=10.0),
         RoadJunctionKM(road=20, non_urban_intersection=2, km=10.0),
+        RoadJunctionKM(road=20, non_urban_intersection=22, km=10.0),
         RoadJunctionKM(road=30, non_urban_intersection=3, km=10.0),
     ]
     segments = [
@@ -90,7 +92,11 @@ def test_get_segment_junctions(self, db):
         actual = sg.get_segment_junctions(2)
         self.assertEqual([1], actual, "2")
         actual = sg.get_segment_junctions(3)
-        self.assertEqual([1, 2, 3], actual, "3")
+        self.assertEqual([1, 2, 22, 3], actual, "3")
+        actual = sg.get_segment_junctions(4)
+        self.assertEqual([3], actual, "4")
+        actual = sg.get_segment_junctions(21)
+        self.assertEqual([2, 22], actual, "5")
 
     def test_get_filter_expression(self):
         actual = get_filter_expression(AccidentMarkerView, "road_segment_name", "seg1")

tests/test_news_flash.py

@@ -144,6 +144,7 @@ def test_twitter_parse():
         for k in raw_fields:
             assert getattr(actual, k) == getattr(expected, k)
 
+        actual.set_critical()
         actual.accident = classify_tweets(actual.description)
         assert actual.accident == expected.accident