Merge branch 'dev' of https://github.com/tkalir/anyway into 2669-Impr…

…ove-location-accuracy-for-street-newsflashes

.github/workflows/Tests.yml

@@ -58,13 +58,15 @@ jobs:
         cd ./anyway-newsflash-infographics/
         bash run.bash
         cd ../
-        docker-compose -f docker-compose.yml up -d --build anyway
+        docker compose -f docker-compose.yml up -d --build anyway
     - name: Waiting for DB startup
       run: |
         bash ./wait_for_postgres.sh
     - name: DB Initialization
       run: |
         docker exec anyway alembic upgrade head
+        docker exec anyway ./main.py process cities
+        docker exec anyway ./main.py process streets
         docker exec anyway ./main.py process registered-vehicles
         docker exec anyway ./main.py process cbs --source local_dir_for_tests_only
         docker exec anyway ./main.py process road-segments

alembic/versions/97740b33407d_add_numeric_indexes.py

@@ -0,0 +1,36 @@
+"""add numeric indexes
+
+Revision ID: 97740b33407d
+Revises: 53d0b00fb750
+Create Date: 2024-08-24 11:19:33.674396
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '97740b33407d'
+down_revision = '53d0b00fb750'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+# import sqlalchemy as sa
+
+
+# pylint: disable=E1101
+def upgrade():
+    for table in ['markers', 'markers_hebrew']:
+        for field in ['yishuv_symbol', 'street1', 'street2']:
+            op.create_index(f'ix_{table}_{field}', table, [field], unique=False)
+    for table in ['markers_hebrew']:
+        for field in ['yishuv_name', 'street1_hebrew', 'street2_hebrew']:
+            op.drop_index(f'ix_{table}_{field}', table_name=table)
+
+
+# pylint: disable=E1101
+def downgrade():
+    for table in ['markers', 'markers_hebrew']:
+        for field in ['yishuv_symbol', 'street1', 'street2']:
+            op.drop_index(f'ix_{table}_{field}', table_name=table)
+    for table in ['markers_hebrew']:
+        for field in ['yishuv_name', 'street1_hebrew', 'street2_hebrew']:
+            op.create_index(f'ix_{table}_{field}', table, [field], unique=False)

alembic/versions/mn9px8cacn24_add_house_number.py

@@ -0,0 +1,31 @@
+"""add house number
+
+Revision ID: mn9px8cacn24
+Revises: 97740b33407d
+Create Date: 2024-06-16 15:05:30.522542
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'mn9px8cacn24'
+down_revision = '97740b33407d'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('involved_markers_hebrew', sa.Column('house_number', sa.Integer(), nullable=True))
+    op.add_column('vehicles_markers_hebrew', sa.Column('house_number', sa.Integer(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('vehicles_markers_hebrew', 'house_number')
+    op.drop_column('involved_markers_hebrew', 'house_number')
+    # ### end Alembic commands ###

anyway/db_views.py

@@ -638,6 +638,7 @@ def create_involved_hebrew_markers_hebrew_view(self):
             AccidentMarkerView.street1_hebrew,
             AccidentMarkerView.street2,
             AccidentMarkerView.street2_hebrew,
+            AccidentMarkerView.house_number,
             AccidentMarkerView.non_urban_intersection,
             AccidentMarkerView.non_urban_intersection_hebrew,
             AccidentMarkerView.non_urban_intersection_by_junction_number,
@@ -836,6 +837,7 @@ def create_vehicles_markers_hebrew_view(self):
             AccidentMarkerView.street1_hebrew,
             AccidentMarkerView.street2,
             AccidentMarkerView.street2_hebrew,
+            AccidentMarkerView.house_number,
             AccidentMarkerView.non_urban_intersection,
             AccidentMarkerView.non_urban_intersection_hebrew,
             AccidentMarkerView.non_urban_intersection_by_junction_number,

anyway/flask_app.py

@@ -63,10 +63,8 @@
 )
 from anyway.request_params import get_request_params_from_request_values
 from anyway.views.news_flash.api import (
-    news_flash,
-    news_flash_new,
     single_news_flash,
-    news_flash_v2,
+    news_flash,
     update_news_flash_qualifying,
     get_downloaded_data,
     DEFAULT_LIMIT_REQ_PARAMETER,
@@ -1094,12 +1092,10 @@ def acc_in_area_query():
     view_func=injured_around_schools_api,
     methods=["GET"],
 )
-app.add_url_rule("/api/news-flash", endpoint=None, view_func=news_flash_v2, methods=["GET", "PATCH", "OPTIONS"])
+app.add_url_rule("/api/news-flash", endpoint=None, view_func=news_flash, methods=["GET", "PATCH", "OPTIONS"])
 app.add_url_rule("/api/comments", endpoint=None, view_func=get_comments, methods=["GET"])
 app.add_url_rule("/api/comments", endpoint=None, view_func=create_comment, methods=["POST"])
 
-app.add_url_rule("/api/v1/news-flash", endpoint=None, view_func=news_flash, methods=["GET"])
-
 nf_parser = reqparse.RequestParser()
 nf_parser.add_argument("id", type=int, help="News flash id")
 nf_parser.add_argument("source", type=str, help="news flash source")
@@ -1176,11 +1172,10 @@ def datetime_to_str(val: datetime.datetime) -> str:
         "road_segment_id": fields.Integer(),
         "newsflash_location_qualification": fields.Integer(),
         "location_qualifying_user": fields.Integer(),
+        "curr_cbs_location_text": fields.String(),
+        "critical": fields.Boolean(),
     },
 )
-news_flash_list_model = api.model(
-    "news_flash_list", {"news_flashes": fields.List(fields.Nested(news_flash_fields_model))}
-)
 
 
 @api.route("/api/news-flash/<int:news_flash_id>", methods=["GET", "PATCH", "OPTIONS"])
@@ -1200,47 +1195,6 @@ def options(self, news_flash_id):
         return single_news_flash(news_flash_id)
 
 
-def filter_json_fields(json_data, fields):
-    return {field: json_data[field] for field in fields if field in json_data}
-
-
-@api.route("/api/news-flash/by-resolution", methods=["GET"])
-class RetrieveNewsFlashByResolution(Resource):
-    @api.doc("get news flash records by resolution")
-    @api.expect(nfbr_parser)
-    @api.response(404, "Parameter value not supported or missing")
-    @api.response(
-        200, "Retrieve news-flash items filtered by given parameters", news_flash_list_model
-    )
-    def get(self):
-        args = nfbr_parser.parse_args()
-        limit = args["limit"] if "limit" in args else None
-        query = search_newsflashes_by_resolution(db.session, args["resolutions"], args["include"], limit)
-        res = query.all()
-        news_flashes_jsons = [n.serialize() for n in res]
-        if not args["fields"]:
-            filtered_jsons = news_flashes_jsons
-        else:
-            filtered_jsons = [filter_json_fields(json_data, args["fields"]) for json_data in news_flashes_jsons]
-        return Response(json.dumps(filtered_jsons, default=str), mimetype="application/json")
-
-
-@api.route("/api/news-flash-new", methods=["GET"])
-class RetrieveNewsFlash(Resource):
-    @api.doc("get news flash records")
-    @api.expect(nf_parser)
-    @api.response(404, "Parameter value not supported or missing")
-    @api.response(
-        200, "Retrieve news-flash items filtered by given parameters", news_flash_list_model
-    )
-    def get(self):
-        args = nf_parser.parse_args()
-        res = news_flash_new(args)
-        for d in res:
-            d["date"] = datetime_to_str(d["date"]) if "date" in d else "None"
-        return {"news_flashes": res}
-
-
 """
     Returns infographics-data API
 """

anyway/localization.py

@@ -3,7 +3,6 @@
 import pandas as pd
 
 from anyway import field_names
-from typing import Optional
 
 import logging
 
@@ -214,11 +213,3 @@ def get_field(field, value=None):
 
 def get_supported_tables():
     return _tables.keys()
-
-
-def get_city_name(symbol_id, lang: str = "he") -> Optional[str]:
-    column_to_fetch = field_names.name if lang == "he" else "ENGLISH_NAME"
-    try:
-        return _cities.loc[symbol_id, column_to_fetch]
-    except Exception:
-        return None