Unit tests/mission classes

daxa/mission/base.py

@@ -758,7 +758,16 @@ def filter_on_positions(self, positions: Union[list, np.ndarray, SkyCoord],
 
         # Checks to see if a list/array of coordinates has been passed, in which case we convert it to a
         #  SkyCoord (or a SkyCoord catalogue).
-        if isinstance(positions, (list, np.ndarray)):
+        # Firstly checking if it is a nested list or a list
+        if isinstance(positions, list):
+            if all(isinstance(i, list) for i in positions):
+                # Then it is a nested list
+                positions = SkyCoord(positions, unit=u.deg, frame=self.coord_frame)
+            else:
+                # Then it is one position in a list
+                positions = SkyCoord(positions[0], positions[1], unit=u.deg, frame=self.coord_frame)
+
+        if isinstance(positions, np.ndarray):
             positions = SkyCoord(positions, unit=u.deg, frame=self.coord_frame)
         # If the input was already a SkyCoord, we should make sure that it is in the same frame as the current
         #  mission's observation position information (honestly probably doesn't make that much of a difference, but
@@ -1218,7 +1227,7 @@ def filter_on_positions_at_time(self, positions: Union[list, np.ndarray, SkyCoor
         """
         # Check that the start and end information is in the same style
         if isinstance(start_datetimes, datetime) != isinstance(end_datetimes, datetime):
-            raise TypeError("The 'start_datetimes' and 'start_datetimes' must either both be individual datetimes, or "
+            raise TypeError("The 'start_datetimes' and 'end_datetimes' must either both be individual datetimes, or "
                             "arrays of datetimes (for multiple positions).")
         # Need to make sure we make the datetimes iterable - even if there is only one position/time period being
         #  investigated
@@ -1231,13 +1240,28 @@ def filter_on_positions_at_time(self, positions: Union[list, np.ndarray, SkyCoor
         if isinstance(positions, list) and not isinstance(positions[0], (list, SkyCoord)):
             positions = [positions]
 
+        # Checking if positions is scalar or not. This is checked for np.ndarrays, lists and skycoord differently
+        if isinstance(positions, SkyCoord):
+            pos_scalar = positions.isscalar
+
+        elif isinstance(positions, list):
+            if len(positions) == 1:
+                pos_scalar = True
+
+            else:
+                pos_scalar = False
+
+        else:
+            # In this indent positions should be an np.ndarray, which should be not scalar
+            pos_scalar = False
+
         # We initially check that the arguments we will be basing the time filtering on are of the right length,
         #  i.e. every position must have corresponding start and end times
-        if not positions.isscalar and (len(start_datetimes) != len(positions) or len(end_datetimes) != len(positions)):
+        if not pos_scalar and (len(start_datetimes) != len(positions) or len(end_datetimes) != len(positions)):
             raise ValueError("The 'start_datetimes' (len={sd}) and 'end_datetimes' (len={ed}) arguments must have one "
                              "entry per position specified by the 'positions' (len={p}) "
                              "arguments.".format(sd=len(start_datetimes), ed=len(end_datetimes), p=len(positions)))
-        elif positions.isscalar and (len(start_datetimes) != 1 or len(end_datetimes) != 1):
+        elif pos_scalar and (len(start_datetimes) != 1 or len(end_datetimes) != 1):
             raise ValueError("The 'start_datetimes' (len={sd}) and 'end_datetimes' (len={ed}) arguments must be "
                              "scalar if a single position is passed".format(sd=len(start_datetimes),
                                                                             ed=len(end_datetimes)))

daxa/mission/erosita.py

@@ -93,12 +93,6 @@ def __init__(self, insts: Union[List[str], str] = None, fields: Union[List[str],
         # Call the name property to set up the name and pretty name attributes
         self.name
 
-        # Runs the method which fetches information on all available RASS observations and stores that
-        #  information in the all_obs_info property
-        self._fetch_obs_info()
-        # Slightly cheesy way of setting the _filter_allowed attribute to be an array identical to the usable
-        #  column of all_obs_info, rather than the initial None value
-        self.reset_filter()
 
     # Defining properties first
     @property
@@ -326,7 +320,9 @@ def all_mission_fields(self) -> List[str]:
         :return: A list of field names.
         :rtype: List[str]
         """
-        return self._miss_poss_fields
+        # _miss_poss_fields returns all the field_name column of EROSITA_CALPV_INFO
+        # so set() is used to remove duplicate field names where obs_ids have the same field name
+        return list(set(self._miss_poss_fields))
 
     @property
     def all_mission_field_types(self) -> List[str]:
@@ -385,9 +381,10 @@ def filter_on_fields(self, fields: Union[str, List[str]]):
         # Said boolean array can be multiplied with the existing filter array (by default all ones, which means
         #  all observations are let through) to produce an updated filter.
         new_filter = self.filter_array*sel_obs_mask
+
         # Then we set the filter array property with that updated mask
-        self.filter_array = new_filter
-    
+        self.filter_array = new_filter    
+
     # Then define user-facing methods
     def _fetch_obs_info(self):
         """
@@ -446,11 +443,11 @@ def _check_chos_fields(self, fields: Union[List[str], str]):
         bad_fields = [f for f in fields if f not in poss_alt_field_names and f not in self._miss_poss_fields
                       and f not in self._miss_poss_field_types and f != 'CRAB']
         if len(bad_fields) != 0:
-            raise ValueError("Some field names or field types {bf} are not associated with this mission, please "
+            raise ValueError("Some field names or field types: {bf} are not associated with this mission, please "
                              "choose from the following fields; {gf} or field types; "
                              "{gft}".format(bf=",".join(bad_fields),
-                                            gf=",".join(self._miss_poss_fields),
-                                            gft=",".join(self._miss_poss_field_types)))
+                                            gf=",".join(list(set(self._miss_poss_fields))),
+                                            gft=",".join(self.all_mission_field_types)))
 
         # Extracting the alt_fields from fields
         alt_fields = [field for field in fields if field in poss_alt_field_names]
@@ -1199,13 +1196,12 @@ def _download_call(obs_id: str, raw_dir: str, download_products: bool, pipeline_
         #  as the user can specify the version (and as we want to use the latest version if they didn't) we need to
         #  see what is available
         vers = list(set([td.split('_')[-1].replace('/', '') for td in top_data]))
-
         if pipeline_version is not None and pipeline_version not in vers:
             raise ValueError("The specified pipeline version ({p}) is not available for "
                              "{oi}".format(p=pipeline_version, oi=obs_id))
         else:
             pipeline_version = vers[np.argmax([int(pv) for pv in vers])]
-
+        
         # Final check that the online archive directory that we're pointing at does actually contain the data
         #  directories we expect it too. Every mission I've implemented I seem to have done this in a slightly
         #  different way, but as eROSITA is an active project things are more liable to change and I think this
@@ -1241,7 +1237,6 @@ def _download_call(obs_id: str, raw_dir: str, download_products: bool, pipeline_
             # Finally we strip anything that doesn't match the file pattern defined by whether the user wants
             #  pre-generated products or not
             to_down = [f for patt in down_patt for f in all_files if patt in f]
-
             # Now we cycle through the files and download them
             for down_file in to_down:
                 down_url = cur_url + down_file

daxa/mission/xmm.py

@@ -177,7 +177,7 @@ def _fetch_obs_info(self):
         #  I'll explain in a second
         count_tab = AQXMMNewton.query_xsa_tap('select count(observation_id) from xsa.v_all_observations')
         # Then I round up to the nearest 1000, probably unnecessary but oh well
-        num_obs = np.ceil(count_tab['count'].tolist()[0] / 1000).astype(int) * 1000
+        num_obs = np.ceil(count_tab['COUNT'].tolist()[0] / 1000).astype(int) * 1000
         # Now I have to be a bit cheesy - If I used select * (which is what I would normally do in an SQL-derived
         #  language to grab every row) it actually only returns the top 2000. I think that * is replaced with TOP 2000
         #  before the query is sent to the server. However if I specify a TOP N, where N is greater than 2000, then it

tests/test_asca.py

@@ -0,0 +1,44 @@
+import unittest
+
+from astropy.coordinates import FK5
+from astropy.units import Quantity
+
+from daxa.mission import ASCA
+
+# Would usually put this in a setUp() function, but it takes some time to instantiate
+# Putting the mission object up here instead saves time when running the tests
+defaults = ASCA()
+
+class TestASCA(unittest.TestCase):
+    def test_valid_inst_selection(self):
+        # Checking that inst argument is working correctly
+        mission = ASCA(insts=['SIS0', 'SIS1'])
+        self.assertEqual(mission.chosen_instruments, ['SIS0', 'SIS1'])
+
+    def test_valid_inst_selection_alt_names(self):
+        # Alternative instrument names should be able to be parsed
+        with self.assertWarns(UserWarning):
+            mission = ASCA(insts=['S0', 'S1'])
+        self.assertEqual(mission.chosen_instruments, ['SIS0', 'SIS1'])
+
+    def test_wrong_insts(self):
+        # Shouldnt be able to declare an invalid instrument
+        with self.assertRaises(ValueError):
+            ASCA(insts=['wrong'])
+
+    # the basic properties of the class are returning what is expected
+    def test_name(self):
+        self.assertEqual(defaults.name, 'asca')
+
+    def test_coord_frame(self):
+        self.assertEqual(defaults.coord_frame, FK5)
+
+    def test_id_regex(self):
+        self.assertEqual(defaults.id_regex, '^[0-9]{8}$')
+
+    def test_fov(self):
+        self.assertEqual(defaults.fov['SIS0'], Quantity(11, 'arcmin'))
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_chandra.py

@@ -0,0 +1,36 @@
+import unittest
+
+from astropy.coordinates import ICRS
+from astropy.units import Quantity
+
+from daxa.mission import Chandra
+
+# Would usually put this in a setUp() function, but it takes some time to instantiate
+# Putting the mission object up here instead saves time when running the tests
+defaults = Chandra()
+
+class TestChandra(unittest.TestCase):
+    def test_valid_inst_selection(self):
+        # Checking that inst argument is working correctly
+        mission = Chandra(insts=['ACIS-I', 'ACIS-S'])
+        self.assertEqual(mission.chosen_instruments, ['ACIS-I', 'ACIS-S'])
+
+    def test_wrong_insts(self):
+        # Shouldnt be able to declare an invalid instrument
+        with self.assertRaises(ValueError):
+            Chandra(insts=['wrong'])
+
+    # the basic properties of the class are returning what is expected
+    def test_name(self):
+        self.assertEqual(defaults.name, 'chandra')
+
+    def test_coord_frame(self):
+        self.assertEqual(defaults.coord_frame, ICRS)
+
+    def test_fov(self):
+        with self.assertWarns(UserWarning):
+            self.assertEqual(defaults.fov['ACIS-I'], Quantity(27.8, 'arcmin'))
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_data/html_responses/134135.txt

@@ -0,0 +1 @@
+'<!DOCTYPE html>\n<html lang="en">\n\n<head>\n  <!--\n    ------------------------------------------------------------\n     eRODat, written by Jeremy Sanders (2022-2023)\n     Please report problems at https://erosita-forum.mpe.mpg.de/\n    ------------------------------------------------------------\n  -->\n\n  <title>Directory listing for /135/134/ | eRODat</title>\n  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css">\n  <link rel="stylesheet" href="/dr1/erodat/static/erodat.css">\n  <script type="text/javascript" src="/dr1/erodat/static/jquery.js"></script>\n  \n\n<link rel="stylesheet" type="text/css" href="/dr1/erodat/static/DataTables/datatables.min.css"/>\n<link rel="stylesheet" type="text/css" href="/dr1/erodat/static/DataTables-Select/css/select.dataTables.min.css"/>\n<script type="text/javascript" src="/dr1/erodat/static/DataTables/datatables.min.js"></script>\n<script type="text/javascript" src="/dr1/erodat/static/DataTables-Select/js/dataTables.select.min.js"></script>\n\n<script type="text/javascript" src="/dr1/erodat/static/cookie.js"></script>\n<script type="text/javascript" src="/dr1/erodat/data/add_to_basket.js"></script>\n\n<script type="text/javascript">\n\n  $(document).ready( function () {\n      var table = $(\'#files\').DataTable({\n          columnDefs: [\n              {\n                  // format size with commas, right-aligned\n                  targets: 3,\n                  render: $.fn.dataTable.render.number(\',\', \'.\', 0, \'\'),\n                  className: \'dt-body-right\',\n              },\n          ],\n          ordering: false,\n          searching: false,\n          paging: false,\n          select: {\n              style: \'multi\',\n              selector: \'tr.isfile\',\n          },\n          initComplete: function( settings, json ) {\n              // hide table when loading and show after to avoid redraw\n              $(\'#files\').show();\n          },\n      });\n\n      // Add selected files to the basket\n      $(\'#AddRows\').click( function() {\n          var selected = $(\'#files\').DataTable().rows({selected: true});\n          if(selected.count() == 0)\n              return;\n\n          const regexfn = /<a.*>(.+)<\\/a>/;  // get filename from url\n          var files = [];\n          for(let i=0; i<selected.count(); i++) {\n              var sel = selected.data()[i][0];\n              var fname = "/135/134/" + sel.match(regexfn)[1];\n\n              var tile = -1\n              var dpart = fname.split(\'/\');\n              if ( dpart.length > 2 ) {\n                  var tile = parseInt(dpart[2]+dpart[1]);\n              }\n              files.push({ptype: \'FILE\', filename: fname, \'skytile\': tile});\n          }\n          add_to_basket(files);\n      });\n\n      // enable/disable add button\n      $(\'#files\').on(\'select.dt deselect.dt\', function () {\n          var count = $(\'#files\').DataTable().rows( { selected: true } ).count();\n          $(\'#AddRows\').prop(\'disabled\', count==0);\n      });\n\n  });\n\n</script>\n\n\n\n  \n  <!-- Matomo -->\n  <script>\n    var _paq = window._paq = window._paq || [];\n    /* tracker methods like "setCustomDimension" should be called before "trackPageView" */\n    _paq.push([\'trackPageView\']);\n    _paq.push([\'enableLinkTracking\']);\n    (function() {\n        var u="//erosita.mpe.mpg.de/html/matomo/";\n        _paq.push([\'setTrackerUrl\', u+\'matomo.php\']);\n        _paq.push([\'setSiteId\', \'16\']);\n        var d=document, g=d.createElement(\'script\'), s=d.getElementsByTagName(\'script\')[0];\n        g.async=true; g.src=u+\'matomo.js\'; s.parentNode.insertBefore(g,s);\n    })();\n  </script>\n  <!-- End Matomo Code -->\n  \n</head>\n\n<body>\n  <img src="/dr1/erodat/static/eROSITA_Logo_Blue.png" width="100%">\n  <h1>eRODat: eROSITA-DE Data Release 1 archive</h1>\n\n  <div class="navbar">\n    <a href="https://erosita.mpe.mpg.de/dr1/">Main DR1 home</a>\n    <a href="/dr1/erodat/">eRODat home</a>\n    <a href="/dr1/erodat/skyview/sky/">Sky view</a>\n    <div class="navdropdown">\n      <button class="navdropbtn">Skytile search <i class="fa fa-caret-down"></i></button>\n      <div class="navdropdown-content">\n        <a href="/dr1/erodat/skyview/skytile_search/">Single position</a>\n        <a href="/dr1/erodat/skyview/skytile_multi_search/">Multiple positions</a>\n        <a href="/dr1/erodat/skyview/skytile_number_search/">By number</a>\n      </div>\n    </div>\n    <div class="navdropdown">\n      <button class="navdropbtn">Catalogue search <i class="fa fa-caret-down"></i></button>\n      <div class="navdropdown-content">\n        <a href="/dr1/erodat/catalogue/search/">Single position</a>\n        <a href="/dr1/erodat/catalogue/search_by_id/">By eROSITA identifier</a>\n      </div>\n    </div>\n    <div class="navdropdown">\n      <button class="navdropbtn">Upper limits <i class="fa fa-caret-down"></i></button>\n      <div class="navdropdown-content">\n        <a href="/dr1/erodat/upperlimit/single/">Single position</a>\n        <a href="/dr1/erodat/upperlimit/multi/">Multiple positions</a>\n      </div>\n    </div>\n    <a href="/dr1/erodat/data/download/">Download area</a>\n    <a href="/dr1/erodat/data/basket/">Basket</a>\n  </div>\n\n  <div id="content">\n    \n\n<h2>Archive directory listing for\n  <a href="../../">/</a><a href="../">135/</a><a href="./">134/</a>\n</h2>\n\n\n\n<div>\n<input title="Adds any selected data files to the basket" type="button" value="Add selected to basket" id="AddRows" style="display:inline;" disabled>\n<span id="BasketMessage"></span>\n</div>\n\n<table id="files" class="display" style="width:100%;display:none">\n  <thead>\n    <tr>\n      <th>Name</th><th>Type</th><th>Date</th><th>Size</th>\n    </tr>\n  </thead>\n\n  <tbody>\n\n  \n  <tr class="isdir"><td><a href="../">Parent directory</a></td><td>Directory</td><td>2023-11-03 12:22:30</td><td></td></tr>\n  \n  <tr class="isdir"><td><a href="DET_010/">DET_010/</a></td><td>Directory</td><td>2023-02-01 19:38:18</td><td></td></tr>\n  \n  <tr class="isdir"><td><a href="EXP_010/">EXP_010/</a></td><td>Directory</td><td>2022-11-25 14:58:38</td><td></td></tr>\n  \n  <tr class="isdir"><td><a href="SOU_010/">SOU_010/</a></td><td>Directory</td><td>2022-09-09 21:37:19</td><td></td></tr>\n  \n  <tr class="isdir"><td><a href="UPP_010/">UPP_010/</a></td><td>Directory</td><td>2023-10-10 08:07:45</td><td></td></tr>\n  \n\n  \n\n  </tbody>\n</table>\n\n\n  </div>\n\n  <footer>\n    <span><a href="https://www.mpe.mpg.de/impressum">Imprint</a></span>\n    <span><a href="https://www.mpe.mpg.de/data-protection">Data Protection</a></span>\n    <span>© eROSITA-DE, MPE</span>\n  </footer>\n\n</body>\n</html>\n'