DataSourceOracle: refactor to use only OPC v1 endpoint (#493)

The /opc/v1/ metadata endpoints[0] are universally available in Oracle
Cloud Infrastructure and the OpenStack endpoints are considered
deprecated, so we can refactor the data source to use the OPC endpoints
exclusively.  This simplifies the datasource code substantially, and
enables use of OPC-specific attributes in future.

[0] https://docs.cloud.oracle.com/en-us/iaas/Content/Compute/Tasks/gettingmetadata.htm

cloudinit/sources/DataSourceOracle.py

@@ -1,22 +1,20 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 """Datasource for Oracle (OCI/Oracle Cloud Infrastructure)
 
-OCI provides a OpenStack like metadata service which provides only
-'2013-10-17' and 'latest' versions..
-
 Notes:
- * This datasource does not support the OCI-Classic. OCI-Classic
-   provides an EC2 lookalike metadata service.
- * The uuid provided in DMI data is not the same as the meta-data provided
+ * This datasource does not support OCI Classic. OCI Classic provides an EC2
+   lookalike metadata service.
+ * The UUID provided in DMI data is not the same as the meta-data provided
    instance-id, but has an equivalent lifespan.
  * We do need to support upgrade from an instance that cloud-init
    identified as OpenStack.
- * Both bare-metal and vms use iscsi root
- * Both bare-metal and vms provide chassis-asset-tag of OracleCloud.com
+ * Bare metal instances use iSCSI root, virtual machine instances do not.
+ * Both bare metal and virtual machine instances provide a chassis-asset-tag of
+   OracleCloud.com.
 """
 
+import base64
 import json
-import re
 
 from cloudinit import log as logging
 from cloudinit import net, sources, util
@@ -26,7 +24,7 @@
     get_interfaces_by_mac,
     is_netfail_master,
 )
-from cloudinit.url_helper import UrlError, combine_url, readurl
+from cloudinit.url_helper import readurl
 
 LOG = logging.getLogger(__name__)
 
@@ -35,8 +33,9 @@
     'configure_secondary_nics': False,
 }
 CHASSIS_ASSET_TAG = "OracleCloud.com"
-METADATA_ENDPOINT = "http://169.254.169.254/openstack/"
-VNIC_METADATA_URL = 'http://169.254.169.254/opc/v1/vnics/'
+METADATA_ROOT = "http://169.254.169.254/opc/v1/"
+METADATA_ENDPOINT = METADATA_ROOT + "instance/"
+VNIC_METADATA_URL = METADATA_ROOT + "vnics/"
 # https://docs.cloud.oracle.com/iaas/Content/Network/Troubleshoot/connectionhang.htm#Overview,
 # indicates that an MTU of 9000 is used within OCI
 MTU = 9000
@@ -189,53 +188,39 @@ def _get_data(self):
         if not self._is_platform_viable():
             return False
 
+        self.system_uuid = _read_system_uuid()
+
         # network may be configured if iscsi root.  If that is the case
         # then read_initramfs_config will return non-None.
         if _is_iscsi_root():
-            data = self.crawl_metadata()
+            data = read_opc_metadata()
         else:
             with dhcp.EphemeralDHCPv4(net.find_fallback_nic()):
-                data = self.crawl_metadata()
+                data = read_opc_metadata()
 
         self._crawled_metadata = data
-        vdata = data['2013-10-17']
-
-        self.userdata_raw = vdata.get('user_data')
-        self.system_uuid = vdata['system_uuid']
-
-        vd = vdata.get('vendor_data')
-        if vd:
-            self.vendordata_pure = vd
-            try:
-                self.vendordata_raw = sources.convert_vendordata(vd)
-            except ValueError as e:
-                LOG.warning("Invalid content in vendor-data: %s", e)
-                self.vendordata_raw = None
-
-        mdcopies = ('public_keys',)
-        md = dict([(k, vdata['meta_data'].get(k))
-                   for k in mdcopies if k in vdata['meta_data']])
-
-        mdtrans = (
-            # oracle meta_data.json name, cloudinit.datasource.metadata name
-            ('availability_zone', 'availability-zone'),
-            ('hostname', 'local-hostname'),
-            ('launch_index', 'launch-index'),
-            ('uuid', 'instance-id'),
-        )
-        for dsname, ciname in mdtrans:
-            if dsname in vdata['meta_data']:
-                md[ciname] = vdata['meta_data'][dsname]
 
-        self.metadata = md
-        return True
+        self.metadata = {
+            "availability-zone": data["ociAdName"],
+            "instance-id": data["id"],
+            "launch-index": 0,
+            "local-hostname": data["hostname"],
+            "name": data["displayName"],
+        }
+
+        if "metadata" in data:
+            user_data = data["metadata"].get("user_data")
+            if user_data:
+                self.userdata_raw = base64.b64decode(user_data)
+            self.metadata["public_keys"] = data["metadata"].get(
+                "ssh_authorized_keys"
+            )
 
-    def crawl_metadata(self):
-        return read_metadata()
+        return True
 
     def _get_subplatform(self):
         """Return the subplatform metadata source details."""
-        return 'metadata (%s)' % METADATA_ENDPOINT
+        return "metadata ({})".format(METADATA_ROOT)
 
     def check_instance_id(self, sys_cfg):
         """quickly check (local only) if self.instance_id is still valid
@@ -292,72 +277,15 @@ def _is_iscsi_root():
     return bool(cmdline.read_initramfs_config())
 
 
-def _load_index(content):
-    """Return a list entries parsed from content.
-
-    OpenStack's metadata service returns a newline delimited list
-    of items.  Oracle's implementation has html formatted list of links.
-    The parser here just grabs targets from <a href="target">
-    and throws away "../".
-
-    Oracle has accepted that to be buggy and may fix in the future
-    to instead return a '\n' delimited plain text list.  This function
-    will continue to work if that change is made."""
-    if not content.lower().startswith("<html>"):
-        return content.splitlines()
-    items = re.findall(
-        r'href="(?P<target>[^"]*)"', content, re.MULTILINE | re.IGNORECASE)
-    return [i for i in items if not i.startswith(".")]
-
-
-def read_metadata(endpoint_base=METADATA_ENDPOINT, sys_uuid=None,
-                  version='2013-10-17'):
-    """Read metadata, return a dictionary.
-
-    Each path listed in the index will be represented in the dictionary.
-    If the path ends in .json, then the content will be decoded and
-    populated into the dictionary.
-
-    The system uuid (/sys/class/dmi/id/product_uuid) is also populated.
-    Example: given paths = ('user_data', 'meta_data.json')
-    This would return:
-      {version: {'user_data': b'blob', 'meta_data': json.loads(blob.decode())
-                 'system_uuid': '3b54f2e0-3ab2-458d-b770-af9926eee3b2'}}
+def read_opc_metadata():
     """
-    endpoint = combine_url(endpoint_base, version) + "/"
-    if sys_uuid is None:
-        sys_uuid = _read_system_uuid()
-    if not sys_uuid:
-        raise sources.BrokenMetadata("Failed to read system uuid.")
-
-    try:
-        resp = readurl(endpoint)
-        if not resp.ok():
-            raise sources.BrokenMetadata(
-                "Bad response from %s: %s" % (endpoint, resp.code))
-    except UrlError as e:
-        raise sources.BrokenMetadata(
-            "Failed to read index at %s: %s" % (endpoint, e))
-
-    entries = _load_index(resp.contents.decode('utf-8'))
-    LOG.debug("index url %s contained: %s", endpoint, entries)
-
-    # meta_data.json is required.
-    mdj = 'meta_data.json'
-    if mdj not in entries:
-        raise sources.BrokenMetadata(
-            "Required field '%s' missing in index at %s" % (mdj, endpoint))
-
-    ret = {'system_uuid': sys_uuid}
-    for path in entries:
-        response = readurl(combine_url(endpoint, path))
-        if path.endswith(".json"):
-            ret[path.rpartition(".")[0]] = (
-                json.loads(response.contents.decode('utf-8')))
-        else:
-            ret[path] = response.contents
+    Fetch metadata from the /opc/ routes.
 
-    return {version: ret}
+    :return:
+        The JSON-decoded value of the /opc/v1/instance/ endpoint on the IMDS.
+    """
+    # retries=1 as requested by Oracle to address a potential race condition
+    return json.loads(readurl(METADATA_ENDPOINT, retries=1)._response.text)
 
 
 # Used to match classes to dependencies
@@ -373,17 +301,21 @@ def get_datasource_list(depends):
 
 if __name__ == "__main__":
     import argparse
-    import os
-
-    parser = argparse.ArgumentParser(description='Query Oracle Cloud Metadata')
-    parser.add_argument("--endpoint", metavar="URL",
-                        help="The url of the metadata service.",
-                        default=METADATA_ENDPOINT)
-    args = parser.parse_args()
-    sys_uuid = "uuid-not-available-not-root" if os.geteuid() != 0 else None
-
-    data = read_metadata(endpoint_base=args.endpoint, sys_uuid=sys_uuid)
-    data['is_platform_viable'] = _is_platform_viable()
-    print(util.json_dumps(data))
+
+    description = """
+        Query Oracle Cloud metadata and emit a JSON object with two keys:
+        `read_opc_metadata` and `_is_platform_viable`.  The values of each are
+        the return values of the corresponding functions defined in
+        DataSourceOracle.py."""
+    parser = argparse.ArgumentParser(description=description)
+    parser.parse_args()
+    print(
+        util.json_dumps(
+            {
+                "read_opc_metadata": read_opc_metadata(),
+                "_is_platform_viable": _is_platform_viable(),
+            }
+        )
+    )
 
 # vi: ts=4 expandtab