Merge branch 'master' into personal/apop5/vs2022

.pytool/Plugin/CompilerPlugin/CompilerPlugin.py

@@ -74,9 +74,10 @@ def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM,
         self._env.SetValue("ACTIVE_PLATFORM", AP_Path, "Set in Compiler Plugin")
 
         # Parse DSC to check for SUPPORTED_ARCHITECTURES
+        build_target = self._env.GetValue("TARGET")
+        input_vars = self._env.GetAllBuildKeyValues(build_target)
         dp = DscParser()
-        dp.SetBaseAbsPath(Edk2pathObj.WorkspacePath)
-        dp.SetPackagePaths(Edk2pathObj.PackagePathList)
+        dp.SetEdk2Path(Edk2pathObj).SetInputVars(input_vars)
         dp.ParseFile(AP_Path)
         if "SUPPORTED_ARCHITECTURES" in dp.LocalVars:
             SUPPORTED_ARCHITECTURES = dp.LocalVars["SUPPORTED_ARCHITECTURES"].split('|')
@@ -85,7 +86,7 @@ def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM,
             # Skip if there is no intersection between SUPPORTED_ARCHITECTURES and TARGET_ARCHITECTURES
             if len(set(SUPPORTED_ARCHITECTURES) & set(TARGET_ARCHITECTURES)) == 0:
                 tc.SetSkipped()
-                tc.LogStdError("No supported architecutres to build")
+                tc.LogStdError("No supported architectures to build")
                 return -1
 
         uefiBuilder = UefiBuilder()

.pytool/Plugin/DscCompleteCheck/DscCompleteCheck.py

@@ -6,9 +6,12 @@
 import logging
 import os
 from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin
+from edk2toollib.uefi.edk2.path_utilities import Edk2Path
 from edk2toollib.uefi.edk2.parsers.dsc_parser import DscParser
 from edk2toollib.uefi.edk2.parsers.inf_parser import InfParser
 from edk2toolext.environment.var_dict import VarDict
+from edk2toollib.gitignore_parser import parse_gitignore_lines
+from pathlib import Path
 
 
 class DscCompleteCheck(ICiBuildPlugin):
@@ -71,38 +74,39 @@ def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM,
 
         # Get INF Files
         INFFiles = self.WalkDirectoryForExtension([".inf"], abs_pkg_path)
-        INFFiles = [Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(
-            x) for x in INFFiles]  # make edk2relative path so can compare with DSC
 
         # remove ignores
-
+        ignored_paths = []
         if "IgnoreInf" in pkgconfig:
-            for a in pkgconfig["IgnoreInf"]:
-                a = a.replace(os.sep, "/")
+            ignore_filter = parse_gitignore_lines(
+                pkgconfig["IgnoreInf"],
+                "DSC Complete Check Config",
+                os.path.dirname(abs_pkg_path))
+
+            # INFFiles must be a list of absolute paths
+            ignored_paths = list(filter(ignore_filter, INFFiles))
+            for a in ignored_paths:
                 try:
                     tc.LogStdOut("Ignoring INF {0}".format(a))
                     INFFiles.remove(a)
-                except:
+                except Exception:
                     tc.LogStdError(
                         "DscCompleteCheck.IgnoreInf -> {0} not found in filesystem.  Invalid ignore file".format(a))
                     logging.info(
                         "DscCompleteCheck.IgnoreInf -> {0} not found in filesystem.  Invalid ignore file".format(a))
 
+        # make edk2relative path so can compare with DSC
+        INFFiles = [Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(x) for x in INFFiles]
+
         # DSC Parser
-        dp = DscParser()
-        dp.SetBaseAbsPath(Edk2pathObj.WorkspacePath)
-        dp.SetPackagePaths(Edk2pathObj.PackagePathList)
+        dp = DscParser().SetEdk2Path(Edk2pathObj)
         dp.SetInputVars(environment.GetAllBuildKeyValues())
         dp.ParseFile(wsr_dsc_path)
 
         # Check if INF in component section
         for INF in INFFiles:
-            if not any(INF.strip() in x for x in dp.ThreeMods) and \
-               not any(INF.strip() in x for x in dp.SixMods) and \
-               not any(INF.strip() in x for x in dp.OtherMods):
-
-                infp = InfParser().SetBaseAbsPath(Edk2pathObj.WorkspacePath)
-                infp.SetPackagePaths(Edk2pathObj.PackagePathList)
+            if not DscCompleteCheck._module_in_dsc(INF, dp, Edk2pathObj):
+                infp = InfParser().SetEdk2Path(Edk2pathObj)
                 infp.ParseFile(INF)
                 if("MODULE_TYPE" not in infp.Dict):
                     tc.LogStdOut(
@@ -131,3 +135,22 @@ def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM,
         else:
             tc.SetSuccess()
         return overall_status
+
+    @staticmethod
+    def _module_in_dsc(inf: str, dsc: DscParser, Edk2pathObj: Edk2Path) -> bool:
+
+        """Checks if the given module (inf) is in the given dsc.
+        Args:
+            inf (str): The inf file to check for
+            dsc (DscParser): The parsed dsc file.
+            Edk2pathObj (Edk2Path): The path object capturing the workspace and package paths.
+        Returns:
+            bool: if the module is in the dsc.
+        """
+        for module_type in (dsc.ThreeMods, dsc.SixMods, dsc.OtherMods):
+            for module in module_type:
+                if Path(module).is_absolute():
+                    module = Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(module)
+                if inf in module:
+                    return True
+        return False

.pytool/Plugin/DscCompleteCheck/Readme.md

@@ -29,4 +29,5 @@ Path to DSC to consider platform dsc
 
 ### IgnoreInf
 
-Ignore error if Inf file is not listed in DSC file
+A list of paths in git ignore syntax to ignore in the check. These can include directory and file paths. The path is
+relative to the directory that contains the package.

BaseTools/Scripts/BinToPcd.py

@@ -10,13 +10,12 @@
 '''
 from __future__ import print_function
 
-import sys
 import argparse
-import re
-import xdrlib
 import io
-import struct
 import math
+import re
+import struct
+import sys
 
 #
 # Globals for help information

BaseTools/Source/Python/Capsule/GenerateCapsule.py

@@ -513,11 +513,15 @@ def Validate(self, args):
                         raise argparse.ArgumentTypeError ('JSON field MonotonicCount must be an integer in range 0x0..0xffffffffffffffff')
                     else:
                         raise argparse.ArgumentTypeError ('--monotonic-count must be an integer in range 0x0..0xffffffffffffffff')
-                if self.UpdateImageIndex >0xFF:
+                if self.UpdateImageIndex < 0x1 or self.UpdateImageIndex > 0xFF:
                     if args.JsonFile:
-                        raise argparse.ArgumentTypeError ('JSON field UpdateImageIndex must be an integer in range 0x0..0xff')
+                        raise argparse.ArgumentTypeError ('JSON field UpdateImageIndex must be an integer in range 0x1..0xff')
                     else:
-                        raise argparse.ArgumentTypeError ('--update-image-index must be an integer in range 0x0..0xff')
+                        raise argparse.ArgumentTypeError ('--update-image-index must be an integer in range 0x1..0xff')
+
+            if args.Decode:
+                if args.OutputFile is None:
+                    raise argparse.ArgumentTypeError ('--decode requires --output')
 
             if self.UseSignTool:
                 if self.SignToolPfxFile is not None:
@@ -576,7 +580,7 @@ def Encode (PayloadDescriptorList, EmbeddedDriverDescriptorList, Buffer):
             try:
                 SinglePayloadDescriptor.Validate (args)
             except Exception as Msg:
-                print ('GenerateCapsule: error:' + str(Msg))
+                print ('GenerateCapsule: error: ' + str(Msg))
                 sys.exit (1)
         for SinglePayloadDescriptor in PayloadDescriptorList:
             ImageCapsuleSupport = 0x0000000000000000
@@ -690,7 +694,7 @@ def Decode (PayloadDescriptorList, PayloadJsonDescriptorList, Buffer):
                                             args.HardwareInstance,
                                             args.UpdateImageIndex,
                                             args.SignToolPfxFile,
-                                            args.SignSubjectName,
+                                            args.SignToolSubjectName,
                                             args.OpenSslSignerPrivateCertFile,
                                             args.OpenSslOtherPublicCertFile,
                                             args.OpenSslTrustedPublicCertFile,
@@ -704,7 +708,7 @@ def Decode (PayloadDescriptorList, PayloadJsonDescriptorList, Buffer):
             try:
                 SinglePayloadDescriptor.Validate (args)
             except Exception as Msg:
-                print ('GenerateCapsule: error:' + str(Msg))
+                print ('GenerateCapsule: error: ' + str(Msg))
                 sys.exit (1)
         try:
             Result = UefiCapsuleHeader.Decode (Buffer)
@@ -827,7 +831,7 @@ def Decode (PayloadDescriptorList, PayloadJsonDescriptorList, Buffer):
                             print ('--------')
                             print ('No EFI_FIRMWARE_IMAGE_AUTHENTICATION')
 
-                    PayloadSignature = struct.unpack ('<I', SinglePayloadDescriptor.Payload[0:4])
+                    (PayloadSignature,) = struct.unpack ('<I', SinglePayloadDescriptor.Payload[0:4])
                     if PayloadSignature != FmpPayloadHeader.Signature:
                         SinglePayloadDescriptor.UseDependency = True
                         try:
@@ -873,8 +877,8 @@ def Decode (PayloadDescriptorList, PayloadJsonDescriptorList, Buffer):
                         print ('GenerateCapsule: error: can not write embedded driver file {File}'.format (File = EmbeddedDriverPath))
                         sys.exit (1)
 
-        except:
-            print ('GenerateCapsule: error: can not decode capsule')
+        except Exception as Msg:
+            print ('GenerateCapsule: error: can not decode capsule: ' + str(Msg))
             sys.exit (1)
         GenerateOutputJson(PayloadJsonDescriptorList)
         PayloadIndex = 0
@@ -914,7 +918,7 @@ def DumpInfo (Buffer, args):
                         print ('--------')
                         print ('No EFI_FIRMWARE_IMAGE_AUTHENTICATION')
 
-                    PayloadSignature = struct.unpack ('<I', Result[0:4])
+                    (PayloadSignature,) = struct.unpack ('<I', Result[0:4])
                     if PayloadSignature != FmpPayloadHeader.Signature:
                         try:
                             Result = CapsuleDependency.Decode (Result)

BaseTools/Source/Python/Common/Uefi/Capsule/FmpCapsuleHeader.py

@@ -92,19 +92,19 @@ def Encode (self):
 
     def Decode (self, Buffer):
         if len (Buffer) < self._StructSize:
-            raise ValueError
+            raise ValueError ('Buffer is too small for decoding')
         (Version, UpdateImageTypeId, UpdateImageIndex, r0, r1, r2, UpdateImageSize, UpdateVendorCodeSize, UpdateHardwareInstance, ImageCapsuleSupport) = \
             struct.unpack (
                      self._StructFormat,
                      Buffer[0:self._StructSize]
                      )
 
         if Version < self.EFI_FIRMWARE_MANAGEMENT_CAPSULE_IMAGE_HEADER_INIT_VERSION:
-            raise ValueError
+            raise ValueError ('Incorrect capsule image header version')
         if UpdateImageIndex < 1:
-            raise ValueError
+            raise ValueError ('Update image index is less than 1')
         if UpdateImageSize + UpdateVendorCodeSize != len (Buffer[self._StructSize:]):
-            raise ValueError
+            raise ValueError ('Non-vendor and vendor parts do not add up')
 
         self.Version                = Version
         self.UpdateImageTypeId      = uuid.UUID (bytes_le = UpdateImageTypeId)
@@ -120,7 +120,7 @@ def Decode (self, Buffer):
 
     def DumpInfo (self):
         if not self._Valid:
-            raise ValueError
+            raise ValueError ('Can not dump an invalid header')
         print ('EFI_FIRMWARE_MANAGEMENT_CAPSULE_IMAGE_HEADER.Version                = {Version:08X}'.format (Version = self.Version))
         print ('EFI_FIRMWARE_MANAGEMENT_CAPSULE_IMAGE_HEADER.UpdateImageTypeId      = {UpdateImageTypeId}'.format (UpdateImageTypeId = str(self.UpdateImageTypeId).upper()))
         print ('EFI_FIRMWARE_MANAGEMENT_CAPSULE_IMAGE_HEADER.UpdateImageIndex       = {UpdateImageIndex:08X}'.format (UpdateImageIndex = self.UpdateImageIndex))
@@ -180,15 +180,15 @@ def AddEmbeddedDriver (self, EmbeddedDriver):
 
     def GetEmbeddedDriver (self, Index):
         if Index > len (self._EmbeddedDriverList):
-            raise ValueError
+            raise ValueError ('Invalid embedded driver index')
         return self._EmbeddedDriverList[Index]
 
     def AddPayload (self, UpdateImageTypeId, Payload = b'', VendorCodeBytes = b'', HardwareInstance = 0, UpdateImageIndex = 1, CapsuleSupport = 0):
         self._PayloadList.append ((UpdateImageTypeId, Payload, VendorCodeBytes, HardwareInstance, UpdateImageIndex, CapsuleSupport))
 
     def GetFmpCapsuleImageHeader (self, Index):
         if Index >= len (self._FmpCapsuleImageHeaderList):
-            raise ValueError
+            raise ValueError ('Invalid capsule image index')
         return self._FmpCapsuleImageHeaderList[Index]
 
     def Encode (self):
@@ -234,14 +234,14 @@ def Encode (self):
 
     def Decode (self, Buffer):
         if len (Buffer) < self._StructSize:
-            raise ValueError
+            raise ValueError ('Buffer is too small for decoding')
         (Version, EmbeddedDriverCount, PayloadItemCount) = \
             struct.unpack (
                      self._StructFormat,
                      Buffer[0:self._StructSize]
                      )
         if Version < self.EFI_FIRMWARE_MANAGEMENT_CAPSULE_HEADER_INIT_VERSION:
-            raise ValueError
+            raise ValueError ('Incorrect capsule header version')
 
         self.Version                    = Version
         self.EmbeddedDriverCount        = EmbeddedDriverCount
@@ -258,7 +258,7 @@ def Decode (self, Buffer):
         for Index in range (0, EmbeddedDriverCount + PayloadItemCount):
             ItemOffset = struct.unpack (self._ItemOffsetFormat, Buffer[Offset:Offset + self._ItemOffsetSize])[0]
             if ItemOffset >= len (Buffer):
-                raise ValueError
+                raise ValueError ('Item offset is outside of buffer')
             self._ItemOffsetList.append (ItemOffset)
             Offset = Offset + self._ItemOffsetSize
         Result = Buffer[Offset:]
@@ -297,7 +297,7 @@ def Decode (self, Buffer):
 
     def DumpInfo (self):
         if not self._Valid:
-            raise ValueError
+            raise ValueError ('Can not dump an invalid header')
         print ('EFI_FIRMWARE_MANAGEMENT_CAPSULE_HEADER.Version             = {Version:08X}'.format (Version = self.Version))
         print ('EFI_FIRMWARE_MANAGEMENT_CAPSULE_HEADER.EmbeddedDriverCount = {EmbeddedDriverCount:08X}'.format (EmbeddedDriverCount = self.EmbeddedDriverCount))
         for EmbeddedDriver in self._EmbeddedDriverList:

BaseTools/Source/Python/Workspace/InfBuildData.py

@@ -592,7 +592,7 @@ def Protocols(self):
         RecordList = self._RawData[MODEL_EFI_PROTOCOL, self._Arch, self._Platform]
         for Record in RecordList:
             CName = Record[0]
-            Value = _ProtocolValue(CName, self.Packages, self.MetaFile.Path)
+            Value = _ProtocolValue(CName, self.Packages, self.MetaFile.OriginalPath.Path)
             if Value is None:
                 PackageList = "\n\t".join(str(P) for P in self.Packages)
                 EdkLogger.error('build', RESOURCE_NOT_AVAILABLE,
@@ -616,7 +616,7 @@ def Ppis(self):
         RecordList = self._RawData[MODEL_EFI_PPI, self._Arch, self._Platform]
         for Record in RecordList:
             CName = Record[0]
-            Value = _PpiValue(CName, self.Packages, self.MetaFile.Path)
+            Value = _PpiValue(CName, self.Packages, self.MetaFile.OriginalPath.Path)
             if Value is None:
                 PackageList = "\n\t".join(str(P) for P in self.Packages)
                 EdkLogger.error('build', RESOURCE_NOT_AVAILABLE,
@@ -640,7 +640,7 @@ def Guids(self):
         RecordList = self._RawData[MODEL_EFI_GUID, self._Arch, self._Platform]
         for Record in RecordList:
             CName = Record[0]
-            Value = GuidValue(CName, self.Packages, self.MetaFile.Path)
+            Value = GuidValue(CName, self.Packages, self.MetaFile.OriginalPath.Path)
             if Value is None:
                 PackageList = "\n\t".join(str(P) for P in self.Packages)
                 EdkLogger.error('build', RESOURCE_NOT_AVAILABLE,
@@ -655,7 +655,7 @@ def Guids(self):
             for TokenSpaceGuid, _, _, _, _, _, LineNo in RecordList:
                 # get the guid value
                 if TokenSpaceGuid not in RetVal:
-                    Value = GuidValue(TokenSpaceGuid, self.Packages, self.MetaFile.Path)
+                    Value = GuidValue(TokenSpaceGuid, self.Packages, self.MetaFile.OriginalPath.Path)
                     if Value is None:
                         PackageList = "\n\t".join(str(P) for P in self.Packages)
                         EdkLogger.error('build', RESOURCE_NOT_AVAILABLE,
@@ -818,11 +818,11 @@ def Depex(self):
                         Value = Token
                     else:
                         # get the GUID value now
-                        Value = _ProtocolValue(Token, self.Packages, self.MetaFile.Path)
+                        Value = _ProtocolValue(Token, self.Packages, self.MetaFile.OriginalPath.Path)
                         if Value is None:
-                            Value = _PpiValue(Token, self.Packages, self.MetaFile.Path)
+                            Value = _PpiValue(Token, self.Packages, self.MetaFile.OriginalPath.Path)
                             if Value is None:
-                                Value = GuidValue(Token, self.Packages, self.MetaFile.Path)
+                                Value = GuidValue(Token, self.Packages, self.MetaFile.OriginalPath.Path)
 
                     if Value is None:
                         PackageList = "\n\t".join(str(P) for P in self.Packages)