BaseTools: Optimize parsing of byte-array PCD values

This change adds a fast path for simple VOID* hexadecimal byte arrays,
avoiding the generic expression tokenizer's repeated processing of large
PCD values. Regression tests for both paths. It does not change the
existing parser for structured and symbolic expressions.

This reduces parsing time for a 28 KiB generated PCD from roughly 2
seconds to 50 milliseconds.

Signed-off-by: Kun Qin <kun.qin@microsoft.com>

2 files changed, 71 insertions, 6 deletions

diff --git a/BaseTools/Source/Python/Common/Expression.py b/BaseTools/Source/Python/Common/Expression.py
index f8b34aaa58..b18161a5ef 100644
--- a/BaseTools/Source/Python/Common/Expression.py
+++ b/BaseTools/Source/Python/Common/Expression.py
@@ -45,6 +45,33 @@ _ReLabel = re.compile(r'LABEL\((\w+)\)')
 _ReOffset = re.compile(r'OFFSET_OF\((\w+)\)')

 PcdPattern = re.compile(r'^[_a-zA-Z][0-9A-Za-z_]*\.[_a-zA-Z][0-9A-Za-z_]*$')

 

+## Fast path for the simple byte array

+#

+#  Simple byte array refers to PCD data in the form of {0x01, 0x02, 0x03}

+#  - Enclosed in {}.

+#  - One or more comma-separated elements.

+#  - Each element is 0x followed by one or two hexadecimal digits.

+#  - Only spaces or tabs surround elements.

+#  - Used only for top-level VOID* real-value evaluation.

+#

+#  Return the stripped original value when valid. Otherwise None.

+def _NormalizeSimpleByteArray(Value):

+    Value = Value.strip()

+    if not Value.startswith('{') or not Value.endswith('}'):

+        return None

+

+    Items = Value[1:-1].split(',')

+    if not Items:

+        return None

+

+    for Item in Items:

+        Item = Item.strip(' \t')

+        if len(Item) < 3 or len(Item) > 4 or Item[:2].lower() != '0x' or \

+           not all(Char in string.hexdigits for Char in Item[2:]):

+            return None

+

+    return Value

+

 ## SplitString

 #  Split string to list according double quote

 #  For example: abc"de\"f"ghi"jkl"mn will be: ['abc', '"de\"f"', 'ghi', '"jkl"', 'mn']

@@ -821,6 +848,10 @@ class ValueExpressionEx(ValueExpression):
 

     def __call__(self, RealValue=False, Depth=0):

         PcdValue = self.PcdValue

+        if RealValue and Depth == 0 and self.PcdType == TAB_VOID and "{CODE(" not in PcdValue:

+            SimpleByteArray = _NormalizeSimpleByteArray(PcdValue)

+            if SimpleByteArray is not None:

+                return SimpleByteArray

         if "{CODE(" not in PcdValue:

             try:

                 PcdValue = ValueExpression.__call__(self, RealValue, Depth)

diff --git a/BaseTools/Tests/TestRegularExpression.py b/BaseTools/Tests/TestRegularExpression.py
index 3e6c5f4463..fbb359e144 100644
--- a/BaseTools/Tests/TestRegularExpression.py
+++ b/BaseTools/Tests/TestRegularExpression.py
@@ -5,19 +5,53 @@
 # SPDX-License-Identifier: BSD-2-Clause-Patent

 

 import unittest

+from Common.DataType import TAB_VOID

+from Common.Expression import ValueExpression, ValueExpressionEx

 from Common.Misc import RemoveCComments

 from Workspace.BuildClassObject import ArrayIndex

 

+

+class TestValueExpressionEx(unittest.TestCase):

+    def test_simple_byte_array_matches_legacy_parser(self):

+        value = ' { 0X01, 0xaB, 0xff } '

+

+        self.assertEqual(ValueExpression(value)(True), ValueExpressionEx(value, TAB_VOID)(True))

+

+    def test_nested_simple_byte_array_uses_legacy_parser(self):

+        value = '{ 0X01, 0x02 }'

+

+        self.assertEqual(ValueExpression(value)(True, 1), ValueExpressionEx(value, TAB_VOID)(True, 1))

+

+    def test_multiline_byte_array_uses_legacy_parser(self):

+        value = '{0x01,\n0x02}'

+

+        self.assertEqual('{0x01, 0x02}', ValueExpressionEx(value, TAB_VOID)(True))

+

+    def test_non_dsc_whitespace_uses_legacy_parser(self):

+        value = '{0x01,\v0x02}'

+

+        self.assertEqual('{0x01, 0x02}', ValueExpressionEx(value, TAB_VOID)(True))

+

+    def test_structured_array_uses_legacy_parser(self):

+        value = '{UINT16(0x1234), 0x56}'

+

+        self.assertEqual('{0x34, 0x12, 0x56}', ValueExpressionEx(value, TAB_VOID)(True))

+

+    def test_trailing_comma_uses_legacy_parser(self):

+        value = '{0x01,}'

+

+        self.assertEqual('{0x01}', ValueExpressionEx(value, TAB_VOID)(True))

+

 class TestRe(unittest.TestCase):

     def test_ccomments(self):

         TestStr1 = """ {0x01,0x02} """

-        self.assertEquals(TestStr1, RemoveCComments(TestStr1))

+        self.assertEqual(TestStr1, RemoveCComments(TestStr1))

 

         TestStr2 = """ L'TestString' """

-        self.assertEquals(TestStr2, RemoveCComments(TestStr2))

+        self.assertEqual(TestStr2, RemoveCComments(TestStr2))

 

         TestStr3 = """ 'TestString' """

-        self.assertEquals(TestStr3, RemoveCComments(TestStr3))

+        self.assertEqual(TestStr3, RemoveCComments(TestStr3))

 

         TestStr4 = """

             {CODE({

@@ -35,14 +69,14 @@ class TestRe(unittest.TestCase):
               {0x01, {0x02, 0x03, 0x04 }},

               })

             }"""

-        self.assertEquals(Expect_TestStr4, RemoveCComments(TestStr4).strip())

+        self.assertEqual(Expect_TestStr4, RemoveCComments(TestStr4).strip())

 

     def Test_ArrayIndex(self):

         TestStr1 = """[1]"""

-        self.assertEquals(['[1]'], ArrayIndex.findall(TestStr1))

+        self.assertEqual(['[1]'], ArrayIndex.findall(TestStr1))

 

         TestStr2 = """[1][2][0x1][0x01][]"""

-        self.assertEquals(['[1]','[2]','[0x1]','[0x01]','[]'], ArrayIndex.findall(TestStr2))

+        self.assertEqual(['[1]','[2]','[0x1]','[0x01]','[]'], ArrayIndex.findall(TestStr2))

 

 if __name__ == '__main__':

     unittest.main()