| #!/usr/bin/env python3 |
| # Copyright (c) 2016 Google Inc. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| """Generates compressed grammar tables from SPIR-V JSON grammar.""" |
| |
| import errno |
| import json |
| import os.path |
| import re |
| import sys |
| from typing import Dict, List, Tuple, Any |
| |
| # Find modules relative to the directory containing this script. |
| # This is needed for hermetic Bazel builds, where the Table files are bundled |
| # together with this script, while keeping their relative locations. |
| sys.path.append(os.path.dirname(os.path.abspath(__file__))) |
| |
| from Table.Context import Context |
| from Table.IndexRange import IndexRange |
| from Table.Operand import Operand |
| |
| class GrammarError(Exception): |
| pass |
| |
| # Extensions to recognize, but which don't necessarily come from the SPIR-V |
| # core or KHR grammar files. Get this list from the SPIR-V registry web page. |
| # NOTE: Only put things on this list if it is not in those grammar files. |
| EXTENSIONS_FROM_SPIRV_REGISTRY_AND_NOT_FROM_GRAMMARS = """ |
| SPV_AMD_gpu_shader_half_float |
| SPV_AMD_gpu_shader_int16 |
| SPV_KHR_non_semantic_info |
| SPV_EXT_relaxed_printf_string_address_space |
| """ |
| |
| class ExtInst(): |
| """ |
| An extended instruction set. |
| |
| Properties: |
| prefix: the string prefix for operand enums. Often an empty string. |
| file: the location of the JSON grammar file |
| name: the name, can be used as an identifier |
| enum_name: the enum name, e.g. SPV_EXT_INST_OPENCL_STD |
| grammar: the JSON object for the grammar, loaded from the file. |
| """ |
| def __init__(self,spec: str): |
| matches = re.fullmatch('^([^,]*),(.*)',spec) |
| if matches is None: |
| raise Exception("Invalid prefix and path: {}".format(spec)) |
| self.prefix = matches[1] |
| if self.prefix is None: |
| self.prefix = "" |
| self.file = matches[2] |
| matches = re.match('.*extinst\\.(.*)\\.grammar.json', self.file) |
| if matches is None: |
| raise Exception("Invalid grammar file name: {}".format(self.file)) |
| self.name = matches[1].replace('-','_').replace('.','_') |
| |
| self.enum_name = 'SPV_EXT_INST_TYPE_{}'.format(self.name).upper() |
| if self.enum_name == "SPV_EXT_INST_TYPE_OPENCL_STD_100": |
| # Live with an old decision, by adjusting this name. |
| self.enum_name = "SPV_EXT_INST_TYPE_OPENCL_STD" |
| |
| self.load() |
| |
| def load(self): |
| """ |
| Populates self.grammar from the file. |
| Applies the self.prefix to operand enums |
| """ |
| with open(self.file) as json_file: |
| self.grammar = json.loads(json_file.read()) |
| if len(self.prefix) > 0: |
| prefix_operand_kind_names(self.prefix, self.grammar) |
| |
| |
| def convert_min_required_version(version): # (version: str | None) -> str |
| """Converts the minimal required SPIR-V version encoded in the grammar to |
| the symbol in SPIRV-Tools.""" |
| if version is None: |
| return 'SPV_SPIRV_VERSION_WORD(1, 0)' |
| if version == 'None': |
| return '0xffffffffu' |
| return 'SPV_SPIRV_VERSION_WORD({})'.format(version.replace('.', ',')) |
| |
| |
| def convert_max_required_version(version): # (version: str | None) -> str |
| """Converts the maximum required SPIR-V version encoded in the grammar to |
| the symbol in SPIRV-Tools.""" |
| if version is None: |
| return '0xffffffffu' |
| return 'SPV_SPIRV_VERSION_WORD({})'.format(version.replace('.', ',')) |
| |
| |
| def c_bool(b: bool) -> str: |
| return 'true' if b else 'false' |
| |
| |
| def ctype(kind: str, quantifier: str) -> str: |
| """Returns the corresponding operand type used in spirv-tools for the given |
| operand kind and quantifier used in the JSON grammar. |
| |
| Arguments: |
| - kind, e.g. 'IdRef' |
| - quantifier, e.g. '', '?', '*' |
| |
| Returns: |
| a string of the enumerant name in spv_operand_type_t |
| """ |
| if kind == '': |
| raise Error("operand JSON object missing a 'kind' field") |
| # The following cases are where we differ between the JSON grammar and |
| # spirv-tools. |
| if kind == 'IdResultType': |
| kind = 'TypeId' |
| elif kind == 'IdResult': |
| kind = 'ResultId' |
| elif kind == 'IdMemorySemantics' or kind == 'MemorySemantics': |
| kind = 'MemorySemanticsId' |
| elif kind == 'IdScope' or kind == 'Scope': |
| kind = 'ScopeId' |
| elif kind == 'IdRef': |
| kind = 'Id' |
| |
| elif kind == 'ImageOperands': |
| kind = 'Image' |
| elif kind == 'Dim': |
| kind = 'Dimensionality' |
| elif kind == 'ImageFormat': |
| kind = 'SamplerImageFormat' |
| elif kind == 'KernelEnqueueFlags': |
| kind = 'KernelEnqFlags' |
| |
| elif kind == 'LiteralExtInstInteger': |
| kind = 'ExtensionInstructionNumber' |
| elif kind == 'LiteralSpecConstantOpInteger': |
| kind = 'SpecConstantOpNumber' |
| elif kind == 'LiteralContextDependentNumber': |
| kind = 'TypedLiteralNumber' |
| |
| elif kind == 'PairLiteralIntegerIdRef': |
| kind = 'LiteralIntegerId' |
| elif kind == 'PairIdRefLiteralInteger': |
| kind = 'IdLiteralInteger' |
| elif kind == 'PairIdRefIdRef': # Used by OpPhi in the grammar |
| kind = 'Id' |
| |
| if kind == 'FPRoundingMode': |
| kind = 'FpRoundingMode' |
| elif kind == 'FPFastMathMode': |
| kind = 'FpFastMathMode' |
| |
| if quantifier == '?': |
| kind = 'Optional{}'.format(kind) |
| elif quantifier == '*': |
| kind = 'Variable{}'.format(kind) |
| |
| return 'SPV_OPERAND_TYPE_{}'.format( |
| re.sub(r'([a-z])([A-Z])', r'\1_\2', kind).upper()) |
| |
| |
| def convert_operand_kind(obj: Dict[str, str]) -> str: |
| """Returns the corresponding operand type used in spirv-tools for the given |
| operand kind and quantifier used in the JSON grammar. |
| |
| Arguments: |
| - obj: an instruction operand, having keys: |
| - 'kind', e.g. 'IdRef' |
| - optionally, a quantifier: '?' or '*' |
| |
| Returns: |
| a string of the enumerant name in spv_operand_type_t |
| """ |
| kind = obj.get('kind', '') |
| quantifier = obj.get('quantifier', '') |
| return ctype(kind, quantifier) |
| |
| |
| def to_safe_identifier(s: str) -> str: |
| """ |
| Returns a new string with all non-letters converted to underscores, |
| and prepending 'k'. |
| The result should be safe to use as a C identifier. |
| """ |
| return 'k' + re.sub(r'[^a-zA-Z0-9]', '_', s) |
| |
| |
| class Grammar(): |
| """ |
| Accumulates string and enum tables. |
| The extensions and operand kinds lists are fixed at creation time. |
| Prints tables for instructions, operand kinds, and underlying string |
| and enum tables. |
| Assumes an index range is emitted by printing an IndexRange object. |
| """ |
| def __init__(self, extensions: List[str], operand_kinds:List[dict], printing_classes: List[str]) -> None: |
| self.context = Context() |
| self.extensions = sorted(extensions) |
| self.operand_kinds = sorted(operand_kinds, key = lambda ok: convert_operand_kind(ok)) |
| self.printing_classes = sorted([to_safe_identifier(x) for x in printing_classes]) |
| |
| # The self.header_ignore_decls are only used to debug the flow. |
| # They are copied into the C++ source code where they are more likely |
| # to be seen by humans. |
| self.header_ignore_decls: List[str] = [self.IndexRangeDecls()] |
| |
| # The self.header_decls content goes into core_tables_header.inc to be |
| # included in a .h file. |
| self.header_decls: List[str] = [] |
| # The self.body_decls content goes into core_tables_body.inc to be included |
| # in a .cpp file. It includes definitions of static variables and |
| # hidden functions. |
| self.body_decls: List[str] = [] |
| |
| if len(self.operand_kinds) == 0: |
| raise Exception("operand_kinds should be a non-empty list") |
| if len(self.extensions) == 0: |
| raise Exception("extensions should be a non-empty list") |
| |
| self.ComputePrintingClassDecls() |
| self.ComputeExtensionDecls() |
| |
| # These operand kinds need to have their optional counterpart to also |
| # be represented in the lookup tables, with the same content. |
| self.operand_kinds_needing_optional_variant = [ |
| 'ImageOperands', |
| 'AccessQualifier', |
| 'MemoryAccess', |
| 'PackedVectorFormat', |
| 'CooperativeMatrixOperands', |
| 'MatrixMultiplyAccumulateOperands', |
| 'RawAccessChainOperands', |
| 'FPEncoding', |
| 'TensorOperands', |
| 'Capability'] |
| |
| def dump(self) -> None: |
| self.context.dump() |
| |
| def IndexRangeDecls(self) -> str: |
| return """ |
| struct IndexRange { |
| uint32_t first = 0; // index of the first element in the range |
| uint32_t count = 0; // number of elements in the range |
| }; |
| constexpr inline IndexRange IR(uint32_t first, uint32_t count) { |
| return {first, count}; |
| } |
| """ |
| |
| def ComputePrintingClassDecls(self) -> str: |
| parts: List[str] = [] |
| parts.append("enum class PrintingClass : uint32_t {"); |
| parts.extend([" {},".format(x) for x in self.printing_classes]) |
| parts.append("};\n") |
| self.header_decls.extend(parts) |
| |
| def ComputeExtensionDecls(self) -> None: |
| parts: List[str] = [] |
| parts.append("enum Extension : uint32_t {"); |
| parts.extend([" {},".format(to_safe_identifier(x)) for x in self.extensions]) |
| parts.append("};\n") |
| self.header_decls.extend(parts) |
| |
| parts = [] |
| parts.append("// Returns the name of an extension, as an index into kStrings") |
| parts.append("IndexRange ExtensionToIndexRange(Extension extension) {\n switch(extension) {") |
| for e in self.extensions: |
| parts.append(' case Extension::k{}: return {};'.format(e,self.context.AddString(e))) |
| parts.append(" default: break;"); |
| parts.append(' }\n return {};\n}\n'); |
| self.body_decls.extend(parts) |
| |
| parts = [] |
| parts.append("""// Extension names and values, ordered by name |
| // The fields in order are: |
| // name, indexing into kStrings |
| // enum value""") |
| parts.append("static const std::array<NameValue,{}> kExtensionNames{{{{".format(len(self.extensions))) |
| for e in self.extensions: |
| parts.append(' {{{}, static_cast<uint32_t>({})}},'.format(self.context.AddString(e), to_safe_identifier(e))) |
| parts.append("}};\n") |
| self.body_decls.extend(parts) |
| |
| def ComputeOperandTables(self) -> None: |
| """ |
| Returns the string for the C definitions of the operand kind tables. |
| |
| An operand kind such as ImageOperands also has an associated |
| operand kind that is an 'optional' variant. |
| These are represented as two distinct operand kinds in spv_operand_type_t. |
| For example, ImageOperands maps to both SPV_OPERAND_TYPE_IMAGE, and also |
| to SPV_OPERAND_TYPE_OPTIONAL_IMAGE. |
| |
| The definitions are: |
| - kOperandsByValue: a 1-dimensional array of all operand descriptions |
| sorted first by operand kind, then by operand value. |
| Only non-optional operand kinds are represented here. |
| |
| - kOperandsByValueRangeByKind: a function mapping from operand kind to |
| the index range into kOperandByValue. |
| This has mappings for both concrete and corresponding optional operand kinds. |
| |
| - kOperandNames: a 1-dimensional array of all operand NameIndex |
| entries, sorted first by operand kinds, then by operand name. |
| The name part is represented by an index range into the string table. |
| The index part is the index of this name's entry into the by-value array. |
| This can have more entries than the by-value array, because names |
| can have string aliases. For example,the MemorySemantics value 0 |
| is named both "Relaxed" and "None". |
| Only non-optional operand kinds are represented here. |
| |
| - kOperandNamesRangeByKind: a mapping from operand kind to the index |
| range into kOperandNames. |
| This has mappings for both concrete and corresponding optional operand kinds. |
| """ |
| |
| self.header_ignore_decls.append( |
| """ |
| struct NameIndex { |
| // Location of the null-terminated name in the global string table. |
| IndexRange name; |
| // Index of this name's entry in in the associated by-value table. |
| uint32_t index; |
| }; |
| struct NameValue { |
| // Location of the null-terminated name in the global string table. |
| IndexRange name; |
| // Enum value in the binary format. |
| uint32_t value; |
| }; |
| // Describes a SPIR-V operand. |
| struct OperandDesc { |
| uint32_t value; |
| IndexRange operands_range; // Indexes kOperandSpans |
| IndexRange name_range; // Indexes kStrings |
| IndexRange aliases_range; // Indexes kAliasSpans |
| IndexRange capabilities_range; // Indexes kCapabilitySpans |
| // A set of extensions that enable this feature. If empty then this operand |
| // value is in core and its availability is subject to minVersion. The |
| // assembler, binary parser, and disassembler ignore this rule, so you can |
| // freely process invalid modules. |
| IndexRange extensions_range; // Indexes kExtensionSpans |
| // Minimal core SPIR-V version required for this feature, if without |
| // extensions. ~0u means reserved for future use. ~0u and non-empty |
| // extension lists means only available in extensions. |
| uint32_t minVersion; |
| uint32_t lastVersion; |
| utils::Span<spv_operand_type_t> operands() const; |
| utils::Span<char> name() const; |
| utils::Span<IndexRange> aliases() const; |
| utils::Span<spv::Capability> capabilities() const; |
| utils::Span<spvtools::Extension> extensions() const; |
| OperandDesc(const OperandDesc&) = delete; |
| OperandDesc(OperandDesc&&) = delete; |
| }; |
| """) |
| |
| def ShouldEmit(operand_kind_json: Dict[str,any]): |
| """ Returns true if we should emit a table for the given |
| operand kind. |
| """ |
| category = operand_kind_json.get('category') |
| return category in ['ValueEnum', 'BitEnum'] |
| |
| # Populate kOperandsByValue |
| operands_by_value: List[str] = [] |
| operands_by_value_by_kind: Dict[str,IndexRange] = {} |
| # Maps the operand kind and value to the index into kOperandsByValue |
| index_by_kind_and_value: Dict[Tuple(str,int),int] = {} |
| index = 0 |
| for operand_kind_json in self.operand_kinds: |
| kind_key: str = convert_operand_kind(operand_kind_json) |
| if ShouldEmit(operand_kind_json): |
| operands = [Operand(o) for o in operand_kind_json['enumerants']] |
| operand_descs: List[str] = [] |
| for o in sorted(operands, key = lambda o: o.value): |
| suboperands = [convert_operand_kind(p) for p in o.parameters] |
| desc = [ |
| o.value, |
| self.context.AddStringList('operand', suboperands), |
| str(self.context.AddString(o.enumerant)) + '/* {} */'.format(o.enumerant), |
| self.context.AddStringList('alias', o.aliases), |
| self.context.AddStringList('capability', o.capabilities), |
| self.context.AddStringList('extension', o.extensions), |
| convert_min_required_version(o.version), |
| convert_max_required_version(o.lastVersion), |
| ] |
| operand_descs.append('{' + ','.join([str(d) for d in desc]) + '}}, // {}'.format(kind_key)) |
| index_by_kind_and_value[(kind_key,o.value)] = index |
| index += 1 |
| operands_by_value_by_kind[kind_key] = IndexRange(len(operands_by_value), len(operand_descs)) |
| operands_by_value.extend(operand_descs) |
| else: |
| pass |
| |
| parts = [] |
| parts.append("""// Operand descriptions, ordered by (operand kind, operand enum value). |
| // The fields in order are: |
| // enum value |
| // operands, an IndexRange into kOperandSpans |
| // name, a character-counting IndexRange into kStrings |
| // aliases, an IndexRange into kAliasSpans |
| // capabilities, an IndexRange into kCapabilitySpans |
| // extensions, as an IndexRange into kExtensionSpans |
| // version, first version of SPIR-V that has it |
| // lastVersion, last version of SPIR-V that has it""") |
| parts.append("static const std::array<OperandDesc, {}> kOperandsByValue{{{{".format(len(operands_by_value))) |
| parts.extend([' ' + str(x) for x in operands_by_value]) |
| parts.append("}};\n") |
| self.body_decls.extend(parts) |
| |
| parts = [] |
| parts.append("""// Maps an operand kind to possible operands for that kind. |
| // The result is an IndexRange into kOperandsByValue, and the operands |
| // are sorted by value within that span. |
| // An optional variant of a kind maps to the details for the corresponding |
| // concrete operand kind.""") |
| parts.append("IndexRange OperandByValueRangeForKind(spv_operand_type_t type) {\n switch(type) {") |
| for kind_key, ir in operands_by_value_by_kind.items(): |
| parts.append(" case {}: return {};".format( |
| kind_key, |
| str(operands_by_value_by_kind[kind_key]))) |
| for kind in self.operand_kinds_needing_optional_variant: |
| non_optional_kind = ctype(kind,'') |
| if non_optional_kind in operands_by_value_by_kind: |
| parts.append(" case {}: return {};".format( |
| ctype(kind, '?'), |
| str(operands_by_value_by_kind[ctype(kind,'')]))) |
| else: |
| raise GrammarError( |
| "error: unknown operand type {}, from JSON grammar operand '{}':".format(non_optional_kind, kind) + |
| " consider updating spv_operand_type_t in spirv-tools/libspirv.h") |
| |
| parts.append(" default: break;"); |
| parts.append(" }\n return IR(0,0);\n}\n") |
| self.body_decls.extend(parts) |
| |
| # Populate kOperandNames |
| operand_names: List[Tuple[IndexRange,int]] = [] |
| name_range_for_kind: Dict[str,IndexRange] = {} |
| for operand_kind_json in self.operand_kinds: |
| kind_key: str = convert_operand_kind(operand_kind_json) |
| if ShouldEmit(operand_kind_json): |
| operands = [Operand(o) for o in operand_kind_json['enumerants']] |
| tuples: List[Tuple[str,int,str]] = [] |
| for o in operands: |
| tuples.append((o.enumerant, o.value, kind_key)) |
| for a in o.aliases: |
| tuples.append((a, o.value, kind_key)) |
| tuples = sorted(tuples, key = lambda t: t[0]) |
| ir_tuples = [(self.context.AddString(t[0]),t[1],t[2]) for t in tuples] |
| name_range_for_kind[kind_key] = IndexRange(len(operand_names), len(ir_tuples)) |
| operand_names.extend(ir_tuples) |
| else: |
| pass |
| operand_name_strings: List[str] = [] |
| for i in range(0, len(operand_names)): |
| ir, value, kind_key = operand_names[i] |
| index = index_by_kind_and_value[(kind_key,value)] |
| operand_name_strings.append('{{{}, {}}}, // {} {} in {}'.format( |
| str(ir),index,i,self.context.GetString(ir),kind_key)) |
| |
| parts: List[str] = [] |
| parts.append("""// Operand names and index into kOperandsByValue, ordered by (operand kind, name) |
| // The fields in order are: |
| // name, either the primary name or an alias, indexing into kStrings |
| // index into the kOperandsByValue array""") |
| parts.append("static const std::array<NameIndex, {}> kOperandNames{{{{".format(len(operand_name_strings))) |
| parts.extend([' ' + str(x) for x in operand_name_strings]) |
| parts.append("}};\n") |
| self.body_decls.extend(parts) |
| |
| parts.append("""// Maps an operand kind to possible names for operands of that kind. |
| // The result is an IndexRange into kOperandNames, and the names |
| // are sorted by name within that span. |
| // An optional variant of a kind maps to the details for the corresponding |
| // concrete operand kind.""") |
| parts = ["IndexRange OperandNameRangeForKind(spv_operand_type_t type) {\n switch(type) {"] |
| for kind_key, ir in name_range_for_kind.items(): |
| parts.append(" case {}: return {};".format( |
| kind_key, |
| str(name_range_for_kind[kind_key]))) |
| for kind in self.operand_kinds_needing_optional_variant: |
| parts.append(" case {}: return {};".format( |
| ctype(kind, '?'), |
| str(name_range_for_kind[ctype(kind,'')]))) |
| parts.append(" default: break;"); |
| parts.append(" }\n return IR(0,0);\n}\n") |
| self.body_decls.extend(parts) |
| |
| |
| def ComputeInstructionTables(self, insts) -> None: |
| """ |
| Creates declarations for instruction tables. |
| Populates self.header_ignore_decls, self.body_decls. |
| |
| Params: |
| insts: an array of instructions objects using the JSON schema |
| """ |
| self.header_ignore_decls.append(""" |
| // Describes an Instruction |
| struct InstructionDesc { |
| const spv::Op value; |
| const bool hasResult; |
| const bool hasType; |
| const IndexRange operands_range; // Indexes kOperandSpans |
| const IndexRange name_range; // Indexes kStrings |
| const IndexRange aliases_range; // Indexes kAliasSpans |
| const IndexRange capabilities_range; // Indexes kCapbilitySpans |
| // A set of extensions that enable this feature. If empty then this operand |
| // value is in core and its availability is subject to minVersion. The |
| // assembler, binary parser, and disassembler ignore this rule, so you can |
| // freely process invalid modules. |
| const IndexRange extensions_range; // Indexes kExtensionSpans |
| // Minimal core SPIR-V version required for this feature, if without |
| // extensions. ~0u means reserved for future use. ~0u and non-empty |
| // extension lists means only available in extensions. |
| uint32_t minVersion; |
| uint32_t lastVersion; |
| PrintingClass printingClass; // Section of SPIR-V spec. e.g. kComposite, kImage |
| utils::Span<spv_operand_type_t> operands() const; |
| utils::Span<char> name() const; |
| utils::Span<IndexRange> aliases() const; |
| utils::Span<spv::Capability> capabilities() const; |
| utils::Span<spvtools::Extension> extensions() const; |
| OperandDesc(const OperandDesc&) = delete; |
| OperandDesc(OperandDesc&&) = delete; |
| }; |
| """) |
| |
| # Create the array of InstructionDesc |
| lines: List[str] = [] |
| # Maps the opcode name (without "Op" prefix) to its index in the table. |
| index_by_opcode: Dict[int,int] = {} |
| # Sort by opcode, so lookup can use binary search |
| for inst in sorted(insts, key = lambda inst: int(inst['opcode'])): |
| parts: List[str] = [] |
| |
| opname: str = inst['opname'] |
| |
| operand_kinds = [convert_operand_kind(o) for o in inst.get('operands',[])] |
| if opname == 'OpExtInst' and operand_kinds[-1] == 'SPV_OPERAND_TYPE_VARIABLE_ID': |
| # The published grammar uses 'sequence of ID' at the |
| # end of the ExtInst operands. But SPIRV-Tools uses |
| # a specific pattern based on the particular opcode. |
| # Drop it here. |
| # See https://github.com/KhronosGroup/SPIRV-Tools/issues/233 |
| operand_kinds.pop() |
| |
| hasResult = 'SPV_OPERAND_TYPE_RESULT_ID' in operand_kinds |
| hasType = 'SPV_OPERAND_TYPE_TYPE_ID' in operand_kinds |
| |
| # Remove the "Op" prefix from opcode alias names |
| aliases = [name[2:] for name in inst.get('aliases',[])] |
| |
| parts.extend([ |
| 'spv::Op::' + opname, |
| c_bool(hasResult), |
| c_bool(hasType), |
| self.context.AddStringList('operand', operand_kinds), |
| self.context.AddString(opname[2:]), |
| self.context.AddStringList('alias', aliases), |
| self.context.AddStringList('capability', inst.get('capabilities',[])), |
| self.context.AddStringList('extension', inst.get('extensions',[])), |
| convert_min_required_version(inst.get('version', None)), |
| convert_max_required_version(inst.get('lastVersion', None)), |
| 'PrintingClass::' + to_safe_identifier(inst.get('class','@exclude')) |
| ]) |
| |
| index_by_opcode[int(inst['opcode'])] = len(lines) |
| lines.append('{{{}}},'.format(', '.join([str(x) for x in parts]))) |
| parts = [] |
| parts.append("""// Instruction descriptions, ordered by opcode. |
| // The fields in order are: |
| // opcode |
| // a boolean indicating if the instruction produces a result ID |
| // a boolean indicating if the instruction result ID has a type |
| // operands, an IndexRange into kOperandSpans |
| // opcode name (without the 'Op' prefix), a character-counting IndexRange into kStrings |
| // aliases, an IndexRange into kAliasSpans |
| // capabilities, an IndexRange into kCapabilitySpans |
| // extensions, as an IndexRange into kExtensionSpans |
| // version, first version of SPIR-V that has it |
| // lastVersion, last version of SPIR-V that has it""") |
| parts.append("static const std::array<InstructionDesc, {}> kInstructionDesc{{{{".format(len(lines))); |
| parts.extend([' ' + l for l in lines]) |
| parts.append("}};\n"); |
| self.body_decls.extend(parts) |
| |
| # Create kInstructionNames. |
| opcode_name_entries: List[str] = [] |
| name_value_pairs: List[Tuple[str,int]] = [] |
| for i in insts: |
| name_value_pairs.append((i['opname'][2:], i['opcode'])) |
| for a in i.get('aliases',[]): |
| name_value_pairs.append((a[2:], i['opcode'])) |
| name_value_pairs = sorted(name_value_pairs) |
| inst_name_strings: List[str] = [] |
| for i in range(0, len(name_value_pairs)): |
| name, value = name_value_pairs[i] |
| ir = self.context.AddString(name) |
| index = index_by_opcode[value] |
| inst_name_strings.append('{{{}, {}}}, // {} {}'.format(str(ir),index,i,name)) |
| parts: List[str] = [] |
| parts.append("""// Opcode strings (without the 'Op' prefix) and opcode values, ordered by name. |
| // The fields in order are: |
| // name, either the primary name or an alias, indexing into kStrings |
| // index into kInstructionDesc""") |
| parts.append("static const std::array<NameIndex, {}> kInstructionNames{{{{".format(len(inst_name_strings))) |
| parts.extend([' ' + str(x) for x in inst_name_strings]) |
| parts.append("}};\n") |
| self.body_decls.extend(parts) |
| |
| |
| def ComputeExtendedInstructions(self, extinsts) -> None: |
| """ |
| Generates tables for extended instructions |
| |
| Args: |
| self |
| extinsts: a list of extinst objects |
| """ |
| |
| """ |
| ExtInstDesc { |
| value: uint32_t |
| name: IndexRange |
| operands: IndexRange |
| capabilities: IndexRange |
| } |
| |
| The definitions are: |
| - kExtInstByValue: a 1-dimensional array of all operand descriptions |
| sorted first by extended instruction enum, then by operand value. |
| |
| - ExtInstByValueRangeForKind: a function mapping from extinst enum to |
| the index range into kExtInstByValue. |
| |
| - kExtInstNames: a 1-dimensional array of all extinst name-index pairs, |
| sorted first by extinst enum, then by operand name. |
| The name part is represented by an index range into the string table. |
| The index part is the index of this name's entry in the kExtInstByValue |
| array. |
| |
| - kExtInstNamesRangeByKind: a mapping from operand kind to the index |
| range into kOperandNames. |
| This has mappings for both concrete and corresponding optional operand kinds. |
| """ |
| |
| # Create kExtInstByValue |
| by_value: List[List[Any]] = [] |
| by_value_by_kind: Dict[str,IndexRange] = {} |
| index_by_kind_and_opcode: Dict[Tuple[str,int],int] = {} |
| index = 0 |
| for e in extinsts: |
| insts_in_set = [] |
| for inst in sorted(e.grammar['instructions'], key = lambda inst: inst['opcode']): |
| operands = [convert_operand_kind(o) for o in inst.get('operands',[])] |
| inst_parts = [ |
| inst['opcode'], |
| self.context.AddStringList('operand', operands), |
| self.context.AddString(inst['opname']), |
| self.context.AddStringList('capability', inst.get('capabilities',[])), |
| ] |
| inst_parts = [str(x) for x in inst_parts] |
| insts_in_set.append(' {{{}}}, // {} in {}'.format( |
| ','.join(inst_parts), inst['opname'], e.name)) |
| index_by_kind_and_opcode[(e.enum_name,int(inst['opcode']))] = index |
| index += 1 |
| by_value_by_kind[e.enum_name] = IndexRange(len(by_value), len(insts_in_set)) |
| by_value.extend(insts_in_set) |
| |
| parts: List[str] = [] |
| parts.append("""// Extended instruction descriptions, ordered by (extinst enum, opcode value). |
| // The fields in order are: |
| // enum value |
| // operands, an IndexRange into kOperandSpans |
| // name, a character-counting IndexRange into kStrings |
| // capabilities, an IndexRange into kCapabilitySpans""") |
| parts.append("static const std::array<ExtInstDesc, {}> kExtInstByValue{{{{".format(len(by_value))) |
| parts.extend(by_value) |
| parts.append('}};\n') |
| self.body_decls.extend(parts) |
| |
| # Create kExtInstByValueRangeForKind |
| parts = [] |
| parts.append("""// Maps an extended instruction enum to possible names for operands of that kind. |
| // The result is an IndexRange into kOperandNames, and the names |
| // are sorted by name within that span. |
| // An optional variant of a kind maps to the details for the corresponding |
| // concrete operand kind.""") |
| parts = ["IndexRange ExtInstByValueRangeForKind(spv_ext_inst_type_t type) {\n switch(type) {"] |
| for name, ir in by_value_by_kind.items(): |
| parts.append(" case {}: return {};".format(name, ir)) |
| parts.append(" default: break;"); |
| parts.append(" }\n return IR(0,0);\n}\n") |
| self.body_decls.extend(parts) |
| |
| # Create kExtInstNames |
| parts = [] |
| by_name: List[List[Any]] = [] |
| by_name_by_kind: Dict[str,IndexRange] = {} |
| for e in extinsts: |
| # Sort by name within a set |
| insts_by_name = sorted(e.grammar['instructions'], key = lambda i: i['opname']) |
| insts_in_set = [] |
| for inst in insts_by_name: |
| index = index_by_kind_and_opcode[(e.enum_name,int(inst['opcode']))] |
| insts_in_set.append( |
| ' {{{}, {}}}, // {} in {}'.format( |
| str(self.context.AddString(inst['opname'])), |
| index, |
| inst['opname'], |
| e.name)) |
| by_name_by_kind[e.enum_name] = IndexRange(len(by_name), len(insts_in_set)) |
| by_name.extend(insts_in_set) |
| parts.append("""// Extended instruction opcode names sorted by extended instruction kind, then opcode name. |
| // The fields in order are: |
| // name |
| // index into kExtInstByValue""") |
| parts.append("static const std::array<NameIndex, {}> kExtInstNames{{{{".format(len(by_name))) |
| parts.extend(by_name) |
| parts.append('}};\n') |
| self.body_decls.extend(parts) |
| |
| # Create kExtInstNameRangeByKind |
| parts = [] |
| parts.append("""// Maps an extended instruction kind to possible names for instructions of that kind. |
| // The result is an IndexRange into kExtInstNames, and the names |
| // are sorted by name within that span.""") |
| parts = ["IndexRange ExtInstNameRangeForKind(spv_ext_inst_type_t type) {\n switch(type) {"] |
| for name, ir in by_name_by_kind.items(): |
| parts.append(" case {}: return {};".format(name, str(ir))) |
| parts.append(" default: break;"); |
| parts.append(" }\n return IR(0,0);\n}\n") |
| self.body_decls.extend(parts) |
| |
| def ComputeLeafTables(self) -> None: |
| """ |
| Generates the tables that the instruction and operand tables point to. |
| The tables are: |
| - the string table |
| - the table of sequences of: |
| - capabilities |
| - extensions |
| - operands |
| |
| This method must be called after computing instruction and operand tables. |
| """ |
| |
| def c_str(s: str): |
| """ |
| Returns the source for a C string literal or the given string, including |
| the explicit null at the end |
| """ |
| return '"{}\\0"'.format(json.dumps(s).strip('"')) |
| |
| parts: List[str] = [] |
| parts.append("// Array of characters, referenced by IndexRanges elsewhere.") |
| parts.append("// Each IndexRange denotes a string.") |
| parts.append('static const char kStrings[] ='); |
| parts.extend([' {} // {}'.format(c_str(s), str(self.context.strings[s])) for s in self.context.string_buffer]) |
| parts.append(';\n'); |
| self.body_decls.extend(parts); |
| |
| parts: List[str] = [] |
| parts.append("""// Array of IndexRanges, where each represents a string by referencing |
| // the kStrings table. |
| // This array contains all sequences of alias strings used in the grammar. |
| // This table is referenced by an IndexRange elsewhere, i.e. by the 'aliases' |
| // field of an instruction or operand description.""") |
| parts.append('static const IndexRange kAliasSpans[] = {'); |
| ranges = self.context.range_buffer['alias'] |
| for i in range(0, len(ranges)): |
| ir = ranges[i] |
| parts.append(' {}, // {} {}'.format(str(ir), i, self.context.GetString(ir))) |
| parts.append('};\n'); |
| self.body_decls.extend(parts); |
| |
| parts = [] |
| parts.append("// Array of capabilities, referenced by IndexRanges elsewhere.") |
| parts.append("// Contains all sequences of capabilities used in the grammar.") |
| parts.append('static const spv::Capability kCapabilitySpans[] = {'); |
| capability_ranges = self.context.range_buffer['capability'] |
| for i in range(0, len(capability_ranges)): |
| ir = capability_ranges[i] |
| cap = self.context.GetString(ir) |
| parts.append(' spv::Capability::{}, // {}'.format(cap, i)) |
| parts.append('};\n'); |
| self.body_decls.extend(parts); |
| |
| parts = [] |
| parts.append("// Array of extensions, referenced by IndexRanges elsewhere.") |
| parts.append("// Contains all sequences of extensions used in the grammar.") |
| parts.append('static const spvtools::Extension kExtensionSpans[] = {'); |
| ranges = self.context.range_buffer['extension'] |
| for i in range(0, len(ranges)): |
| ir = ranges[i] |
| name = self.context.GetString(ir) |
| parts.append(' spvtools::Extension::k{}, // {}'.format(name, i)) |
| parts.append('};\n'); |
| self.body_decls.extend(parts); |
| |
| parts = [] |
| parts.append("// Array of operand types, referenced by IndexRanges elsewhere.") |
| parts.append("// Contains all sequences of operand types used in the grammar.") |
| parts.append('static const spv_operand_type_t kOperandSpans[] = {'); |
| ranges = self.context.range_buffer['operand'] |
| for i in range(0, len(ranges)): |
| ir = ranges[i] |
| name = self.context.GetString(ir) |
| parts.append(' {}, // {}'.format(name, i)) |
| parts.append('};\n'); |
| self.body_decls.extend(parts) |
| |
| |
| def make_path_to_file(f: str) -> None: |
| """Makes all ancestor directories to the given file, if they don't yet |
| exist. |
| |
| Arguments: |
| f: The file whose ancestor directories are to be created. |
| """ |
| dir = os.path.dirname(os.path.abspath(f)) |
| try: |
| os.makedirs(dir) |
| except OSError as e: |
| if e.errno == errno.EEXIST and os.path.isdir(dir): |
| pass |
| else: |
| raise |
| |
| |
| def get_extension_list(instructions, operand_kinds): |
| """Returns extensions as an alphabetically sorted list of strings. |
| |
| Args: |
| instructions: list of instruction objects, using the JSON grammar file schema |
| operand_kinds: list of operand_kind objects, using the JSON grammar file schema |
| """ |
| |
| things_with_an_extensions_field = [item for item in instructions] |
| |
| enumerants = sum([item.get('enumerants', []) |
| for item in operand_kinds], []) |
| |
| things_with_an_extensions_field.extend(enumerants) |
| |
| extensions = sum([item.get('extensions', []) |
| for item in things_with_an_extensions_field |
| if item.get('extensions')], []) |
| |
| for item in EXTENSIONS_FROM_SPIRV_REGISTRY_AND_NOT_FROM_GRAMMARS.split(): |
| # If it's already listed in a grammar, then don't put it in the |
| # special exceptions list. |
| assert item not in extensions, 'Extension %s is already in a grammar file' % item |
| |
| extensions.extend( |
| EXTENSIONS_FROM_SPIRV_REGISTRY_AND_NOT_FROM_GRAMMARS.split()) |
| |
| # Validator would ignore type declaration unique check. Should only be used |
| # for legacy autogenerated test files containing multiple instances of the |
| # same type declaration, if fixing the test by other methods is too |
| # difficult. Shouldn't be used for any other reasons. |
| extensions.append('SPV_VALIDATOR_ignore_type_decl_unique') |
| |
| return sorted(set(extensions)) |
| |
| |
| def prefix_operand_kind_names(prefix, json_dict): |
| """Modifies json_dict, by prefixing all the operand kind names |
| with the given prefix. Also modifies their uses in the instructions |
| to match. |
| """ |
| |
| old_to_new = {} |
| for operand_kind in json_dict["operand_kinds"]: |
| old_name = operand_kind["kind"] |
| new_name = prefix + old_name |
| operand_kind["kind"] = new_name |
| old_to_new[old_name] = new_name |
| |
| for instruction in json_dict["instructions"]: |
| for operand in instruction.get("operands", []): |
| replacement = old_to_new.get(operand["kind"]) |
| if replacement is not None: |
| operand["kind"] = replacement |
| |
| |
| def main(): |
| import argparse |
| parser = argparse.ArgumentParser(description='Generate SPIR-V info tables') |
| |
| parser.add_argument('--spirv-core-grammar', metavar='<path>', |
| type=str, required=False, |
| help='input JSON grammar file for core SPIR-V ' |
| 'instructions') |
| parser.add_argument('--extinst', metavar='<path>', |
| type=str, action='append', required=False, default=None, |
| help='extended instruction info: an enum prefix, then a comma, then' |
| ' the file location of the JSON grammar') |
| |
| parser.add_argument('--core-tables-body-output', metavar='<path>', |
| type=str, required=False, default=None, |
| help='output file for core SPIR-V grammar tables to be included in .cpp') |
| parser.add_argument('--core-tables-header-output', metavar='<path>', |
| type=str, required=False, default=None, |
| help='output file for core SPIR-V grammar tables to be included in .h') |
| |
| args = parser.parse_args() |
| |
| if args.spirv_core_grammar is None: |
| print('error: missing --spirv-core-grammar ') |
| sys.exit(1) |
| if (args.core_tables_body_output is None) and (args.core_tables_header_output is None): |
| print('error: need at least one of --core-tables-body-output --core-tables-header-output ') |
| sys.exit(1) |
| if len(args.extinst) < 1: |
| print('error: missing --extinst ') |
| sys.exit(1) |
| |
| # Load the JSON grammar files. |
| extinsts = sorted([ExtInst(e) for e in args.extinst], key = lambda e: e.name) |
| with open(args.spirv_core_grammar) as json_file: |
| core_grammar = json.loads(json_file.read()) |
| printing_class: List[str] = [e['tag'] for e in core_grammar['instruction_printing_class']] |
| |
| # Collect all operand kinds and instructions, so we can generate |
| # extension lists, capability lists, and alias lists. |
| # Make a copy to avoid polluting the instruction list. |
| instructions = [x for x in core_grammar['instructions']] |
| operand_kinds = [x for x in core_grammar['operand_kinds']] |
| for e in extinsts: |
| instructions.extend(e.grammar.get('instructions',[])) |
| operand_kinds.extend(e.grammar.get('operand_kinds',[])) |
| |
| extensions = get_extension_list(instructions, operand_kinds) |
| |
| g = Grammar(extensions, operand_kinds, printing_class) |
| g.ComputeOperandTables() |
| g.ComputeInstructionTables(core_grammar['instructions']) |
| g.ComputeExtendedInstructions(extinsts) |
| g.ComputeLeafTables() |
| |
| if args.core_tables_body_output is not None: |
| make_path_to_file(args.core_tables_body_output) |
| with open(args.core_tables_body_output, 'w') as f: |
| f.write('\n'.join(g.body_decls)) |
| if args.core_tables_header_output is not None: |
| make_path_to_file(args.core_tables_header_output) |
| with open(args.core_tables_header_output, 'w') as f: |
| f.write('\n'.join(g.header_decls)) |
| sys.exit(0) |
| |
| |
| if __name__ == '__main__': |
| try: |
| main() |
| except GrammarError as ge: |
| print(ge) |
| sys.exit(1) |
