#!/usr/bin/env python3 import os import sys import argparse import hashlib import re import yaml whoami = os.path.basename(sys.argv[0]) BANNER = f'''// // This file is automatically generated by {whoami}. // Edits will be automatically overwritten if the build is // run in maintainer mode. //''' def warn(*args, **kwargs): print(*args, file=sys.stderr, **kwargs) class Main: SOURCES = [ whoami, 'manual/_ext/qpdf.py', 'job.yml', 'manual/cli.rst', ] DESTS = { 'decl': 'libqpdf/qpdf/auto_job_decl.hh', 'init': 'libqpdf/qpdf/auto_job_init.hh', 'help': 'libqpdf/qpdf/auto_job_help.hh', } SUMS = 'job.sums' def main(self, args=sys.argv[1:], prog=whoami): options = self.parse_args(args, prog) self.top(options) def parse_args(self, args, prog): parser = argparse.ArgumentParser( prog=prog, description='Generate files for QPDFJob', ) mxg = parser.add_mutually_exclusive_group(required=True) mxg.add_argument('--check', help='update checksums if files are not up to date', action='store_true', default=False) mxg.add_argument('--generate', help='generate files from sources', action='store_true', default=False) return parser.parse_args(args) def top(self, options): if options.check: self.check() elif options.generate: self.generate() else: exit(f'{whoami} unknown mode') def get_hashes(self): hashes = {} for i in sorted([*self.SOURCES, *self.DESTS.values()]): m = hashlib.sha256() try: with open(i, 'rb') as f: m.update(f.read()) hashes[i] = m.hexdigest() except FileNotFoundError: pass return hashes def check(self): hashes = self.get_hashes() match = False try: old_hashes = {} with open(self.SUMS, 'r') as f: for line in f.readlines(): m = re.match(r'^(\S+) (\S+)\s*$', line) if m: old_hashes[m.group(1)] = m.group(2) match = old_hashes == hashes except Exception: pass if not match: exit(f'{whoami}: auto job inputs have changed') def update_hashes(self): hashes = self.get_hashes() with open(self.SUMS, 'w') as f: print(f'# Generated by {whoami}', file=f) for k, v in hashes.items(): print(f'{k} {v}', file=f) def generate_doc(self, df, f): st_top = 0 st_topic = 1 st_option = 2 st_option_help = 3 state = st_top indent = None topic = None option = None short_text = None long_text = None # Generate a bunch of short static functions rather than a big # member function for help. Some compilers have problems with # very large member functions in classes in anonymous # namespaces. help_files = 0 help_lines = 0 self.all_topics = set(self.options_without_help) self.referenced_topics = set() def set_indent(x): nonlocal indent indent = ' ' * len(x) def append_long_text(line, topic): nonlocal indent, long_text if line == '\n': long_text += '\n' elif line.startswith(indent): long_text += line[len(indent):] else: long_text = long_text.strip() if long_text == '': raise Exception(f'missing long text for {topic}') long_text += '\n' for i in re.finditer(r'--help=([^\.\s]+)', long_text): self.referenced_topics.add(i.group(1)) return True return False lineno = 0 for line in df.readlines(): if help_lines == 0: if help_files > 0: print('}', file=f) help_files += 1 help_lines += 1 print(f'static void add_help_{help_files}(QPDFArgParser& ap)\n' '{', file=f) lineno += 1 if state == st_top: m = re.match(r'^(\s*\.\. )help-topic (\S+): (.*)$', line) if m: set_indent(m.group(1)) topic = m.group(2) short_text = m.group(3) long_text = '' state = st_topic continue m = re.match( r'^(\s*\.\. )qpdf:option:: (([^=\s]+)([= ](.+))?)$', line) if m: if topic is None: raise Exception('option seen before topic') set_indent(m.group(1)) option = m.group(3) synopsis = m.group(2) if synopsis.endswith('`'): raise Exception( f'stray ` at end of option line (line {lineno})') if synopsis != option: long_text = synopsis + '\n' else: long_text = '' state = st_option continue elif state == st_topic: if append_long_text(line, topic): self.all_topics.add(topic) print(f'ap.addHelpTopic("{topic}", "{short_text}",' f' R"({long_text})");', file=f) help_lines += 1 state = st_top elif state == st_option: if line == '\n' or line.startswith(indent): m = re.match(r'^(\s*\.\. )help: (.*)$', line) if m: set_indent(m.group(1)) short_text = m.group(2) state = st_option_help else: raise Exception('option without help text') state = st_top elif state == st_option_help: if append_long_text(line, option): if option in self.options_without_help: self.options_without_help.remove(option) else: raise Exception( f'help for unknown option {option},' f' lineno={lineno}') print(f'ap.addOptionHelp("{option}", "{topic}",' f' "{short_text}", R"({long_text})");', file=f) help_lines += 1 state = st_top if help_lines == 20: help_lines = 0 print('}', file=f) print('static void add_help(QPDFArgParser& ap)\n{', file=f) for i in range(help_files): print(f' add_help_{i+1}(ap);', file=f) print('ap.addHelpFooter("For detailed help, visit' ' the qpdf manual: https://qpdf.readthedocs.io\\n");', file=f) print('}\n', file=f) for i in self.referenced_topics: if i not in self.all_topics: raise Exception(f'help text referenced --help={i}') for i in self.options_without_help: raise Exception( 'Options without help: ' + ', '.join(self.options_without_help)) def generate(self): warn(f'{whoami}: regenerating auto job files') with open('job.yml', 'r') as f: data = yaml.safe_load(f.read()) self.validate(data) self.options_without_help = set( ['--completion-bash', '--completion-zsh', '--help'] ) with open(self.DESTS['decl'], 'w') as f: print(BANNER, file=f) self.generate_decl(data, f) with open(self.DESTS['init'], 'w') as f: print(BANNER, file=f) self.generate_init(data, f) with open(self.DESTS['help'], 'w') as f: with open('manual/cli.rst', 'r') as df: print(BANNER, file=f) self.generate_doc(df, f) # Update hashes last to ensure that this will be rerun in the # event of a failure. self.update_hashes() # DON'T ADD CODE TO generate AFTER update_hashes def check_keys(self, what, d, exp): if not isinstance(d, dict): exit(f'{what} is not a dictionary') actual = set(d.keys()) extra = actual - exp if extra: exit(f'{what}: unknown keys = {extra}') def validate(self, data): self.check_keys('top', data, set(['choices', 'options'])) for o in data['options']: self.check_keys('top', o, set( ['table', 'prefix', 'bare', 'positional', 'optional_parameter', 'required_parameter', 'required_choices', 'optional_choices', 'from_table'])) def to_identifier(self, label, prefix, const): identifier = re.sub(r'[^a-zA-Z0-9]', '_', label) if const: identifier = identifier.upper() else: identifier = identifier.lower() identifier = re.sub(r'(?:^|_)([a-z])', lambda x: x.group(1).upper(), identifier).replace('_', '') return prefix + identifier def generate_decl(self, data, f): for o in data['options']: table = o['table'] if table in ('main', 'help'): continue i = self.to_identifier(table, 'O_', True) print(f'static constexpr char const* {i} = "{table}";', file=f) print('', file=f) for o in data['options']: table = o['table'] prefix = 'arg' + o.get('prefix', '') if o.get('positional', False): print(f'void {prefix}Positional(char*);', file=f) for i in o.get('bare', []): identifier = self.to_identifier(i, prefix, False) print(f'void {identifier}();', file=f) for i in o.get('optional_parameter', []): identifier = self.to_identifier(i, prefix, False) print(f'void {identifier}(char *);', file=f) for i in o.get('required_parameter', {}): identifier = self.to_identifier(i, prefix, False) print(f'void {identifier}(char *);', file=f) for i in o.get('required_choices', {}): identifier = self.to_identifier(i, prefix, False) print(f'void {identifier}(char *);', file=f) for i in o.get('optional_choices', {}): identifier = self.to_identifier(i, prefix, False) print(f'void {identifier}(char *);', file=f) if table not in ('main', 'help'): identifier = self.to_identifier(table, 'argEnd', False) print(f'void {identifier}();', file=f) def generate_init(self, data, f): print('auto b = [this](void (ArgParser::*f)()) {', file=f) print(' return QPDFArgParser::bindBare(f, this);', file=f) print('};', file=f) print('auto p = [this](void (ArgParser::*f)(char *)) {', file=f) print(' return QPDFArgParser::bindParam(f, this);', file=f) print('};', file=f) print('', file=f) for k, v in data['choices'].items(): print(f'char const* {k}_choices[] = {{', file=f, end='') for i in v: print(f'"{i}", ', file=f, end='') print('0};', file=f) print('', file=f) for o in data['options']: table = o['table'] if table == 'main': print('this->ap.selectMainOptionTable();', file=f) elif table == 'help': print('this->ap.selectHelpOptionTable();', file=f) else: identifier = self.to_identifier(table, 'argEnd', False) print(f'this->ap.registerOptionTable("{table}",' f' b(&ArgParser::{identifier}));', file=f) prefix = 'arg' + o.get('prefix', '') if o.get('positional', False): print('this->ap.addPositional(' f'p(&ArgParser::{prefix}Positional));', file=f) for i in o.get('bare', []): self.options_without_help.add(f'--{i}') identifier = self.to_identifier(i, prefix, False) print(f'this->ap.addBare("{i}", ' f'b(&ArgParser::{identifier}));', file=f) for i in o.get('optional_parameter', []): self.options_without_help.add(f'--{i}') identifier = self.to_identifier(i, prefix, False) print(f'this->ap.addOptionalParameter("{i}", ' f'p(&ArgParser::{identifier}));', file=f) for k, v in o.get('required_parameter', {}).items(): self.options_without_help.add(f'--{k}') identifier = self.to_identifier(k, prefix, False) print(f'this->ap.addRequiredParameter("{k}", ' f'p(&ArgParser::{identifier})' f', "{v}");', file=f) for k, v in o.get('required_choices', {}).items(): self.options_without_help.add(f'--{k}') identifier = self.to_identifier(k, prefix, False) print(f'this->ap.addChoices("{k}", ' f'p(&ArgParser::{identifier})' f', true, {v}_choices);', file=f) for k, v in o.get('optional_choices', {}).items(): self.options_without_help.add(f'--{k}') identifier = self.to_identifier(k, prefix, False) print(f'this->ap.addChoices("{k}", ' f'p(&ArgParser::{identifier})' f', false, {v}_choices);', file=f) for o in data['options']: table = o['table'] if 'from_table' not in o: continue if table == 'main': print('this->ap.selectMainOptionTable();', file=f) elif table == 'help': print('this->ap.selectHelpOptionTable();', file=f) else: print(f'this->ap.selectOptionTable("{table}");', file=f) ft = o['from_table'] other_table = ft['table'] for j in ft['options']: print('this->ap.copyFromOtherTable' f'("{j}", "{other_table}");', file=f) if __name__ == '__main__': try: os.chdir(os.path.dirname(os.path.realpath(__file__))) Main().main() except KeyboardInterrupt: exit(130)