admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support running gn_check_autofix.py on a local build dir (e.g. out/Default)

Browse Source 
Bug: webrtc:42226242
Change-Id: I8afedb8c316ab86a7219b07774f93da782f80a4d
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/355000
Commit-Queue: Björn Terelius <terelius@webrtc.org>
Reviewed-by: Harald Alvestrand <hta@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#42513}
This commit is contained in:
Björn Terelius 2024-06-18 15:29:11 +02:00 committed by WebRTC LUCI CQ
parent aefed55c25
commit d5238b0998
1 changed files with 143 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

259
tools_webrtc/gn_check_autofix.py
View File

@ -10,15 +10,19 @@
""" """
This tool tries to fix (some) errors reported by `gn gen --check` or This tool tries to fix (some) errors reported by `gn gen --check` or
`gn check`. `gn check`.
It will run `mb gen` in a temporary directory and it is really useful to If a command line flag `-C out/<dir>` is supplied, it will run `gn gen --check`
check for different configurations. in that directory. Otherwise it will run `mb gen` in a temporary directory
which is useful to check for different configurations.
Usage: Usage:
$ vpython3 tools_webrtc/gn_check_autofix.py -C out/Default
or
$ vpython3 tools_webrtc/gn_check_autofix.py -m some_mater -b some_bot $ vpython3 tools_webrtc/gn_check_autofix.py -m some_mater -b some_bot
or or
$ vpython3 tools_webrtc/gn_check_autofix.py -c some_mb_config $ vpython3 tools_webrtc/gn_check_autofix.py -c some_mb_config
""" """
import argparse
import os import os
import re import re
import shutil import shutil
@ -39,70 +43,71 @@ TARGET_RE = re.compile(
class TemporaryDirectory: class TemporaryDirectory:
def __init__(self):
self._closed = False
self._name = None
self._name = tempfile.mkdtemp()
def __enter__(self): def __init__(self):
return self._name self._closed = False
self._name = None
self._name = tempfile.mkdtemp()
def __exit__(self, exc, value, _tb): def __enter__(self):
if self._name and not self._closed: return self._name
shutil.rmtree(self._name)
self._closed = True def __exit__(self, exc, value, _tb):
if self._name and not self._closed:
shutil.rmtree(self._name)
self._closed = True
def Run(cmd): def Run(cmd):
print('Running:', ' '.join(cmd)) print('Running:', ' '.join(cmd))
sub = subprocess.Popen(cmd, sub = subprocess.Popen(cmd,
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, stderr=subprocess.PIPE,
universal_newlines=True) universal_newlines=True)
return sub.communicate() return sub.communicate()
def FixErrors(filename, missing_deps, deleted_sources): def fix_errors(filename, missing_deps, deleted_sources):
with open(filename) as f: with open(filename) as file:
lines = f.readlines() lines = file.readlines()
fixed_file = '' fixed_file = ''
indentation_level = None indentation_level = None
for line in lines: for line in lines:
match = TARGET_RE.match(line) match = TARGET_RE.match(line)
if match: if match:
target = match.group('target_name') target = match.group('target_name')
if target in missing_deps: if target in missing_deps:
indentation_level = match.group('indentation_level') indentation_level = match.group('indentation_level')
elif indentation_level is not None: elif indentation_level is not None:
match = re.match(indentation_level + '}$', line) match = re.match(indentation_level + '}$', line)
if match: if match:
line = ('deps = [\n' + ''.join(' "' + dep + '",\n' line = ('deps = [\n' + ''.join(' "' + dep + '",\n'
for dep in missing_deps[target]) + for dep in missing_deps[target])
']\n') + line + ']\n') + line
indentation_level = None indentation_level = None
elif line.strip().startswith('deps = ['): elif line.strip().startswith('deps = ['):
joined_deps = ''.join(' "' + dep + '",\n' joined_deps = ''.join(' "' + dep + '",\n'
for dep in missing_deps[target]) for dep in missing_deps[target])
line = line.replace('deps = [', 'deps = [' + joined_deps) line = line.replace('deps = [', 'deps = [' + joined_deps)
indentation_level = None indentation_level = None
if line.strip() not in deleted_sources: if line.strip() not in deleted_sources:
fixed_file += line fixed_file += line
with open(filename, 'w') as f: with open(filename, 'w') as file:
f.write(fixed_file) file.write(fixed_file)
Run(['gn', 'format', filename]) Run(['gn', 'format', filename])
def FirstNonEmpty(iterable): def first_non_empty(iterable):
"""Return first item which evaluates to True, or fallback to None.""" """Return first item which evaluates to True, or fallback to None."""
return next((x for x in iterable if x), None) return next((x for x in iterable if x), None)
def Rebase(base_path, dependency_path, dependency): def rebase(base_path, dependency_path, dependency):
"""Adapt paths so they work both in stand-alone WebRTC and Chromium tree. """Adapt paths so they work both in stand-alone WebRTC and Chromium tree.
To cope with varying top-level directory (WebRTC VS Chromium), we use: To cope with varying top-level directory (WebRTC VS Chromium), we use:
* relative paths for WebRTC modules. * relative paths for WebRTC modules.
@ -119,83 +124,105 @@ def Rebase(base_path, dependency_path, dependency):
Full target path (E.g. '../rtc_base/time:timestamp_extrapolator'). Full target path (E.g. '../rtc_base/time:timestamp_extrapolator').
""" """
root = FirstNonEmpty(dependency_path.split('/')) root = first_non_empty(dependency_path.split('/'))
if root in CHROMIUM_DIRS: if root in CHROMIUM_DIRS:
# Chromium paths must remain absolute. E.g. //third_party//abseil-cpp... # Chromium paths must remain absolute. E.g.
rebased = dependency_path # //third_party//abseil-cpp...
else: rebased = dependency_path
base_path = base_path.split(os.path.sep) else:
dependency_path = dependency_path.split(os.path.sep) base_path = base_path.split(os.path.sep)
dependency_path = dependency_path.split(os.path.sep)
first_difference = None first_difference = None
shortest_length = min(len(dependency_path), len(base_path)) shortest_length = min(len(dependency_path), len(base_path))
for i in range(shortest_length): for i in range(shortest_length):
if dependency_path[i] != base_path[i]: if dependency_path[i] != base_path[i]:
first_difference = i first_difference = i
break break
first_difference = first_difference or shortest_length first_difference = first_difference or shortest_length
base_path = base_path[first_difference:] base_path = base_path[first_difference:]
dependency_path = dependency_path[first_difference:] dependency_path = dependency_path[first_difference:]
rebased = os.path.sep.join((['..'] * len(base_path)) + dependency_path) rebased = os.path.sep.join((['..'] * len(base_path)) + dependency_path)
return rebased + ':' + dependency return rebased + ':' + dependency
def main(): def main():
deleted_sources = set() helptext = """
errors_by_file = defaultdict(lambda: defaultdict(set)) This tool tries to fix (some) errors reported by `gn gen --check`.
with TemporaryDirectory() as tmp_dir: If a command line flag `-C out/<dir>` is supplied, it will run `gn gen --check`
mb_script_path = os.path.join(SCRIPT_DIR, 'mb', 'mb.py') in that directory. Otherwise it will run `mb gen` in a temporary directory
mb_config_file_path = os.path.join(SCRIPT_DIR, 'mb', 'mb_config.pyl') with all other command line arguments forwarded to `mb gen`. This mode is
mb_gen_command = ([ useful to check for different configurations."""
mb_script_path,
'gen',
tmp_dir,
'--config-file',
mb_config_file_path,
] + sys.argv[1:])
mb_output = Run(mb_gen_command) parser = argparse.ArgumentParser(
errors = mb_output[0].split('ERROR')[1:] description=helptext,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('-C',
dest='local_build_dir',
help='Path lo a local build dir, e.g. out/Default')
(flags, argv_to_forward) = parser.parse_known_args(sys.argv[1:])
if mb_output[1]: deleted_sources = set()
print(mb_output[1]) errors_by_file = defaultdict(lambda: defaultdict(set))
return 1
for error in errors: if flags.local_build_dir:
error = error.split('\n') mb_output = Run(["gn", "gen", "--check", flags.local_build_dir])
target_msg = 'The target:'
if target_msg not in error:
target_msg = 'It is not in any dependency of'
if target_msg not in error:
print('\n'.join(error))
continue
index = error.index(target_msg) + 1
path, target = error[index].strip().split(':')
if error[index + 1] in ('is including a file from the target:',
'The include file is in the target(s):'):
dep = error[index + 2].strip()
dep_path, dep = dep.split(':')
dep = Rebase(path, dep_path, dep)
# Replacing /target:target with /target
dep = re.sub(r'/(\w+):(\1)$', r'/\1', dep)
# Replacing target:target with target
dep = re.sub(r'^(\w+):(\1)$', r'\1', dep)
path = os.path.join(path[2:], 'BUILD.gn')
errors_by_file[path][target].add(dep)
elif error[index + 1] == 'has a source file:':
deleted_file = '"' + os.path.basename(error[index + 2].strip()) + '",'
deleted_sources.add(deleted_file)
else: else:
print('\n'.join(error)) with TemporaryDirectory() as tmp_dir:
continue mb_script_path = os.path.join(SCRIPT_DIR, 'mb', 'mb.py')
mb_config_file_path = os.path.join(SCRIPT_DIR, 'mb',
'mb_config.pyl')
mb_gen_command = ([
mb_script_path,
'gen',
tmp_dir,
'--config-file',
mb_config_file_path,
] + argv_to_forward)
mb_output = Run(mb_gen_command)
for path, missing_deps in list(errors_by_file.items()): errors = mb_output[0].split('ERROR')[1:]
FixErrors(path, missing_deps, deleted_sources)
return 0 if mb_output[1]:
print(mb_output[1])
return 1
for error in errors:
error = error.split('\n')
target_msg = 'The target:'
if target_msg not in error:
target_msg = 'It is not in any dependency of'
if target_msg not in error:
print('\n'.join(error))
continue
index = error.index(target_msg) + 1
path, target = error[index].strip().split(':')
if error[index + 1] in ('is including a file from the target:',
'The include file is in the target(s):'):
dep = error[index + 2].strip()
dep_path, dep = dep.split(':')
dep = rebase(path, dep_path, dep)
# Replacing /target:target with /target
dep = re.sub(r'/(\w+):(\1)$', r'/\1', dep)
# Replacing target:target with target
dep = re.sub(r'^(\w+):(\1)$', r'\1', dep)
path = os.path.join(path[2:], 'BUILD.gn')
errors_by_file[path][target].add(dep)
elif error[index + 1] == 'has a source file:':
deleted_file = '"' + os.path.basename(
error[index + 2].strip()) + '",'
deleted_sources.add(deleted_file)
else:
print('\n'.join(error))
continue
for path, missing_deps in list(errors_by_file.items()):
fix_errors(path, missing_deps, deleted_sources)
return 0
if __name__ == '__main__': if __name__ == '__main__':
sys.exit(main()) sys.exit(main())