KApiDox

preprocessing.py
1 # -*- coding: utf-8 -*-
2 #
3 # SPDX-FileCopyrightText: 2016 Olivier Churlaud <[email protected]>
4 # SPDX-FileCopyrightText: 2014 Alex Merry <[email protected]>
5 # SPDX-FileCopyrightText: 2014 Aurélien Gâteau <[email protected]>
6 # SPDX-FileCopyrightText: 2014 Alex Turbov <[email protected]>
7 #
8 # SPDX-License-Identifier: BSD-2-Clause
9 
10 import logging
11 import os
12 import sys
13 from typing import Any, Dict, Optional
14 
15 from urllib.request import Request, urlopen
16 from urllib.error import HTTPError
17 
18 import yaml
19 
20 from kapidox import utils
21 from kapidox.models import Library, Product
22 
23 __all__ = (
24  "create_metainfo",
25  "parse_tree")
26 
27 
28 PLATFORM_ALL = "All"
29 PLATFORM_UNKNOWN = "UNKNOWN"
30 
31 
32 ## @package kapidox.preprocessing
33 #
34 # Preprocessing of the needed information.
35 #
36 # The module allow to walk through folders, read metainfo files and create
37 # products, subgroups and libraries representing the projects.
38 #
39 
40 def expand_platform_all(dct, available_platforms):
41  """If one of the keys of dct is `PLATFORM_ALL` (or `PLATFORM_UNKNOWN`),
42  remove it and add entries for all available platforms to dct
43 
44  Args:
45  dct: (dictionary) dictionary to expand
46  available_platforms: (list of string) name of platforms
47  """
48 
49  add_all_platforms = False
50  if PLATFORM_ALL in dct:
51  note = dct[PLATFORM_ALL]
52  add_all_platforms = True
53  del dct[PLATFORM_ALL]
54  if PLATFORM_UNKNOWN in dct:
55  add_all_platforms = True
56  note = dct[PLATFORM_UNKNOWN]
57  del dct[PLATFORM_UNKNOWN]
58  if add_all_platforms:
59  for platform in available_platforms:
60  if platform not in dct:
61  dct[platform] = note
62 
63 
64 def create_metainfo(path) -> Optional[Dict[str,Any]]:
65  """Look for a `metadata.yaml` file and create a dictionary out it.
66 
67  Args:
68  path: (string) the current path to search.
69  Returns:
70  A dictionary containing all the parsed information, or `None` if it
71  did not fulfill some conditions.
72  """
73 
74  metainfo: Optional[Dict[str,Any]]
75 
76  if not os.path.isdir(path):
77  return None
78 
79  try:
80  metainfo_file = os.path.join(path, 'metainfo.yaml')
81  except UnicodeDecodeError as e:
82  logging.warning('Unusual base path {!r} for metainfo.yaml'.format(path))
83  return None
84  if not os.path.isfile(metainfo_file):
85  return None
86 
87  try:
88  metainfo = yaml.safe_load(open(metainfo_file))
89  except Exception as e:
90  print(e)
91  logging.warning('Could not load metainfo.yaml for {}, skipping it'
92  .format(path))
93  return None
94 
95  if metainfo is None:
96  logging.warning('Empty metainfo.yaml for {}, skipping it'
97  .format(path))
98  return None
99 
100  if 'subgroup' in metainfo and 'group' not in metainfo:
101  logging.warning('Subgroup but no group in {}, skipping it'
102  .format(path))
103  return None
104 
105  # Suppose we get a relative path passed in (e.g. on the command-line,
106  # path .. because we're building the dox in a subdirectory of a source
107  # checkout) then we don't want dirname to be "..", but the name that
108  # that resolves to.
109  dirname = os.path.basename(os.path.abspath(path))
110  if 'fancyname' in metainfo:
111  fancyname = metainfo['fancyname']
112  else:
113  fancyname = utils.parse_fancyname(path)
114 
115  if not fancyname:
116  logging.warning('Could not find fancy name for {}, skipping it'
117  .format(path))
118  return None
119  # A fancyname has 1st char capitalized
120  fancyname = fancyname[0].capitalize() + fancyname[1:]
121 
122  if 'repo_id' in metainfo:
123  repo_id = metainfo['repo_id']
124  else:
125  repo_id = dirname
126 
127  qdoc: bool = False
128 
129  if 'qdoc' in metainfo:
130  qdoc = metainfo['qdoc']
131 
132  metainfo.update({
133  'fancyname': fancyname,
134  'name': dirname,
135  'repo_id': repo_id,
136  'public_lib': metainfo.get('public_lib', False),
137  'dependency_diagram': None,
138  'path': path,
139  'qdoc': qdoc,
140  })
141 
142  # replace legacy platform names
143  if 'platforms' in metainfo:
144  platforms = metainfo['platforms']
145  for index, x in enumerate(platforms):
146  if x['name'] == "MacOSX":
147  x['name'] = "macOS";
148  platforms[index] = x
149  logging.warning('{} uses outdated platform name, please replace "MacOSX" with "macOS".'
150  .format(metainfo['fancyname']))
151  metainfo.update({ 'platforms': platforms })
152  if 'group_info' in metainfo:
153  group_info = metainfo['group_info']
154  if 'platforms' in group_info:
155  platforms = group_info['platforms']
156  for index, x in enumerate(platforms):
157  if "MacOSX" in x:
158  x = x.replace("MacOSX", "macOS");
159  platforms[index] = x
160  logging.warning('Group {} uses outdated platform name, please replace "MacOSX" with "macOS".'
161  .format(group_info['fancyname']))
162  group_info.update({ 'platforms': platforms })
163 
164  return metainfo
165 
166 
167 def parse_tree(rootdir):
168  """Recursively call create_metainfo() in subdirs of rootdir
169 
170  Args:
171  rootdir: (string) Top level directory containing the libraries.
172 
173  Returns:
174  A list of metainfo dictionary (see create_metainfo()).
175 
176  """
177  metalist = []
178  for path, dirs, _ in os.walk(rootdir, topdown=True):
179  # We don't want to do the recursion in the dotdirs
180  dirs[:] = [d for d in dirs if not d[0] == '.']
181  metainfo = create_metainfo(path)
182  if metainfo is not None:
183  # There was a metainfo.yaml, which means it was
184  # the top of a checked-out repository. Stop processing,
185  # because we do not support having repo B checked out (even
186  # as a submodule) inside repo A.
187  #
188  # There are exceptions: messagelib (KDE PIM) contains
189  # multiple subdirectories with their own metainfo.yaml,
190  # which are listed as public sources.
191  dirs[:] = [d for d in dirs if d in metainfo.get('public_source_dirs', [])]
192  if metainfo['public_lib'] or 'group_info' in metainfo:
193  metalist.append(metainfo)
194  else:
195  logging.warning("{} has no public libraries"
196  .format(metainfo['name']))
197 
198  return metalist
199 
200 
201 def sort_metainfo(metalist, all_maintainers):
202  """Extract the structure (Product/Subproduct/Library) from the metainfo
203  list.
204 
205  Args:
206  metalist: (list of dict) lists of the metainfo extracted in
207  parse_tree().
208  all_maintainers: (dict of dict) all possible maintainers.
209 
210  Returns:
211  A list of Products, a list of groups (which are products containing
212  several libraries), a list of Libraries and the available platforms.
213  """
214  products = dict()
215  groups = []
216  libraries = []
217  available_platforms = set(['Windows', 'macOS', 'Linux', 'Android', 'FreeBSD'])
218 
219  # First extract the structural info
220  for metainfo in metalist:
221  product = extract_product(metainfo, all_maintainers)
222  if product is not None:
223  products[product.name] = product
224 
225  # Second extract the libraries
226  for metainfo in metalist:
227  try:
228  platforms = metainfo['platforms']
229  platform_lst = [x['name'] for x in platforms
230  if x['name'] not in (PLATFORM_ALL,
231  PLATFORM_UNKNOWN)]
232 
233  available_platforms.update(set(platform_lst))
234  except (KeyError, TypeError):
235  logging.warning('{} library lacks valid platform definitions'
236  .format(metainfo['fancyname']))
237  platforms = [dict(name=PLATFORM_UNKNOWN)]
238 
239  dct = dict((x['name'], x.get('note', '')) for x in platforms)
240 
241  expand_platform_all(dct, available_platforms)
242  platforms = dct
243 
244  if metainfo['public_lib']:
245  lib = Library(metainfo, products, platforms, all_maintainers)
246  libraries.append(lib)
247 
248  groups = []
249  for key in products.copy():
250  if len(products[key].libraries) == 0:
251  del products[key]
252  elif products[key].part_of_group:
253  groups.append(products[key])
254 
255  return list(products.values()), groups, libraries, available_platforms
256 
257 
258 def extract_product(metainfo, all_maintainers):
259  """Extract a product from a metainfo dictionary.
260 
261  Args:
262  metainfo: (dict) metainfo created by the create_metainfo() function.
263  all_maintainer: (dict of dict) all possible maintainers
264 
265  Returns:
266  A Product or None if the metainfo does not describe a product.
267  """
268 
269  if 'group_info' not in metainfo and 'group' in metainfo:
270  # This is not a product but a simple lib
271  return None
272 
273  try:
274  product = Product(metainfo, all_maintainers)
275  return product
276  except ValueError as e:
277  logging.error(e)
278  return None
def create_metainfo(path)
Look for a metadata.yaml file and create a dictionary out it.
def sort_metainfo(metalist, all_maintainers)
Extract the structure (Product/Subproduct/Library) from the metainfo list.
def parse_tree(rootdir)
Recursively call create_metainfo() in subdirs of rootdir.
def extract_product(metainfo, all_maintainers)
Extract a product from a metainfo dictionary.
def expand_platform_all(dct, available_platforms)
If one of the keys of dct is PLATFORM_ALL (or PLATFORM_UNKNOWN), remove it and add entries for all av...
Contains the classes representing the objects used by kapidox.
Definition: models.py:1
This file is part of the KDE documentation.
Documentation copyright © 1996-2021 The KDE developers.
Generated on Tue Oct 26 2021 22:55:54 by doxygen 1.8.11 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.