KApiDox

preprocessing.py
1 # -*- coding: utf-8 -*-
2 #
3 # SPDX-FileCopyrightText: 2016 Olivier Churlaud <[email protected]>
4 # SPDX-FileCopyrightText: 2014 Alex Merry <[email protected]>
5 # SPDX-FileCopyrightText: 2014 Aurélien Gâteau <[email protected]>
6 # SPDX-FileCopyrightText: 2014 Alex Turbov <[email protected]>
7 #
8 # SPDX-License-Identifier: BSD-2-Clause
9 
10 import logging
11 import os
12 import sys
13 
14 from urllib.request import Request, urlopen
15 from urllib.error import HTTPError
16 
17 import yaml
18 
19 from kapidox import utils
20 from kapidox.models import Library, Product
21 
22 __all__ = (
23  "create_metainfo",
24  "parse_tree")
25 
26 
27 PLATFORM_ALL = "All"
28 PLATFORM_UNKNOWN = "UNKNOWN"
29 
30 
31 ## @package kapidox.preprocessing
32 #
33 # Preprocessing of the needed information.
34 #
35 # The module allow to walk through folders, read metainfo files and create
36 # products, subgroups and libraries representing the projects.
37 #
38 
39 def expand_platform_all(dct, available_platforms):
40  """If one of the keys of dct is `PLATFORM_ALL` (or `PLATFORM_UNKNOWN`),
41  remove it and add entries for all available platforms to dct
42 
43  Args:
44  dct: (dictionary) dictionary to expand
45  available_platforms: (list of string) name of platforms
46  """
47 
48  add_all_platforms = False
49  if PLATFORM_ALL in dct:
50  note = dct[PLATFORM_ALL]
51  add_all_platforms = True
52  del dct[PLATFORM_ALL]
53  if PLATFORM_UNKNOWN in dct:
54  add_all_platforms = True
55  note = dct[PLATFORM_UNKNOWN]
56  del dct[PLATFORM_UNKNOWN]
57  if add_all_platforms:
58  for platform in available_platforms:
59  if platform not in dct:
60  dct[platform] = note
61 
62 
63 def create_metainfo(path):
64  """Look for a `metadata.yaml` file and create a dictionary out it.
65 
66  Args:
67  path: (string) the current path to search.
68  Returns:
69  A dictionary containing all the parsed information, or `None` if it
70  did not fulfill some conditions.
71  """
72 
73  if not os.path.isdir(path):
74  return None
75 
76  try:
77  metainfo_file = os.path.join(path, 'metainfo.yaml')
78  except UnicodeDecodeError as e:
79  logging.warning('Unusual base path {!r} for metainfo.yaml'.format(path))
80  return None
81  if not os.path.isfile(metainfo_file):
82  return None
83 
84  try:
85  metainfo = yaml.safe_load(open(metainfo_file))
86  except Exception as e:
87  print(e)
88  logging.warning('Could not load metainfo.yaml for {}, skipping it'
89  .format(path))
90  return None
91 
92  if metainfo is None:
93  logging.warning('Empty metainfo.yaml for {}, skipping it'
94  .format(path))
95  return None
96 
97  if 'subgroup' in metainfo and 'group' not in metainfo:
98  logging.warning('Subgroup but no group in {}, skipping it'
99  .format(path))
100  return None
101 
102  dirname = os.path.basename(path)
103  if 'fancyname' in metainfo:
104  fancyname = metainfo['fancyname']
105  else:
106  fancyname = utils.parse_fancyname(path)
107 
108  if not fancyname:
109  logging.warning('Could not find fancy name for {}, skipping it'
110  .format(path))
111  return None
112  # A fancyname has 1st char capitalized
113  fancyname = fancyname[0].capitalize() + fancyname[1:]
114 
115  if 'repo_id' in metainfo:
116  repo_id = metainfo['repo_id']
117  else:
118  repo_id = dirname
119 
120  metainfo.update({
121  'fancyname': fancyname,
122  'name': dirname,
123  'repo_id': repo_id,
124  'public_lib': metainfo.get('public_lib', False),
125  'dependency_diagram': None,
126  'path': path,
127  })
128 
129  # replace legacy platform names
130  if 'platforms' in metainfo:
131  platforms = metainfo['platforms']
132  for index, x in enumerate(platforms):
133  if x['name'] == "MacOSX":
134  x['name'] = "macOS";
135  platforms[index] = x
136  logging.warning('{} uses outdated platform name, please replace "MacOSX" with "macOS".'
137  .format(metainfo['fancyname']))
138  metainfo.update({ 'platforms': platforms })
139  if 'group_info' in metainfo:
140  group_info = metainfo['group_info']
141  if 'platforms' in group_info:
142  platforms = group_info['platforms']
143  for index, x in enumerate(platforms):
144  if "MacOSX" in x:
145  x = x.replace("MacOSX", "macOS");
146  platforms[index] = x
147  logging.warning('Group {} uses outdated platform name, please replace "MacOSX" with "macOS".'
148  .format(group_info['fancyname']))
149  group_info.update({ 'platforms': platforms })
150 
151  return metainfo
152 
153 
154 def parse_tree(rootdir):
155  """Recursively call create_metainfo() in subdirs of rootdir
156 
157  Args:
158  rootdir: (string) Top level directory containing the libraries.
159 
160  Returns:
161  A list of metainfo dictionary (see create_metainfo()).
162 
163  """
164  metalist = []
165  for path, dirs, _ in os.walk(rootdir):
166  # We don't want to do the recursion in the dotdirs
167  dirs[:] = [d for d in dirs if not d[0] == '.']
168  metainfo = create_metainfo(path)
169  if metainfo is not None:
170  if metainfo['public_lib'] or 'group_info' in metainfo:
171  metalist.append(metainfo)
172  else:
173  logging.warning("{} has no public libraries"
174  .format(metainfo['name']))
175 
176  return metalist
177 
178 
179 def sort_metainfo(metalist, all_maintainers):
180  """Extract the structure (Product/Subproduct/Library) from the metainfo
181  list.
182 
183  Args:
184  metalist: (list of dict) lists of the metainfo extracted in
185  parse_tree().
186  all_maintainers: (dict of dict) all possible maintainers.
187 
188  Returns:
189  A list of Products, a list of groups (which are products containing
190  several libraries), a list of Libraries and the available platforms.
191  """
192  products = dict()
193  groups = []
194  libraries = []
195  available_platforms = set(['Windows', 'macOS', 'Linux', 'Android', 'FreeBSD'])
196 
197  # First extract the structural info
198  for metainfo in metalist:
199  product = extract_product(metainfo, all_maintainers)
200  if product is not None:
201  products[product.name] = product
202 
203  # Second extract the libraries
204  for metainfo in metalist:
205  try:
206  platforms = metainfo['platforms']
207  platform_lst = [x['name'] for x in platforms
208  if x['name'] not in (PLATFORM_ALL,
209  PLATFORM_UNKNOWN)]
210 
211  available_platforms.update(set(platform_lst))
212  except (KeyError, TypeError):
213  logging.warning('{} library lacks valid platform definitions'
214  .format(metainfo['fancyname']))
215  platforms = [dict(name=PLATFORM_UNKNOWN)]
216 
217  dct = dict((x['name'], x.get('note', '')) for x in platforms)
218 
219  expand_platform_all(dct, available_platforms)
220  platforms = dct
221 
222  if metainfo['public_lib']:
223  lib = Library(metainfo, products, platforms, all_maintainers)
224  libraries.append(lib)
225 
226  groups = []
227  for key in products.copy():
228  if len(products[key].libraries) == 0:
229  del products[key]
230  elif products[key].part_of_group:
231  groups.append(products[key])
232 
233  return list(products.values()), groups, libraries, available_platforms
234 
235 
236 def extract_product(metainfo, all_maintainers):
237  """Extract a product from a metainfo dictionary.
238 
239  Args:
240  metainfo: (dict) metainfo created by the create_metainfo() function.
241  all_maintainer: (dict of dict) all possible maintainers
242 
243  Returns:
244  A Product or None if the metainfo does not describe a product.
245  """
246 
247  if 'group_info' not in metainfo and 'group' in metainfo:
248  # This is not a product but a simple lib
249  return None
250 
251  try:
252  product = Product(metainfo, all_maintainers)
253  return product
254  except ValueError as e:
255  logging.error(e)
256  return None
def create_metainfo(path)
Look for a metadata.yaml file and create a dictionary out it.
def sort_metainfo(metalist, all_maintainers)
Extract the structure (Product/Subproduct/Library) from the metainfo list.
def parse_tree(rootdir)
Recursively call create_metainfo() in subdirs of rootdir.
def extract_product(metainfo, all_maintainers)
Extract a product from a metainfo dictionary.
def expand_platform_all(dct, available_platforms)
If one of the keys of dct is PLATFORM_ALL (or PLATFORM_UNKNOWN), remove it and add entries for all av...
Contains the classes representing the objects used by kapidox.
Definition: models.py:1
This file is part of the KDE documentation.
Documentation copyright © 1996-2020 The KDE developers.
Generated on Mon Nov 30 2020 22:54:36 by doxygen 1.8.11 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.