Ticket #1175: bundle_builder-1.py

File bundle_builder-1.py, 25.3 KB (added by tic20@…, 7 years ago)

Added by email2trac

Line 
1# vim: set expandtab ts=4 sw=4:
2
3def distlib_hack(_func):
4 # This hack is needed because distlib and wheel do not yet
5 # agree on the metadata file name.
6 def hack(*args, **kw):
7 # import distutils.core
8 # distutils.core.DEBUG = True
9 # TODO: remove distlib monkey patch when the wheel package
10 # implements PEP 426's pydist.json
11 from distlib import metadata
12 save = metadata.METADATA_FILENAME
13 metadata.METADATA_FILENAME = "metadata.json"
14 try:
15 return _func(*args, **kw)
16 finally:
17 metadata.METADATA_FILENAME = save
18 return hack
19
20
21class BundleBuilder:
22
23 def __init__(self, logger, bundle_path=None):
24 import os, os.path
25 self.logger = logger
26 if bundle_path is None:
27 bundle_path = os.getcwd()
28 self.path = bundle_path
29 info_file = os.path.join(bundle_path, "bundle_info.xml")
30 if not os.path.exists(info_file):
31 raise IOError("Bundle info file %s is missing" % repr(info_file))
32 self._read_bundle_info(info_file)
33 self._make_paths()
34 self._make_setup_arguments()
35
36 @distlib_hack
37 def make_wheel(self, test=True, debug=False):
38 # HACK: distutils uses a cache to track created directories
39 # for a single setup() run. We want to run setup() multiple
40 # times which can remove/create the same directories.
41 # So we need to flush the cache before each run.
42 import distutils.dir_util
43 try:
44 distutils.dir_util._path_created.clear()
45 except AttributeError:
46 pass
47 import os.path
48 for lib in self.c_libraries:
49 self.datafiles[self.package].append(
50 lib.compile(self.logger, self.dependencies, debug=debug))
51 setup_args = ["--no-user-cfg", "build"]
52 if debug:
53 setup_args.append("--debug")
54 if test:
55 setup_args.append("test")
56 setup_args.extend(["bdist_wheel"])
57 built = self._run_setup(setup_args)
58 if not built or not os.path.exists(self.wheel_path):
59 raise RuntimeError("Building wheel failed")
60 else:
61 print("Distribution is in %s" % self.wheel_path)
62
63 @distlib_hack
64 def make_install(self, session, test=True, debug=False, user=None):
65 self.make_wheel(test=test, debug=debug)
66 from chimerax.core.commands import run
67 cmd = "toolshed install %r reinstall true" % self.wheel_path
68 if user is not None:
69 if user:
70 cmd += " user true"
71 else:
72 cmd += " user false"
73 run(session, cmd)
74
75 @distlib_hack
76 def make_clean(self):
77 import os.path
78 self._rmtree(os.path.join(self.path, "build"))
79 self._rmtree(os.path.join(self.path, "dist"))
80 self._rmtree(os.path.join(self.path, "src/__pycache__"))
81 self._rmtree(self.egg_info)
82
83 def dump(self):
84 for a in dir(self):
85 if a.startswith('_'):
86 continue
87 v = getattr(self, a)
88 if not callable(v):
89 print("%s: %s" % (a, repr(v)))
90
91 def _rmtree(self, path):
92 import shutil
93 shutil.rmtree(path, ignore_errors=True)
94
95 _mac_platforms = ["mac", "macos", "darwin"]
96 _windows_platforms = ["windows", "win32"]
97 _linux_platforms = ["linux"]
98
99 def _read_bundle_info(self, bundle_info):
100 # Setup platform variable so we can skip non-matching elements
101 import sys
102 if sys.platform == "darwin":
103 # Tested with macOS 10.12
104 self._platform_names = self._mac_platforms
105 elif sys.platform == "win32":
106 # Tested with Cygwin
107 self._platform_names = self._windows_platforms
108 else:
109 # Presumably Linux
110 # Tested with Ubuntu 16.04 LTS running in
111 # a singularity container on CentOS 7.3
112 self._platform_names = self._linux_platforms
113 # Read data from XML file
114 self._used_elements = set()
115 from xml.dom.minidom import parse
116 doc = parse(bundle_info)
117 bi = doc.documentElement
118 self._get_identifiers(bi)
119 self._get_categories(bi)
120 self._get_descriptions(bi)
121 self._get_datafiles(bi)
122 self._get_dependencies(bi)
123 self._get_c_modules(bi)
124 self._get_c_libraries(bi)
125 self._get_packages(bi)
126 self._get_classifiers(bi)
127 self._check_unused_elements(bi)
128
129 def _get_identifiers(self, bi):
130 self.name = bi.getAttribute("name")
131 self.version = bi.getAttribute("version")
132 self.package = bi.getAttribute("package")
133 self.min_session = bi.getAttribute("minSessionVersion")
134 self.max_session = bi.getAttribute("maxSessionVersion")
135 self.custom_init = bi.getAttribute("customInit")
136 self.pure_python = bi.getAttribute("purePython")
137
138 def _get_categories(self, bi):
139 self.categories = []
140 deps = self._get_singleton(bi, "Categories")
141 for e in self._get_elements(deps, "Category"):
142 self.categories.append(e.getAttribute("name"))
143
144 def _get_descriptions(self, bi):
145 self.author = self._get_singleton_text(bi, "Author")
146 self.email = self._get_singleton_text(bi, "Email")
147 self.url = self._get_singleton_text(bi, "URL")
148 self.synopsis = self._get_singleton_text(bi, "Synopsis")
149 self.description = self._get_singleton_text(bi, "Description")
150 try:
151 self.license = self._get_singleton_text(bi, "License")
152 except ValueError:
153 self.license = None
154
155 def _get_datafiles(self, bi):
156 self.datafiles = {}
157 for dfs in self._get_elements(bi, "DataFiles"):
158 pkg_name = dfs.getAttribute("package")
159 files = []
160 for e in self._get_elements(dfs, "DataFile"):
161 filename = self._get_element_text(e)
162 files.append(filename)
163 if files:
164 if not pkg_name:
165 pkg_name = self.package
166 self.datafiles[pkg_name] = files
167
168 def _get_dependencies(self, bi):
169 self.dependencies = []
170 try:
171 deps = self._get_singleton(bi, "Dependencies")
172 except ValueError:
173 # Dependencies is optional, although
174 # ChimeraXCore *should* always be present
175 return
176 for e in self._get_elements(deps, "Dependency"):
177 pkg = e.getAttribute("name")
178 ver = e.getAttribute("version")
179 self.dependencies.append("%s %s" % (pkg, ver))
180
181 def _get_c_modules(self, bi):
182 self.c_modules = []
183 for cm in self._get_elements(bi, "CModule"):
184 mod_name = cm.getAttribute("name")
185 try:
186 major = int(cm.getAttribute("major_version"))
187 except ValueError:
188 major = 0
189 try:
190 minor = int(cm.getAttribute("minor_version"))
191 except ValueError:
192 minor = 1
193 uses_numpy = cm.getAttribute("usesNumpy") == "true"
194 c = _CModule(mod_name, uses_numpy, major, minor)
195 self._add_c_options(c, cm)
196 self.c_modules.append(c)
197
198 def _get_c_libraries(self, bi):
199 self.c_libraries = []
200 for lib in self._get_elements(bi, "CLibrary"):
201 c = _CLibrary(lib.getAttribute("name"),
202 lib.getAttribute("usesNumpy") == "true",
203 lib.getAttribute("static") == "true")
204 self._add_c_options(c, lib)
205 self.c_libraries.append(c)
206
207 def _add_c_options(self, c, ce):
208 for e in self._get_elements(ce, "Requires"):
209 c.add_require(self._get_element_text(e))
210 for e in self._get_elements(ce, "SourceFile"):
211 c.add_source_file(self._get_element_text(e))
212 for e in self._get_elements(ce, "IncludeDir"):
213 c.add_include_dir(self._get_element_text(e))
214 for e in self._get_elements(ce, "Library"):
215 c.add_library(self._get_element_text(e))
216 for e in self._get_elements(ce, "LibraryDir"):
217 c.add_library_dir(self._get_element_text(e))
218 for e in self._get_elements(ce, "LinkArgument"):
219 c.add_link_argument(self._get_element_text(e))
220 for e in self._get_elements(ce, "Framework"):
221 c.add_framework(self._get_element_text(e))
222 for e in self._get_elements(ce, "FrameworkDir"):
223 c.add_framework_dir(self._get_element_text(e))
224 for e in self._get_elements(ce, "Define"):
225 edef = self._get_element_text(e).split("=")
226 if len(edef) > 2:
227 raise TypeError(
228 "Too many arguments for macro definition: {}".format(edef))
229 elif len(edef) == 1:
230 edef.append(None)
231 c.add_macro_define(*edef)
232 for e in self._get_elements(ce, "Undefine"):
233 c.add_macro_undef(self._get_element_text(e))
234
235 def _get_packages(self, bi):
236 self.packages = []
237 try:
238 pkgs = self._get_singleton(bi, "AdditionalPackages")
239 except ValueError:
240 # AdditionalPackages is optional
241 return
242 for pkg in self._get_elements(pkgs, "Package"):
243 pkg_name = pkg.getAttribute("name")
244 pkg_folder = pkg.getAttribute("folder")
245 self.packages.append((pkg_name, pkg_folder))
246
247 def _get_classifiers(self, bi):
248 self.python_classifiers = [
249 "Framework :: ChimeraX",
250 "Intended Audience :: Science/Research",
251 "Programming Language :: Python :: 3",
252 "Topic :: Scientific/Engineering :: Visualization",
253 "Topic :: Scientific/Engineering :: Chemistry",
254 "Topic :: Scientific/Engineering :: Bio-Informatics",
255 ]
256 cls = self._get_singleton(bi, "Classifiers")
257 for e in self._get_elements(cls, "PythonClassifier"):
258 self.python_classifiers.append(self._get_element_text(e))
259 self.chimerax_classifiers = [
260 ("ChimeraX :: Bundle :: " + ','.join(self.categories) +
261 " :: " + self.min_session + "," + self.max_session +
262 " :: " + self.package + " :: :: " + self.custom_init)
263 ]
264 for e in self._get_elements(cls, "ChimeraXClassifier"):
265 self.chimerax_classifiers.append(self._get_element_text(e))
266
267 def _is_pure_python(self):
268 return (not self.c_modules and not self.c_libraries
269 and self.pure_python != "false")
270
271 def _make_setup_arguments(self):
272 def add_argument(name, value):
273 if value:
274 self.setup_arguments[name] = value
275 self.setup_arguments = {"name": self.name,
276 "python_requires": ">= 3.6"}
277 add_argument("version", self.version)
278 add_argument("description", self.synopsis)
279 add_argument("long_description", self.description)
280 add_argument("author", self.author)
281 add_argument("author_email", self.email)
282 add_argument("url", self.url)
283 add_argument("install_requires", self.dependencies)
284 add_argument("license", self.license)
285 add_argument("package_data", self.datafiles)
286 # We cannot call find_packages unless we are already
287 # in the right directory, and that will not happen
288 # until run_setup. So we do the package stuff there.
289 ext_mods = [em for em in [cm.ext_mod(self.logger, self.package,
290 self.dependencies)
291 for cm in self.c_modules]
292 if em is not None]
293 if not self._is_pure_python():
294 import sys
295 if sys.platform == "darwin":
296 env = "Environment :: MacOS X :: Aqua",
297 op_sys = "Operating System :: MacOS :: MacOS X"
298 elif sys.platform == "win32":
299 env = "Environment :: Win32 (MS Windows)"
300 op_sys = "Operating System :: Microsoft :: Windows :: Windows 10"
301 else:
302 env = "Environment :: X11 Applications"
303 op_sys = "Operating System :: POSIX :: Linux"
304 platform_classifiers = [env, op_sys]
305 if not ext_mods:
306 # From https://stackoverflow.com/questions/35112511/pip-setup-py-bdist-wheel-no-longer-builds-forced-non-pure-wheels
307 from setuptools.dist import Distribution
308 class BinaryDistribution(Distribution):
309 def has_ext_modules(foo):
310 return True
311 self.setup_arguments["distclass"] = BinaryDistribution
312 else:
313 # pure Python
314 platform_classifiers = [
315 "Environment :: MacOS X :: Aqua",
316 "Environment :: Win32 (MS Windows)",
317 "Environment :: X11 Applications",
318 "Operating System :: MacOS :: MacOS X",
319 "Operating System :: Microsoft :: Windows :: Windows 10",
320 "Operating System :: POSIX :: Linux",
321 ]
322 self.python_classifiers.extend(platform_classifiers)
323 self.setup_arguments["ext_modules"] = ext_mods
324 self.setup_arguments["classifiers"] = (self.python_classifiers +
325 self.chimerax_classifiers)
326
327 def _make_package_arguments(self):
328 from setuptools import find_packages
329 def add_package(base_package, folder):
330 package_dir[base_package] = folder
331 packages.append(base_package)
332 packages.extend([base_package + "." + sub_pkg
333 for sub_pkg in find_packages(folder)])
334 package_dir = {}
335 packages = []
336 add_package(self.package, "src")
337 for name, folder in self.packages:
338 add_package(name, folder)
339 return package_dir, packages
340
341 def _make_paths(self):
342 import os.path
343 from .wheel_tag import tag
344 self.tag = tag(self._is_pure_python())
345 self.bundle_base_name = self.name.replace("ChimeraX-", "")
346 bundle_wheel_name = self.name.replace("-", "_")
347 wheel = "%s-%s-%s.whl" % (bundle_wheel_name, self.version, self.tag)
348 self.wheel_path = os.path.join(self.path, "dist", wheel)
349 self.egg_info = os.path.join(self.path, bundle_wheel_name + ".egg-info")
350
351 def _run_setup(self, cmd):
352 import os, sys, setuptools
353 cwd = os.getcwd()
354 save = sys.argv
355 try:
356 os.chdir(self.path)
357 kw = self.setup_arguments.copy()
358 kw["package_dir"], kw["packages"] = self._make_package_arguments()
359 sys.argv = ["setup.py"] + cmd
360 setuptools.setup(**kw)
361 return True
362 except:
363 import traceback
364 traceback.print_exc()
365 return False
366 finally:
367 sys.argv = save
368 os.chdir(cwd)
369
370 #
371 # Utility functions dealing with XML tree
372 #
373 def _get_elements(self, e, tag):
374 tagged_elements = e.getElementsByTagName(tag)
375 # Mark element as used even for non-applicable platform
376 self._used_elements.update(tagged_elements)
377 elements = []
378 for se in tagged_elements:
379 platform = se.getAttribute("platform")
380 if not platform or platform in self._platform_names:
381 elements.append(se)
382 return elements
383
384 def _get_element_text(self, e):
385 text = ""
386 for node in e.childNodes:
387 if node.nodeType == node.TEXT_NODE:
388 text += node.data
389 return text.strip()
390
391 def _get_singleton(self, bi, tag):
392 elements = bi.getElementsByTagName(tag)
393 self._used_elements.update(elements)
394 if len(elements) > 1:
395 raise ValueError("too many %s elements" % repr(tag))
396 elif len(elements) == 0:
397 raise ValueError("%s element is missing" % repr(tag))
398 return elements[0]
399
400 def _get_singleton_text(self, bi, tag):
401 return self._get_element_text(self._get_singleton(bi, tag))
402
403 def _check_unused_elements(self, bi):
404 for node in bi.childNodes:
405 if node.nodeType != node.ELEMENT_NODE:
406 continue
407 if node not in self._used_elements:
408 print("WARNING: unsupported element:", node.nodeName)
409
410
411class _CompiledCode:
412
413 def __init__(self, name, uses_numpy):
414 self.name = name
415 self.uses_numpy = uses_numpy
416 self.requires = []
417 self.source_files = []
418 self.frameworks = []
419 self.libraries = []
420 self.link_arguments = []
421 self.include_dirs = []
422 self.library_dirs = []
423 self.framework_dirs = []
424 self.macros = []
425
426 def add_require(self, req):
427 self.requires.append(req)
428
429 def add_source_file(self, f):
430 self.source_files.append(f)
431
432 def add_include_dir(self, d):
433 self.include_dirs.append(d)
434
435 def add_library(self, l):
436 self.libraries.append(l)
437
438 def add_library_dir(self, d):
439 self.library_dirs.append(d)
440
441 def add_link_argument(self, a):
442 self.link_arguments.append(a)
443
444 def add_framework(self, f):
445 self.frameworks.append(f)
446
447 def add_framework_dir(self, d):
448 self.framework_dirs.append(d)
449
450 def add_macro_define(self, m, val):
451 # 2-tuple defines (set val to None to define without a value)
452 self.macros.append((m, val))
453
454 def add_macro_undef(self, m):
455 # 1-tuple of macro name undefines
456 self.macros.append((m,))
457
458 def _compile_options(self, logger, dependencies):
459 import sys, os.path
460 for req in self.requires:
461 if not os.path.exists(req):
462 raise ValueError("unused on this platform")
463 # platform-specific
464 # Assume Python executable is in ROOT/bin/python
465 # and make include directory be ROOT/include
466 root = os.path.dirname(os.path.dirname(sys.executable))
467 inc_dirs = [os.path.join(root, "include")]
468 lib_dirs = [os.path.join(root, "lib")]
469 if self.uses_numpy:
470 from numpy.distutils.misc_util import get_numpy_include_dirs
471 inc_dirs.extend(get_numpy_include_dirs())
472 if sys.platform == "darwin":
473 libraries = self.libraries
474 # Unfortunately, clang on macOS (for now) exits
475 # when receiving a -std=c++11 option when compiling
476 # a C (not C++) source file, which is why this value
477 # is named "cpp_flags" not "compile_flags"
478 cpp_flags = ["-std=c++11", "-stdlib=libc++"]
479 extra_link_args = ["-F" + d for d in self.framework_dirs]
480 for fw in self.frameworks:
481 extra_link_args.extend(["-framework", fw])
482 elif sys.platform == "win32":
483 libraries = []
484 for lib in self.libraries:
485 if lib.lower().endswith(".lib"):
486 # Strip the .lib since suffixes are handled automatically
487 libraries.append(lib[:-4])
488 else:
489 libraries.append("lib" + lib)
490 cpp_flags = []
491 extra_link_args = []
492 else:
493 libraries = self.libraries
494 cpp_flags = ["-std=c++11"]
495 extra_link_args = []
496 for req in self.requires:
497 if not os.path.exists(req):
498 return None
499 inc_dirs.extend(self.include_dirs)
500 lib_dirs.extend(self.library_dirs)
501 for dep in dependencies:
502 d_inc, d_lib = self._get_bundle_dirs(logger, dep)
503 if d_inc:
504 inc_dirs.append(d_inc)
505 if d_lib:
506 lib_dirs.append(d_lib)
507 extra_link_args.extend(self.link_arguments)
508 return inc_dirs, lib_dirs, self.macros, extra_link_args, libraries, cpp_flags
509
510 def _get_bundle_dirs(self, logger, dep):
511 from chimerax.core import toolshed
512 # It's either pulling unsupported class from pip or roll our own.
513 from pip.req.req_install import InstallRequirement
514 ir = InstallRequirement.from_line(dep)
515 if not ir.check_if_exists():
516 raise RuntimeError("unsatisfied dependency: %s" % dep)
517 ts = toolshed.get_toolshed()
518 bundle = ts.find_bundle(ir.name, logger)
519 if not bundle:
520 return None, None
521 inc = bundle.include_dir()
522 lib = bundle.library_dir()
523 return inc, lib
524
525
526class _CModule(_CompiledCode):
527
528 def __init__(self, name, uses_numpy, major, minor):
529 super().__init__(name, uses_numpy)
530 self.major = major
531 self.minor = minor
532
533 def ext_mod(self, logger, package, dependencies):
534 from setuptools import Extension
535 try:
536 (inc_dirs, lib_dirs, macros, extra_link_args,
537 libraries, cpp_flags) = self._compile_options(logger, dependencies)
538 macros.extend([("MAJOR_VERSION", self.major),
539 ("MINOR_VERSION", self.minor)])
540 except ValueError:
541 return None
542 import sys
543 if sys.platform == "linux":
544 extra_link_args.append("-Wl,-rpath,$ORIGIN")
545 return Extension(package + '.' + self.name,
546 define_macros=macros,
547 extra_compile_args=cpp_flags,
548 include_dirs=inc_dirs,
549 library_dirs=lib_dirs,
550 libraries=libraries,
551 extra_link_args=extra_link_args,
552 sources=self.source_files)
553
554
555class _CLibrary(_CompiledCode):
556
557 def __init__(self, name, uses_numpy, static):
558 super().__init__(name, uses_numpy)
559 self.static = static
560
561 def compile(self, logger, dependencies, debug=False):
562 import sys, os, os.path, distutils.ccompiler, distutils.sysconfig
563 import distutils.log
564 distutils.log.set_verbosity(1)
565 try:
566 (inc_dirs, lib_dirs, macros, extra_link_args,
567 libraries, cpp_flags) = self._compile_options(logger, dependencies)
568 except ValueError:
569 print("Error on {}".format(self.name))
570 return None
571 output_dir = "src"
572 compiler = distutils.ccompiler.new_compiler()
573 distutils.sysconfig.customize_compiler(compiler)
574 if inc_dirs:
575 compiler.set_include_dirs(inc_dirs)
576 if lib_dirs:
577 compiler.set_library_dirs(lib_dirs)
578 if libraries:
579 compiler.set_libraries(libraries)
580 compiler.add_include_dir(distutils.sysconfig.get_python_inc())
581 if sys.platform == "win32":
582 # Link library directory for Python on Windows
583 compiler.add_library_dir(os.path.join(sys.exec_prefix, 'libs'))
584 lib_name = "lib" + self.name
585 else:
586 lib_name = self.name
587 if not self.static:
588 macros.append(("DYNAMIC_LIBRARY", 1))
589 # compiler.define_macro("DYNAMIC_LIBRARY", 1)
590 compiler.compile(self.source_files, extra_preargs=cpp_flags, macros=macros, debug=debug)
591 objs = compiler.object_filenames(self.source_files)
592 compiler.mkpath(output_dir)
593 if self.static:
594 lib = compiler.library_filename(lib_name, lib_type="static")
595 compiler.create_static_lib(objs, lib_name, output_dir=output_dir,
596 debug=debug)
597 else:
598 if sys.platform == "darwin":
599 # On Mac, we only need the .dylib and it MUST be compiled
600 # with "-dynamiclib", not "-bundle". Hence the giant hack:
601 try:
602 n = compiler.linker_so.index("-bundle")
603 except ValueError:
604 pass
605 else:
606 compiler.linker_so[n] = "-dynamiclib"
607 lib = compiler.library_filename(lib_name, lib_type="dylib")
608 extra_link_args.append("-Wl,-install_name,@loader_path/%s" % lib)
609 compiler.link_shared_object(objs, lib, output_dir=output_dir,
610 extra_postargs=extra_link_args,
611 debug=debug)
612 elif sys.platform == "win32":
613 # On Windows, we need both .dll and .lib
614 link_lib = compiler.library_filename(lib_name, lib_type="static")
615 extra_link_args.append("/LIBPATH:%s" % link_lib)
616 lib = compiler.shared_object_filename(lib_name)
617 compiler.link_shared_object(objs, lib, output_dir=output_dir,
618 extra_postargs=extra_link_args,
619 debug=debug)
620 else:
621 # On Linux, we only need the .so
622 lib = compiler.library_filename(lib_name, lib_type="shared")
623 compiler.link_shared_object(objs, lib, output_dir=output_dir,
624 extra_postargs=extra_link_args,
625 debug=debug)
626 return lib
627
628if __name__ == "__main__" or __name__.startswith("ChimeraX_sandbox"):
629 import sys
630 bb = BundleBuilder()
631 for cmd in sys.argv[1:]:
632 if cmd == "wheel":
633 bb.make_wheel()
634 elif cmd == "install":
635 try:
636 bb.make_install(session)
637 except NameError:
638 print("%s only works from ChimeraX, not Python" % repr(cmd))
639 elif cmd == "clean":
640 bb.make_clean()
641 elif cmd == "dump":
642 bb.dump()
643 else:
644 print("unknown command: %s" % repr(cmd))
645 raise SystemExit(0)