1 """
2 CSB build related tools and programs.
3
4 When executed as a program, this module will run the CSB Build Console and
5 build the source tree it belongs to. The source tree is added at the
6 B{beginning} of sys.path to make sure that all subsequent imports from the
7 Test and Doc consoles will import the right thing (think of multiple CSB
8 packages installed on the same server).
9
10 Here is how to build, test and package the whole project::
11
12 $ hg clone https://hg.codeplex.com/csb CSB
13 $ CSB/csb/build.py -o <output directory>
14
15 The Console can also be imported and instantiated as a regular Python class.
16 In this case the Console again builds the source tree it is part of, but
17 sys.path will remain intact. Therefore, the Console will assume that all
18 modules currently in memory, as well as those that can be subsequently imported
19 by the Console itself, belong to the same CSB package.
20
21 @note: The CSB build services no longer support the option to build external
22 source trees.
23 @see: [CSB 0000038]
24 """
25 from __future__ import print_function
26
27 import os
28 import sys
29 import getopt
30 import traceback
31
32 if os.path.basename(__file__) == '__init__.py':
33 PARENT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
34 else:
35 PARENT = os.path.abspath(os.path.dirname(__file__))
36
37 ROOT = 'csb'
38 SOURCETREE = os.path.abspath(os.path.join(PARENT, ".."))
39
40 if __name__ == '__main__':
41
42
43
44 for path in sys.path:
45 if path.startswith(SOURCETREE):
46 sys.path.remove(path)
47
48 import io
49 assert hasattr(io, 'BufferedIOBase')
50
51 sys.path = [SOURCETREE] + sys.path
52
53
54 """
55 It is now safe to import any modules
56 """
57 import imp
58 import shutil
59 import tarfile
60
61 import csb
62
63 from abc import ABCMeta, abstractmethod
64 from csb.io import Shell
68 """
69 Enumeration of build types.
70 """
71
72 SOURCE = 'source'
73 BINARY = 'binary'
74
75 _du = { SOURCE: 'sdist', BINARY: 'bdist' }
76
77 @staticmethod
79 try:
80 return BuildTypes._du[key]
81 except KeyError:
82 raise ValueError('Unhandled build type: {0}'.format(key))
83
86 """
87 CSB Build Bot. Run with -h for usage.
88
89 @param output: build output directory
90 @type output: str
91 @param verbosity: verbosity level
92 @type verbosity: int
93
94 @note: The build console automatically detects and builds the csb package
95 it belongs to. You cannot build a different source tree with it.
96 See the module documentation for more info.
97 """
98
99 PROGRAM = __file__
100
101 USAGE = r"""
102 CSB Build Console: build, test and package the entire csb project.
103
104 Usage:
105 python {program} -o output [-v verbosity] [-t type] [-h]
106
107 Options:
108 -o output Build output directory
109 -v verbosity Verbosity level, default is 1
110 -t type Build type:
111 source - build source code distribution (default)
112 binary - build executable
113 -h, --help Display this help
114 """
115
117
118 self._input = None
119 self._output = None
120 self._temp = None
121 self._docs = None
122 self._apidocs = None
123 self._root = None
124 self._verbosity = None
125 self._type = buildtype
126 self._dist = BuildTypes.get(buildtype)
127
128 if os.path.join(SOURCETREE, ROOT) != PARENT:
129 raise IOError('{0} must be a sub-package or sub-module of {1}'.format(__file__, ROOT))
130 self._input = SOURCETREE
131
132 self.output = output
133 self.verbosity = verbosity
134
135 @property
138
139 @property
142 @output.setter
144
145 self._output = os.path.abspath(value)
146 self._temp = os.path.join(self._output, 'build')
147 self._docs = os.path.join(self._temp, 'docs')
148 self._apidocs = os.path.join(self._docs, 'api')
149 self._root = os.path.join(self._temp, ROOT)
150
151 @property
153 return self._verbosity
154 @verbosity.setter
156 self._verbosity = int(value)
157
159 """
160 Run the console.
161 """
162 self.log('\n# Building package {0} from {1}\n'.format(ROOT, SOURCETREE))
163
164 self._init()
165 v = self._revision()
166 self._doc(v)
167 self._test()
168 vn = self._package()
169
170 self.log('\n# Done ({0}).\n'.format(vn.full))
171
172 - def log(self, message, level=1, ending='\n'):
173
174 if self._verbosity >= level:
175 sys.stdout.write(message)
176 sys.stdout.write(ending)
177 sys.stdout.flush()
178
180 """
181 Collect all required stuff in the output folder.
182 """
183 self.log('# Preparing the file system...')
184
185 if not os.path.exists(self._output):
186 self.log('Creating output directory {0}'.format(self._output), level=2)
187 os.mkdir(self._output)
188
189 if os.path.exists(self._temp):
190 self.log('Deleting existing temp directory {0}'.format(self._temp), level=2)
191 shutil.rmtree(self._temp)
192
193 self.log('Copying the source tree to temp directory {0}'.format(self._temp), level=2)
194 shutil.copytree(self._input, self._temp)
195
196 if os.path.exists(self._apidocs):
197 self.log('Deleting existing API docs directory {0}'.format(self._apidocs), level=2)
198 shutil.rmtree(self._apidocs)
199 if not os.path.isdir(self._docs):
200 self.log('Creating docs directory {0}'.format(self._docs), level=2)
201 os.mkdir(self._docs)
202 self.log('Creating API docs directory {0}'.format(self._apidocs), level=2)
203 os.mkdir(self._apidocs)
204
223
251
252 - def _doc(self, version):
253 """
254 Build documentation in the output folder.
255 """
256 self.log('\n# Generating API documentation...')
257 try:
258 import epydoc.cli
259 except ImportError:
260 self.log('\n Skipped: epydoc is missing')
261 return
262
263 self.log('\n# Emulating ARGV for the Doc Builder...', level=2)
264 argv = sys.argv
265 sys.argv = ['epydoc', '--html', '-o', self._apidocs,
266 '--name', '{0} v{1}'.format(ROOT.upper(), version),
267 '--no-private', '--introspect-only', '--exclude', 'csb.test.cases',
268 '--css', os.path.join(self._temp, 'epydoc.css'),
269 '--fail-on-error', '--fail-on-warning', '--fail-on-docstring-warning',
270 self._root]
271
272 if self._verbosity > 0:
273 sys.argv.append('-v')
274
275 try:
276 epydoc.cli.cli()
277 sys.exit(0)
278 except SystemExit as ex:
279 if ex.code is 0:
280 self.log('\n Passed all doc tests')
281 else:
282 if ex.code == 2:
283 self.log('\n DID NOT PASS: The docs might be broken')
284 else:
285 self.log('\n FAIL: Epydoc returned "#{0.code}: {0}"'.format(ex))
286
287 self.log('\n# Restoring the previous ARGV...', level=2)
288 sys.argv = argv
289
291 """
292 Make package.
293 """
294 self.log('\n# Configuring CWD and ARGV for the Setup...', level=2)
295 cwd = os.curdir
296 os.chdir(self._temp)
297
298 if self._verbosity > 1:
299 verbosity = '-v'
300 else:
301 verbosity = '-q'
302 argv = sys.argv
303 sys.argv = ['setup.py', verbosity, self._dist, '-d', self._output]
304
305 self.log('\n# Building {0} distribution...'.format(self._type))
306 try:
307 setup = imp.load_source('setupcsb', 'setup.py')
308 d = setup.build()
309 version = setup.VERSION
310 package = d.dist_files[0][2]
311
312 if self._type == BuildTypes.BINARY:
313 self._strip_source(package)
314
315 except SystemExit as ex:
316 if ex.code is not 0:
317 package = 'FAIL'
318 self.log('\n FAIL: Setup returned: \n\n{0}\n'.format(ex))
319
320 self.log('\n# Restoring the previous CWD and ARGV...', level=2)
321 os.chdir(cwd)
322 sys.argv = argv
323
324 self.log(' Packaged ' + package)
325 return version
326
328 """
329 Delete plain text source code files from the package.
330 """
331 cwd = os.getcwd()
332
333 try:
334 tmp = os.path.join(self.output, 'tmp')
335 os.mkdir(tmp)
336
337 self.log('\n# Entering {1} in order to delete .py files from {0}...'.format(package, tmp), level=2)
338 os.chdir(tmp)
339
340 oldtar = tarfile.open(package, mode='r:gz')
341 oldtar.extractall(tmp)
342 oldtar.close()
343
344 newtar = tarfile.open(package, mode='w:gz')
345
346 try:
347 for i in os.walk('.'):
348 for fn in i[2]:
349 if fn.endswith('.py'):
350 module = os.path.join(i[0], fn);
351 if not os.path.isfile(module.replace('.py', '.pyc')):
352 raise ValueError('Missing bytecode for module {0}'.format(module))
353 else:
354 os.remove(os.path.join(i[0], fn))
355
356 for i in os.listdir('.'):
357 newtar.add(i)
358 finally:
359 newtar.close()
360
361 finally:
362 self.log('\n# Restoring the previous CWD...', level=2)
363 os.chdir(cwd)
364 if os.path.exists(tmp):
365 shutil.rmtree(tmp)
366
367 @staticmethod
368 - def exit(message=None, code=0, usage=True):
376
377 @staticmethod
378 - def run(argv=None):
379
380 if argv is None:
381 argv = sys.argv[1:]
382
383 output = None
384 verb = 1
385 buildtype = BuildTypes.SOURCE
386
387 try:
388 options, dummy = getopt.getopt(argv, 'o:v:t:h', ['output=', 'verbosity=', 'type=', 'help'])
389
390 for option, value in options:
391 if option in('-h', '--help'):
392 Console.exit(message=None, code=0)
393 if option in('-o', '--output'):
394 if not os.path.isdir(value):
395 Console.exit(message='E: Output directory not found "{0}".'.format(value), code=3)
396 output = value
397 if option in('-v', '--verbosity'):
398 try:
399 verb = int(value)
400 except ValueError:
401 Console.exit(message='E: Verbosity must be an integer.', code=4)
402 if option in('-t', '--type'):
403 if value not in [BuildTypes.SOURCE, BuildTypes.BINARY]:
404 Console.exit(message='E: Invalid build type "{0}".'.format(value), code=5)
405 buildtype = value
406 except getopt.GetoptError as oe:
407 Console.exit(message='E: ' + str(oe), code=1)
408
409 if not output:
410 Console.exit(code=1, usage=True)
411 else:
412 try:
413 Console(output, verbosity=verb, buildtype=buildtype).build()
414 except Exception as ex:
415 msg = 'Unexpected Error: {0}\n\n{1}'.format(ex, traceback.format_exc())
416 Console.exit(message=msg, code=99, usage=False)
417
426
428 """
429 Determines the current repository revision number of a working copy.
430
431 @param path: a local checkout path to be examined
432 @type path: str
433 @param sc: name of the source control program
434 @type sc: str
435 """
436
438
439 self._path = None
440 self._sc = None
441
442 if os.path.exists(path):
443 self._path = path
444 else:
445 raise IOError('Path not found: {0}'.format(path))
446 if Shell.run([sc, 'help']).code is 0:
447 self._sc = sc
448 else:
449 raise RevisionError('Source control binary probe failed', None, None)
450
451 @property
454
455 @property
458
459 @abstractmethod
461 """
462 Return the current revision information.
463 @rtype: L{RevisionInfo}
464 """
465 pass
466
467 - def write(self, revision, sourcefile):
468 """
469 Finalize the __version__ = major.minor.micro.{revision} tag.
470 Overwrite C{sourcefile} in place by substituting the {revision} macro.
471
472 @param revision: revision number to write to the source file.
473 @type revision: int
474 @param sourcefile: python source file with a __version__ tag, typically
475 "csb/__init__.py"
476 @type sourcefile: str
477
478 @return: sourcefile.__version__
479 """
480 content = open(sourcefile).readlines()
481
482 with open(sourcefile, 'w') as src:
483 for line in content:
484 if line.startswith('__version__'):
485 src.write(line.format(revision=revision))
486 else:
487 src.write(line)
488
489 self._delcache(sourcefile)
490 return imp.load_source('____source', sourcefile).__version__
491
492 - def _run(self, cmd):
493
494 si = Shell.run(cmd)
495 if si.code > 0:
496 raise RevisionError('SC failed ({0.code}): {0.stderr}'.format(si), si.code, si.cmd)
497
498 return si.stdout.splitlines()
499
501
502 compiled = os.path.splitext(sourcefile)[0] + '.pyc'
503 if os.path.isfile(compiled):
504 os.remove(compiled)
505
506 pycache = os.path.join(os.path.dirname(compiled), '__pycache__')
507 if os.path.isdir(pycache):
508 shutil.rmtree(pycache)
509
514
516
517 cmd = '{0.sc} info {0.path} -R'.format(self)
518 maxrevision = None
519
520 for line in self._run(cmd):
521 if line.startswith('Revision:'):
522 rev = int(line[9:] .strip())
523 if rev > maxrevision:
524 maxrevision = rev
525
526 if maxrevision is None:
527 raise RevisionError('No revision number found', code=0, cmd=cmd)
528
529 return RevisionInfo(self.path, maxrevision)
530
532
539
541
542 wd = os.getcwd()
543 os.chdir(self.path)
544
545 try:
546 cmd = '{0.sc} log -r tip'.format(self)
547
548 revision = None
549 changeset = ''
550
551 for line in self._run(cmd):
552 if line.startswith('changeset:'):
553 items = line[10:].split(':')
554 revision = int(items[0])
555 changeset = items[1].strip()
556 break
557
558 if revision is None:
559 raise RevisionError('No revision number found', code=0, cmd=cmd)
560
561 return RevisionInfo(self.path, revision, changeset)
562
563 finally:
564 os.chdir(wd)
565
567
568 - def __init__(self, item, revision, id=None):
573
577
578
579 if __name__ == '__main__':
580
581 main()
582