1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 """Classes to handle advanced configuration in simple to complex applications.
19
20 Allows to load the configuration from a file or from command line
21 options, to generate a sample configuration file or to display
22 program's usage. Fills the gap between optik/optparse and ConfigParser
23 by adding data types (which are also available as a standalone optik
24 extension in the `optik_ext` module).
25
26
27 Quick start: simplest usage
28 ---------------------------
29
30 .. python ::
31
32 >>> import sys
33 >>> from logilab.common.configuration import Configuration
34 >>> options = [('dothis', {'type':'yn', 'default': True, 'metavar': '<y or n>'}),
35 ... ('value', {'type': 'string', 'metavar': '<string>'}),
36 ... ('multiple', {'type': 'csv', 'default': ('yop',),
37 ... 'metavar': '<comma separated values>',
38 ... 'help': 'you can also document the option'}),
39 ... ('number', {'type': 'int', 'default':2, 'metavar':'<int>'}),
40 ... ]
41 >>> config = Configuration(options=options, name='My config')
42 >>> print config['dothis']
43 True
44 >>> print config['value']
45 None
46 >>> print config['multiple']
47 ('yop',)
48 >>> print config['number']
49 2
50 >>> print config.help()
51 Usage: [options]
52
53 Options:
54 -h, --help show this help message and exit
55 --dothis=<y or n>
56 --value=<string>
57 --multiple=<comma separated values>
58 you can also document the option [current: none]
59 --number=<int>
60
61 >>> f = open('myconfig.ini', 'w')
62 >>> f.write('''[MY CONFIG]
63 ... number = 3
64 ... dothis = no
65 ... multiple = 1,2,3
66 ... ''')
67 >>> f.close()
68 >>> config.load_file_configuration('myconfig.ini')
69 >>> print config['dothis']
70 False
71 >>> print config['value']
72 None
73 >>> print config['multiple']
74 ['1', '2', '3']
75 >>> print config['number']
76 3
77 >>> sys.argv = ['mon prog', '--value', 'bacon', '--multiple', '4,5,6',
78 ... 'nonoptionargument']
79 >>> print config.load_command_line_configuration()
80 ['nonoptionargument']
81 >>> print config['value']
82 bacon
83 >>> config.generate_config()
84 # class for simple configurations which don't need the
85 # manager / providers model and prefer delegation to inheritance
86 #
87 # configuration values are accessible through a dict like interface
88 #
89 [MY CONFIG]
90
91 dothis=no
92
93 value=bacon
94
95 # you can also document the option
96 multiple=4,5,6
97
98 number=3
99 >>>
100 """
101 __docformat__ = "restructuredtext en"
102
103 __all__ = ('OptionsManagerMixIn', 'OptionsProviderMixIn',
104 'ConfigurationMixIn', 'Configuration',
105 'OptionsManager2ConfigurationAdapter')
106
107 import os
108 import sys
109 import re
110 from os.path import exists, expanduser
111 from copy import copy
112 from ConfigParser import ConfigParser, NoOptionError, NoSectionError, \
113 DuplicateSectionError
114 from warnings import warn
115
116 from logilab.common.compat import callable, raw_input, str_encode as _encode
117
118 from logilab.common.textutils import normalize_text, unquote
119 from logilab.common import optik_ext as optparse
120
121 OptionError = optparse.OptionError
122
123 REQUIRED = []
124
126 """raised by set_option when it doesn't know what to do for an action"""
127
128
130 encoding = encoding or getattr(stream, 'encoding', None)
131 if not encoding:
132 import locale
133 encoding = locale.getpreferredencoding()
134 return encoding
135
136
137
138
140 """validate and return a converted value for option of type 'choice'
141 """
142 if not value in optdict['choices']:
143 msg = "option %s: invalid value: %r, should be in %s"
144 raise optparse.OptionValueError(msg % (name, value, optdict['choices']))
145 return value
146
148 """validate and return a converted value for option of type 'choice'
149 """
150 choices = optdict['choices']
151 values = optparse.check_csv(None, name, value)
152 for value in values:
153 if not value in choices:
154 msg = "option %s: invalid value: %r, should be in %s"
155 raise optparse.OptionValueError(msg % (name, value, choices))
156 return values
157
159 """validate and return a converted value for option of type 'csv'
160 """
161 return optparse.check_csv(None, name, value)
162
164 """validate and return a converted value for option of type 'yn'
165 """
166 return optparse.check_yn(None, name, value)
167
169 """validate and return a converted value for option of type 'named'
170 """
171 return optparse.check_named(None, name, value)
172
174 """validate and return a filepath for option of type 'file'"""
175 return optparse.check_file(None, name, value)
176
178 """validate and return a valid color for option of type 'color'"""
179 return optparse.check_color(None, name, value)
180
182 """validate and return a string for option of type 'password'"""
183 return optparse.check_password(None, name, value)
184
186 """validate and return a mx DateTime object for option of type 'date'"""
187 return optparse.check_date(None, name, value)
188
190 """validate and return a time object for option of type 'time'"""
191 return optparse.check_time(None, name, value)
192
194 """validate and return an integer for option of type 'bytes'"""
195 return optparse.check_bytes(None, name, value)
196
197
198 VALIDATORS = {'string': unquote,
199 'int': int,
200 'float': float,
201 'file': file_validator,
202 'font': unquote,
203 'color': color_validator,
204 'regexp': re.compile,
205 'csv': csv_validator,
206 'yn': yn_validator,
207 'bool': yn_validator,
208 'named': named_validator,
209 'password': password_validator,
210 'date': date_validator,
211 'time': time_validator,
212 'bytes': bytes_validator,
213 'choice': choice_validator,
214 'multiple_choice': multiple_choice_validator,
215 }
216
218 if opttype not in VALIDATORS:
219 raise Exception('Unsupported type "%s"' % opttype)
220 try:
221 return VALIDATORS[opttype](optdict, option, value)
222 except TypeError:
223 try:
224 return VALIDATORS[opttype](value)
225 except optparse.OptionValueError:
226 raise
227 except:
228 raise optparse.OptionValueError('%s value (%r) should be of type %s' %
229 (option, value, opttype))
230
231
232
241
245
257 return input_validator
258
259 INPUT_FUNCTIONS = {
260 'string': input_string,
261 'password': input_password,
262 }
263
264 for opttype in VALIDATORS.keys():
265 INPUT_FUNCTIONS.setdefault(opttype, _make_input_function(opttype))
266
268 """monkey patch OptionParser.expand_default since we have a particular
269 way to handle defaults to avoid overriding values in the configuration
270 file
271 """
272 if self.parser is None or not self.default_tag:
273 return option.help
274 optname = option._long_opts[0][2:]
275 try:
276 provider = self.parser.options_manager._all_options[optname]
277 except KeyError:
278 value = None
279 else:
280 optdict = provider.get_option_def(optname)
281 optname = provider.option_name(optname, optdict)
282 value = getattr(provider.config, optname, optdict)
283 value = format_option_value(optdict, value)
284 if value is optparse.NO_DEFAULT or not value:
285 value = self.NO_DEFAULT_VALUE
286 return option.help.replace(self.default_tag, str(value))
287
288
289 -def convert(value, optdict, name=''):
290 """return a validated value for an option according to its type
291
292 optional argument name is only used for error message formatting
293 """
294 try:
295 _type = optdict['type']
296 except KeyError:
297
298 return value
299 return _call_validator(_type, optdict, name, value)
300
305
322
337
356
364
381
382 format_section = ini_format_section
383
403
404
406 """MixIn to handle a configuration from both a configuration file and
407 command line options
408 """
409
410 - def __init__(self, usage, config_file=None, version=None, quiet=0):
411 self.config_file = config_file
412 self.reset_parsers(usage, version=version)
413
414 self.options_providers = []
415
416 self._all_options = {}
417 self._short_options = {}
418 self._nocallback_options = {}
419 self._mygroups = dict()
420
421 self.quiet = quiet
422 self._maxlevel = 0
423
425
426 self.cfgfile_parser = ConfigParser()
427
428 self.cmdline_parser = optparse.OptionParser(usage=usage, version=version)
429 self.cmdline_parser.options_manager = self
430 self._optik_option_attrs = set(self.cmdline_parser.option_class.ATTRS)
431
433 """register an options provider"""
434 assert provider.priority <= 0, "provider's priority can't be >= 0"
435 for i in range(len(self.options_providers)):
436 if provider.priority > self.options_providers[i].priority:
437 self.options_providers.insert(i, provider)
438 break
439 else:
440 self.options_providers.append(provider)
441 non_group_spec_options = [option for option in provider.options
442 if 'group' not in option[1]]
443 groups = getattr(provider, 'option_groups', ())
444 if own_group and non_group_spec_options:
445 self.add_option_group(provider.name.upper(), provider.__doc__,
446 non_group_spec_options, provider)
447 else:
448 for opt, optdict in non_group_spec_options:
449 self.add_optik_option(provider, self.cmdline_parser, opt, optdict)
450 for gname, gdoc in groups:
451 gname = gname.upper()
452 goptions = [option for option in provider.options
453 if option[1].get('group', '').upper() == gname]
454 self.add_option_group(gname, gdoc, goptions, provider)
455
457 """add an option group including the listed options
458 """
459 assert options
460
461 if group_name in self._mygroups:
462 group = self._mygroups[group_name]
463 else:
464 group = optparse.OptionGroup(self.cmdline_parser,
465 title=group_name.capitalize())
466 self.cmdline_parser.add_option_group(group)
467 group.level = provider.level
468 self._mygroups[group_name] = group
469
470 if group_name != "DEFAULT":
471 self.cfgfile_parser.add_section(group_name)
472
473 for opt, optdict in options:
474 self.add_optik_option(provider, group, opt, optdict)
475
477 if 'inputlevel' in optdict:
478 warn('[0.50] "inputlevel" in option dictionary for %s is deprecated,'
479 ' use "level"' % opt, DeprecationWarning)
480 optdict['level'] = optdict.pop('inputlevel')
481 args, optdict = self.optik_option(provider, opt, optdict)
482 option = optikcontainer.add_option(*args, **optdict)
483 self._all_options[opt] = provider
484 self._maxlevel = max(self._maxlevel, option.level or 0)
485
487 """get our personal option definition and return a suitable form for
488 use with optik/optparse
489 """
490 optdict = copy(optdict)
491 others = {}
492 if 'action' in optdict:
493 self._nocallback_options[provider] = opt
494 else:
495 optdict['action'] = 'callback'
496 optdict['callback'] = self.cb_set_provider_option
497
498
499 if 'default' in optdict:
500 if (optparse.OPTPARSE_FORMAT_DEFAULT and 'help' in optdict and
501 optdict.get('default') is not None and
502 not optdict['action'] in ('store_true', 'store_false')):
503 optdict['help'] += ' [current: %default]'
504 del optdict['default']
505 args = ['--' + str(opt)]
506 if 'short' in optdict:
507 self._short_options[optdict['short']] = opt
508 args.append('-' + optdict['short'])
509 del optdict['short']
510
511 for key in optdict.keys():
512 if not key in self._optik_option_attrs:
513 optdict.pop(key)
514 return args, optdict
515
517 """optik callback for option setting"""
518 if opt.startswith('--'):
519
520 opt = opt[2:]
521 else:
522
523 opt = self._short_options[opt[1:]]
524
525 if value is None:
526 value = 1
527 self.global_set_option(opt, value)
528
530 """set option on the correct option provider"""
531 self._all_options[opt].set_option(opt, value)
532
534 """write a configuration file according to the current configuration
535 into the given stream or stdout
536 """
537 options_by_section = {}
538 sections = []
539 for provider in self.options_providers:
540 for section, options in provider.options_by_section():
541 if section is None:
542 section = provider.name
543 if section in skipsections:
544 continue
545 options = [(n, d, v) for (n, d, v) in options
546 if d.get('type') is not None]
547 if not options:
548 continue
549 if not section in sections:
550 sections.append(section)
551 alloptions = options_by_section.setdefault(section, [])
552 alloptions += options
553 stream = stream or sys.stdout
554 encoding = _get_encoding(encoding, stream)
555 printed = False
556 for section in sections:
557 if printed:
558 print >> stream, '\n'
559 format_section(stream, section.upper(), options_by_section[section],
560 encoding)
561 printed = True
562
563 - def generate_manpage(self, pkginfo, section=1, stream=None):
564 """write a man page for the current configuration into the given
565 stream or stdout
566 """
567 self._monkeypatch_expand_default()
568 try:
569 optparse.generate_manpage(self.cmdline_parser, pkginfo,
570 section, stream=stream or sys.stdout,
571 level=self._maxlevel)
572 finally:
573 self._unmonkeypatch_expand_default()
574
575
576
578 """initialize configuration using default values"""
579 for provider in self.options_providers:
580 provider.load_defaults()
581
586
588 """read the configuration file but do not load it (i.e. dispatching
589 values to each options provider)
590 """
591 helplevel = 1
592 while helplevel <= self._maxlevel:
593 opt = '-'.join(['long'] * helplevel) + '-help'
594 if opt in self._all_options:
595 break
596 def helpfunc(option, opt, val, p, level=helplevel):
597 print self.help(level)
598 sys.exit(0)
599 helpmsg = '%s verbose help.' % ' '.join(['more'] * helplevel)
600 optdict = {'action' : 'callback', 'callback' : helpfunc,
601 'help' : helpmsg}
602 provider = self.options_providers[0]
603 self.add_optik_option(provider, self.cmdline_parser, opt, optdict)
604 provider.options += ( (opt, optdict), )
605 helplevel += 1
606 if config_file is None:
607 config_file = self.config_file
608 if config_file is not None:
609 config_file = expanduser(config_file)
610 if config_file and exists(config_file):
611 parser = self.cfgfile_parser
612 parser.read([config_file])
613
614 for sect, values in parser._sections.items():
615 if not sect.isupper() and values:
616 parser._sections[sect.upper()] = values
617 elif not self.quiet:
618 msg = 'No config file found, using default configuration'
619 print >> sys.stderr, msg
620 return
621
639
641 """dispatch values previously read from a configuration file to each
642 options provider)
643 """
644 parser = self.cfgfile_parser
645 for provider in self.options_providers:
646 for section, option, optdict in provider.all_options():
647 try:
648 value = parser.get(section, option)
649 provider.set_option(option, value, optdict=optdict)
650 except (NoSectionError, NoOptionError), ex:
651 continue
652
654 """override configuration according to given parameters
655 """
656 for opt, opt_value in kwargs.items():
657 opt = opt.replace('_', '-')
658 provider = self._all_options[opt]
659 provider.set_option(opt, opt_value)
660
662 """override configuration according to command line parameters
663
664 return additional arguments
665 """
666 self._monkeypatch_expand_default()
667 try:
668 if args is None:
669 args = sys.argv[1:]
670 else:
671 args = list(args)
672 (options, args) = self.cmdline_parser.parse_args(args=args)
673 for provider in self._nocallback_options.keys():
674 config = provider.config
675 for attr in config.__dict__.keys():
676 value = getattr(options, attr, None)
677 if value is None:
678 continue
679 setattr(config, attr, value)
680 return args
681 finally:
682 self._unmonkeypatch_expand_default()
683
684
685
686
688 """add a dummy option section for help purpose """
689 group = optparse.OptionGroup(self.cmdline_parser,
690 title=title.capitalize(),
691 description=description)
692 group.level = level
693 self._maxlevel = max(self._maxlevel, level)
694 self.cmdline_parser.add_option_group(group)
695
697
698 try:
699 self.__expand_default_backup = optparse.HelpFormatter.expand_default
700 optparse.HelpFormatter.expand_default = expand_default
701 except AttributeError:
702
703 pass
705
706 if hasattr(optparse.HelpFormatter, 'expand_default'):
707
708 optparse.HelpFormatter.expand_default = self.__expand_default_backup
709
710 - def help(self, level=0):
711 """return the usage string for available options """
712 self.cmdline_parser.formatter.output_level = level
713 self._monkeypatch_expand_default()
714 try:
715 return self.cmdline_parser.format_help()
716 finally:
717 self._unmonkeypatch_expand_default()
718
719
721 """used to ease late binding of default method (so you can define options
722 on the class using default methods on the configuration instance)
723 """
725 self.method = methname
726 self._inst = None
727
728 - def bind(self, instance):
729 """bind the method to its instance"""
730 if self._inst is None:
731 self._inst = instance
732
734 assert self._inst, 'unbound method'
735 return getattr(self._inst, self.method)(*args, **kwargs)
736
737
739 """Mixin to provide options to an OptionsManager"""
740
741
742 priority = -1
743 name = 'default'
744 options = ()
745 level = 0
746
748 self.config = optparse.Values()
749 for option in self.options:
750 try:
751 option, optdict = option
752 except ValueError:
753 raise Exception('Bad option: %r' % option)
754 if isinstance(optdict.get('default'), Method):
755 optdict['default'].bind(self)
756 elif isinstance(optdict.get('callback'), Method):
757 optdict['callback'].bind(self)
758 self.load_defaults()
759
761 """initialize the provider using default values"""
762 for opt, optdict in self.options:
763 action = optdict.get('action')
764 if action != 'callback':
765
766 default = self.option_default(opt, optdict)
767 if default is REQUIRED:
768 continue
769 self.set_option(opt, default, action, optdict)
770
772 """return the default value for an option"""
773 if optdict is None:
774 optdict = self.get_option_def(opt)
775 default = optdict.get('default')
776 if callable(default):
777 default = default()
778 return default
779
781 """get the config attribute corresponding to opt
782 """
783 if optdict is None:
784 optdict = self.get_option_def(opt)
785 return optdict.get('dest', opt.replace('-', '_'))
786
788 """get the current value for the given option"""
789 return getattr(self.config, self.option_name(opt), None)
790
791 - def set_option(self, opt, value, action=None, optdict=None):
792 """method called to set an option (registered in the options list)
793 """
794
795 if optdict is None:
796 optdict = self.get_option_def(opt)
797 if value is not None:
798 value = convert(value, optdict, opt)
799 if action is None:
800 action = optdict.get('action', 'store')
801 if optdict.get('type') == 'named':
802 optname = self.option_name(opt, optdict)
803 currentvalue = getattr(self.config, optname, None)
804 if currentvalue:
805 currentvalue.update(value)
806 value = currentvalue
807 if action == 'store':
808 setattr(self.config, self.option_name(opt, optdict), value)
809 elif action in ('store_true', 'count'):
810 setattr(self.config, self.option_name(opt, optdict), 0)
811 elif action == 'store_false':
812 setattr(self.config, self.option_name(opt, optdict), 1)
813 elif action == 'append':
814 opt = self.option_name(opt, optdict)
815 _list = getattr(self.config, opt, None)
816 if _list is None:
817 if isinstance(value, (list, tuple)):
818 _list = value
819 elif value is not None:
820 _list = []
821 _list.append(value)
822 setattr(self.config, opt, _list)
823 elif isinstance(_list, tuple):
824 setattr(self.config, opt, _list + (value,))
825 else:
826 _list.append(value)
827 elif action == 'callback':
828 optdict['callback'](None, opt, value, None)
829 else:
830 raise UnsupportedAction(action)
831
852
854 """return the dictionary defining an option given it's name"""
855 assert self.options
856 for option in self.options:
857 if option[0] == opt:
858 return option[1]
859 raise OptionError('no such option %s in section %r'
860 % (opt, self.name), opt)
861
862
864 """return an iterator on available options for this provider
865 option are actually described by a 3-uple:
866 (section, option name, option dictionary)
867 """
868 for section, options in self.options_by_section():
869 if section is None:
870 if self.name is None:
871 continue
872 section = self.name.upper()
873 for option, optiondict, value in options:
874 yield section, option, optiondict
875
877 """return an iterator on options grouped by section
878
879 (section, [list of (optname, optdict, optvalue)])
880 """
881 sections = {}
882 for optname, optdict in self.options:
883 sections.setdefault(optdict.get('group'), []).append(
884 (optname, optdict, self.option_value(optname)))
885 if None in sections:
886 yield None, sections.pop(None)
887 for section, options in sections.items():
888 yield section.upper(), options
889
895
896
898 """basic mixin for simple configurations which don't need the
899 manager / providers model
900 """
917
926
929
931 return iter(self.config.__dict__.iteritems())
932
934 try:
935 return getattr(self.config, self.option_name(key))
936 except (optparse.OptionValueError, AttributeError):
937 raise KeyError(key)
938
941
942 - def get(self, key, default=None):
943 try:
944 return getattr(self.config, self.option_name(key))
945 except (OptionError, AttributeError):
946 return default
947
948
950 """class for simple configurations which don't need the
951 manager / providers model and prefer delegation to inheritance
952
953 configuration values are accessible through a dict like interface
954 """
955
956 - def __init__(self, config_file=None, options=None, name=None,
957 usage=None, doc=None, version=None):
965
966
968 """Adapt an option manager to behave like a
969 `logilab.common.configuration.Configuration` instance
970 """
972 self.config = provider
973
975 return getattr(self.config, key)
976
978 provider = self.config._all_options[key]
979 try:
980 return getattr(provider.config, provider.option_name(key))
981 except AttributeError:
982 raise KeyError(key)
983
986
987 - def get(self, key, default=None):
988 provider = self.config._all_options[key]
989 try:
990 return getattr(provider.config, provider.option_name(key))
991 except AttributeError:
992 return default
993
994
996 """initialize newconfig from a deprecated configuration file
997
998 possible changes:
999 * ('renamed', oldname, newname)
1000 * ('moved', option, oldgroup, newgroup)
1001 * ('typechanged', option, oldtype, newvalue)
1002 """
1003
1004 changesindex = {}
1005 for action in changes:
1006 if action[0] == 'moved':
1007 option, oldgroup, newgroup = action[1:]
1008 changesindex.setdefault(option, []).append((action[0], oldgroup, newgroup))
1009 continue
1010 if action[0] == 'renamed':
1011 oldname, newname = action[1:]
1012 changesindex.setdefault(newname, []).append((action[0], oldname))
1013 continue
1014 if action[0] == 'typechanged':
1015 option, oldtype, newvalue = action[1:]
1016 changesindex.setdefault(option, []).append((action[0], oldtype, newvalue))
1017 continue
1018 if action[1] in ('added', 'removed'):
1019 continue
1020 raise Exception('unknown change %s' % action[0])
1021
1022 options = []
1023 for optname, optdef in newconfig.options:
1024 for action in changesindex.pop(optname, ()):
1025 if action[0] == 'moved':
1026 oldgroup, newgroup = action[1:]
1027 optdef = optdef.copy()
1028 optdef['group'] = oldgroup
1029 elif action[0] == 'renamed':
1030 optname = action[1]
1031 elif action[0] == 'typechanged':
1032 oldtype = action[1]
1033 optdef = optdef.copy()
1034 optdef['type'] = oldtype
1035 options.append((optname, optdef))
1036 if changesindex:
1037 raise Exception('unapplied changes: %s' % changesindex)
1038 oldconfig = Configuration(options=options, name=newconfig.name)
1039
1040 oldconfig.load_file_configuration(configfile)
1041
1042 changes.reverse()
1043 done = set()
1044 for action in changes:
1045 if action[0] == 'renamed':
1046 oldname, newname = action[1:]
1047 newconfig[newname] = oldconfig[oldname]
1048 done.add(newname)
1049 elif action[0] == 'typechanged':
1050 optname, oldtype, newvalue = action[1:]
1051 newconfig[optname] = newvalue
1052 done.add(optname)
1053 for optname, optdef in newconfig.options:
1054 if optdef.get('type') and not optname in done:
1055 newconfig.set_option(optname, oldconfig[optname], optdict=optdef)
1056
1057
1059 """preprocess options to remove duplicate"""
1060 alloptions = {}
1061 options = list(options)
1062 for i in range(len(options)-1, -1, -1):
1063 optname, optdict = options[i]
1064 if optname in alloptions:
1065 options.pop(i)
1066 alloptions[optname].update(optdict)
1067 else:
1068 alloptions[optname] = optdict
1069 return tuple(options)
1070