1 """Classes to handle advanced configuration in simple to complex applications.
2
3 Allows to load the configuration from a file or from command line
4 options, to generate a sample configuration file or to display
5 program's usage. Fills the gap between optik/optparse and ConfigParser
6 by adding data types (which are also available as a standalone optik
7 extension in the `optik_ext` module).
8
9
10 Quick start: simplest usage
11 ---------------------------
12
13 .. python ::
14
15 >>> import sys
16 >>> from logilab.common.configuration import Configuration
17 >>> options = [('dothis', {'type':'yn', 'default': True, 'metavar': '<y or n>'}),
18 ... ('value', {'type': 'string', 'metavar': '<string>'}),
19 ... ('multiple', {'type': 'csv', 'default': ('yop',),
20 ... 'metavar': '<comma separated values>',
21 ... 'help': 'you can also document the option'}),
22 ... ('number', {'type': 'int', 'default':2, 'metavar':'<int>'}),
23 ... ]
24 >>> config = Configuration(options=options, name='My config')
25 >>> print config['dothis']
26 True
27 >>> print config['value']
28 None
29 >>> print config['multiple']
30 ('yop',)
31 >>> print config['number']
32 2
33 >>> print config.help()
34 Usage: [options]
35
36 Options:
37 -h, --help show this help message and exit
38 --dothis=<y or n>
39 --value=<string>
40 --multiple=<comma separated values>
41 you can also document the option [current: none]
42 --number=<int>
43
44 >>> f = open('myconfig.ini', 'w')
45 >>> f.write('''[MY CONFIG]
46 ... number = 3
47 ... dothis = no
48 ... multiple = 1,2,3
49 ... ''')
50 >>> f.close()
51 >>> config.load_file_configuration('myconfig.ini')
52 >>> print config['dothis']
53 False
54 >>> print config['value']
55 None
56 >>> print config['multiple']
57 ['1', '2', '3']
58 >>> print config['number']
59 3
60 >>> sys.argv = ['mon prog', '--value', 'bacon', '--multiple', '4,5,6',
61 ... 'nonoptionargument']
62 >>> print config.load_command_line_configuration()
63 ['nonoptionargument']
64 >>> print config['value']
65 bacon
66 >>> config.generate_config()
67 # class for simple configurations which don't need the
68 # manager / providers model and prefer delegation to inheritance
69 #
70 # configuration values are accessible through a dict like interface
71 #
72 [MY CONFIG]
73
74 dothis=no
75
76 value=bacon
77
78 # you can also document the option
79 multiple=4,5,6
80
81 number=3
82 >>>
83
84
85 :copyright: 2003-2009 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
86 :contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr
87 :license: General Public License version 2 - http://www.gnu.org/licenses
88 """
89 from __future__ import generators
90 __docformat__ = "restructuredtext en"
91
92 __all__ = ('OptionsManagerMixIn', 'OptionsProviderMixIn',
93 'ConfigurationMixIn', 'Configuration',
94 'OptionsManager2ConfigurationAdapter')
95
96 import os
97 import sys
98 import re
99 from os.path import exists, expanduser
100 from copy import copy
101 from ConfigParser import ConfigParser, NoOptionError, NoSectionError, \
102 DuplicateSectionError
103
104 from logilab.common.compat import set
105 from logilab.common.textutils import normalize_text, unquote
106 from logilab.common.deprecation import deprecated
107 from logilab.common import optik_ext as opt
108
109 REQUIRED = []
110
111 check_csv = deprecated('use lgc.optik_ext.check_csv')(opt.check_csv)
112
114 """raised by set_option when it doesn't know what to do for an action"""
115
116
118 encoding = encoding or getattr(stream, 'encoding', None)
119 if not encoding:
120 import locale
121 encoding = locale.getpreferredencoding()
122 return encoding
123
125 if isinstance(string, unicode):
126 return string.encode(encoding)
127 return str(string)
128
129
130
131
133 """validate and return a converted value for option of type 'choice'
134 """
135 if not value in opt_dict['choices']:
136 msg = "option %s: invalid value: %r, should be in %s"
137 raise opt.OptionValueError(msg % (name, value, opt_dict['choices']))
138 return value
139
141 """validate and return a converted value for option of type 'choice'
142 """
143 choices = opt_dict['choices']
144 values = opt.check_csv(None, name, value)
145 for value in values:
146 if not value in choices:
147 msg = "option %s: invalid value: %r, should be in %s"
148 raise opt.OptionValueError(msg % (name, value, choices))
149 return values
150
152 """validate and return a converted value for option of type 'csv'
153 """
154 return opt.check_csv(None, name, value)
155
157 """validate and return a converted value for option of type 'yn'
158 """
159 return opt.check_yn(None, name, value)
160
162 """validate and return a converted value for option of type 'named'
163 """
164 return opt.check_named(None, name, value)
165
167 """validate and return a filepath for option of type 'file'"""
168 return opt.check_file(None, name, value)
169
171 """validate and return a valid color for option of type 'color'"""
172 return opt.check_color(None, name, value)
173
175 """validate and return a string for option of type 'password'"""
176 return opt.check_password(None, name, value)
177
179 """validate and return a mx DateTime object for option of type 'date'"""
180 return opt.check_date(None, name, value)
181
183 """validate and return a time object for option of type 'time'"""
184 return opt.check_time(None, name, value)
185
187 """validate and return an integer for option of type 'bytes'"""
188 return opt.check_bytes(None, name, value)
189
190
191 VALIDATORS = {'string' : unquote,
192 'int' : int,
193 'float': float,
194 'file': file_validator,
195 'font': unquote,
196 'color': color_validator,
197 'regexp': re.compile,
198 'csv': csv_validator,
199 'yn': yn_validator,
200 'bool': yn_validator,
201 'named': named_validator,
202 'password': password_validator,
203 'date': date_validator,
204 'time': time_validator,
205 'bytes': bytes_validator,
206 'choice': choice_validator,
207 'multiple_choice': multiple_choice_validator,
208 }
209
223
224
225
234
238
250 return input_validator
251
252 INPUT_FUNCTIONS = {
253 'string': input_string,
254 'password': input_password,
255 }
256
257 for opttype in VALIDATORS.keys():
258 INPUT_FUNCTIONS.setdefault(opttype, _make_input_function(opttype))
259
261 """monkey patch OptionParser.expand_default since we have a particular
262 way to handle defaults to avoid overriding values in the configuration
263 file
264 """
265 if self.parser is None or not self.default_tag:
266 return option.help
267 optname = option._long_opts[0][2:]
268 try:
269 provider = self.parser.options_manager._all_options[optname]
270 except KeyError:
271 value = None
272 else:
273 optdict = provider.get_option_def(optname)
274 optname = provider.option_name(optname, optdict)
275 value = getattr(provider.config, optname, optdict)
276 value = format_option_value(optdict, value)
277 if value is opt.NO_DEFAULT or not value:
278 value = self.NO_DEFAULT_VALUE
279 return option.help.replace(self.default_tag, str(value))
280
281
282 -def convert(value, opt_dict, name=''):
283 """return a validated value for an option according to its type
284
285 optional argument name is only used for error message formatting
286 """
287 try:
288 _type = opt_dict['type']
289 except KeyError:
290
291 return value
292 return _call_validator(_type, opt_dict, name, value)
293
298
317
338
339 format_section = ini_format_section
340
360
361
363 """MixIn to handle a configuration from both a configuration file and
364 command line options
365 """
366
367 - def __init__(self, usage, config_file=None, version=None, quiet=0):
368 self.config_file = config_file
369 self.reset_parsers(usage, version=version)
370
371 self.options_providers = []
372
373 self._all_options = {}
374 self._short_options = {}
375 self._nocallback_options = {}
376 self._mygroups = set()
377
378 self.quiet = quiet
379
381
382 self._config_parser = ConfigParser()
383
384 self._optik_parser = opt.OptionParser(usage=usage, version=version)
385 self._optik_parser.options_manager = self
386
388 """register an options provider"""
389 assert provider.priority <= 0, "provider's priority can't be >= 0"
390 for i in range(len(self.options_providers)):
391 if provider.priority > self.options_providers[i].priority:
392 self.options_providers.insert(i, provider)
393 break
394 else:
395 self.options_providers.append(provider)
396 non_group_spec_options = [option for option in provider.options
397 if 'group' not in option[1]]
398 groups = getattr(provider, 'option_groups', ())
399 if own_group:
400 self.add_option_group(provider.name.upper(), provider.__doc__,
401 non_group_spec_options, provider)
402 else:
403 for opt_name, opt_dict in non_group_spec_options:
404 args, opt_dict = self.optik_option(provider, opt_name, opt_dict)
405 self._optik_parser.add_option(*args, **opt_dict)
406
407 self._all_options[opt_name] = provider
408 for gname, gdoc in groups:
409 gname = gname.upper()
410 goptions = [option for option in provider.options
411 if option[1].get('group', '').upper() == gname]
412 self.add_option_group(gname, gdoc, goptions, provider)
413
415 """add an option group including the listed options
416 """
417
418 if group_name != "DEFAULT":
419 try:
420 self._config_parser.add_section(group_name)
421 except DuplicateSectionError:
422
423
424
425 if group_name in self._mygroups:
426 raise
427 self._mygroups.add(group_name)
428
429 if options:
430 group = opt.OptionGroup(self._optik_parser,
431 title=group_name.capitalize())
432 self._optik_parser.add_option_group(group)
433
434 for opt_name, opt_dict in options:
435 args, opt_dict = self.optik_option(provider, opt_name, opt_dict)
436 group.add_option(*args, **opt_dict)
437 self._all_options[opt_name] = provider
438
440 """get our personal option definition and return a suitable form for
441 use with optik/optparse
442 """
443 opt_dict = copy(opt_dict)
444 if 'action' in opt_dict:
445 self._nocallback_options[provider] = opt_name
446 else:
447 opt_dict['action'] = 'callback'
448 opt_dict['callback'] = self.cb_set_provider_option
449
450
451 if 'default' in opt_dict:
452 if (opt.OPTPARSE_FORMAT_DEFAULT and 'help' in opt_dict and
453 opt_dict.get('default') is not None and
454 not opt_dict['action'] in ('store_true', 'store_false')):
455 opt_dict['help'] += ' [current: %default]'
456 del opt_dict['default']
457 args = ['--' + opt_name]
458 if 'short' in opt_dict:
459 self._short_options[opt_dict['short']] = opt_name
460 args.append('-' + opt_dict['short'])
461 del opt_dict['short']
462 available_keys = set(self._optik_parser.option_class.ATTRS)
463
464 for key in opt_dict.keys():
465 if not key in available_keys:
466 opt_dict.pop(key)
467 return args, opt_dict
468
470 """optik callback for option setting"""
471 if opt_name.startswith('--'):
472
473 opt_name = opt_name[2:]
474 else:
475
476 opt_name = self._short_options[opt_name[1:]]
477
478 if value is None:
479 value = 1
480 self.global_set_option(opt_name, value)
481
483 """set option on the correct option provider"""
484 self._all_options[opt_name].set_option(opt_name, value)
485
487 """write a configuration file according to the current configuration
488 into the given stream or stdout
489 """
490 stream = stream or sys.stdout
491 encoding = _get_encoding(encoding, stream)
492 printed = False
493 for provider in self.options_providers:
494 default_options = []
495 sections = {}
496 for section, options in provider.options_by_section():
497 if section in skipsections:
498 continue
499 options = [(n, d, v) for (n, d, v) in options
500 if d.get('type') is not None]
501 if not options:
502 continue
503 if section is None:
504 section = provider.name
505 doc = provider.__doc__
506 else:
507 doc = None
508 if printed:
509 print >> stream, '\n'
510 format_section(stream, section.upper(), options, encoding, doc)
511 printed = True
512
513 - def generate_manpage(self, pkginfo, section=1, stream=None):
514 """write a man page for the current configuration into the given
515 stream or stdout
516 """
517 self._monkeypatch_expand_default()
518 try:
519 opt.generate_manpage(self._optik_parser, pkginfo,
520 section, stream=stream or sys.stdout)
521 finally:
522 self._unmonkeypatch_expand_default()
523
524
525
527 """initialize configuration using default values"""
528 for provider in self.options_providers:
529 provider.load_defaults()
530
535
537 """read the configuration file but do not load it (i.e. dispatching
538 values to each options provider)
539 """
540 if config_file is None:
541 config_file = self.config_file
542 if config_file is not None:
543 config_file = expanduser(config_file)
544 if config_file and exists(config_file):
545 parser = self._config_parser
546 parser.read([config_file])
547
548 for sect, values in parser._sections.items():
549 if not sect.isupper() and values:
550 parser._sections[sect.upper()] = values
551 elif not self.quiet:
552 msg = 'No config file found, using default configuration'
553 print >> sys.stderr, msg
554 return
555
570
572 """dispatch values previously read from a configuration file to each
573 options provider)
574 """
575 parser = self._config_parser
576 for provider in self.options_providers:
577 for section, option, optdict in provider.all_options():
578 try:
579 value = parser.get(section, option)
580 provider.set_option(option, value, opt_dict=optdict)
581 except (NoSectionError, NoOptionError), ex:
582 continue
583
585 """override configuration according to given parameters
586 """
587 for opt_name, opt_value in kwargs.items():
588 opt_name = opt_name.replace('_', '-')
589 provider = self._all_options[opt_name]
590 provider.set_option(opt_name, opt_value)
591
593 """override configuration according to command line parameters
594
595 return additional arguments
596 """
597 self._monkeypatch_expand_default()
598 try:
599 if args is None:
600 args = sys.argv[1:]
601 else:
602 args = list(args)
603 (options, args) = self._optik_parser.parse_args(args=args)
604 for provider in self._nocallback_options.keys():
605 config = provider.config
606 for attr in config.__dict__.keys():
607 value = getattr(options, attr, None)
608 if value is None:
609 continue
610 setattr(config, attr, value)
611 return args
612 finally:
613 self._unmonkeypatch_expand_default()
614
615
616
617
619 """add a dummy option section for help purpose """
620 group = opt.OptionGroup(self._optik_parser,
621 title=title.capitalize(),
622 description=description)
623 self._optik_parser.add_option_group(group)
624
626
627 try:
628 self.__expand_default_backup = opt.HelpFormatter.expand_default
629 opt.HelpFormatter.expand_default = expand_default
630 except AttributeError:
631
632 pass
634
635 if hasattr(opt.HelpFormatter, 'expand_default'):
636
637 opt.HelpFormatter.expand_default = self.__expand_default_backup
638
640 """return the usage string for available options """
641 self._monkeypatch_expand_default()
642 try:
643 return self._optik_parser.format_help()
644 finally:
645 self._unmonkeypatch_expand_default()
646
647
649 """used to ease late binding of default method (so you can define options
650 on the class using default methods on the configuration instance)
651 """
653 self.method = methname
654 self._inst = None
655
656 - def bind(self, instance):
657 """bind the method to its instance"""
658 if self._inst is None:
659 self._inst = instance
660
662 assert self._inst, 'unbound method'
663 return getattr(self._inst, self.method)(*args, **kwargs)
664
665
667 """Mixin to provide options to an OptionsManager"""
668
669
670 priority = -1
671 name = 'default'
672 options = ()
673
675 self.config = opt.Values()
676 for option in self.options:
677 try:
678 option, optdict = option
679 except ValueError:
680 raise Exception('Bad option: %r' % option)
681 if isinstance(optdict.get('default'), Method):
682 optdict['default'].bind(self)
683 elif isinstance(optdict.get('callback'), Method):
684 optdict['callback'].bind(self)
685 self.load_defaults()
686
688 """initialize the provider using default values"""
689 for opt_name, opt_dict in self.options:
690 action = opt_dict.get('action')
691 if action != 'callback':
692
693 default = self.option_default(opt_name, opt_dict)
694 if default is REQUIRED:
695 continue
696 self.set_option(opt_name, default, action, opt_dict)
697
699 """return the default value for an option"""
700 if opt_dict is None:
701 opt_dict = self.get_option_def(opt_name)
702 default = opt_dict.get('default')
703 if callable(default):
704 default = default()
705 return default
706
708 """get the config attribute corresponding to opt_name
709 """
710 if opt_dict is None:
711 opt_dict = self.get_option_def(opt_name)
712 return opt_dict.get('dest', opt_name.replace('-', '_'))
713
715 """get the current value for the given option"""
716 return getattr(self.config, self.option_name(opt_name), None)
717
718 - def set_option(self, opt_name, value, action=None, opt_dict=None):
719 """method called to set an option (registered in the options list)
720 """
721
722 if opt_dict is None:
723 opt_dict = self.get_option_def(opt_name)
724 if value is not None:
725 value = convert(value, opt_dict, opt_name)
726 if action is None:
727 action = opt_dict.get('action', 'store')
728 if opt_dict.get('type') == 'named':
729 optname = self.option_name(opt_name, opt_dict)
730 currentvalue = getattr(self.config, optname, None)
731 if currentvalue:
732 currentvalue.update(value)
733 value = currentvalue
734 if action == 'store':
735 setattr(self.config, self.option_name(opt_name, opt_dict), value)
736 elif action in ('store_true', 'count'):
737 setattr(self.config, self.option_name(opt_name, opt_dict), 0)
738 elif action == 'store_false':
739 setattr(self.config, self.option_name(opt_name, opt_dict), 1)
740 elif action == 'append':
741 opt_name = self.option_name(opt_name, opt_dict)
742 _list = getattr(self.config, opt_name, None)
743 if _list is None:
744 if type(value) in (type(()), type([])):
745 _list = value
746 elif value is not None:
747 _list = []
748 _list.append(value)
749 setattr(self.config, opt_name, _list)
750 elif type(_list) is type(()):
751 setattr(self.config, opt_name, _list + (value,))
752 else:
753 _list.append(value)
754 elif action == 'callback':
755 opt_dict['callback'](None, opt_name, value, None)
756 else:
757 raise UnsupportedAction(action)
758
780
782 """return the dictionary defining an option given it's name"""
783 assert self.options
784 for opt in self.options:
785 if opt[0] == opt_name:
786 return opt[1]
787 raise opt.OptionError('no such option in section %r' % self.name, opt_name)
788
789
791 """return an iterator on available options for this provider
792 option are actually described by a 3-uple:
793 (section, option name, option dictionary)
794 """
795 for section, options in self.options_by_section():
796 if section is None:
797 section = self.name.upper()
798 for option, optiondict, value in options:
799 yield section, option, optiondict
800
802 """return an iterator on options grouped by section
803
804 (section, [list of (optname, optdict, optvalue)])
805 """
806 sections = {}
807 for optname, optdict in self.options:
808 sections.setdefault(optdict.get('group'), []).append(
809 (optname, optdict, self.option_value(optname)))
810 if None in sections:
811 yield None, sections.pop(None)
812 for section, options in sections.items():
813 yield section.upper(), options
814
815
817 """basic mixin for simple configurations which don't need the
818 manager / providers model
819 """
836
845
848
850 return iter(self.config.__dict__.iteritems())
851
853 try:
854 return getattr(self.config, self.option_name(key))
855 except (opt.OptionValueError, AttributeError):
856 raise KeyError(key)
857
860
861 - def get(self, key, default=None):
862 try:
863 return getattr(self.config, self.option_name(key))
864 except (opt.OptionError, AttributeError):
865 return default
866
867
869 """class for simple configurations which don't need the
870 manager / providers model and prefer delegation to inheritance
871
872 configuration values are accessible through a dict like interface
873 """
874
875 - def __init__(self, config_file=None, options=None, name=None,
876 usage=None, doc=None, version=None):
884
885
887 """Adapt an option manager to behave like a
888 `logilab.common.configuration.Configuration` instance
889 """
891 self.config = provider
892
894 return getattr(self.config, key)
895
897 provider = self.config._all_options[key]
898 try:
899 return getattr(provider.config, provider.option_name(key))
900 except AttributeError:
901 raise KeyError(key)
902
905
906 - def get(self, key, default=None):
907 provider = self.config._all_options[key]
908 try:
909 return getattr(provider.config, provider.option_name(key))
910 except AttributeError:
911 return default
912
913
915 """initialize newconfig from a deprecated configuration file
916
917 possible changes:
918 * ('renamed', oldname, newname)
919 * ('moved', option, oldgroup, newgroup)
920 * ('typechanged', option, oldtype, newvalue)
921 """
922
923 changesindex = {}
924 for action in changes:
925 if action[0] == 'moved':
926 option, oldgroup, newgroup = action[1:]
927 changesindex.setdefault(option, []).append((action[0], oldgroup, newgroup))
928 continue
929 if action[0] == 'renamed':
930 oldname, newname = action[1:]
931 changesindex.setdefault(newname, []).append((action[0], oldname))
932 continue
933 if action[0] == 'typechanged':
934 option, oldtype, newvalue = action[1:]
935 changesindex.setdefault(option, []).append((action[0], oldtype, newvalue))
936 continue
937 if action[1] in ('added', 'removed'):
938 continue
939 raise Exception('unknown change %s' % action[0])
940
941 options = []
942 for optname, optdef in newconfig.options:
943 for action in changesindex.pop(optname, ()):
944 if action[0] == 'moved':
945 oldgroup, newgroup = action[1:]
946 optdef = optdef.copy()
947 optdef['group'] = oldgroup
948 elif action[0] == 'renamed':
949 optname = action[1]
950 elif action[0] == 'typechanged':
951 oldtype = action[1]
952 optdef = optdef.copy()
953 optdef['type'] = oldtype
954 options.append((optname, optdef))
955 if changesindex:
956 raise Exception('unapplied changes: %s' % changesindex)
957 oldconfig = Configuration(options=options, name=newconfig.name)
958
959 oldconfig.load_file_configuration(configfile)
960
961 changes.reverse()
962 done = set()
963 for action in changes:
964 if action[0] == 'renamed':
965 oldname, newname = action[1:]
966 newconfig[newname] = oldconfig[oldname]
967 done.add(newname)
968 elif action[0] == 'typechanged':
969 optname, oldtype, newvalue = action[1:]
970 newconfig[optname] = newvalue
971 done.add(optname)
972 for optname, optdef in newconfig.options:
973 if not optname in done:
974 newconfig.set_option(optname, oldconfig[optname], opt_dict=optdef)
975
976
978 """preprocess options to remove duplicate"""
979 alloptions = {}
980 options = list(options)
981 for i in range(len(options)-1, -1, -1):
982 optname, optdict = options[i]
983 if optname in alloptions:
984 options.pop(i)
985 alloptions[optname].update(optdict)
986 else:
987 alloptions[optname] = optdict
988 return tuple(options)
989