config.py 52.2 KB
Newer Older
1
2
3
"""
Universe configuration builder.
"""
4
5
# absolute_import needed for tool_shed package.
from __future__ import absolute_import
6

7
8
9
import ConfigParser
import logging
import logging.config
10
11
import os
import re
12
13
import socket
import string
14
15
import sys
import tempfile
John Chilton's avatar
John Chilton committed
16
import threading
17
from datetime import timedelta
18
from galaxy.exceptions import ConfigurationError
19
from galaxy.util import listify
20
from galaxy.util import string_as_bool
21
from galaxy.util.dbkeys import GenomeBuilds
22
from galaxy.web.formatting import expand_pretty_datetime_format
23
from .version import VERSION_MAJOR
24

25
26
log = logging.getLogger( __name__ )

27
28
29
30
31
# The uwsgi module is automatically injected by the parent uwsgi
# process and only exists that way.  If anything works, this is a
# uwsgi-managed process.
try:
    import uwsgi
32
    if hasattr(uwsgi, "numproc"):
33
        process_is_uwsgi = True
34
35
    else:
        process_is_uwsgi = False
36
37
38
39
except ImportError:
    # This is not a uwsgi process, or something went horribly wrong.
    process_is_uwsgi = False

40

41
42
def resolve_path( path, root ):
    """If 'path' is relative make absolute by prepending 'root'"""
Nicola Soranzo's avatar
Nicola Soranzo committed
43
    if not os.path.isabs( path ):
44
45
        path = os.path.join( root, path )
    return path
46

47

48
class Configuration( object ):
49
    deprecated_options = ( 'database_file', )
50

51
52
53
    def __init__( self, **kwargs ):
        self.config_dict = kwargs
        self.root = kwargs.get( 'root_dir', '.' )
54
55
56
57

        # Resolve paths of other config files
        self.__parse_config_file_options( kwargs )

58
        # Collect the umask and primary gid from the environment
59
60
61
        self.umask = os.umask( 077 )  # get the current umask
        os.umask( self.umask )  # can't get w/o set, so set it back
        self.gid = os.getgid()  # if running under newgrp(1) we'll need to fix the group of data created on the cluster
62

63
        self.version_major = VERSION_MAJOR
64
        # Database related configuration
65
        self.database = resolve_path( kwargs.get( "database_file", "database/universe.sqlite" ), self.root )
66
        self.database_connection = kwargs.get( "database_connection", False )
67
        self.database_engine_options = get_database_engine_options( kwargs )
68
        self.database_create_tables = string_as_bool( kwargs.get( "database_create_tables", "True" ) )
69
        self.database_query_profiling_proxy = string_as_bool( kwargs.get( "database_query_profiling_proxy", "False" ) )
70

71
72
73
        # Don't set this to true for production databases, but probably should
        # default to True for sqlite databases.
        self.database_auto_migrate = string_as_bool( kwargs.get( "database_auto_migrate", "False" ) )
74
75
76
77
78

        # Install database related configuration (if different).
        self.install_database_connection = kwargs.get( "install_database_connection", None )
        self.install_database_engine_options = get_database_engine_options( kwargs, model_prefix="install_" )

79
        # Where dataset files are stored
80
        self.file_path = resolve_path( kwargs.get( "file_path", "database/files" ), self.root )
81
        self.new_file_path = resolve_path( kwargs.get( "new_file_path", "database/tmp" ), self.root )
82
        tempfile.tempdir = self.new_file_path
Nate Coraor's avatar
Nate Coraor committed
83
        self.openid_consumer_cache_path = resolve_path( kwargs.get( "openid_consumer_cache_path", "database/openid_consumer_cache" ), self.root )
84
        self.cookie_path = kwargs.get( "cookie_path", "/" )
85
        # Galaxy OpenID settings
Nate Coraor's avatar
Nate Coraor committed
86
        self.enable_openid = string_as_bool( kwargs.get( 'enable_openid', False ) )
Nate Coraor's avatar
Nate Coraor committed
87
        self.enable_quotas = string_as_bool( kwargs.get( 'enable_quotas', False ) )
88
        self.enable_unique_workflow_defaults = string_as_bool( kwargs.get( 'enable_unique_workflow_defaults', False ) )
89
        self.tool_path = resolve_path( kwargs.get( "tool_path", "tools" ), self.root )
90
        self.tool_data_path = resolve_path( kwargs.get( "tool_data_path", "tool-data" ), os.getcwd() )
91
        self.builds_file_path = resolve_path( kwargs.get( "builds_file_path", os.path.join( self.tool_data_path, 'shared', 'ucsc', 'builds.txt') ), self.root )
92
        self.len_file_path = resolve_path( kwargs.get( "len_file_path", os.path.join( self.tool_data_path, 'shared', 'ucsc', 'chrom') ), self.root )
93
        self.test_conf = resolve_path( kwargs.get( "test_conf", "" ), self.root )
94
95
        # The value of migrated_tools_config is the file reserved for containing only those tools that have been eliminated from the distribution
        # and moved to the tool shed.
96
        self.integrated_tool_panel_config = resolve_path( kwargs.get( 'integrated_tool_panel_config', 'integrated_tool_panel.xml' ), self.root )
97
98
99
100
101
        integrated_tool_panel_tracking_directory = kwargs.get( 'integrated_tool_panel_tracking_directory', None )
        if integrated_tool_panel_tracking_directory:
            self.integrated_tool_panel_tracking_directory = resolve_path( integrated_tool_panel_tracking_directory, self.root )
        else:
            self.integrated_tool_panel_tracking_directory = None
102
        self.toolbox_filter_base_modules = listify( kwargs.get( "toolbox_filter_base_modules", "galaxy.tools.filters,galaxy.tools.toolbox.filters" ) )
103
104
105
106
107
108
109
110
        self.tool_filters = listify( kwargs.get( "tool_filters", [] ), do_strip=True )
        self.tool_label_filters = listify( kwargs.get( "tool_label_filters", [] ), do_strip=True )
        self.tool_section_filters = listify( kwargs.get( "tool_section_filters", [] ), do_strip=True )

        self.user_tool_filters = listify( kwargs.get( "user_tool_filters", [] ), do_strip=True )
        self.user_label_filters = listify( kwargs.get( "user_tool_label_filters", [] ), do_strip=True )
        self.user_section_filters = listify( kwargs.get( "user_tool_section_filters", [] ), do_strip=True )

111
        self.tour_config_dir = resolve_path( kwargs.get("tour_config_dir", "config/plugins/tours"), self.root)
Dannon Baker's avatar
Tours.    
Dannon Baker committed
112

113
114
115
        self.expose_user_name = kwargs.get( "expose_user_name", False )
        self.expose_user_email = kwargs.get( "expose_user_email", False )

116
117
118
        # Check for tools defined in the above non-shed tool configs (i.e., tool_conf.xml) tht have
        # been migrated from the Galaxy code distribution to the Tool Shed.
        self.check_migrate_tools = string_as_bool( kwargs.get( 'check_migrate_tools', True ) )
119
        self.shed_tool_data_path = kwargs.get( "shed_tool_data_path", None )
120
        self.x_frame_options = kwargs.get( "x_frame_options", "SAMEORIGIN" )
121
122
123
124
        if self.shed_tool_data_path:
            self.shed_tool_data_path = resolve_path( self.shed_tool_data_path, self.root )
        else:
            self.shed_tool_data_path = self.tool_data_path
125
        self.manage_dependency_relationships = string_as_bool( kwargs.get( 'manage_dependency_relationships', False ) )
126
        self.running_functional_tests = string_as_bool( kwargs.get( 'running_functional_tests', False ) )
127
        self.hours_between_check = kwargs.get( 'hours_between_check', 12 )
128
        self.enable_tool_shed_check = string_as_bool( kwargs.get( 'enable_tool_shed_check', False ) )
129
130
        if isinstance( self.hours_between_check, basestring ):
            self.hours_between_check = float( self.hours_between_check )
131
        try:
132
133
134
135
136
137
138
139
140
141
142
143
            if isinstance( self.hours_between_check, int ):
                if self.hours_between_check < 1 or self.hours_between_check > 24:
                    self.hours_between_check = 12
            elif isinstance( self.hours_between_check, float ):
                # If we're running functional tests, the minimum hours between check should be reduced to 0.001, or 3.6 seconds.
                if self.running_functional_tests:
                    if self.hours_between_check < 0.001 or self.hours_between_check > 24.0:
                        self.hours_between_check = 12.0
                else:
                    if self.hours_between_check < 1.0 or self.hours_between_check > 24.0:
                        self.hours_between_check = 12.0
            else:
144
145
                self.hours_between_check = 12
        except:
146
            self.hours_between_check = 12
147
        self.update_integrated_tool_panel = kwargs.get( "update_integrated_tool_panel", True )
148
        self.enable_data_manager_user_view = string_as_bool( kwargs.get( "enable_data_manager_user_view", "False" ) )
149
        self.galaxy_data_manager_data_path = kwargs.get( 'galaxy_data_manager_data_path', self.tool_data_path )
150
        self.tool_secret = kwargs.get( "tool_secret", "" )
151
        self.id_secret = kwargs.get( "id_secret", "USING THE DEFAULT IS NOT SECURE!" )
152
        self.retry_metadata_internally = string_as_bool( kwargs.get( "retry_metadata_internally", "True" ) )
153
        self.max_metadata_value_size = int( kwargs.get( "max_metadata_value_size", 5242880 ) )
Nate Coraor's avatar
Nate Coraor committed
154
        self.use_remote_user = string_as_bool( kwargs.get( "use_remote_user", "False" ) )
155
        self.normalize_remote_user_email = string_as_bool( kwargs.get( "normalize_remote_user_email", "False" ) )
Nate Coraor's avatar
Nate Coraor committed
156
        self.remote_user_maildomain = kwargs.get( "remote_user_maildomain", None )
157
        self.remote_user_header = kwargs.get( "remote_user_header", 'HTTP_REMOTE_USER' )
158
        self.remote_user_logout_href = kwargs.get( "remote_user_logout_href", None )
159
        self.remote_user_secret = kwargs.get( "remote_user_secret", None )
160
161
        self.require_login = string_as_bool( kwargs.get( "require_login", "False" ) )
        self.allow_user_creation = string_as_bool( kwargs.get( "allow_user_creation", "True" ) )
162
        self.allow_user_deletion = string_as_bool( kwargs.get( "allow_user_deletion", "False" ) )
163
        self.allow_user_dataset_purge = string_as_bool( kwargs.get( "allow_user_dataset_purge", "False" ) )
164
        self.allow_user_impersonation = string_as_bool( kwargs.get( "allow_user_impersonation", "False" ) )
165
        self.new_user_dataset_access_role_default_private = string_as_bool( kwargs.get( "new_user_dataset_access_role_default_private", "False" ) )
166
        self.collect_outputs_from = [ x.strip() for x in kwargs.get( 'collect_outputs_from', 'new_file_path,job_working_directory' ).lower().split(',') ]
167
        self.template_path = resolve_path( kwargs.get( "template_path", "templates" ), self.root )
168
        self.template_cache = resolve_path( kwargs.get( "template_cache_path", "database/compiled_templates" ), self.root )
169
        self.local_job_queue_workers = int( kwargs.get( "local_job_queue_workers", "5" ) )
170
        self.cluster_job_queue_workers = int( kwargs.get( "cluster_job_queue_workers", "3" ) )
171
        self.job_queue_cleanup_interval = int( kwargs.get("job_queue_cleanup_interval", "5") )
172
        self.cluster_files_directory = os.path.abspath( kwargs.get( "cluster_files_directory", "database/pbs" ) )
173
        self.job_working_directory = resolve_path( kwargs.get( "job_working_directory", "database/job_working_directory" ), self.root )
174
        self.default_job_shell = kwargs.get( "default_job_shell", "/bin/sh" )
175
        self.cleanup_job = kwargs.get( "cleanup_job", "always" )
176
        self.container_image_cache_path = self.resolve_path( kwargs.get( "container_image_cache_path", "database/container_images" ) )
177
        self.outputs_to_working_directory = string_as_bool( kwargs.get( 'outputs_to_working_directory', False ) )
178
        self.output_size_limit = int( kwargs.get( 'output_size_limit', 0 ) )
179
        self.retry_job_output_collection = int( kwargs.get( 'retry_job_output_collection', 0 ) )
180
        self.check_job_script_integrity = string_as_bool( kwargs.get( "check_job_script_integrity", True ) )
181
        self.job_walltime = kwargs.get( 'job_walltime', None )
182
183
184
185
        self.job_walltime_delta = None
        if self.job_walltime is not None:
            h, m, s = [ int( v ) for v in self.job_walltime.split( ':' ) ]
            self.job_walltime_delta = timedelta( 0, s, 0, 0, m, h )
186
        self.admin_users = kwargs.get( "admin_users", "" )
187
        self.admin_users_list = [u.strip() for u in self.admin_users.split(',') if u]
188
        self.mailing_join_addr = kwargs.get('mailing_join_addr', 'galaxy-announce-join@bx.psu.edu')
189
        self.error_email_to = kwargs.get( 'error_email_to', None )
190
191
192
        # activation_email was used until release_15.03
        activation_email = kwargs.get( 'activation_email', None )
        self.email_from = kwargs.get( 'email_from', activation_email )
Martin Cech's avatar
Martin Cech committed
193
        self.user_activation_on = string_as_bool( kwargs.get( 'user_activation_on', False ) )
194
195
        self.activation_grace_period = kwargs.get( 'activation_grace_period', None )
        self.inactivity_box_content = kwargs.get( 'inactivity_box_content', None )
196
197
        self.terms_url = kwargs.get( 'terms_url', None )
        self.instance_resource_url = kwargs.get( 'instance_resource_url', None )
198
        self.registration_warning_message = kwargs.get( 'registration_warning_message', None )
199
        self.ga_code = kwargs.get( 'ga_code', None )
200
        self.session_duration = int(kwargs.get( 'session_duration', 0 ))
Martin Cech's avatar
Martin Cech committed
201
202
203
204
205
206
        #  Get the disposable email domains blacklist file and its contents
        self.blacklist_location = kwargs.get( 'blacklist_file', None )
        self.blacklist_content = None
        if self.blacklist_location is not None:
            self.blacklist_file = resolve_path( kwargs.get( 'blacklist_file', None ), self.root )
            try:
207
208
                with open( self.blacklist_file ) as blacklist:
                    self.blacklist_content = [ line.rstrip() for line in blacklist.readlines() ]
Martin Cech's avatar
Martin Cech committed
209
            except IOError:
210
                log.error( "CONFIGURATION ERROR: Can't open supplied blacklist file from path: " + str( self.blacklist_file ) )
211
        self.smtp_server = kwargs.get( 'smtp_server', None )
212
213
        self.smtp_username = kwargs.get( 'smtp_username', None )
        self.smtp_password = kwargs.get( 'smtp_password', None )
214
        self.smtp_ssl = kwargs.get( 'smtp_ssl', None )
215
        self.track_jobs_in_database = string_as_bool( kwargs.get( 'track_jobs_in_database', 'True') )
216
        self.start_job_runners = listify(kwargs.get( 'start_job_runners', '' ))
217
        self.expose_dataset_path = string_as_bool( kwargs.get( 'expose_dataset_path', 'False' ) )
218
219
        # External Service types used in sample tracking
        self.external_service_type_path = resolve_path( kwargs.get( 'external_service_type_path', 'external_service_types' ), self.root )
Dannon Baker's avatar
Dannon Baker committed
220
221
        # Tasked job runner.
        self.use_tasked_jobs = string_as_bool( kwargs.get( 'use_tasked_jobs', False ) )
222
        self.local_task_queue_workers = int(kwargs.get("local_task_queue_workers", 2))
223
        self.commands_in_new_shell = string_as_bool( kwargs.get( 'enable_beta_tool_command_isolation', "True" ) )
224
225
        self.tool_submission_burst_threads = int( kwargs.get( 'tool_submission_burst_threads', '1' ) )
        self.tool_submission_burst_at = int( kwargs.get( 'tool_submission_burst_at', '10' ) )
226
227
        # The transfer manager and deferred job queue
        self.enable_beta_job_managers = string_as_bool( kwargs.get( 'enable_beta_job_managers', 'False' ) )
228
        # These workflow modules should not be considered part of Galaxy's
229
        # public API yet - the module state definitions may change and
230
231
232
        # workflows built using these modules may not function in the
        # future.
        self.enable_beta_workflow_modules = string_as_bool( kwargs.get( 'enable_beta_workflow_modules', 'False' ) )
233
234
235
        # These are not even beta - just experiments - don't use them unless
        # you want yours tools to be broken in the future.
        self.enable_beta_tool_formats = string_as_bool( kwargs.get( 'enable_beta_tool_formats', 'False' ) )
236
237
238
239
240
241

        # Certain modules such as the pause module will automatically cause
        # workflows to be scheduled in job handlers the way all workflows will
        # be someday - the following two properties can also be used to force this
        # behavior in under conditions - namely for workflows that have a minimum
        # number of steps or that consume collections.
242
        self.force_beta_workflow_scheduled_min_steps = int( kwargs.get( 'force_beta_workflow_scheduled_min_steps', '250' ) )
243
244
        self.force_beta_workflow_scheduled_for_collections = string_as_bool( kwargs.get( 'force_beta_workflow_scheduled_for_collections', 'False' ) )

245
        # Per-user Job concurrency limitations
246
        self.cache_user_job_count = string_as_bool( kwargs.get( 'cache_user_job_count', False ) )
247
        self.user_job_limit = int( kwargs.get( 'user_job_limit', 0 ) )
248
249
        self.registered_user_job_limit = int( kwargs.get( 'registered_user_job_limit', self.user_job_limit ) )
        self.anonymous_user_job_limit = int( kwargs.get( 'anonymous_user_job_limit', self.user_job_limit ) )
250
        self.default_cluster_job_runner = kwargs.get( 'default_cluster_job_runner', 'local:///' )
251
        self.pbs_application_server = kwargs.get('pbs_application_server', "" )
252
253
        self.pbs_dataset_server = kwargs.get('pbs_dataset_server', "" )
        self.pbs_dataset_path = kwargs.get('pbs_dataset_path', "" )
254
        self.pbs_stage_path = kwargs.get('pbs_stage_path', "" )
ichorny's avatar
ichorny committed
255
256
257
        self.drmaa_external_runjob_script = kwargs.get('drmaa_external_runjob_script', None )
        self.drmaa_external_killjob_script = kwargs.get('drmaa_external_killjob_script', None)
        self.external_chown_script = kwargs.get('external_chown_script', None)
258
        self.environment_setup_file = kwargs.get( 'environment_setup_file', None )
259
        self.use_heartbeat = string_as_bool( kwargs.get( 'use_heartbeat', 'False' ) )
260
261
        self.log_actions = string_as_bool( kwargs.get( 'log_actions', 'False' ) )
        self.log_events = string_as_bool( kwargs.get( 'log_events', 'False' ) )
262
        self.sanitize_all_html = string_as_bool( kwargs.get( 'sanitize_all_html', True ) )
263
        self.sanitize_whitelist_file = resolve_path( kwargs.get( 'sanitize_whitelist_file', "config/sanitize_whitelist.txt" ), self.root )
264
        self.serve_xss_vulnerable_mimetypes = string_as_bool( kwargs.get( 'serve_xss_vulnerable_mimetypes', False ) )
265
        self.trust_ipython_notebook_conversion = string_as_bool( kwargs.get( 'trust_ipython_notebook_conversion', False ) )
266
        self.enable_old_display_applications = string_as_bool( kwargs.get( "enable_old_display_applications", "True" ) )
Nate Coraor's avatar
Nate Coraor committed
267
        self.brand = kwargs.get( 'brand', None )
268
        self.welcome_url = kwargs.get( 'welcome_url', '/static/welcome.html' )
269
        self.show_welcome_with_login = string_as_bool( kwargs.get( "show_welcome_with_login", "False" ) )
270
271
272
273
        # Configuration for the message box directly below the masthead.
        self.message_box_visible = kwargs.get( 'message_box_visible', False )
        self.message_box_content = kwargs.get( 'message_box_content', None )
        self.message_box_class = kwargs.get( 'message_box_class', 'info' )
Nicola Soranzo's avatar
Nicola Soranzo committed
274
        self.support_url = kwargs.get( 'support_url', 'https://wiki.galaxyproject.org/Support' )
275
        self.wiki_url = kwargs.get( 'wiki_url', 'https://wiki.galaxyproject.org/' )
276
277
        self.blog_url = kwargs.get( 'blog_url', None )
        self.screencasts_url = kwargs.get( 'screencasts_url', None )
278
        self.library_import_dir = kwargs.get( 'library_import_dir', None )
279
        self.user_library_import_dir = kwargs.get( 'user_library_import_dir', None )
280
281
282
283
        # Searching data libraries
        self.enable_lucene_library_search = string_as_bool( kwargs.get( 'enable_lucene_library_search', False ) )
        self.enable_whoosh_library_search = string_as_bool( kwargs.get( 'enable_whoosh_library_search', False ) )
        self.whoosh_index_dir = resolve_path( kwargs.get( "whoosh_index_dir", "database/whoosh_indexes" ), self.root )
284
        self.ftp_upload_dir = kwargs.get( 'ftp_upload_dir', None )
285
        self.ftp_upload_dir_identifier = kwargs.get( 'ftp_upload_dir_identifier', 'email' )  # attribute on user - email, username, id, etc...
286
        self.ftp_upload_site = kwargs.get( 'ftp_upload_site', None )
287
        self.allow_library_path_paste = kwargs.get( 'allow_library_path_paste', False )
288
        self.disable_library_comptypes = kwargs.get( 'disable_library_comptypes', '' ).lower().split( ',' )
289
        self.watch_tools = kwargs.get( 'watch_tools', 'false' )
290
291
292
        # On can mildly speed up Galaxy startup time by disabling index of help,
        # not needed on production systems but useful if running many functional tests.
        self.index_tool_help = string_as_bool( kwargs.get( "index_tool_help", True ) )
293
294
295
296
297
        self.tool_name_boost = kwargs.get( "tool_name_boost", 9 )
        self.tool_section_boost = kwargs.get( "tool_section_boost", 3 )
        self.tool_description_boost = kwargs.get( "tool_description_boost", 2 )
        self.tool_help_boost = kwargs.get( "tool_help_boost", 0.5 )
        self.tool_search_limit = kwargs.get( "tool_search_limit", 20 )
298
        # Location for tool dependencies.
299
300
        if 'tool_dependency_dir' in kwargs:
            self.tool_dependency_dir = resolve_path( kwargs.get( "tool_dependency_dir" ), self.root )
301
302
303
            # Setting the following flag to true will ultimately cause tool dependencies
            # to be located in the shell environment and used by the job that is executing
            # the tool.
304
305
306
            self.use_tool_dependencies = True
        else:
            self.tool_dependency_dir = None
307
            self.use_tool_dependencies = os.path.exists(self.dependency_resolvers_config_file)
308
        # Configuration options for taking advantage of nginx features
309
        self.upstream_gzip = string_as_bool( kwargs.get( 'upstream_gzip', False ) )
310
        self.apache_xsendfile = string_as_bool( kwargs.get( 'apache_xsendfile', False ) )
311
312
313
314
        self.nginx_x_accel_redirect_base = kwargs.get( 'nginx_x_accel_redirect_base', False )
        self.nginx_x_archive_files_base = kwargs.get( 'nginx_x_archive_files_base', False )
        self.nginx_upload_store = kwargs.get( 'nginx_upload_store', False )
        self.nginx_upload_path = kwargs.get( 'nginx_upload_path', False )
315
316
        self.nginx_upload_job_files_store = kwargs.get( 'nginx_upload_job_files_store', False )
        self.nginx_upload_job_files_path = kwargs.get( 'nginx_upload_job_files_path', False )
317
318
        if self.nginx_upload_store:
            self.nginx_upload_store = os.path.abspath( self.nginx_upload_store )
319
        self.object_store = kwargs.get( 'object_store', 'disk' )
320
        self.object_store_check_old_style = string_as_bool( kwargs.get( 'object_store_check_old_style', False ) )
321
        self.object_store_cache_path = resolve_path( kwargs.get( "object_store_cache_path", "database/object_store_cache" ), self.root )
322
323
        # Handle AWS-specific config options for backward compatibility
        if kwargs.get( 'aws_access_key', None) is not None:
324
325
326
            self.os_access_key = kwargs.get( 'aws_access_key', None )
            self.os_secret_key = kwargs.get( 'aws_secret_key', None )
            self.os_bucket_name = kwargs.get( 's3_bucket', None )
327
328
329
330
331
332
            self.os_use_reduced_redundancy = kwargs.get( 'use_reduced_redundancy', False )
        else:
            self.os_access_key = kwargs.get( 'os_access_key', None )
            self.os_secret_key = kwargs.get( 'os_secret_key', None )
            self.os_bucket_name = kwargs.get( 'os_bucket_name', None )
            self.os_use_reduced_redundancy = kwargs.get( 'os_use_reduced_redundancy', False )
333
334
335
336
        self.os_host = kwargs.get( 'os_host', None )
        self.os_port = kwargs.get( 'os_port', None )
        self.os_is_secure = string_as_bool( kwargs.get( 'os_is_secure', True ) )
        self.os_conn_path = kwargs.get( 'os_conn_path', '/' )
337
        self.object_store_cache_size = float(kwargs.get( 'object_store_cache_size', -1 ))
338
        self.distributed_object_store_config_file = kwargs.get( 'distributed_object_store_config_file', None )
Nate Coraor's avatar
Nate Coraor committed
339
340
        if self.distributed_object_store_config_file is not None:
            self.distributed_object_store_config_file = resolve_path( self.distributed_object_store_config_file, self.root )
341
342
        self.irods_root_collection_path = kwargs.get( 'irods_root_collection_path', None )
        self.irods_default_resource = kwargs.get( 'irods_default_resource', None )
343
        # Parse global_conf and save the parser
344
345
        global_conf = kwargs.get( 'global_conf', None )
        global_conf_parser = ConfigParser.ConfigParser()
346
        self.config_file = None
347
        self.global_conf_parser = global_conf_parser
348
        if global_conf and "__file__" in global_conf:
349
            self.config_file = global_conf['__file__']
350
            global_conf_parser.read(global_conf['__file__'])
351
        # Heartbeat log file name override
352
353
        if global_conf is not None:
            self.heartbeat_log = global_conf.get( 'heartbeat_log', 'heartbeat.log' )
354
355
356
357
358
359
        # Determine which 'server:' this is
        self.server_name = 'main'
        for arg in sys.argv:
            # Crummy, but PasteScript does not give you a way to determine this
            if arg.lower().startswith('--server-name='):
                self.server_name = arg.split('=', 1)[-1]
360
361
362
        # Allow explicit override of server name in confg params
        if "server_name" in kwargs:
            self.server_name = kwargs.get("server_name")
363
364
365
366
367
        # Store all configured server names
        self.server_names = []
        for section in global_conf_parser.sections():
            if section.startswith('server:'):
                self.server_names.append(section.replace('server:', '', 1))
368
369
370

        # Default URL (with schema http/https) of the Galaxy instance within the
        # local network - used to remotely communicate with the Galaxy API.
371
372
        web_port = kwargs.get("galaxy_infrastructure_web_port", None)
        self.galaxy_infrastructure_web_port = web_port
373
374
375
376
377
378
379
        galaxy_infrastructure_url = kwargs.get( 'galaxy_infrastructure_url', None )
        galaxy_infrastructure_url_set = True
        if galaxy_infrastructure_url is None:
            # Still provide a default but indicate it was not explicitly set
            # so dependending on the context a better default can be used (
            # request url in a web thread, Docker parent in IE stuff, etc...)
            galaxy_infrastructure_url = "http://localhost"
380
381
382
            web_port = self.galaxy_infrastructure_web_port or self.guess_galaxy_port()
            if web_port:
                galaxy_infrastructure_url += ":%s" % (web_port)
383
            galaxy_infrastructure_url_set = False
384
385
386
387
        if "HOST_IP" in galaxy_infrastructure_url:
            galaxy_infrastructure_url = string.Template(galaxy_infrastructure_url).safe_substitute({
                'HOST_IP': socket.gethostbyname(socket.gethostname())
            })
388
389
390
        self.galaxy_infrastructure_url = galaxy_infrastructure_url
        self.galaxy_infrastructure_url_set = galaxy_infrastructure_url_set

391
392
393
        # Store advanced job management config
        self.job_manager = kwargs.get('job_manager', self.server_name).strip()
        self.job_handlers = [ x.strip() for x in kwargs.get('job_handlers', self.server_name).split(',') ]
394
        self.default_job_handlers = [ x.strip() for x in kwargs.get('default_job_handlers', ','.join( self.job_handlers ) ).split(',') ]
395
        # Store per-tool runner configs
396
397
398
399
400
401
402
403
404
405
        self.tool_handlers = self.__read_tool_job_config( global_conf_parser, 'galaxy:tool_handlers', 'name' )
        self.tool_runners = self.__read_tool_job_config( global_conf_parser, 'galaxy:tool_runners', 'url' )
        # Galaxy messaging (AMQP) configuration options
        self.amqp = {}
        try:
            amqp_config = global_conf_parser.items("galaxy_amqp")
        except ConfigParser.NoSectionError:
            amqp_config = {}
        for k, v in amqp_config:
            self.amqp[k] = v
406
407
408
409
410
411
412
413
        # Galaxy internal control queue configuration.
        # If specified in universe, use it, otherwise we use whatever 'real'
        # database is specified.  Lastly, we create and use new sqlite database
        # (to minimize locking) as a final option.
        if 'amqp_internal_connection' in kwargs:
            self.amqp_internal_connection = kwargs.get('amqp_internal_connection')
            # TODO Get extra amqp args as necessary for ssl
        elif 'database_connection' in kwargs:
414
            self.amqp_internal_connection = "sqlalchemy+" + self.database_connection
415
416
        else:
            self.amqp_internal_connection = "sqlalchemy+sqlite:///%s?isolation_level=IMMEDIATE" % resolve_path( "database/control.sqlite", self.root )
417
418
419
        self.biostar_url = kwargs.get( 'biostar_url', None )
        self.biostar_key_name = kwargs.get( 'biostar_key_name', None )
        self.biostar_key = kwargs.get( 'biostar_key', None )
420
        self.biostar_enable_bug_reports = string_as_bool( kwargs.get( 'biostar_enable_bug_reports', True ) )
421
        self.biostar_never_authenticate = string_as_bool( kwargs.get( 'biostar_never_authenticate', False ) )
422
        self.pretty_datetime_format = expand_pretty_datetime_format( kwargs.get( 'pretty_datetime_format', '$locale (UTC)' ) )
423
        self.master_api_key = kwargs.get( 'master_api_key', None )
424
        if self.master_api_key == "changethis":  # default in sample config file
425
            raise ConfigurationError("Insecure configuration, please change master_api_key to something other than default (changethis)")
426

427
        # Experimental: This will not be enabled by default and will hide
428
        # nonproduction code.
429
        # The api_folders refers to whether the API exposes the /folders section.
430
        self.api_folders = string_as_bool( kwargs.get( 'api_folders', False ) )
431
432
        # This is for testing new library browsing capabilities.
        self.new_lib_browse = string_as_bool( kwargs.get( 'new_lib_browse', False ) )
433
434
        # Error logging with sentry
        self.sentry_dsn = kwargs.get( 'sentry_dsn', None )
435
436
        # Statistics and profiling with statsd
        self.statsd_host = kwargs.get( 'statsd_host', '')
Dannon Baker's avatar
Dannon Baker committed
437
        self.statsd_port = int( kwargs.get( 'statsd_port', 8125 ) )
438
439
440
441
        # Logging with fluentd
        self.fluent_log = string_as_bool( kwargs.get( 'fluent_log', False ) )
        self.fluent_host = kwargs.get( 'fluent_host', 'localhost' )
        self.fluent_port = int( kwargs.get( 'fluent_port', 24224 ) )
442
        # directory where the visualization registry searches for plugins
443
444
        self.visualization_plugins_directory = kwargs.get(
            'visualization_plugins_directory', 'config/plugins/visualizations' )
445
446
447
448
449
450
        ie_dirs = kwargs.get( 'interactive_environment_plugins_directory', None )
        if ie_dirs and not self.visualization_plugins_directory:
            self.visualization_plugins_directory = ie_dirs
        elif ie_dirs:
            self.visualization_plugins_directory += ",%s" % ie_dirs

451
        self.proxy_session_map = self.resolve_path( kwargs.get( "dynamic_proxy_session_map", "database/session_map.sqlite" ) )
452
        self.manage_dynamic_proxy = string_as_bool( kwargs.get( "dynamic_proxy_manage", "True" ) )  # Set to false if being launched externally
453
        self.dynamic_proxy_debug = string_as_bool( kwargs.get( "dynamic_proxy_debug", "False" ) )
454
455
        self.dynamic_proxy_bind_port = int( kwargs.get( "dynamic_proxy_bind_port", "8800" ) )
        self.dynamic_proxy_bind_ip = kwargs.get( "dynamic_proxy_bind_ip", "0.0.0.0" )
456
        self.dynamic_proxy_external_proxy = string_as_bool( kwargs.get( "dynamic_proxy_external_proxy", "False" ) )
457
        self.dynamic_proxy_prefix = kwargs.get( "dynamic_proxy_prefix", "gie_proxy" )
458

459
460
        # Default chunk size for chunkable datatypes -- 64k
        self.display_chunk_size = int( kwargs.get( 'display_chunk_size', 65536) )
461

462
463
464
465
        self.citation_cache_type = kwargs.get( "citation_cache_type", "file" )
        self.citation_cache_data_dir = self.resolve_path( kwargs.get( "citation_cache_data_dir", "database/citations/data" ) )
        self.citation_cache_lock_dir = self.resolve_path( kwargs.get( "citation_cache_lock_dir", "database/citations/locks" ) )

466
467
468
    @property
    def sentry_dsn_public( self ):
        """
469
        Sentry URL with private key removed for use in client side scripts,
470
471
472
473
474
475
476
        sentry server will need to be configured to accept events
        """
        if self.sentry_dsn:
            return re.sub( r"^([^:/?#]+:)?//(\w+):(\w+)", r"\1//\2", self.sentry_dsn )
        else:
            return None

477
    def reload_sanitize_whitelist( self, explicit=True ):
478
479
480
481
482
483
484
        self.sanitize_whitelist = []
        try:
            with open(self.sanitize_whitelist_file, 'rt') as f:
                for line in f.readlines():
                    if not line.startswith("#"):
                        self.sanitize_whitelist.append(line.strip())
        except IOError:
485
486
            if explicit:
                log.warning("Sanitize log file explicitly specified as '%s' but does not exist, continuing with no tools whitelisted.", self.sanitize_whitelist_file)
487

488
489
490
491
492
    def __parse_config_file_options( self, kwargs ):
        """
        Backwards compatibility for config files moved to the config/ dir.
        """
        defaults = dict(
Nicola Soranzo's avatar
Nicola Soranzo committed
493
            auth_config_file=[ 'config/auth_conf.xml', 'config/auth_conf.xml.sample' ],
494
495
496
497
            data_manager_config_file=[ 'config/data_manager_conf.xml', 'data_manager_conf.xml', 'config/data_manager_conf.xml.sample' ],
            datatypes_config_file=[ 'config/datatypes_conf.xml', 'datatypes_conf.xml', 'config/datatypes_conf.xml.sample' ],
            external_service_type_config_file=[ 'config/external_service_types_conf.xml', 'external_service_types_conf.xml', 'config/external_service_types_conf.xml.sample' ],
            job_config_file=[ 'config/job_conf.xml', 'job_conf.xml' ],
Martin Cech's avatar
Martin Cech committed
498
            job_metrics_config_file=[ 'config/job_metrics_conf.xml', 'job_metrics_conf.xml', 'config/job_metrics_conf.xml.sample' ],
499
500
501
502
503
504
505
506
            dependency_resolvers_config_file=[ 'config/dependency_resolvers_conf.xml', 'dependency_resolvers_conf.xml' ],
            job_resource_params_file=[ 'config/job_resource_params_conf.xml', 'job_resource_params_conf.xml' ],
            migrated_tools_config=[ 'migrated_tools_conf.xml', 'config/migrated_tools_conf.xml' ],
            object_store_config_file=[ 'config/object_store_conf.xml', 'object_store_conf.xml' ],
            openid_config_file=[ 'config/openid_conf.xml', 'openid_conf.xml', 'config/openid_conf.xml.sample' ],
            shed_data_manager_config_file=[ 'shed_data_manager_conf.xml', 'config/shed_data_manager_conf.xml' ],
            shed_tool_data_table_config=[ 'shed_tool_data_table_conf.xml', 'config/shed_tool_data_table_conf.xml' ],
            tool_sheds_config_file=[ 'config/tool_sheds_conf.xml', 'tool_sheds_conf.xml', 'config/tool_sheds_conf.xml.sample' ],
507
            workflow_schedulers_config_file=['config/workflow_schedulers_conf.xml', 'config/workflow_schedulers_conf.xml.sample'],
508
509
510
        )

        listify_defaults = dict(
511
            tool_data_table_config_path=[ 'config/tool_data_table_conf.xml', 'tool_data_table_conf.xml', 'config/tool_data_table_conf.xml.sample' ],
512
513
514
515
516
517
518
519
520
521
            # rationale:
            # [0]: user has explicitly created config/tool_conf.xml but did not
            #      move their existing shed_tool_conf.xml, don't use
            #      config/shed_tool_conf.xml, which is probably the empty
            #      version copied from the sample, or else their shed tools
            #      will disappear
            # [1]: user has created config/tool_conf.xml and, having passed
            #      [0], probably moved their shed_tool_conf.xml as well
            # [2]: user has done nothing, use the old files
            # [3]: fresh install
522
523
524
525
            tool_config_file=[ 'config/tool_conf.xml,shed_tool_conf.xml',
                               'config/tool_conf.xml,config/shed_tool_conf.xml',
                               'tool_conf.xml,shed_tool_conf.xml',
                               'config/tool_conf.xml.sample,config/shed_tool_conf.xml' ]
526
527
528
529
        )

        for var, defaults in defaults.items():
            if kwargs.get( var, None ) is not None:
530
                path = kwargs.get( var )
531
532
533
534
535
536
537
538
539
540
541
542
            else:
                for default in defaults:
                    if os.path.exists( resolve_path( default, self.root ) ):
                        path = default
                        break
                else:
                    path = defaults[-1]
            setattr( self, var, resolve_path( path, self.root ) )

        for var, defaults in listify_defaults.items():
            paths = []
            if kwargs.get( var, None ) is not None:
543
                paths = listify( kwargs.get( var ) )
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
            else:
                for default in defaults:
                    for path in listify( default ):
                        if not os.path.exists( resolve_path( path, self.root ) ):
                            break
                    else:
                        paths = listify( default )
                        break
                else:
                    paths = listify( defaults[-1] )
            setattr( self, var, [ resolve_path( x, self.root ) for x in paths ] )

        # Backwards compatibility for names used in too many places to fix
        self.datatypes_config = self.datatypes_config_file
        self.tool_configs = self.tool_config_file

560
    def __read_tool_job_config( self, global_conf_parser, section, key ):
561
        try:
562
            tool_runners_config = global_conf_parser.items( section )
563

564
            # Process config to group multiple configs for the same tool.
565
            rval = {}
566
            for entry in tool_runners_config:
567
                tool_config, val = entry
568
569
570
571
572
573
574
575
576
577
578
579
                tool = None
                runner_dict = {}
                if tool_config.find("[") != -1:
                    # Found tool with additional params; put params in dict.
                    tool, params = tool_config[:-1].split( "[" )
                    param_dict = {}
                    for param in params.split( "," ):
                        name, value = param.split( "@" )
                        param_dict[ name ] = value
                    runner_dict[ 'params' ] = param_dict
                else:
                    tool = tool_config
580

581
                # Add runner URL.
582
                runner_dict[ key ] = val
583

584
                # Create tool entry if necessary.
585
586
                if tool not in rval:
                    rval[ tool ] = []
587

588
                # Add entry to runners.
589
                rval[ tool ].append( runner_dict )
590

591
            return rval
592
        except ConfigParser.NoSectionError:
593
            return {}
594

595
596
    def get( self, key, default ):
        return self.config_dict.get( key, default )
597

598
599
    def get_bool( self, key, default ):
        if key in self.config_dict:
Nate Coraor's avatar
Nate Coraor committed
600
            return string_as_bool( self.config_dict[key] )
601
602
        else:
            return default
603

604
605
606
607
608
609
610
611
612
613
    def ensure_tempdir( self ):
        self._ensure_directory( self.new_file_path )

    def _ensure_directory( self, path ):
        if path not in [ None, False ] and not os.path.isdir( path ):
            try:
                os.makedirs( path )
            except Exception, e:
                raise ConfigurationError( "Unable to create missing directory: %s\n%s" % ( path, e ) )

614
    def check( self ):
615
        paths_to_check = [ self.root, self.tool_path, self.tool_data_path, self.template_path ]
616
        # Check that required directories exist
617
618
619
620
621
622
        for path in paths_to_check:
            if path not in [ None, False ] and not os.path.isdir( path ):
                try:
                    os.makedirs( path )
                except Exception, e:
                    raise ConfigurationError( "Unable to create missing directory: %s\n%s" % ( path, e ) )
623
        # Create the directories that it makes sense to create
624
625
626
627
        if self.object_store_config_file is None:
            for path in (self.file_path, self.job_working_directory):
                self._ensure_directory( path )
        for path in (self.new_file_path, self.template_cache, self.ftp_upload_dir,
628
                     self.library_import_dir, self.user_library_import_dir,
629
630
                     self.nginx_upload_store, self.whoosh_index_dir,
                     self.object_store_cache_path):
631
            self._ensure_directory( path )
632
        # Check that required files exist
633
634
635
636
        tool_configs = self.tool_configs
        if self.migrated_tools_config not in tool_configs:
            tool_configs.append( self.migrated_tools_config )
        for path in tool_configs:
637
            if not os.path.exists( path ):
638
                raise ConfigurationError("Tool config file not found: %s" % path )
639
640
641
        for datatypes_config in listify( self.datatypes_config ):
            if not os.path.isfile( datatypes_config ):
                raise ConfigurationError("Datatypes config file not found: %s" % datatypes_config )
642
643
644
        # Check for deprecated options.
        for key in self.config_dict.keys():
            if key in self.deprecated_options:
645
646
                log.warning( "Config option '%s' is deprecated and will be removed in a future release.  Please consult the latest version of the sample configuration file." % key )

647
    def is_admin_user( self, user ):
648
649
        """
        Determine if the provided user is listed in `admin_users`.
650

651
652
653
        NOTE: This is temporary, admin users will likely be specified in the
              database in the future.
        """
654
        admin_users = [ x.strip() for x in self.get( "admin_users", "" ).split( "," ) ]
Nicola Soranzo's avatar
Nicola Soranzo committed
655
        return user is not None and user.email in admin_users
656

657
658
659
660
661
    def resolve_path( self, path ):
        """ Resolve a path relative to Galaxy's root.
        """
        return resolve_path( path, self.root )

662
663
664
665
666
667
668
669
670
671
672
673
674
    def guess_galaxy_port(self):
        # Code derived from IPython work ie.mako
        config = ConfigParser.SafeConfigParser({'port': '8080'})
        if self.config_file:
            config.read( self.config_file )

        try:
            port = config.getint('server:%s' % self.server_name, 'port')
        except:
            # uWSGI galaxy installations don't use paster and only speak uWSGI not http
            port = None
        return port

675

676
def get_database_engine_options( kwargs, model_prefix='' ):
677
678
    """
    Allow options for the SQLAlchemy database engine to be passed by using
679
    the prefix "database_engine_option".
680
    """
681
    conversions = {
682
683
684
685
686
687
688
        'convert_unicode': string_as_bool,
        'pool_timeout': int,
        'echo': string_as_bool,
        'echo_pool': string_as_bool,
        'pool_recycle': int,
        'pool_size': int,
        'max_overflow': int,
689
690
        'pool_threadlocal': string_as_bool,
        'server_side_cursors': string_as_bool
691
    }
692
    prefix = "%sdatabase_engine_option_" % model_prefix
693
694
695
696
697
698
699
700
701
702
    prefix_len = len( prefix )
    rval = {}
    for key, value in kwargs.iteritems():
        if key.startswith( prefix ):
            key = key[prefix_len:]
            if key in conversions:
                value = conversions[key](value)
            rval[ key  ] = value
    return rval

703

704
705
def configure_logging( config ):
    """
706
    Allow some basic logging configuration to be read from ini file.
707
708
709
    """
    # Get root logger
    root = logging.getLogger()
710
711
    # PasteScript will have already configured the logger if the
    # 'loggers' section was found in the config file, otherwise we do
712
    # some simple setup using the 'log_*' values from the config.
713
714
715
    paste_configures_logging = config.global_conf_parser.has_section( "loggers" )
    auto_configure_logging = not paste_configures_logging and string_as_bool( config.get( "auto_configure_logging", "True" ) )
    if auto_configure_logging:
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
        format = config.get( "log_format", "%(name)s %(levelname)s %(asctime)s %(message)s" )
        level = logging._levelNames[ config.get( "log_level", "DEBUG" ) ]
        destination = config.get( "log_destination", "stdout" )
        log.info( "Logging at '%s' level to '%s'" % ( level, destination ) )
        # Set level
        root.setLevel( level )
        # Turn down paste httpserver logging
        if level <= logging.DEBUG:
            logging.getLogger( "paste.httpserver.ThreadPool" ).setLevel( logging.WARN )
        # Remove old handlers
        for h in root.handlers[:]:
            root.removeHandler(h)
        # Create handler
        if destination == "stdout":
            handler = logging.StreamHandler( sys.stdout )
        else:
            handler = logging.FileHandler( destination )
        # Create formatter
        formatter = logging.Formatter( format )
        # Hook everything up
        handler.setFormatter( formatter )
        root.addHandler( handler )
738
739
740
741
742
743
    # If sentry is configured, also log to it
    if config.sentry_dsn:
        from raven.handlers.logging import SentryHandler
        sentry_handler = SentryHandler( config.sentry_dsn )
        sentry_handler.setLevel( logging.WARN )
        root.addHandler( sentry_handler )
744

745
746
747
748
749

class ConfiguresGalaxyMixin:
    """ Shared code for configuring Galaxy-like app objects.
    """

750
751
752
    def _configure_genome_builds( self, data_table_name="__dbkeys__", load_old_style=True ):
        self.genome_builds = GenomeBuilds( self, data_table_name=data_table_name, load_old_style=load_old_style )

753
    def reload_toolbox(self):
754
        # Initialize the tools, making sure the list of tool configs includes the reserved migrated_tools_conf.xml file.
755

756
757
758
        tool_configs = self.config.tool_configs
        if self.config.migrated_tools_config not in tool_configs:
            tool_configs.append( self.config.migrated_tools_config )
759

760
        from galaxy import tools
John Chilton's avatar
John Chilton committed
761
762
763
764
765
766
        with self._toolbox_lock:
            old_toolbox = self.toolbox
            self.toolbox = tools.ToolBox( tool_configs, self.config.tool_path, self )
            self.reindex_tool_search()
            if old_toolbox:
                old_toolbox.shutdown()
767

768
    def _configure_toolbox( self ):
769
770
771
        from galaxy.managers.citations import CitationsManager
        self.citations_manager = CitationsManager( self )

772
773
774
        from galaxy.tools.toolbox.cache import ToolCache
        self.tool_cache = ToolCache()

John Chilton's avatar
John Chilton committed
775
776
        self._toolbox_lock = threading.Lock()
        self.toolbox = None
777
        self.reload_toolbox()
778

779
780
781
782
783
784
        from galaxy.tools.deps import containers
        galaxy_root_dir = os.path.abspath(self.config.root)
        file_path = os.path.abspath(getattr(self.config, "file_path"))
        app_info = containers.AppInfo(
            galaxy_root_dir,
            default_file_path=file_path,
785
786
            outputs_to_working_directory=self.config.outputs_to_working_directory,
            container_image_cache_path=self.config.container_image_cache_path,
787
        )
788
789
790
791
792
793
794
        self.container_finder = containers.ContainerFinder(app_info)

    def reindex_tool_search( self ):
        # Call this when tools are added or removed.
        import galaxy.tools.search
        index_help = getattr( self.config, "index_tool_help", True )
        self.toolbox_search = galaxy.tools.search.ToolBoxSearch( self.toolbox, index_help )
795

796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
    def _configure_tool_data_tables( self, from_shed_config ):
        from galaxy.tools.data import ToolDataTableManager

        # Initialize tool data tables using the config defined by self.config.tool_data_table_config_path.
        self.tool_data_tables = ToolDataTableManager( tool_data_path=self.config.tool_data_path,
                                                      config_filename=self.config.tool_data_table_config_path )
        # Load additional entries defined by self.config.shed_tool_data_table_config into tool data tables.
        self.tool_data_tables.load_from_config_file( config_filename=self.config.shed_tool_data_table_config,
                                                     tool_data_path=self.tool_data_tables.tool_data_path,
                                                     from_shed_config=from_shed_config )

    def _configure_datatypes_registry( self, installed_repository_manager=None ):
        from galaxy.datatypes import registry
        # Create an empty datatypes registry.
        self.datatypes_registry = registry.Registry()
        if installed_repository_manager:
            # Load proprietary datatypes defined in datatypes_conf.xml files in all installed tool shed repositories.  We
            # load proprietary datatypes before datatypes in the distribution because Galaxy's default sniffers include some
            # generic sniffers (eg text,xml) which catch anything, so it's impossible for proprietary sniffers to be used.
            # However, if there is a conflict (2 datatypes with the same extension) between a proprietary datatype and a datatype
            # in the Galaxy distribution, the datatype in the Galaxy distribution will take precedence.  If there is a conflict
            # between 2 proprietary datatypes, the datatype from the repository that was installed earliest will take precedence.
            installed_repository_manager.load_proprietary_datatypes()
        # Load the data types in the Galaxy distribution, which are defined in self.config.datatypes_config.
820
821
822
823
824
825
        datatypes_configs = self.config.datatypes_config
        for datatypes_config in listify( datatypes_configs ):
            # Setting override=False would make earlier files would take
            # precedence - but then they wouldn't override tool shed
            # datatypes.
            self.datatypes_registry.load_datatypes( self.config.root, datatypes_config, override=True )
826
827
828
829
830
831
832
833
834
835
836
837
838

    def _configure_object_store( self, **kwds ):
        from galaxy.objectstore import build_object_store_from_config
        self.object_store = build_object_store_from_config( self.config, **kwds )

    def _configure_security( self ):
        from galaxy.web import security
        self.security = security.SecurityHelper( id_secret=self.config.id_secret )

    def _configure_tool_shed_registry( self ):
        import tool_shed.tool_shed_registry

        # Set up the tool sheds registry
839
840
        if os.path.isfile( self.config.tool_sheds_config_file ):
            self.tool_shed_registry = tool_shed.tool_shed_registry.Registry( self.config.root, self.config.tool_sheds_config_file )
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
        else:
            self.tool_shed_registry = None

    def _configure_models( self, check_migrate_databases=False, check_migrate_tools=False, config_file=None ):
        """
        Preconditions: object_store must be set on self.
        """
        if self.config.database_connection:
            db_url = self.config.database_connection
        else:
            db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database
        install_db_url = self.config.install_database_connection
        # TODO: Consider more aggressive check here that this is not the same
        # database file under the hood.
        combined_install_database = not( install_db_url and install_db_url != db_url )
        install_db_url = install_db_url or db_url

        if check_migrate_databases:
            # Initialize database / check for appropriate schema version.  # If this
            # is a new installation, we'll restrict the tool migration messaging.
            from galaxy.model.migrate.check import create_or_verify_database
            create_or_verify_database( db_url, config_file, self.config.database_engine_options, app=self )
            if not combined_install_database:
                from galaxy.model.tool_shed_install.migrate.check import create_or_verify_database as tsi_create_or_verify_database
                tsi_create_or_verify_database( install_db_url, self.config.install_database_engine_options, app=self )

        if check_migrate_tools:
            # Alert the Galaxy admin to tools that have been moved from the distribution to the tool shed.
            from tool_shed.galaxy_install.migrate.check import verify_tools