Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
genotoul-bioinfo
ng6
Commits
da53fd08
Commit
da53fd08
authored
Apr 30, 2019
by
Celine Noirot
Committed by
Romain Therville
Jul 26, 2019
Browse files
Add retention date management
parent
09e41229
Changes
6
Hide whitespace changes
Inline
Side-by-side
src/ng6/analysis.py
View file @
da53fd08
...
...
@@ -134,6 +134,7 @@ class Analysis (Component):
self
.
project
=
None
self
.
is_editable
=
False
self
.
space_id
=
space_id
self
.
retention_date
=
None
# Set the temp folder to the ng6 temp folder
ng6conf
=
NG6ConfigReader
()
...
...
@@ -182,8 +183,8 @@ class Analysis (Component):
if
not
os
.
path
.
isdir
(
save_dir
)
and
not
os
.
path
.
isdir
(
work_dir
):
break
directory_name
=
uuid
.
uuid4
().
hex
[:
9
]
self
.
directory
=
"/"
+
os
.
path
.
join
(
ng6conf
.
get_space_directory
(
self
.
space_id
),
self
.
DIRECTORIES_STRUCTURE
,
directory_name
)
self
.
retention_date
=
ng6conf
.
get_retention_date
(
self
.
space_id
)
# first create the output directory
if
not
os
.
path
.
isdir
(
self
.
output_directory
):
...
...
@@ -320,7 +321,8 @@ class Analysis (Component):
logging
.
getLogger
(
"Analysis.save"
).
debug
(
"Doing the add analysis"
)
self
.
id
=
t3mysql
.
add_analysis
(
self
.
get_template
(),
self
.
name
,
self
.
description
,
self
.
admin_login
,
datetime
.
date
.
today
(),
self
.
directory
,
self
.
software
,
self
.
version
,
self
.
options
,
self
.
is_editable
,
parent_uid
=
parent_uid
)
self
.
options
,
self
.
retention_date
,
self
.
retention_date
,
self
.
is_editable
,
parent_uid
=
parent_uid
)
logging
.
getLogger
(
"Analysis.save"
).
debug
(
"add_analysis done"
)
# Then add all results link to this analysis
for
file
in
self
.
results
:
...
...
src/ng6/config_reader.py
View file @
da53fd08
...
...
@@ -26,6 +26,11 @@ import os
from
configparser
import
ConfigParser
from
.utils
import
*
import
logging
from
dateutil.relativedelta
import
relativedelta
import
datetime
import
time
class
NG6ConfigReader
(
object
):
"""
Class NG6ConfigReader: this object read the config file and return the different configuration values
...
...
@@ -56,7 +61,7 @@ class NG6ConfigReader(object):
db_params
.
append
(
self
.
reader
.
get
(
'database'
,
'dbname'
))
return
db_params
except
:
raise
E
rror
(
"Failed when parsing the config file !"
)
raise
E
xception
(
"Failed when parsing the config file
- get_typo3_db_params
!"
)
def
get_pid
(
self
):
...
...
@@ -75,7 +80,7 @@ class NG6ConfigReader(object):
try
:
return
self
.
reader
.
get
(
'storage'
,
'work_directory'
)
except
:
raise
E
rror
(
"Failed when parsing the config file
!
"
)
raise
E
xception
(
"Failed when parsing the config file
- get_work_directory!
"
)
def
get_tmp_directory
(
self
):
...
...
@@ -86,7 +91,7 @@ class NG6ConfigReader(object):
try
:
return
self
.
reader
.
get
(
'storage'
,
'tmp_directory'
)
except
:
raise
E
rror
(
"Failed when parsing the config file !"
)
raise
E
xception
(
"Failed when parsing the config file
get_tmp_directory
!"
)
def
get_save_directory
(
self
):
...
...
@@ -97,7 +102,7 @@ class NG6ConfigReader(object):
try
:
return
self
.
reader
.
get
(
'storage'
,
'save_directory'
)
except
:
raise
E
rror
(
"Failed when parsing the config file !"
)
raise
E
xception
(
"Failed when parsing the config file
get_save_directory
!"
)
def
get_space_directory
(
self
,
space_id
=
"default"
):
"""
...
...
@@ -106,9 +111,28 @@ class NG6ConfigReader(object):
"""
try
:
return
self
.
reader
.
get
(
'space_'
+
space_id
,
'dir_name'
).
strip
(
"/"
)
except
:
raise
E
rror
(
"Failed when parsing the config file !"
)
except
:
raise
E
xception
(
"Failed when parsing the config file !"
)
def
get_retention_date
(
self
,
space_id
=
"default"
):
"""
return the directory corresponding to space_id
@return: space_dir
"""
date
=
None
try
:
nb_month
=
self
.
reader
.
getint
(
'space_'
+
space_id
,
"retention_period"
)
retention_date
=
time
.
mktime
((
datetime
.
datetime
.
today
()
+
relativedelta
(
months
=
nb_month
)).
timetuple
())
return
retention_date
except
:
try
:
(
d
,
m
,
y
)
=
self
.
reader
.
get
(
'space_'
+
space_id
,
"retention_date"
).
split
(
'/'
)
date
=
time
.
mktime
(
datetime
.
date
(
int
(
y
),
int
(
m
),
int
(
d
)).
timetuple
())
return
date
except
:
raise
Exception
(
"Failed when parsing the config file !"
)
raise
ValueError
(
"Failed while gnerating retention date!"
)
def
get_log_file_path
(
self
):
"""
return the log file path
...
...
@@ -117,7 +141,7 @@ class NG6ConfigReader(object):
try
:
return
self
.
reader
.
get
(
'storage'
,
'log_file'
)
except
:
raise
E
rror
(
"Failed when parsing the config file !"
)
raise
E
xception
(
"Failed when parsing the config file !"
)
def
get_log_level
(
self
):
"""
return the level of log
...
...
@@ -126,7 +150,7 @@ class NG6ConfigReader(object):
try
:
return
self
.
reader
.
get
(
'storage'
,
'log_level'
)
except
:
raise
E
rror
(
"Failed when parsing the config file !"
)
raise
E
xception
(
"Failed when parsing the config file !"
)
def
get_454_mids
(
self
):
...
...
@@ -141,7 +165,7 @@ class NG6ConfigReader(object):
mid_array
[
mid
[
0
]]
=
mid
[
1
]
return
mid_array
except
:
raise
E
rror
(
"Failed when parsing the config file !"
)
raise
E
xception
(
"Failed when parsing the config file
get_454_mids
!"
)
def
get_workflow_filters
(
self
):
"""
...
...
@@ -162,7 +186,7 @@ class NG6ConfigReader(object):
try
:
return
self
.
reader
.
get
(
'ui'
,
'user_base_directory'
)
except
:
raise
E
rror
(
"Could not retrieve server directory !"
)
raise
E
xception
(
"Could not retrieve server directory
get_user_base_directory
!"
)
def
get_server_parameters
(
self
):
...
...
@@ -170,14 +194,15 @@ class NG6ConfigReader(object):
return the server parameters
@return: server_socket_host, server_socket_port, server_socket_protocol
"""
# try:
server_params
=
[]
server_params
.
append
(
self
.
reader
.
get
(
'global'
,
'server_socket_host'
))
server_params
.
append
(
self
.
reader
.
get
(
'global'
,
'server_socket_port'
))
server_params
.
append
(
self
.
reader
.
get
(
"global"
,
'server_socket_protocol'
))
return
server_params
# except :
# raise Error("Failed when parsing the config file !")
try
:
server_params
=
[]
server_params
.
append
(
self
.
reader
.
get
(
'global'
,
'server_socket_host'
))
server_params
.
append
(
self
.
reader
.
get
(
'global'
,
'server_socket_port'
))
server_params
.
append
(
self
.
reader
.
get
(
"global"
,
'server_socket_protocol'
))
return
server_params
except
:
raise
Exception
(
"Failed when parsing the config file get_server_parameters !"
)
def
get_retention_time
(
self
):
"""
return the retention time parameter
...
...
@@ -186,4 +211,4 @@ class NG6ConfigReader(object):
try
:
return
self
.
reader
.
get
(
'global'
,
'retention_time'
)
except
:
raise
E
rror
(
"Could not retrieve
retention_time
!"
)
raise
E
xception
(
"Failed when parsing the config file get_
retention_time!"
)
src/ng6/run.py
View file @
da53fd08
...
...
@@ -67,6 +67,8 @@ class Run(object):
self
.
raw_files
=
[]
self
.
admin_login
=
None
self
.
space_id
=
space_id
self
.
retention_date
=
None
# Set the temp folder to the ng6 temp folder
ng6conf
=
NG6ConfigReader
()
tempfile
.
tempdir
=
ng6conf
.
get_tmp_directory
()
...
...
@@ -95,6 +97,7 @@ class Run(object):
# Then make directories
os
.
makedirs
(
work_dir
,
0o751
)
self
.
directory
=
"/"
+
os
.
path
.
join
(
ng6conf
.
get_space_directory
(
self
.
space_id
),
self
.
DIRECTORIES_STRUCTURE
,
directory_name
)
self
.
retention_date
=
ng6conf
.
get_retention_date
(
self
.
space_id
)
def
process_raw_files
(
self
,
total_size
,
total_seq
):
"""
...
...
@@ -366,7 +369,7 @@ class Run(object):
# First add the run into the database
t3mysql
=
t3MySQLdb
()
self
.
id
=
t3mysql
.
add_run
(
self
.
name
,
self
.
date
,
self
.
directory
,
self
.
species
,
self
.
data_nature
,
self
.
type
,
0
,
0
,
self
.
description
,
self
.
sequencer
,
self
.
admin_login
)
self
.
type
,
0
,
0
,
self
.
description
,
self
.
sequencer
,
self
.
admin_login
,
self
.
retention_date
)
return
self
.
id
def
__get_work_directory
(
self
):
...
...
src/ng6/t3MySQLdb.py
View file @
da53fd08
...
...
@@ -22,7 +22,9 @@ __version__ = '1.0'
__email__
=
'support.bioinfo.genotoul@inra.fr'
__status__
=
'beta'
import
datetime
,
time
,
calendar
,
logging
import
time
import
calendar
import
logging
import
pymysql
import
pymysql.cursors
import
collections
...
...
@@ -293,7 +295,7 @@ class t3MySQLdb(object):
# Run functions
def
add_run
(
self
,
name
,
date
,
directory
,
species
,
data_nature
,
type
,
nb_sequences
,
full_seq_size
,
description
,
sequencer
,
cruser_login
,
hidden
=
1
):
type
,
nb_sequences
,
full_seq_size
,
description
,
sequencer
,
cruser_login
,
retention_date
,
hidden
=
1
):
"""
Add to the database a new run
@param name : the run name
...
...
@@ -307,12 +309,12 @@ class t3MySQLdb(object):
@param description : the run description
@param sequencer : the sequencer used
@param cruser_login : the create user login
@param retention_date: the retention date for storage.
@param hidden : is the analysis hidden
"""
logging
.
getLogger
(
"t3MySQLdb.add_run"
).
debug
(
"Inserting run with date of run ="
+
str
(
date
))
cruser_id
=
self
.
get_user_id
(
cruser_login
)
crdate
=
str
(
time
.
time
()).
split
(
"."
)[
0
]
retention_date
=
time
.
mktime
((
datetime
.
date
.
today
()
+
datetime
.
timedelta
(
days
=
365
*
int
(
self
.
cfg_reader
.
get_retention_time
()))).
timetuple
())
sql
=
"INSERT INTO tx_nG6_run (pid, tstamp, crdate, cruser_id, name, date, directory, species, data_nature,
\
type, nb_sequences, full_seq_size, description, hidden, sequencer, retention_date, data_state, purged_date, purged_size, mail_sent_date) VALUES ('%s', '%s', '%s', %s, '%s', '%s','%s',
\
'%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s')"
%
(
self
.
cfg_reader
.
get_pid
(),
crdate
,
crdate
,
cruser_id
,
self
.
esc_q
(
name
),
...
...
@@ -671,7 +673,7 @@ class t3MySQLdb(object):
#--------------------------------------------------------------------
# Analyze functions
def
add_analysis
(
self
,
aclass
,
name
,
description
,
cruser_login
,
date
,
directory
,
software
,
version
,
params
,
is_editable
,
hidden
=
1
,
parent_uid
=
0
):
def
add_analysis
(
self
,
aclass
,
name
,
description
,
cruser_login
,
date
,
directory
,
software
,
version
,
params
,
is_editable
,
retention_date
,
hidden
=
1
,
parent_uid
=
0
):
"""
Update an Analyze
@param aclass : the class of the analysis
...
...
@@ -683,13 +685,13 @@ class t3MySQLdb(object):
@param software : the software name used for the analysis
@param version : the software version used for the analysis
@param params : the software params used
@param retention_date:the retention date for storage period
@param hidden : is the analysis hidden
@param parent_uid : the analysis parent uid
@param is_editable : True if analysis has been added by an user
"""
cruser_id
=
self
.
get_user_id
(
cruser_login
)
crdate
=
str
(
time
.
time
()).
split
(
"."
)[
0
]
retention_date
=
time
.
mktime
((
datetime
.
date
.
today
()
+
datetime
.
timedelta
(
days
=
365
*
int
(
self
.
cfg_reader
.
get_retention_time
()))).
timetuple
())
if
is_editable
:
analysis_is_editable
=
1
else
:
analysis_is_editable
=
0
...
...
workflows/addproject/__init__.py
View file @
da53fd08
...
...
@@ -19,7 +19,7 @@ from jflow.workflow import Workflow
from
ng6.project
import
Project
from
workflows.types
import
ng6adminlogin
,
uniqproject
,
ng6sspace
from
workflows.types
import
ng6adminlogin
,
uniqproject
class
AddProject
(
Workflow
):
...
...
@@ -30,7 +30,7 @@ class AddProject (Workflow):
self
.
add_parameter
(
"admin_login"
,
"The login of a ng6 administrator"
,
required
=
True
,
type
=
ng6adminlogin
)
self
.
add_parameter
(
"project_name"
,
"Give a name to your project (has to be unique)"
,
flag
=
"--name"
,
required
=
True
,
type
=
uniqproject
)
self
.
add_parameter
(
"project_description"
,
"Give a description to your project"
,
flag
=
"--description"
,
required
=
True
)
self
.
add_parameter
(
"space"
,
"Where to store data"
,
flag
=
"--space"
,
default
=
"default"
,
type
=
ng6sspace
)
self
.
add_parameter
(
"space"
,
"Where to store data"
,
flag
=
"--space"
,
default
=
"default"
)
def
process
(
self
):
project
=
Project
(
self
.
project_name
,
self
.
project_description
,
self
.
admin_login
,
None
,
self
.
space
)
...
...
workflows/types.py
View file @
da53fd08
...
...
@@ -91,12 +91,6 @@ def existingprojectid(id):
except
:
raise
argparse
.
ArgumentTypeError
(
"The project id '%s' does not exists"
%
id
)
def
ng6sspace
(
val
):
try
:
NG6ConfigReader
.
get_space_directory
(
self
.
val
)
except
:
raise
argparse
.
ArgumentTypeError
(
"The space name '%s' is not define in application.properties"
%
val
)
def
existingrun
(
id
):
try
:
t3mysql
=
t3MySQLdb
()
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment