Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
genotoul-bioinfo
ng6
Commits
aad58bf9
Commit
aad58bf9
authored
Nov 13, 2015
by
Penom Nom
Browse files
fix download
parent
79ab67bb
Changes
10
Expand all
Hide whitespace changes
Inline
Side-by-side
src/ng6/ng6workflow.py
View file @
aad58bf9
...
...
@@ -117,12 +117,13 @@ class BasicNG6Workflow (Workflow):
self
.
set_to_address
(
infos
[
'email'
])
class
DownloadWorkflow
(
BasicNG6
Workflow
):
class
DownloadWorkflow
(
Workflow
):
"""
Base class for download workflows classes
"""
def
__init__
(
self
,
args
=
{},
id
=
None
,
function
=
"process"
):
BasicNG6Workflow
.
__init__
(
self
,
args
,
id
,
function
)
Workflow
.
__init__
(
self
,
args
,
id
,
function
)
self
.
add_parameter
(
"admin_login"
,
"Who is the project administrator"
,
type
=
'ng6userlogin'
,
display_name
=
"Admin login"
)
self
.
add_parameter_list
(
'data_id'
,
'Ids of a run from which rawdata will be retrieved'
,
type
=
'existingrun'
)
self
.
add_parameter_list
(
'run_id'
,
'Ids of run from which all data will be retrieved'
,
type
=
'existingrun'
)
self
.
add_parameter_list
(
'analysis_id'
,
'Ids of analysis to retrieve'
,
type
=
'existinganalysis'
)
...
...
src/ng6/t3MySQLdb.py
View file @
aad58bf9
...
...
@@ -389,7 +389,7 @@ class t3MySQLdb(object):
'tx_nG6_run.date DESC'
]
sql
=
" "
.
join
(
parts
)
qresult
=
self
.
execute
(
sql
)
qresult
=
self
.
execute
(
sql
,
dictc
=
True
)
row
=
qresult
.
rows
[
0
]
return
{
'id'
:
row
[
'run_id'
],
...
...
@@ -816,7 +816,7 @@ class t3MySQLdb(object):
]
sql
=
" "
.
join
(
parts
)
qresult
=
self
.
execute
(
sql
)
qresult
=
self
.
execute
(
sql
,
dictc
=
True
)
# project analysis, add project informations
if
qresult
.
rowcount
>
0
:
...
...
@@ -871,7 +871,7 @@ class t3MySQLdb(object):
sql
=
" "
.
join
(
parts
)
qresult
=
self
.
execute
(
sql
)
qresult
=
self
.
execute
(
sql
,
dictc
=
True
)
row
=
qresult
.
rows
[
0
]
return
{
...
...
src/ng6/utils.py
View file @
aad58bf9
...
...
@@ -32,13 +32,6 @@ import base64
import
pty
import
sys
def
b_decode
(
func
):
def
b_decode_wrapper
(
name
):
if
isinstance
(
name
,
bytes
)
:
# defualt decodeto utf-8
return
func
(
name
).
decode
()
return
func
(
name
)
return
b_decode_wrapper
class
Utils
(
object
):
"""
...
...
@@ -335,15 +328,20 @@ class Utils(object):
@
staticmethod
def
encode_passwd
(
clear
):
return
base64
.
urlsafe_b64encode
(
clear
.
encode
()).
decode
()
"""
enc = []
for i in range(len(clear)):
key_c = Utils.PASS_KEY[i % len(Utils.PASS_KEY)]
enc_c = chr((ord(clear[i]) + ord(key_c)) % 256)
enc.append(enc_c)
return base64.urlsafe_b64encode("".join(enc))
"""
@
staticmethod
def
decode_passwd
(
enc
):
return
base64
.
urlsafe_b64decode
(
enc
.
encode
()).
decode
()
"""
dec = []
enc = base64.urlsafe_b64decode(enc)
for i in range(len(enc)):
...
...
@@ -351,6 +349,7 @@ class Utils(object):
dec_c = chr((256 + ord(enc[i]) - ord(key_c)) % 256)
dec.append(dec_c)
return "".join(dec)
"""
@
staticmethod
def
get_directories_structure_and_content
(
ng6_username
,
data_folder
,
output_folder
,
prefixed_ids
):
...
...
@@ -442,13 +441,13 @@ class SSH(object):
return
(
pid
,
f
)
def
_read
(
self
,
f
):
x
=
''
x
=
b
''
try
:
x
=
os
.
read
(
f
,
1024
)
except
Exception
as
e
:
# this always fails with io error
pass
return
x
return
x
.
decode
()
def
ssh_results
(
self
,
pid
,
f
):
output
=
''
...
...
@@ -456,7 +455,7 @@ class SSH(object):
m
=
re
.
search
(
'assword:'
,
got
)
if
m
:
# send passwd
os
.
write
(
f
,
self
.
password
+
'
\n
'
)
os
.
write
(
f
,
(
self
.
password
+
'
\n
'
)
.
encode
()
)
# read two lines
tmp
=
self
.
_read
(
f
)
tmp
+=
self
.
_read
(
f
)
...
...
ui/nG6/res/js/tx_nG6_pi5.js
View file @
aad58bf9
This diff is collapsed.
Click to expand it.
workflows/download_archive/__init__.py
View file @
aad58bf9
...
...
@@ -26,5 +26,5 @@ class DownloadArchive (DownloadWorkflow):
self
.
add_parameter
(
'archive_mail'
,
'Provide a valid email address to be warned when the archive is created'
,
display_name
=
"User mail"
,
type
=
"email"
,
required
=
True
)
def
process
(
self
):
self
.
add_component
(
'CreateArchive'
,
[
self
.
admin_login
,
self
.
archive_mail
,
self
.
data_id
,
self
.
run_id
,
self
.
analysis_id
])
self
.
add_component
(
'CreateArchive'
,
[
self
.
archive_mail
,
self
.
data_id
,
self
.
run_id
,
self
.
analysis_id
,
self
.
admin_login
])
workflows/download_archive/components/createarchive.py
View file @
aad58bf9
...
...
@@ -22,7 +22,7 @@ from jflow.component import Component
from
weaver.function
import
PythonFunction
def
create_archive
(
ng6_usernam
e
,
email
,
output_folder
,
ids_dump_path
,
web_archive
,
archive_fil
e
):
def
create_archive
(
archive_fil
e
,
email
,
output_folder
,
ids_dump_path
,
web_archive
,
ng6_username
=
Non
e
):
import
os
import
pickle
import
shutil
...
...
@@ -42,6 +42,8 @@ def create_archive( ng6_username, email, output_folder, ids_dump_path, web_archi
fh
.
close
()
prefixed_ids
=
";"
.
join
(
prefixed_ids
)
if
ng6_username
in
[
''
,
'None'
,
'none'
,
'null'
,
None
]
:
ng6_username
=
None
src_directories
,
dest_directories
=
Utils
.
get_directories_structure_and_content
(
ng6_username
,
data_folder
,
output_folder
,
prefixed_ids
)
for
i
,
source_dir
in
enumerate
(
src_directories
):
...
...
@@ -73,7 +75,7 @@ def create_archive( ng6_username, email, output_folder, ids_dump_path, web_archi
class
CreateArchive
(
Component
):
def
define_parameters
(
self
,
login
,
email
,
data_ids
=
[],
run_ids
=
[],
analysis_ids
=
[]):
def
define_parameters
(
self
,
email
,
data_ids
=
[],
run_ids
=
[],
analysis_ids
=
[]
,
login
=
None
):
self
.
add_parameter
(
'login'
,
'The login of the user in ng6'
,
type
=
'ng6userlogin'
,
default
=
login
)
self
.
add_parameter
(
'email'
,
'The email where the url to the archive will be sent'
,
default
=
email
,
required
=
True
)
self
.
add_parameter_list
(
'data_ids'
,
'Ids of a run from which rawdata will be retrieved'
,
default
=
data_ids
)
...
...
@@ -94,8 +96,8 @@ class CreateArchive (Component):
fh
.
close
()
web_archive
=
self
.
_webify_outputs
(
self
.
archive
)
archive
=
PythonFunction
(
create_archive
,
cmd_format
=
"{EXE}
{ARG}
{IN} {OUT}
"
)
archive
(
arguments
=
[
self
.
login
,
self
.
email
,
output_folder
,
ids_dump_path
,
web_archive
],
outputs
=
self
.
archive
)
archive
=
PythonFunction
(
create_archive
,
cmd_format
=
"{EXE} {IN} {OUT}
{ARG}"
)
archive
(
arguments
=
[
self
.
email
,
output_folder
,
ids_dump_path
,
web_archive
,
self
.
login
],
outputs
=
self
.
archive
)
def
_webify_outputs
(
self
,
path
):
...
...
workflows/download_symlink/__init__.py
View file @
aad58bf9
...
...
@@ -30,9 +30,9 @@ class DownloadSymlink (DownloadWorkflow):
self
.
add_parameter
(
'username'
,
'A valid user username on the server'
,
type
=
"localuser"
,
required
=
True
)
self
.
add_parameter
(
'password'
,
'The connection password for the username on the server'
,
type
=
"password"
,
required
=
True
)
self
.
add_parameter
(
'user_directory'
,
"""The directory where the links will be created. This directory must exists on the server and the
provided user must have write permission on this directory"""
,
default
=
config
.
get_ser
ver
_directory
(),
provided user must have write permission on this directory"""
,
default
=
config
.
get_
u
ser
_base
_directory
(),
required
=
True
,
type
=
"localdirectory"
)
def
process
(
self
):
self
.
add_component
(
'CreateSymlink'
,
[
self
.
admin_login
,
self
.
username
,
self
.
password
,
self
.
user_directory
,
self
.
data_id
,
self
.
run_id
,
self
.
analysis_id
])
self
.
add_component
(
'CreateSymlink'
,
[
self
.
username
,
self
.
password
,
self
.
user_directory
,
self
.
data_id
,
self
.
run_id
,
self
.
analysis_id
,
self
.
admin_login
])
workflows/download_symlink/components/createsymlink.py
View file @
aad58bf9
...
...
@@ -23,7 +23,7 @@ from ng6.utils import Utils
from
weaver.function
import
PythonFunction
def
create_symbolik_link
(
username
,
password
,
ng6_username
,
output_folder
,
ids_dump_path
,
output_list
,
user_script
):
def
create_symbolik_link
(
ids_dump_path
,
output_list
,
user_script
,
username
,
password
,
output_folder
,
ng6_username
=
None
):
import
os
import
pickle
import
subprocess
...
...
@@ -31,11 +31,12 @@ def create_symbolik_link (username, password, ng6_username, output_folder, ids_d
from
ng6.config_reader
import
NG6ConfigReader
from
ng6.utils
import
Utils
,
SSH
config
=
NG6ConfigReader
()
data_folder
=
config
.
get_save_directory
()
password
=
Utils
.
decode_passwd
(
password
)
if
ng6_username
in
[
''
,
'None'
,
'none'
,
'null'
,
None
]
:
ng6_username
=
None
try
:
fh
=
open
(
ids_dump_path
,
"rb"
)
prefixed_ids
=
pickle
.
load
(
fh
)
...
...
@@ -72,7 +73,7 @@ def create_symbolik_link (username, password, ng6_username, output_folder, ids_d
class
CreateSymlink
(
Component
):
def
define_parameters
(
self
,
login
,
username
,
password
,
user_directory
,
data_ids
=
[],
run_ids
=
[],
analysis_ids
=
[]):
def
define_parameters
(
self
,
username
,
password
,
user_directory
,
data_ids
=
[],
run_ids
=
[],
analysis_ids
=
[]
,
login
=
None
):
self
.
add_parameter
(
'login'
,
'The login of the user in ng6'
,
type
=
'ng6userlogin'
,
default
=
login
)
self
.
add_parameter
(
'username'
,
'Valid username on the server'
,
required
=
True
,
default
=
username
)
self
.
add_parameter
(
'password'
,
'A password for the usernmae on the server'
,
required
=
True
,
default
=
Utils
.
encode_passwd
(
password
))
...
...
@@ -93,8 +94,8 @@ class CreateSymlink (Component):
pickle
.
dump
(
run_ids
+
data_ids
+
analysis_ids
,
fh
)
fh
.
close
()
createlinks
=
PythonFunction
(
create_symbolik_link
,
cmd_format
=
"{EXE}
{ARG}
{IN} {OUT} "
)
createlinks
(
arguments
=
[
self
.
username
,
"'"
+
self
.
password
+
"'"
,
self
.
login
,
self
.
user_directory
],
createlinks
=
PythonFunction
(
create_symbolik_link
,
cmd_format
=
"{EXE} {IN} {OUT}
{ARG}
"
)
createlinks
(
arguments
=
[
self
.
username
,
"'"
+
self
.
password
+
"'"
,
self
.
user_directory
,
self
.
login
],
inputs
=
ids_dump_path
,
outputs
=
[
self
.
output_list
,
self
.
user_script
],
local
=
True
)
\ No newline at end of file
workflows/download_url/__init__.py
View file @
aad58bf9
...
...
@@ -26,5 +26,5 @@ class DownloadURL (DownloadWorkflow):
pass
def
process
(
self
):
self
.
add_component
(
'GetURLS'
,
[
self
.
admin_login
,
self
.
data_id
,
self
.
run_id
,
self
.
analysis_id
])
self
.
add_component
(
'GetURLS'
,
[
self
.
data_id
,
self
.
run_id
,
self
.
analysis_id
,
self
.
admin_login
])
workflows/download_url/components/geturls.py
View file @
aad58bf9
...
...
@@ -22,7 +22,7 @@ from jflow.component import Component
from
weaver.function
import
PythonFunction
def
get_urls
(
ng6_username
,
ids_dump_path
,
output_url
):
def
get_urls
(
output_url
,
ids_dump_path
,
ng6_username
=
None
):
import
os
import
pickle
import
subprocess
...
...
@@ -34,7 +34,9 @@ def get_urls( ng6_username, ids_dump_path, output_url):
ng6config
=
NG6ConfigReader
()
data_directory
=
ng6config
.
get_save_directory
()
if
ng6_username
in
[
''
,
'None'
,
'none'
,
'null'
,
None
]
:
ng6_username
=
None
fh
=
open
(
ids_dump_path
,
"rb"
)
prefixed_ids
=
pickle
.
load
(
fh
)
fh
.
close
()
...
...
@@ -56,7 +58,7 @@ def get_urls( ng6_username, ids_dump_path, output_url):
class
GetURLS
(
Component
):
def
define_parameters
(
self
,
login
,
data_ids
=
[],
run_ids
=
[],
analysis_ids
=
[]):
def
define_parameters
(
self
,
data_ids
=
[],
run_ids
=
[],
analysis_ids
=
[]
,
login
=
None
):
self
.
add_parameter
(
'login'
,
'The login of the user in ng6'
,
type
=
'ng6userlogin'
,
default
=
login
)
self
.
add_parameter_list
(
'data_ids'
,
'Ids of a run from which rawdata will be retrieved'
,
default
=
data_ids
)
self
.
add_parameter_list
(
'run_ids'
,
'Ids of run from which all data will be retrieved'
,
default
=
run_ids
)
...
...
@@ -74,7 +76,7 @@ class GetURLS (Component):
pickle
.
dump
(
run_ids
+
data_ids
+
analysis_ids
,
fh
)
fh
.
close
()
fn
=
PythonFunction
(
get_urls
,
cmd_format
=
"{EXE}
{ARG}
{IN} {OUT} "
)
fn
(
arguments
=
[
self
.
login
,
ids_dump_path
],
outputs
=
self
.
output_url
,
local
=
True
)
fn
=
PythonFunction
(
get_urls
,
cmd_format
=
"{EXE} {IN} {OUT}
{ARG}
"
)
fn
(
arguments
=
[
ids_dump_path
,
self
.
login
],
outputs
=
self
.
output_url
,
local
=
True
)
\ No newline at end of file
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment