Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
Menu
Open sidebar
Strand Lab
Strand Lab Cellxgene Gateway
Commits
5404a8b0
Commit
5404a8b0
authored
Mar 28, 2021
by
Alok Saldanha
Browse files
removed trailing slash from render_annotations
parent
6f2b372d
Changes
7
Hide whitespace changes
Inline
Side-by-side
.pre-commit-config.yaml
View file @
5404a8b0
...
...
@@ -7,12 +7,6 @@ repos:
language
:
system
types
:
[
python
]
stages
:
[
commit
]
-
id
:
flake8
name
:
flake8
language
:
system
entry
:
flake8
types
:
[
python
]
stages
:
[
commit
]
-
id
:
black
language_version
:
python3.6+
name
:
black
...
...
cellxgene_gateway/cache_entry.py
View file @
5404a8b0
...
...
@@ -8,14 +8,14 @@
# the specific language governing permissions and limitations under the License.
import
datetime
import
logging
from
flask.helpers
import
url_for
from
flask.wrappers
import
Response
import
re
from
enum
import
Enum
import
psutil
from
enum
import
Enum
from
flask
import
make_response
,
render_template
,
request
from
flask.helpers
import
url_for
from
flask.wrappers
import
Response
from
requests
import
get
,
post
,
put
import
re
from
cellxgene_gateway
import
env
from
cellxgene_gateway.cellxgene_exception
import
CellxgeneException
...
...
@@ -103,9 +103,7 @@ class CacheEntry:
terminated
.
append
(
p
.
pid
)
p
.
terminate
()
psutil
.
wait_procs
([
p
],
callback
=
on_terminate
)
logging
.
getLogger
(
"cellxgene_gateway"
).
info
(
f
"terminated
{
terminated
}
"
)
logging
.
getLogger
(
"cellxgene_gateway"
).
info
(
f
"terminated
{
terminated
}
"
)
self
.
status
=
CacheEntryStatus
.
terminated
def
rewrite_text_content
(
self
,
cellxgene_content
):
...
...
@@ -182,9 +180,7 @@ class CacheEntry:
data
=
request
.
data
,
)
else
:
raise
CellxgeneException
(
f
"Unexpected method
{
request
.
method
}
"
,
400
)
raise
CellxgeneException
(
f
"Unexpected method
{
request
.
method
}
"
,
400
)
content_type
=
cellxgene_response
.
headers
[
"content-type"
]
if
"text"
in
content_type
:
gateway_content
=
self
.
rewrite_text_content
(
...
...
cellxgene_gateway/env.py
View file @
5404a8b0
...
...
@@ -29,15 +29,11 @@ enable_upload = os.environ.get("GATEWAY_ENABLE_UPLOAD", "").lower() in [
"true"
,
"1"
,
]
enable_annotations
=
os
.
environ
.
get
(
"GATEWAY_ENABLE_ANNOTATIONS"
,
""
).
lower
()
in
[
enable_annotations
=
os
.
environ
.
get
(
"GATEWAY_ENABLE_ANNOTATIONS"
,
""
).
lower
()
in
[
"true"
,
"1"
,
]
enable_backed_mode
=
os
.
environ
.
get
(
"GATEWAY_ENABLE_BACKED_MODE"
,
""
).
lower
()
in
[
enable_backed_mode
=
os
.
environ
.
get
(
"GATEWAY_ENABLE_BACKED_MODE"
,
""
).
lower
()
in
[
"true"
,
"1"
,
]
...
...
cellxgene_gateway/filecrawl.py
View file @
5404a8b0
...
...
@@ -8,14 +8,12 @@
# the specific language governing permissions and limitations under the License.
import
os
from
cellxgene_gateway
import
env
from
cellxgene_gateway.dir_util
import
(
make_h5ad
,
make_annotations
,
annotations_suffix
,
)
from
flask
import
url_for
from
cellxgene_gateway
import
env
from
cellxgene_gateway.dir_util
import
annotations_suffix
,
make_annotations
,
make_h5ad
def
recurse_dir
(
path
):
if
not
os
.
path
.
exists
(
path
):
...
...
@@ -45,19 +43,18 @@ def recurse_dir(path):
"name"
:
x
[:
-
13
]
if
(
len
(
x
)
>
13
and
x
[
-
13
]
in
[
"-"
,
"_"
])
else
(
x
[:
-
4
]
if
x
.
endswith
(
".csv"
)
else
x
),
"path"
:
os
.
path
.
join
(
full_path
,
x
).
replace
(
"path"
:
os
.
path
.
join
(
full_path
,
x
+
"/"
).
replace
(
env
.
cellxgene_data
,
""
),
}
for
x
in
sorted
(
os
.
listdir
(
full_path
))
if
x
.
endswith
(
".csv"
)
and
os
.
path
.
isfile
(
os
.
path
.
join
(
full_path
,
x
))
if
x
.
endswith
(
".csv"
)
and
os
.
path
.
isfile
(
os
.
path
.
join
(
full_path
,
x
))
]
return
[
{
"name"
:
"new"
,
"class"
:
"new"
,
"path"
:
full_path
.
replace
(
env
.
cellxgene_data
,
""
),
"path"
:
full_path
.
replace
(
env
.
cellxgene_data
,
""
)
.
rstrip
(
"/"
)
,
}
]
+
entries
...
...
@@ -92,7 +89,7 @@ def render_entries(entries):
def
get_url
(
entry
):
return
url_for
(
"do_view"
,
path
=
entry
[
"path"
].
lstrip
(
"/"
)
+
"/"
)
return
url_for
(
"do_view"
,
path
=
entry
[
"path"
].
lstrip
(
"/"
))
def
get_class
(
entry
):
...
...
@@ -113,7 +110,7 @@ def render_annotations(entry):
def
render_entry
(
entry
):
if
entry
[
"type"
]
==
"file"
:
return
f
"<li> <a href='
{
get_url
(
entry
)
}
'>
{
entry
[
'name'
]
}
</a>
{
render_annotations
(
entry
)
}
</li>"
return
f
"<li> <a href='
{
get_url
(
entry
)
}
/
'>
{
entry
[
'name'
]
}
</a>
{
render_annotations
(
entry
)
}
</li>"
elif
entry
[
"type"
]
==
"directory"
:
url
=
f
"/filecrawl/
{
entry
[
'path'
].
lstrip
(
'/'
)
}
"
return
f
"<li><a href='
{
url
}
'>
{
entry
[
'name'
]
}
</a>
{
render_entries
(
entry
[
'children'
])
}
</li>"
...
...
cellxgene_gateway/gateway.py
View file @
5404a8b0
...
...
@@ -9,8 +9,6 @@
import
json
import
logging
# import BaseHTTPServer
import
os
from
threading
import
Lock
,
Thread
...
...
@@ -24,8 +22,9 @@ from flask import (
url_for
,
)
from
flask_api
import
status
from
werkzeug.utils
import
secure_filename
from
werkzeug.middleware.proxy_fix
import
ProxyFix
from
werkzeug.utils
import
secure_filename
from
cellxgene_gateway
import
env
from
cellxgene_gateway.backend_cache
import
BackendCache
from
cellxgene_gateway.cache_entry
import
CacheEntryStatus
...
...
@@ -163,11 +162,7 @@ def upload_file():
if
"file"
in
request
.
files
:
f
=
request
.
files
[
"file"
]
if
f
and
f
.
filename
.
endswith
(
".h5ad"
):
f
.
save
(
os
.
path
.
join
(
full_upload_path
,
secure_filename
(
f
.
filename
)
)
)
f
.
save
(
os
.
path
.
join
(
full_upload_path
,
secure_filename
(
f
.
filename
)))
return
redirect
(
url_for
(
"filecrawl"
),
code
=
302
)
else
:
raise
CellxgeneException
(
...
...
@@ -180,21 +175,15 @@ def upload_file():
status
.
HTTP_400_BAD_REQUEST
,
)
else
:
raise
CellxgeneException
(
"Invalid directory."
,
status
.
HTTP_400_BAD_REQUEST
)
raise
CellxgeneException
(
"Invalid directory."
,
status
.
HTTP_400_BAD_REQUEST
)
return
redirect
(
url_for
(
"index"
),
code
=
302
)
if
env
.
enable_upload
:
app
.
add_url_rule
(
"/make_user"
,
"make_user"
,
make_user
,
methods
=
[
"POST"
])
app
.
add_url_rule
(
"/make_subdir"
,
"make_subdir"
,
make_subdir
,
methods
=
[
"POST"
]
)
app
.
add_url_rule
(
"/upload_file"
,
"upload_file"
,
upload_file
,
methods
=
[
"POST"
]
)
app
.
add_url_rule
(
"/make_subdir"
,
"make_subdir"
,
make_subdir
,
methods
=
[
"POST"
])
app
.
add_url_rule
(
"/upload_file"
,
"upload_file"
,
upload_file
,
methods
=
[
"POST"
])
def
set_no_cache
(
resp
):
...
...
cellxgene_gateway/prune_process_cache.py
View file @
5404a8b0
...
...
@@ -10,8 +10,7 @@
import
logging
import
time
from
cellxgene_gateway
import
env
from
cellxgene_gateway
import
util
from
cellxgene_gateway
import
env
,
util
logger
=
logging
.
getLogger
(
__name__
)
...
...
@@ -29,9 +28,7 @@ class PruneProcessCache:
def
prune
(
self
):
timestamp
=
util
.
current_time_stamp
()
cutoff
=
timestamp
-
self
.
expire_seconds
processes_to_delete
=
[
p
for
p
in
self
.
cache
.
entry_list
if
p
.
timestamp
<
cutoff
]
processes_to_delete
=
[
p
for
p
in
self
.
cache
.
entry_list
if
p
.
timestamp
<
cutoff
]
processes_to_keep
=
[
p
for
p
in
self
.
cache
.
entry_list
if
not
p
.
timestamp
<
cutoff
]
...
...
@@ -42,9 +39,7 @@ class PruneProcessCache:
for
process
in
processes_to_delete
:
try
:
logger
.
info
(
f
"pruning process
{
process
.
pid
}
(
{
process
.
key
.
dataset
}
)"
)
logger
.
info
(
f
"pruning process
{
process
.
pid
}
(
{
process
.
key
.
dataset
}
)"
)
self
.
cache
.
prune
(
process
)
except
Exception
:
logger
.
exception
(
...
...
cellxgene_gateway/subprocess_backend.py
View file @
5404a8b0
...
...
@@ -11,14 +11,11 @@ import logging
import
subprocess
from
flask_api
import
status
from
cellxgene_gateway.cache_entry
import
CacheEntryStatus
from
cellxgene_gateway.dir_util
import
make_annotations
from
cellxgene_gateway.env
import
cellxgene_args
,
enable_annotations
,
enable_backed_mode
from
cellxgene_gateway.path_util
import
get_annotation_file_path
,
get_file_path
from
cellxgene_gateway.env
import
(
enable_annotations
,
enable_backed_mode
,
cellxgene_args
,
)
from
cellxgene_gateway.process_exception
import
ProcessException
...
...
@@ -26,14 +23,10 @@ class SubprocessBackend:
def
__init__
(
self
):
pass
def
create_cmd
(
self
,
cellxgene_loc
,
file_path
,
port
,
scripts
,
annotation_file_path
):
def
create_cmd
(
self
,
cellxgene_loc
,
file_path
,
port
,
scripts
,
annotation_file_path
):
if
enable_annotations
and
not
annotation_file_path
is
None
:
if
annotation_file_path
==
""
:
extra_args
=
(
f
" --annotations-dir
{
make_annotations
(
file_path
)
}
"
)
extra_args
=
f
" --annotations-dir
{
make_annotations
(
file_path
)
}
"
else
:
extra_args
=
f
" --annotations-file
{
annotation_file_path
}
"
else
:
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment