Unverified Commit 25755d3f authored by Alokito's avatar Alokito Committed by GitHub
Browse files

Merge pull request #41 from Novartis/grst_master

Grst master
parents 942410bb 893092f1
......@@ -60,6 +60,7 @@ Here's what the environment variables mean:
* `CELLXGENE_LOCATION` - the location of the cellxgene executable, e.g. `~/anaconda2/envs/cellxgene/bin/cellxgene`
* `CELLXGENE_DATA` - a directory that can contain subdirectories with `.h5ad` data files, *without* trailing slash, e.g. `/mnt/cellxgene_data`
Optional environment variables:
* `CELLXGENE_ARGS` - catch-all variable that can be used to pass additional command line args to cellxgene server
* `EXTERNAL_HOST` - the hostname and port from the perspective of the web browser, typically `localhost:5005` if running locally. Defaults to "localhost:{GATEWAY_PORT}"
......@@ -69,7 +70,14 @@ Optional environment variables:
* `GATEWAY_EXTRA_SCRIPTS` - JSON array of script paths, will be embedded into each page and forwarded with `--scripts` to cellxgene server
* `GATEWAY_ENABLE_UPLOAD` - Set to `true` or `1` to enable HTTP uploads. This is not recommended for a public server.
* `GATEWAY_ENABLE_ANNOTATIONS` - Set to `true` or to `1` to enable cellxgene annotations.
* `GATEWAY_ENABLE_BACKED_MODE` - Set to `true` or to `1` to load AnnData in file-backed mode. This saves memory and speeds up launch time but may reduce overall performance.
* `GATEWAY_ENABLE_BACKED_MODE` - Set to `true` or to `1` to load AnnData in file-backed mode. This saves memory and speeds up launch time but may reduce overall performance.
If any of the following optional variables are set, [ProxyFix](https://werkzeug.palletsprojects.com/en/1.0.x/middleware/proxy_fix/) will be used.
* `PROXY_FIX_FOR` - Number of upstream proxies setting X-Forwarded-For
* `PROXY_FIX_PROTO` - Number of upstream proxies setting X-Forwarded-Proto
* `PROXY_FIX_HOST` - Number of upstream proxies setting X-Forwarded-Host
* `PROXY_FIX_PORT` - Number of upstream proxies setting X-Forwarded-Port
* `PROXY_FIX_PREFIX` - Number of upstream proxies setting X-Forwarded-Prefix
The defaults should be fine if you set up a venv and cellxgene_data folder as above.
......
......@@ -8,6 +8,8 @@
# the specific language governing permissions and limitations under the License.
import datetime
import logging
from flask.helpers import url_for
from flask.wrappers import Response
import psutil
from enum import Enum
......@@ -120,7 +122,7 @@ class CacheEntry:
return gateway_content
def gateway_basepath(self):
return f"{env.external_protocol}://{env.external_host}/view/{self.key.pathpart}/"
return url_for("do_view", path=self.key.pathpart) + "/"
def cellxgene_basepath(self):
return f"http://127.0.0.1:{self.port}"
......@@ -130,7 +132,7 @@ class CacheEntry:
subpath = path[len(self.key.pathpart) :] # noqa: E203
if len(subpath) == 0:
r = make_response(f"Redirect to {gateway_basepath}\n", 301)
r = make_response(f"Redirect to {gateway_basepath}\n", 302)
r.headers["location"] = gateway_basepath + querystring()
return r
elif self.status == CacheEntryStatus.loading:
......
......@@ -47,6 +47,12 @@ env_vars = {
"CELLXGENE_DATA": cellxgene_data,
}
proxy_fix_for = int(os.environ.get("PROXY_FIX_FOR", "0"))
proxy_fix_proto = int(os.environ.get("PROXY_FIX_PROTO", "0"))
proxy_fix_host = int(os.environ.get("PROXY_FIX_HOST", "0"))
proxy_fix_port = int(os.environ.get("PROXY_FIX_PORT", "0"))
proxy_fix_prefix = int(os.environ.get("PROXY_FIX_PREFIX", "0"))
optional_env_vars = {
"EXTERNAL_HOST": external_host,
"EXTERNAL_PROTOCOL": external_protocol,
......@@ -58,6 +64,11 @@ optional_env_vars = {
"GATEWAY_ENABLE_ANNOTATIONS": enable_annotations,
"GATEWAY_ENABLE_BACKED_MODE": enable_backed_mode,
"CELLXGENE_ARGS": cellxgene_args,
"PROXY_FIX_FOR": proxy_fix_for,
"PROXY_FIX_PROTO": proxy_fix_proto,
"PROXY_FIX_HOST": proxy_fix_host,
"PROXY_FIX_PORT": proxy_fix_port,
"PROXY_FIX_PREFIX": proxy_fix_prefix,
}
......
......@@ -14,6 +14,7 @@ from cellxgene_gateway.dir_util import (
make_annotations,
annotations_suffix,
)
from flask import url_for
def recurse_dir(path):
......@@ -91,7 +92,7 @@ def render_entries(entries):
def get_url(entry):
return f"/view/{ entry['path'].lstrip('/') }/"
return url_for("do_view", path=entry["path"].lstrip("/") + "/")
def get_class(entry):
......
......@@ -25,7 +25,7 @@ from flask import (
)
from flask_api import status
from werkzeug.utils import secure_filename
from werkzeug.middleware.proxy_fix import ProxyFix
from cellxgene_gateway import env
from cellxgene_gateway.backend_cache import BackendCache
from cellxgene_gateway.cache_entry import CacheEntryStatus
......@@ -50,9 +50,23 @@ def _force_https(app):
app.wsgi_app = _force_https(app.wsgi_app)
if (
env.proxy_fix_for > 0
or env.proxy_fix_proto > 0
or env.proxy_fix_host > 0
or env.proxy_fix_port > 0
or env.proxy_fix_prefix > 0
):
app.wsgi_app = ProxyFix(
app.wsgi_app,
x_for=env.proxy_fix_for,
x_proto=env.proxy_fix_proto,
x_host=env.proxy_fix_host,
x_port=env.proxy_fix_port,
x_prefix=env.proxy_fix_prefix,
)
cache = BackendCache()
location = f"{env.external_protocol}://{env.external_host}"
@app.errorhandler(CellxgeneException)
......@@ -126,7 +140,7 @@ def make_user():
create_dir(env.cellxgene_data, dir_name)
return redirect(location, code=302)
return redirect(url_for("index"), code=302)
def make_subdir():
......@@ -135,7 +149,7 @@ def make_subdir():
create_dir(parent_path, dir_name)
return redirect(location, code=302)
return redirect(url_for("index"), code=302)
def upload_file():
......@@ -154,7 +168,7 @@ def upload_file():
full_upload_path, secure_filename(f.filename)
)
)
return redirect("/filecrawl.html", code=302)
return redirect(url_for("filecrawl"), code=302)
else:
raise CellxgeneException(
"Uploaded file must be in anndata (.h5ad) format.",
......@@ -170,7 +184,7 @@ def upload_file():
"Invalid directory.", status.HTTP_400_BAD_REQUEST
)
return redirect(location, code=302)
return redirect(url_for("index"), code=302)
if env.enable_upload:
......
......@@ -10,14 +10,16 @@
import logging
import time
from cellxgene_gateway.env import ttl
from cellxgene_gateway.util import current_time_stamp
from cellxgene_gateway import env
from cellxgene_gateway import util
logger = logging.getLogger(__name__)
class PruneProcessCache:
def __init__(self, cache):
self.cache = cache
self.expire_seconds = 3600 if ttl is None else int(ttl)
self.expire_seconds = 3600 if env.ttl is None else int(env.ttl)
def __call__(self):
while True:
......@@ -25,7 +27,7 @@ class PruneProcessCache:
self.prune()
def prune(self):
timestamp = current_time_stamp()
timestamp = util.current_time_stamp()
cutoff = timestamp - self.expire_seconds
processes_to_delete = [
p for p in self.cache.entry_list if p.timestamp < cutoff
......@@ -33,9 +35,9 @@ class PruneProcessCache:
processes_to_keep = [
p for p in self.cache.entry_list if not p.timestamp < cutoff
]
logger = logging.getLogger("cellxgene_gateway")
logger.debug(
f"Cutoff {cutoff} = timestamp {timestamp} - expire seconds {self.expire_seconds} , keeping {processes_to_keep}"
f"Cutoff {cutoff} = timestamp {timestamp} - expire seconds {self.expire_seconds} , keeping {processes_to_keep}, pruning {processes_to_delete}"
)
for process in processes_to_delete:
......
......@@ -28,11 +28,11 @@
<h4>{{ message }}</h4>
<a href="/filecrawl.html">
<a href="{{ url_for('filecrawl') }}">
Please click here to be redirected to the file directory.
</a>
<br>
<a href="/">
<a href="{{ url_for('index') }}">
Please click here to return to the homepage.
</a>
</div>
......
......@@ -36,10 +36,10 @@
Navigation:
<ul>
{% if path %}
<li><a href="/filecrawl.html">top level</a></li>
<li><a href="{{ url_for('filecrawl') }}">top level</a></li>
{% else %}
{% endif %}
<li><a href="/">homepage</a></li>
<li><a href="{{ url_for('index') }}">homepage</a></li>
</ul>
</p>
<script>
......
......@@ -35,12 +35,12 @@
Links:
</h1>
<div class="list-group" style="width:50%;padding-left:65px">
<a href="/filecrawl.html" class="list-group-item list-group-item-action">
<a href="{{ url_for('filecrawl') }}" class="list-group-item list-group-item-action">
<u>File Crawler: Allows you to view all uploaded data.</u></a>
</div>
<div class="list-group" style="width:50%;padding-left:65px">
<a href="/cache_status" class="list-group-item list-group-item-action">
<a href="{{ url_for('do_GET_status') }}" class="list-group-item list-group-item-action">
<u>Cache Status: view status of launched cellxgene servers.</u></a>
</div>
......
......@@ -35,11 +35,11 @@
The page will refresh shortly.
</p>
<a href="/filecrawl.html">
<a href="{{ url_for('filecrawl') }}">
Please click here to be redirected to the file directory.
</a>
<br>
<a href="/">
<a href="{{ url_for('index') }}">
Please click here to return to the homepage.
</a>
</div>
......
......@@ -39,10 +39,10 @@
<li><a href="{{url_for('do_relaunch', path=dataset)}}">
Attempt to relaunch the cellxgene server.
</a></li>
<li><a href="/filecrawl.html">
<li><a href="{{ url_for('filecrawl') }}">
Return to the file directory.
</a></li>
<li><a href="/">
<li><a href="{{ url_for('index') }}">
Return to the homepage.
</a></li>
</ul>
......
import unittest
from flask import Flask
from cellxgene_gateway.cache_entry import CacheEntry, CacheEntryStatus
from cellxgene_gateway.cache_key import CacheKey
from cellxgene_gateway.gateway import app
key = CacheKey("czi/pbmc3k.h5ad", "pbmc3k.h5ad", "tmp.csv")
class TestRenderEntry(unittest.TestCase):
def setUp(self):
self.app = app
self.app_context = self.app.test_request_context()
self.app_context.push()
self.client = self.app.test_client()
def test_GIVEN_key_and_port_THEN_returns_loading_CacheEntry(self):
entry = CacheEntry.for_key("some-key", 1)
self.assertEqual(entry.status, CacheEntryStatus.loading)
......@@ -14,16 +22,14 @@ class TestRenderEntry(unittest.TestCase):
actual = CacheEntry.for_key(key, 8000).rewrite_text_content(
"src:url(/static/assets/"
)
expected = (
"src:url(http://localhost:5005/view/czi/pbmc3k.h5ad/static/assets/"
)
expected = "src:url(/view/czi/pbmc3k.h5ad/static/assets/"
self.assertEqual(actual, expected)
def test_GIVEN_absolute_src_THEN_include_path(self):
actual = CacheEntry.for_key(key, 8000).rewrite_text_content(
'<link rel="shortcut icon" href="/static/assets/favicon.ico">'
)
expected = '<link rel="shortcut icon" href="http://localhost:5005/view/czi/pbmc3k.h5ad/static/assets/favicon.ico">'
expected = '<link rel="shortcut icon" href="/view/czi/pbmc3k.h5ad/static/assets/favicon.ico">'
self.assertEqual(actual, expected)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment