Commit 928194ed authored by root's avatar root

New minimal schema

parents fecb5cb7 7fa427dd
......@@ -17,7 +17,7 @@
</div>
<div>
<ul class="nav navbar-nav navbar-right">
<li><a href="https://pypi.python.org/pypi/transfer-client/0.0.6">Try our new CLI</a></li>
<li><a href="https://pypi.python.org/pypi/transfer-client/">Try our CLI</a></li>
<li><a href="#" id="link-user"><span class="glyphicon glyphicon-user" aria-hidden="true">&nbsp;</span><span id="user-name"></span></a></li>
<li><a href="#" id="link-signout"><span class="glyphicon glyphicon-log-out" aria-hidden="true">&nbsp;</span>Sign out</a></li>
</ul>
......
......@@ -198,8 +198,8 @@ def initiate_transfer(user):
if not os.path.exists(primary_path):
try:
os.makedirs(primary_path, mode=0770)
os.chmod(primary_path, 0770)
original_umask = os.umask(0)
os.makedirs(primary_path, 02770)
except (OSError, IOError) as excp:
transfer.status = TRANSFER_FAILED
db.session.commit()
......@@ -207,6 +207,8 @@ def initiate_transfer(user):
primary_path +
' Exception: ' + excp.strerror)
return jsonify({'error': 'file saving failed'}), 400
finally:
os.umask(original_umask)
else:
transfer.status = TRANSFER_FAILED
db.session.commit()
......@@ -220,8 +222,8 @@ def initiate_transfer(user):
if not os.path.exists(secondary_path):
try:
os.makedirs(secondary_path, mode=0770)
os.chmod(secondary_path, 0770)
original_umask = os.umask(0)
os.makedirs(secondary_path, 02770)
except (OSError, IOError) as excp:
# Clean up.
transfer.status = TRANSFER_FAILED
......@@ -235,6 +237,8 @@ def initiate_transfer(user):
logging.error('Could not remove ' + primary_path +
' Exception: ' + excp1.strerror)
return jsonify({'error': 'file saving failed'}), 400
finally:
os.umask(original_umask)
else:
# Clean up.
transfer.status = TRANSFER_FAILED
......
.cache/
dist/
transfer_client.egg-info/
*.pyc
Copyright (c) 2017 Leiden University Medical Center <humgen@lumc.nl>
Copyright (c) 2017 Jeroen F.J. Laros <J.F.J.Laros@lumc.nl>
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
# Transfer client
This package provides functions for communication with the Restful Transfer
Server.
## Installation
Via [pypi](https://pypi.python.org/pypi/transfer-client):
pip install transfer_client
From source:
git clone https://git.lumc.nl/j.k.vis/transfer.git
cd transfer/transfer_client
pip install .
### Server certificate
If the server uses a self-signed certificate, the administrator of the server
needs to add this certificate to the list of trusted authorities (change
`server_name` and `domain`):
openssl s_client -connect server_name.domain:443 < /dev/null | \
openssl x509 > /usr/local/share/ca-certificates/server_name.crt
update-ca-certificates
## Command line interface
Use the command `transfer_client -h` for a list of subcommands. For every
subcommand a separate help is available, e.g.,
```
$ transfer_client users -h
usage: transfer_client users [-h] [-o OUTPUT] SERVER USER
Gives a JSON object of a user together with its transfers.
positional arguments:
SERVER server name
USER user id
optional arguments:
-h, --help show this help message and exit
-o OUTPUT output file
-n disable ssl certificate check
```
### Typical usage
The three high level subcommands are probably the only ones needed.
To transfer a list of files, use the `transfer` subcommand. We assume that the
server name is `server_name.domain` and the user ID is `xxxxxxxx`.
transfer_client transfer server_name.domain xxxxxxxx *.gz
If a transfer is interrupted for some reason, it can be resumed with the
`resume` subcommand.
transfer_client resume server_name.domain xxxxxxxx
To cancel an interrupted transfer, the `cancel` subcommand can be used.
transfer_client cancel server_name.domain xxxxxxxx
## Library
The library implements an interface to the API of the Restful Transfer server.
Full documentation can be found [here](https://git.lumc.nl/j.k.vis/transfer).
### Usage
To use the interface, we first make a class instance of `TransferClient`, then
we can use the API endpoints.
```python
>>> from transfer_client.transfer_client import TransferClient
>>>
>>> # Create a class instance for a server running on server_name.domain.
>>> transfer_client = TransferClient('server_name.domain')
>>>
>>> # Get user information together with its transfers.
>>> transfer_client.users('xxxxxxxx')
{ ... }
```
{
"required": [
"files"
],
"type": "object",
"properties": {
"files": {
"minItems": 1,
"items": {
"required": [
"filename",
"md5"
],
"type": "object",
"properties": {
"md5": {
"pattern": "^[a-f0-9]{32}$",
"type": "string"
},
"filename": {
"pattern": ".+",
"type": "string"
}
}
},
"uniqueItems": true,
"type": "array"
},
"title": {
"pattern": ".+",
"type": "string"
}
},
"title": "Minimal transfer metadata"
}
jsonschema
pyyaml
requests
requests-toolbelt
urllib3
import os
import sys
from setuptools import setup
package = 'transfer_client'
package_name = 'transfer-client'
description = '{}: Transfer client library and CLI.'.format(package_name)
documentation = 'README.md'
license = 'MIT License'
keywords = []
dependencies = [
'jsonschema', 'pyyaml', 'requests', 'requests-toolbelt', 'urllib3']
develop_dependencies = ['fake-open', 'pytest', 'tox']
supported = [(2, 7), (3, 3), (3, 4)]
classifiers = [
'Development Status :: 3 - Alpha',
'Intended Audience :: Science/Research',
'Intended Audience :: Developers',
'Topic :: Scientific/Engineering',
]
if sys.version_info < supported[0]:
raise Exception('{} requires Python {}.{} or higher.'.format(
package, *supported[0]))
if sys.version_info[:2] == supported[0]:
dependencies.extend(['argparse', 'importlib'])
# This is quite the hack, but we don't want to import our package from here
# since that's recipe for disaster (it might have some uninstalled
# dependencies, or we might import another already installed version).
distmeta = {}
for line in open(os.path.join(package, '__init__.py')):
try:
field, value = (x.strip() for x in line.split('='))
except ValueError:
continue
if field == '__version_info__':
value = value.strip('[]()')
value = '.'.join(x.strip(' \'"') for x in value.split(','))
else:
value = value.strip('\'"')
distmeta[field] = value
try:
with open(documentation) as readme:
long_description = readme.read()
except IOError:
long_description = 'See ' + distmeta['__homepage__']
language_string = 'Programming Language :: Python'
classifiers += [
'License :: OSI Approved :: {}'.format(license),
'Operating System :: OS Independent',
language_string,
'{} :: {}'.format(language_string, supported[0][0]),
'{} :: {}'.format(language_string, supported[-1][0])] + \
['{} :: {}.{}'.format(language_string, *version) for version in supported]
setup(
name=package_name,
version=distmeta['__version_info__'],
description=description,
long_description=long_description,
author=distmeta['__author__'],
author_email=distmeta['__contact__'],
url=distmeta['__homepage__'],
license=license,
platforms=['any'],
packages=[package],
install_requires=dependencies,
tests_require=develop_dependencies,
entry_points={
'console_scripts': ['{0} = {0}.cli:main'.format(package)]
},
classifiers=classifiers,
keywords=' '.join(keywords)
)
"""Tests for the transfer_client CLI.
"""
from __future__ import unicode_literals
from io import StringIO
import json
from fake_open import FakeOpen, make_fake_file, md5_check
from transfer_client import cli, TransferClient
class FakeReply(object):
def __init__(self):
self._replies = []
def add(self, reply):
self._replies = [reply] + self._replies
def pop(self):
if self._replies:
return self._replies.pop()
return None
class TestCLI(object):
def setup(self):
opener = FakeOpen()
self._handles = opener.handles
cli.open = opener.open
self._reply = FakeReply()
self._file_handle = make_fake_file('upload.bin', '')
self._file_handles = [
make_fake_file('a', 'a\n'), make_fake_file('b', 'b\n')]
self._metadata_handle = make_fake_file('metadata.json', 'null\n')
self._output_handle = StringIO()
self._log_handle = StringIO()
TransferClient._request = lambda *args, **kwargs: self._reply.pop()
def test_users(self):
cli.users(self._output_handle, '127.0.0.1', '123', False)
assert self._output_handle.getvalue() == 'null\n'
def test_schema(self):
cli.schema(self._output_handle, '127.0.0.1', '123', False)
assert self._output_handle.getvalue() == 'null\n'
def test_transfers(self):
cli.transfers(
self._output_handle, self._metadata_handle,
'127.0.0.1', '123', False)
assert self._output_handle.getvalue() == 'null\n'
def test_status(self):
cli.status(self._output_handle, '127.0.0.1', '123', '456', False)
assert self._output_handle.getvalue() == 'null\n'
def test_update(self):
cli.update(
self._output_handle, '127.0.0.1', '123', '456', 'cancelled', False)
assert self._output_handle.getvalue() == 'null\n'
def test_uploads(self):
cli.uploads(
self._output_handle, self._file_handle,
'127.0.0.1', '123', '456', False)
assert self._output_handle.getvalue() == 'null\n'
def test_completed(self):
cli.completed(self._output_handle, '127.0.0.1', '789', False)
assert self._output_handle.getvalue() == 'null\n'
def test_make_metadata_1(self):
metadata = cli._make_metadata(
self._log_handle, self._file_handles, 'test')
assert metadata['title'] == 'test'
assert metadata['files'][0]['filename'] == 'a'
assert (
metadata['files'][1]['md5'] == '3b5d5c3712955042212316173ccf37be')
def test_make_metadata_2(self):
cli.make_metadata(
self._output_handle, self._log_handle, self._file_handles, 'test')
assert md5_check(
self._output_handle.getvalue(), 'ee7ca88f46ca4c7f58375279a2fbd62a')
assert md5_check(
self._log_handle.getvalue(), 'c16aad07df999e5a1a5bcd066c1f692f')
def test_transfer(self):
self._reply.add(json.loads(open('data/schema.json').read()))
self._reply.add({'id': 0})
cli.transfer(
self._log_handle, self._file_handles,
'127.0.0.1', '123', 'test', False)
assert md5_check(
self._log_handle.getvalue(), '307fc32a1709cb2505c5c4525aa6c34e')
def test_interrupted_transfer_1(self):
self._reply.add({'transfers': []})
try:
cli._interrupted_transfer(
self._log_handle, TransferClient('127.0.0.1', False), '123')
except ValueError as error:
assert error.message == 'no interrupted transfers found'
else:
assert False
def test_interrupted_transfer_2(self):
self._reply.add({'transfers': [{'id': 0, 'status': 'initiated'}]})
cli._interrupted_transfer(
self._log_handle, TransferClient('127.0.0.1', False), '123')
assert md5_check(
self._log_handle.getvalue(), 'b6a19671cc29806d6468b84695121344')
def test_resume(self):
self._reply.add({
'transfers': [
{
'id': 0,
'status': 'initiated',
'files': [
{
'status': 'pending',
'filename': 'a'
}
]
}
]
})
cli.resume(self._log_handle, '127.0.0.1', '123', False)
assert md5_check(
self._log_handle.getvalue(), '641eeda3b448cc99a9c5f74c8343d589')
def test_cancel(self):
self._reply.add({
'transfers': [
{
'id': 0,
'status': 'initiated'
}
]
})
cli.cancel(self._log_handle, '127.0.0.1', '123', False)
assert md5_check(
self._log_handle.getvalue(), 'b6a19671cc29806d6468b84695121344')
def test_reformat_transfer_1(self):
readable_transfer = cli._reformat_transfer(
{'files': [], 'status': 'completed'})
assert readable_transfer['number_of_files'] == 0
def test_reformat_transfer_2(self):
readable_transfer = cli._reformat_transfer(
{'files': [{'status': 'uploaded'}], 'status': 'initiated'})
assert readable_transfer['uploaded'] == 1
def test_reformat_transfer_3(self):
readable_transfer = cli._reformat_transfer({
'files': [],
'status': 'completed',
'start_date': '2016-11-25 09:54:57.088350',
'end_date': '2016-11-25 09:55:57.088350'
})
assert readable_transfer['duration'] == '0:01:00'
def test_transfers_summary(self):
self._reply.add({
'transfers': [
{
'files': [],
'status': 'completed'
}
]
})
cli.transfers_summary(self._output_handle, '127.0.0.1', '123', False)
assert md5_check(
self._output_handle.getvalue(), '69f8fe82668ffe4f1b1d3fc47906c8d4')
def test_last_transfer_summary(self):
self._reply.add({
'transfers': [
{
'files': [],
'status': 'completed'
}
]
})
cli.last_transfer_summary(
self._output_handle, '127.0.0.1', '123', False)
assert md5_check(
self._output_handle.getvalue(), '3f3384fc1041333b147cdb1f46608256')
def test_transfer_summary(self):
self._reply.add({
'transfers': [
{
'id': '456',
'files': [],
'status': 'completed'
}
]
})
cli.transfer_summary(
self._output_handle, '127.0.0.1', '123', '456', False)
assert md5_check(
self._output_handle.getvalue(), 'c38201114fb64755abb2747bcd40e88d')
def test_check_metadata(self):
self._reply.add({
'type': 'object',
'properties': {
'id': {'type': 'string'}
}
})
cli.check_metadata(StringIO(
'{"id": "test"}\n'), '127.0.0.1', '123', False)
"""Tests for the transfer_client library.
"""
from __future__ import unicode_literals
from fake_open import make_fake_file
from transfer_client import TransferClient
class TestLibrary(object):
def setup(self):
TransferClient._request = lambda *args, **kwargs: None
self._transfer_client = TransferClient('127.0.0.1', False)
self._input_handle = make_fake_file('upload.bin', '')
def test_users(self):
assert self._transfer_client.users('123') == None
def test_schema(self):
assert self._transfer_client.schema('123') == None
def test_transfers(self):
assert self._transfer_client.transfers(
'123', {}, 'metadata.json') == None
def test_status(self):
assert self._transfer_client.status('123', '456') == None
def test_update(self):
assert self._transfer_client.update('123', '456', 'cancelled') == None
def test_uploads(self):
assert self._transfer_client.uploads(
'123', '456', self._input_handle) == None
def test_completed(self):
assert self._transfer_client.completed('789') == None
"""Transfer server client CLI.
Copyright (c) 2017 Leiden University Medical Center <humgen@lumc.nl>
Copyright (c) 2017 Jeroen F.J. Laros <J.F.J.Laros@lumc.nl>
Licensed under the MIT license, see the LICENSE file.
"""
from .transfer_client import TransferClient
__version_info__ = ('0', '0', '7')
__version__ = '.'.join(__version_info__)
__author__ = 'LUMC, Jeroen F.J. Laros'
__contact__ = 'J.F.J.Laros@lumc.nl'
__homepage__ = 'https://git.lumc.nl/j.k.vis/transfer'
usage = __doc__.split('\n\n\n')
def doc_split(func):
return func.__doc__.split('\n\n')[0]
def version(name):
return '{} version {}\n\nAuthor : {} <{}>\nHomepage : {}'.format(
name, __version__, __author__, __contact__, __homepage__)
This diff is collapsed.
import json
import requests
import requests_toolbelt
import sys
import urllib3
class TransferClient(object):
def __init__(self, server_name, verify=True):
"""Initialise the class.
:arg str server_name: Name or IP of the transfer server.
:arg bool verify: SSL server certificate verification.
"""
self.server_name = server_name
self._verify = verify
if not self._verify:
try:
requests.packages.urllib3.disable_warnings()
except AttributeError:
try:
urllib3.disable_warnings()
except AttributeError:
sys.stderr = open('/dev/null')
def _request(self, method, endpoint, headers=None, data=None, json=None):
"""Handle a request.
:arg str method: Method for the request.
:arg str endpoint: API endpoint.
:arg dict headers: Dictionary of HTTP Headers to send with the request.
:arg dict data: Dictionary of file like objects for multipart encoding
upload.
:arg dict json: JSON data to send in the body of the request.
:returns dict: JSON encoded content of the response.
"""
try:
response = requests.request(
method, 'https://{}/{}'.format(self.server_name, endpoint),
headers=headers, data=data, json=json, verify=self._verify)
except requests.exceptions.SSLError:
raise OSError('SSL error, no server certificate installed?')
if not response.ok:
raise ValueError(response.json()['error'])
return response.json()
def users(self, user_id):
"""Gives a JSON object of a user together with its transfers.
:arg str user_id: User ID.
:returns dict: A JSON object of a user together with its transfers.
"""
return self._request('get', 'users', {'User-Id': user_id})
def schema(self, user_id):
"""Gives the JSON schema for a user.
:arg str user_id: User ID.
:returns dict: JSON schema.
"""
return self._request('get', 'users/schema', {'User-Id': user_id})
def transfers(self, user_id, metadata, name):
"""Initiates a new transfer.
:arg str user_id: User ID.
:arg dict metadata: Metadata.
:arg str name: Name of the metadata file.
:returns dict: Transfer JSON object.
"""
multipart = requests_toolbelt.MultipartEncoder(
fields={'metadata': (name, json.dumps(metadata))})
return self._request(
'post', 'transfers',
{'User-Id': user_id, 'Content-Type': multipart.content_type},
multipart)
def status(self, user_id, transfer_id):
"""Gives a JSON object of a transfer.
:arg str user_id: User ID.
:arg str transfer_id: Transfer ID.
:returns dict: Transfer JSON object.
"""
return self._request(
'get', 'transfers/{}'.format(transfer_id), {'User-Id': user_id})
def update(self, user_id, transfer_id, status):
"""Updates a transfer.
:arg str user_id: User ID.
:arg str transfer_id: Transfer ID.
:arg str status: New transfer status.
:returns dict: Transfer JSON object.
"""
return self._request(
'put', 'transfers/{}'.format(transfer_id), {'User-Id': user_id},
json={'status': status})
def uploads(self, user_id, transfer_id, file_handle):
"""Uploads a file to a transfer.
:arg str user_id: User ID.
:arg str transfer_id: Transfer ID.
:arg handle file_handle: Open readable file handle.
:returns dict: Transfer JSON object.
"""
multipart = requests_toolbelt.MultipartEncoder(
fields={'upload': (file_handle.name, file_handle)})
return self._request(
'post', 'transfers/{}/uploads'.format(transfer_id),
{'User-Id': user_id, 'Content-Type': multipart.content_type},
multipart)
def completed(self, client_id):
"""Gives a JSON object of all transfers for this client, i.e., a list
of transfer ids.
:arg str client_id: Client ID.
:returns dict: JSON object of all transfers.
"""
return self._request('get', 'completed', {'Client-Id': client_id})
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment