Commit a9770390 authored by jkvis's avatar jkvis

Merge branch 'master' of git.lumc.nl:j.k.vis/transfer

parents 612d1552 ac7eb771
......@@ -6,7 +6,7 @@ Overview
A transfer consists of a collection of (large) files that belong together
that should be transferred (moved) from one location to another. We
consider a transfer to be successful iff all associated files have been
copied and checked for consistancy successfully. A user is associated as
copied and checked for consistency successfully. A user is associated as
owner of a transfer. Only registered users can initiate transfers. Only
one transfer can be active for a given user at any time.
A global overview of a transfer is as follows:
......
.cache/
dist/
transfer_client.egg-info/
*.pyc
transfer_client.egg-info
dist
......@@ -3,7 +3,7 @@ This package provides functions for communication with the Restful Transfer
Server.
## Installation
Via pypi:
Via [pypi](https://pypi.python.org/pypi/transfer-client):
pip install transfer_client
......@@ -13,6 +13,15 @@ From source:
cd transfer/transfer_client
pip install .
### Server certificate
If the server uses a self-signed certificate, the administrator of the server
needs to add this certificate to the list of trusted authorities (change
`server_name` and `domain`):
openssl s_client -connect server_name.domain:443 < /dev/null | \
openssl x509 > /usr/local/share/ca-certificates/server_name.crt
update-ca-certificates
## Command line interface
Use the command `transfer_client -h` for a list of subcommands. For every
subcommand a separate help is available, e.g.,
......@@ -30,24 +39,25 @@ positional arguments:
optional arguments:
-h, --help show this help message and exit
-o OUTPUT output file
-n disable ssl certificate check
```
### Typical usage
The three high level subcommands are probably the only ones needed.
To transfer a list of files, use the `transfer` subcommand. We assume that the
server name is `server.domain` and the user ID is `xxxxxxxx`.
server name is `server_name.domain` and the user ID is `xxxxxxxx`.
transfer_client transfer server.domain xxxxxxxx *.gz
transfer_client transfer server_name.domain xxxxxxxx *.gz
If a transfer is interrupted for some reason, it can be resumed with the
`resume` subcommand.
transfer_client resume server.domain xxxxxxxx
transfer_client resume server_name.domain xxxxxxxx
To cancel an interrupted transfer, the `cancel` subcommand can be used.
transfer_client cancel server.domain xxxxxxxx
transfer_client cancel server_name.domain xxxxxxxx
## Library
The library implements an interface to the API of the Restful Transfer server.
......@@ -60,8 +70,8 @@ we can use the API endpoints.
```python
>>> from transfer_client.transfer_client import TransferClient
>>>
>>> # Create a class instance for a server running on server.domain.
>>> transfer_client = TransferClient('server.domain')
>>> # Create a class instance for a server running on server_name.domain.
>>> transfer_client = TransferClient('server_name.domain')
>>>
>>> # Get user information together with its transfers.
>>> transfer_client.users('xxxxxxxx')
......
jsonschema
pyyaml
requests
requests-toolbelt
urllib3
......@@ -11,7 +11,9 @@ documentation = 'README.md'
license = 'MIT License'
keywords = []
dependencies = ['requests']
dependencies = [
'jsonschema', 'pyyaml', 'requests', 'requests-toolbelt', 'urllib3']
develop_dependencies = ['fake-open', 'pytest', 'tox']
supported = [(2, 7), (3, 3), (3, 4)]
classifiers = [
'Development Status :: 3 - Alpha',
......@@ -71,6 +73,7 @@ setup(
platforms=['any'],
packages=[package],
install_requires=dependencies,
tests_require=develop_dependencies,
entry_points={
'console_scripts': ['{0} = {0}.cli:main'.format(package)]
},
......
"""
Tests for the transfer_client CLI.
"""
from __future__ import unicode_literals
from io import StringIO
from fake_open import FakeOpen, make_fake_file, md5_check
from transfer_client import cli, TransferClient
class TestCLI(object):
def setup(self):
opener = FakeOpen()
self._handles = opener.handles
cli.open = opener.open
self._fake_reply = None
self._file_handle = make_fake_file('upload.bin', '')
self._file_handles = [
make_fake_file('a', 'a\n'), make_fake_file('b', 'b\n')]
self._metadata_handle = make_fake_file('metadata.json', 'null\n')
self._output_handle = StringIO()
self._log_handle = StringIO()
TransferClient._request = lambda *args, **kwargs: self._fake_reply
def test_users(self):
cli.users(self._output_handle, '127.0.0.1', '123', False)
assert self._output_handle.getvalue() == 'null\n'
def test_schema(self):
cli.schema(self._output_handle, '127.0.0.1', '123', False)
assert self._output_handle.getvalue() == 'null\n'
def test_transfers(self):
cli.transfers(
self._output_handle, self._metadata_handle,
'127.0.0.1', '123', False)
assert self._output_handle.getvalue() == 'null\n'
def test_status(self):
cli.status(self._output_handle, '127.0.0.1', '123', '456', False)
assert self._output_handle.getvalue() == 'null\n'
def test_update(self):
cli.update(
self._output_handle, '127.0.0.1', '123', '456', 'cancelled', False)
assert self._output_handle.getvalue() == 'null\n'
def test_uploads(self):
cli.uploads(
self._output_handle, self._file_handle,
'127.0.0.1', '123', '456', False)
assert self._output_handle.getvalue() == 'null\n'
def test_completed(self):
cli.completed(self._output_handle, '127.0.0.1', '789', False)
assert self._output_handle.getvalue() == 'null\n'
def test_make_metadata_1(self):
metadata = cli._make_metadata(
self._log_handle, self._file_handles, 'test')
assert metadata['title'] == 'test'
assert metadata['files'][0]['filename'] == 'a'
assert (
metadata['files'][1]['md5'] == '3b5d5c3712955042212316173ccf37be')
def test_make_metadata_2(self):
cli.make_metadata(
self._output_handle, self._log_handle, self._file_handles, 'test')
assert md5_check(
self._output_handle.getvalue(), 'ee7ca88f46ca4c7f58375279a2fbd62a')
assert md5_check(
self._log_handle.getvalue(), 'c16aad07df999e5a1a5bcd066c1f692f')
def test_transfer(self):
self._fake_reply = {'id': 0}
cli.transfer(
self._log_handle, self._file_handles,
'127.0.0.1', '123', 'test', False)
assert md5_check(
self._log_handle.getvalue(), '307fc32a1709cb2505c5c4525aa6c34e')
def test_interrupted_transfer_1(self):
self._fake_reply = {'transfers': []}
try:
cli._interrupted_transfer(
self._log_handle, TransferClient('127.0.0.1', False), '123')
except ValueError as error:
assert error.message == 'no interrupted transfers found'
else:
assert False
def test_interrupted_transfer_2(self):
self._fake_reply = {'transfers': [{'id': 0, 'status': 'initiated'}]}
cli._interrupted_transfer(
self._log_handle, TransferClient('127.0.0.1', False), '123')
assert md5_check(
self._log_handle.getvalue(), 'b6a19671cc29806d6468b84695121344')
def test_resume(self):
self._fake_reply = {
'transfers': [
{
'id': 0,
'status': 'initiated',
'files': [
{
'status': 'pending',
'filename': 'a'
}
]
}
]
}
cli.resume(self._log_handle, '127.0.0.1', '123', False)
assert md5_check(
self._log_handle.getvalue(), '641eeda3b448cc99a9c5f74c8343d589')
def test_cancel(self):
self._fake_reply = {
'transfers': [
{
'id': 0,
'status': 'initiated'
}
]
}
cli.cancel(self._log_handle, '127.0.0.1', '123', False)
assert md5_check(
self._log_handle.getvalue(), 'b6a19671cc29806d6468b84695121344')
def test_reformat_transfer_1(self):
readable_transfer = cli._reformat_transfer(
{'files': [], 'status': 'completed'})
assert readable_transfer['number_of_files'] == 0
def test_reformat_transfer_2(self):
readable_transfer = cli._reformat_transfer(
{'files': [{'status': 'uploaded'}], 'status': 'initiated'})
assert readable_transfer['uploaded'] == 1
def test_reformat_transfer_3(self):
readable_transfer = cli._reformat_transfer({
'files': [],
'status': 'completed',
'start_date': '2016-11-25 09:54:57.088350',
'end_date': '2016-11-25 09:55:57.088350'
})
assert readable_transfer['duration'] == '0:01:00'
def test_transfers_summary(self):
self._fake_reply = {
'transfers': [
{
'files': [],
'status': 'completed'
}
]
}
cli.transfers_summary(self._output_handle, '127.0.0.1', '123', False)
assert md5_check(
self._output_handle.getvalue(), '69f8fe82668ffe4f1b1d3fc47906c8d4')
def test_last_transfer_summary(self):
self._fake_reply = {
'transfers': [
{
'files': [],
'status': 'completed'
}
]
}
cli.last_transfer_summary(
self._output_handle, '127.0.0.1', '123', False)
assert md5_check(
self._output_handle.getvalue(), '3f3384fc1041333b147cdb1f46608256')
def test_transfer_summary(self):
self._fake_reply = {
'transfers': [
{
'id': '456',
'files': [],
'status': 'completed'
}
]
}
cli.transfer_summary(
self._output_handle, '127.0.0.1', '123', '456', False)
assert md5_check(
self._output_handle.getvalue(), 'c38201114fb64755abb2747bcd40e88d')
def test_check_metadata(self):
self._fake_reply = {
'type': 'object',
'properties': {
'id': {'type': 'string'}
}
}
cli.check_metadata(StringIO(
'{"id": "test"}\n'), '127.0.0.1', '123', False)
"""
Tests for the transfer_client library.
"""
from __future__ import unicode_literals
from fake_open import make_fake_file
from transfer_client import TransferClient
class TestLibrary(object):
def setup(self):
TransferClient._request = lambda *args, **kwargs: None
self._transfer_client = TransferClient('127.0.0.1', False)
self._input_handle = make_fake_file('upload.bin', '')
def test_users(self):
assert self._transfer_client.users('123') == None
def test_schema(self):
assert self._transfer_client.schema('123') == None
def test_transfers(self):
assert self._transfer_client.transfers(
'123', make_fake_file('metadata.json', 'null\n')) == None
def test_status(self):
assert self._transfer_client.status('123', '456') == None
def test_update(self):
assert self._transfer_client.update('123', '456', 'cancelled') == None
def test_uploads(self):
assert self._transfer_client.uploads(
'123', '456', self._input_handle) == None
def test_completed(self):
assert self._transfer_client.completed('789') == None
......@@ -7,20 +7,23 @@ Copyright (c) 2016 Jeroen F.J. Laros <J.F.J.Laros@lumc.nl>
Licensed under the MIT license, see the LICENSE file.
"""
from .transfer_client import TransferClient
__version_info__ = ('0', '0', '2')
__version_info__ = ('0', '0', '6')
__version__ = '.'.join(__version_info__)
__author__ = 'LUMC, Jeroen F.J. Laros'
__contact__ = 'J.F.J.Laros@lumc.nl'
__homepage__ = 'https://git.lumc.nl/j.k.vis/transfer'
usage = __doc__.split("\n\n\n")
usage = __doc__.split('\n\n\n')
def doc_split(func):
return func.__doc__.split("\n\n")[0]
return func.__doc__.split('\n\n')[0]
def version(name):
return "%s version %s\n\nAuthor : %s <%s>\nHomepage : %s" % (name,
__version__, __author__, __contact__, __homepage__)
return '{} version {}\n\nAuthor : {} <{}>\nHomepage : {}'.format(
name, __version__, __author__, __contact__, __homepage__)
This diff is collapsed.
import json
import requests
import requests_toolbelt
import sys
import urllib3
class TransferClient(object):
def __init__(self, server_name):
def __init__(self, server_name, verify=True):
"""
Initialise the class.
:arg str server_name: Name or IP of the transfer server.
"""
self.server_name = server_name
self._verify = False
self._verify = verify
def _request(self, method, endpoint, headers=None, files=None, json=None):
if not self._verify:
try:
requests.packages.urllib3.disable_warnings()
except AttributeError:
try:
urllib3.disable_warnings()
except AttributeError:
sys.stderr = open('/dev/null')
def _request(self, method, endpoint, headers=None, data=None, json=None):
"""
Handle a request.
......@@ -25,9 +37,12 @@ class TransferClient(object):
:returns dict: JSON encoded content of the response.
"""
response = requests.request(method,
'https://{}/{}'.format(self.server_name, endpoint),
headers=headers, files=files, json=json, verify=self._verify)
try:
response = requests.request(
method, 'https://{}/{}'.format(self.server_name, endpoint),
headers=headers, data=data, json=json, verify=self._verify)
except requests.exceptions.SSLError:
raise OSError('SSL error, no server certificate installed?')
if not response.ok:
raise ValueError(response.json()['error'])
return response.json()
......@@ -52,7 +67,7 @@ class TransferClient(object):
"""
return self._request('get', 'users/schema', {'User-Id': user_id})
def transfers(self, user_id, metadata):
def transfers(self, user_id, metadata_handle):
"""
Initiates a new transfer.
......@@ -60,9 +75,12 @@ class TransferClient(object):
:returns dict: Transfer JSON object.
"""
multipart = requests_toolbelt.MultipartEncoder(
fields={'metadata': (metadata_handle.name, metadata_handle)})
return self._request(
'post', 'transfers', {'User-Id': user_id},
{'metadata': json.dumps(metadata)})
'post', 'transfers',
{'User-Id': user_id, 'Content-Type': multipart.content_type},
multipart)
def status(self, user_id, transfer_id):
"""
......@@ -95,9 +113,12 @@ class TransferClient(object):
:returns dict: Transfer JSON object.
"""
multipart = requests_toolbelt.MultipartEncoder(
fields={'upload': (file_handle.name, file_handle)})
return self._request(
'post', 'transfers/{}/uploads'.format(transfer_id),
{'User-Id': user_id}, {'upload': file_handle})
{'User-Id': user_id, 'Content-Type': multipart.content_type},
multipart)
def completed(self, client_id):
"""
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment