Commit 950bf964 authored by jkvis's avatar jkvis
Browse files

Merge branch 'master' of git.lumc.nl:j.k.vis/transfer

parents 4e915cc7 0a2f67ff
.cache/
dist/
transfer_client.egg-info/
*.pyc
Copyright (c) 2017 Leiden University Medical Center <humgen@lumc.nl>
Copyright (c) 2017 Jeroen F.J. Laros <J.F.J.Laros@lumc.nl>
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
# Transfer client
This package provides functions for communication with the Restful Transfer
Server.
## Installation
Via [pypi](https://pypi.python.org/pypi/transfer-client):
pip install transfer_client
From source:
git clone https://git.lumc.nl/j.k.vis/transfer.git
cd transfer/transfer_client
pip install .
### Server certificate
If the server uses a self-signed certificate, the administrator of the server
needs to add this certificate to the list of trusted authorities (change
`server_name` and `domain`):
openssl s_client -connect server_name.domain:443 < /dev/null | \
openssl x509 > /usr/local/share/ca-certificates/server_name.crt
update-ca-certificates
## Command line interface
Use the command `transfer_client -h` for a list of subcommands. For every
subcommand a separate help is available, e.g.,
```
$ transfer_client users -h
usage: transfer_client users [-h] [-o OUTPUT] SERVER USER
Gives a JSON object of a user together with its transfers.
positional arguments:
SERVER server name
USER user id
optional arguments:
-h, --help show this help message and exit
-o OUTPUT output file
-n disable ssl certificate check
```
### Typical usage
The three high level subcommands are probably the only ones needed.
To transfer a list of files, use the `transfer` subcommand. We assume that the
server name is `server_name.domain` and the user ID is `xxxxxxxx`.
transfer_client transfer server_name.domain xxxxxxxx *.gz
If a transfer is interrupted for some reason, it can be resumed with the
`resume` subcommand.
transfer_client resume server_name.domain xxxxxxxx
To cancel an interrupted transfer, the `cancel` subcommand can be used.
transfer_client cancel server_name.domain xxxxxxxx
## Library
The library implements an interface to the API of the Restful Transfer server.
Full documentation can be found [here](https://git.lumc.nl/j.k.vis/transfer).
### Usage
To use the interface, we first make a class instance of `TransferClient`, then
we can use the API endpoints.
```python
>>> from transfer_client.transfer_client import TransferClient
>>>
>>> # Create a class instance for a server running on server_name.domain.
>>> transfer_client = TransferClient('server_name.domain')
>>>
>>> # Get user information together with its transfers.
>>> transfer_client.users('xxxxxxxx')
{ ... }
```
{
"required": [
"files"
],
"type": "object",
"properties": {
"files": {
"minItems": 1,
"items": {
"required": [
"filename",
"md5"
],
"type": "object",
"properties": {
"md5": {
"pattern": "^[a-f0-9]{32}$",
"type": "string"
},
"filename": {
"pattern": ".+",
"type": "string"
}
}
},
"uniqueItems": true,
"type": "array"
},
"title": {
"pattern": ".+",
"type": "string"
}
},
"title": "Minimal transfer metadata"
}
jsonschema
pyyaml
requests
requests-toolbelt
urllib3
import os
import sys
from setuptools import setup
package = 'transfer_client'
package_name = 'transfer-client'
description = '{}: Transfer client library and CLI.'.format(package_name)
documentation = 'README.md'
license = 'MIT License'
keywords = []
dependencies = [
'jsonschema', 'pyyaml', 'requests', 'requests-toolbelt', 'urllib3']
develop_dependencies = ['fake-open', 'pytest', 'tox']
supported = [(2, 7), (3, 3), (3, 4)]
classifiers = [
'Development Status :: 3 - Alpha',
'Intended Audience :: Science/Research',
'Intended Audience :: Developers',
'Topic :: Scientific/Engineering',
]
if sys.version_info < supported[0]:
raise Exception('{} requires Python {}.{} or higher.'.format(
package, *supported[0]))
if sys.version_info[:2] == supported[0]:
dependencies.extend(['argparse', 'importlib'])
# This is quite the hack, but we don't want to import our package from here
# since that's recipe for disaster (it might have some uninstalled
# dependencies, or we might import another already installed version).
distmeta = {}
for line in open(os.path.join(package, '__init__.py')):
try:
field, value = (x.strip() for x in line.split('='))
except ValueError:
continue
if field == '__version_info__':
value = value.strip('[]()')
value = '.'.join(x.strip(' \'"') for x in value.split(','))
else:
value = value.strip('\'"')
distmeta[field] = value
try:
with open(documentation) as readme:
long_description = readme.read()
except IOError:
long_description = 'See ' + distmeta['__homepage__']
language_string = 'Programming Language :: Python'
classifiers += [
'License :: OSI Approved :: {}'.format(license),
'Operating System :: OS Independent',
language_string,
'{} :: {}'.format(language_string, supported[0][0]),
'{} :: {}'.format(language_string, supported[-1][0])] + \
['{} :: {}.{}'.format(language_string, *version) for version in supported]
setup(
name=package_name,
version=distmeta['__version_info__'],
description=description,
long_description=long_description,
author=distmeta['__author__'],
author_email=distmeta['__contact__'],
url=distmeta['__homepage__'],
license=license,
platforms=['any'],
packages=[package],
install_requires=dependencies,
tests_require=develop_dependencies,
entry_points={
'console_scripts': ['{0} = {0}.cli:main'.format(package)]
},
classifiers=classifiers,
keywords=' '.join(keywords)
)
"""Tests for the transfer_client CLI.
"""
from __future__ import unicode_literals
from io import StringIO
import json
from fake_open import FakeOpen, make_fake_file, md5_check
from transfer_client import cli, TransferClient
class FakeReply(object):
def __init__(self):
self._replies = []
def add(self, reply):
self._replies = [reply] + self._replies
def pop(self):
if self._replies:
return self._replies.pop()
return None
class TestCLI(object):
def setup(self):
opener = FakeOpen()
self._handles = opener.handles
cli.open = opener.open
self._reply = FakeReply()
self._file_handle = make_fake_file('upload.bin', '')
self._file_handles = [
make_fake_file('a', 'a\n'), make_fake_file('b', 'b\n')]
self._metadata_handle = make_fake_file('metadata.json', 'null\n')
self._output_handle = StringIO()
self._log_handle = StringIO()
TransferClient._request = lambda *args, **kwargs: self._reply.pop()
def test_users(self):
cli.users(self._output_handle, '127.0.0.1', '123', False)
assert self._output_handle.getvalue() == 'null\n'
def test_schema(self):
cli.schema(self._output_handle, '127.0.0.1', '123', False)
assert self._output_handle.getvalue() == 'null\n'
def test_transfers(self):
cli.transfers(
self._output_handle, self._metadata_handle,
'127.0.0.1', '123', False)
assert self._output_handle.getvalue() == 'null\n'
def test_status(self):
cli.status(self._output_handle, '127.0.0.1', '123', '456', False)
assert self._output_handle.getvalue() == 'null\n'
def test_update(self):
cli.update(
self._output_handle, '127.0.0.1', '123', '456', 'cancelled', False)
assert self._output_handle.getvalue() == 'null\n'
def test_uploads(self):
cli.uploads(
self._output_handle, self._file_handle,
'127.0.0.1', '123', '456', False)
assert self._output_handle.getvalue() == 'null\n'
def test_completed(self):
cli.completed(self._output_handle, '127.0.0.1', '789', False)
assert self._output_handle.getvalue() == 'null\n'
def test_make_metadata_1(self):
metadata = cli._make_metadata(
self._log_handle, self._file_handles, 'test')
assert metadata['title'] == 'test'
assert metadata['files'][0]['filename'] == 'a'
assert (
metadata['files'][1]['md5'] == '3b5d5c3712955042212316173ccf37be')
def test_make_metadata_2(self):
cli.make_metadata(
self._output_handle, self._log_handle, self._file_handles, 'test')
assert md5_check(
self._output_handle.getvalue(), 'ee7ca88f46ca4c7f58375279a2fbd62a')
assert md5_check(
self._log_handle.getvalue(), 'c16aad07df999e5a1a5bcd066c1f692f')
def test_transfer(self):
self._reply.add(json.loads(open('data/schema.json').read()))
self._reply.add({'id': 0})
cli.transfer(
self._log_handle, self._file_handles,
'127.0.0.1', '123', 'test', False)
assert md5_check(
self._log_handle.getvalue(), '307fc32a1709cb2505c5c4525aa6c34e')
def test_interrupted_transfer_1(self):
self._reply.add({'transfers': []})
try:
cli._interrupted_transfer(
self._log_handle, TransferClient('127.0.0.1', False), '123')
except ValueError as error:
assert error.message == 'no interrupted transfers found'
else:
assert False
def test_interrupted_transfer_2(self):
self._reply.add({'transfers': [{'id': 0, 'status': 'initiated'}]})
cli._interrupted_transfer(
self._log_handle, TransferClient('127.0.0.1', False), '123')
assert md5_check(
self._log_handle.getvalue(), 'b6a19671cc29806d6468b84695121344')
def test_resume(self):
self._reply.add({
'transfers': [
{
'id': 0,
'status': 'initiated',
'files': [
{
'status': 'pending',
'filename': 'a'
}
]
}
]
})
cli.resume(self._log_handle, '127.0.0.1', '123', False)
assert md5_check(
self._log_handle.getvalue(), '641eeda3b448cc99a9c5f74c8343d589')
def test_cancel(self):
self._reply.add({
'transfers': [
{
'id': 0,
'status': 'initiated'
}
]
})
cli.cancel(self._log_handle, '127.0.0.1', '123', False)
assert md5_check(
self._log_handle.getvalue(), 'b6a19671cc29806d6468b84695121344')
def test_reformat_transfer_1(self):
readable_transfer = cli._reformat_transfer(
{'files': [], 'status': 'completed'})
assert readable_transfer['number_of_files'] == 0
def test_reformat_transfer_2(self):
readable_transfer = cli._reformat_transfer(
{'files': [{'status': 'uploaded'}], 'status': 'initiated'})
assert readable_transfer['uploaded'] == 1
def test_reformat_transfer_3(self):
readable_transfer = cli._reformat_transfer({
'files': [],
'status': 'completed',
'start_date': '2016-11-25 09:54:57.088350',
'end_date': '2016-11-25 09:55:57.088350'
})
assert readable_transfer['duration'] == '0:01:00'
def test_transfers_summary(self):
self._reply.add({
'transfers': [
{
'files': [],
'status': 'completed'
}
]
})
cli.transfers_summary(self._output_handle, '127.0.0.1', '123', False)
assert md5_check(
self._output_handle.getvalue(), '69f8fe82668ffe4f1b1d3fc47906c8d4')
def test_last_transfer_summary(self):
self._reply.add({
'transfers': [
{
'files': [],
'status': 'completed'
}
]
})
cli.last_transfer_summary(
self._output_handle, '127.0.0.1', '123', False)
assert md5_check(
self._output_handle.getvalue(), '3f3384fc1041333b147cdb1f46608256')
def test_transfer_summary(self):
self._reply.add({
'transfers': [
{
'id': '456',
'files': [],
'status': 'completed'
}
]
})
cli.transfer_summary(
self._output_handle, '127.0.0.1', '123', '456', False)
assert md5_check(
self._output_handle.getvalue(), 'c38201114fb64755abb2747bcd40e88d')
def test_check_metadata(self):
self._reply.add({
'type': 'object',
'properties': {
'id': {'type': 'string'}
}
})
cli.check_metadata(StringIO(
'{"id": "test"}\n'), '127.0.0.1', '123', False)
"""Tests for the transfer_client library.
"""
from __future__ import unicode_literals
from fake_open import make_fake_file
from transfer_client import TransferClient
class TestLibrary(object):
def setup(self):
TransferClient._request = lambda *args, **kwargs: None
self._transfer_client = TransferClient('127.0.0.1', False)
self._input_handle = make_fake_file('upload.bin', '')
def test_users(self):
assert self._transfer_client.users('123') == None
def test_schema(self):
assert self._transfer_client.schema('123') == None
def test_transfers(self):
assert self._transfer_client.transfers(
'123', {}, 'metadata.json') == None
def test_status(self):
assert self._transfer_client.status('123', '456') == None
def test_update(self):
assert self._transfer_client.update('123', '456', 'cancelled') == None
def test_uploads(self):
assert self._transfer_client.uploads(
'123', '456', self._input_handle) == None
def test_completed(self):
assert self._transfer_client.completed('789') == None
"""Transfer server client CLI.
Copyright (c) 2017 Leiden University Medical Center <humgen@lumc.nl>
Copyright (c) 2017 Jeroen F.J. Laros <J.F.J.Laros@lumc.nl>
Licensed under the MIT license, see the LICENSE file.
"""
from .transfer_client import TransferClient
__version_info__ = ('0', '0', '7')
__version__ = '.'.join(__version_info__)
__author__ = 'LUMC, Jeroen F.J. Laros'
__contact__ = 'J.F.J.Laros@lumc.nl'
__homepage__ = 'https://git.lumc.nl/j.k.vis/transfer'
usage = __doc__.split('\n\n\n')
def doc_split(func):
return func.__doc__.split('\n\n')[0]
def version(name):
return '{} version {}\n\nAuthor : {} <{}>\nHomepage : {}'.format(
name, __version__, __author__, __contact__, __homepage__)
from __future__ import unicode_literals
from os.path import basename
import argparse
import datetime
import getpass
import hashlib
import json
import socket
import sys
from jsonschema.exceptions import ValidationError
import jsonschema
import yaml
from . import doc_split, usage, version
from .transfer_client import TransferClient
def _write(output_handle, data):
output_handle.write('{}\n'.format(json.dumps(data, indent=4)))
def users(output_handle, server_name, user_id, ssl_check):
"""Gives a JSON object of a user together with its transfers.
:arg stream output_handle: Open writeable handle to a file.
:arg str server_name: Name of the transfer server.
:arg str user_id: User ID.
:arg bool ssl_check: Check server SSL certificate.
"""
_write(
output_handle, TransferClient(server_name, ssl_check).users(user_id))
def schema(output_handle, server_name, user_id, ssl_check):
"""Gives the JSON schema for a user.
:arg stream output_handle: Open writeable handle to a file.
:arg str server_name: Name of the transfer server.
:arg str user_id: User ID.
:arg bool ssl_check: Check server SSL certificate.
"""
_write(
output_handle, TransferClient(server_name, ssl_check).schema(user_id))
def transfers(output_handle, metadata_handle, server_name, user_id, ssl_check):
"""Initiates a new transfer.
:arg stream output_handle: Open writeable handle to a file.
:arg stream metadata_handle: Open readable handle to a metadata file.
:arg str server_name: Name of the transfer server.
:arg str user_id: User ID.
:arg bool ssl_check: Check server SSL certificate.
"""
_write(output_handle, TransferClient(server_name, ssl_check).transfers(
user_id, json.loads(metadata_handle.read()), metadata_handle.name))
def status(output_handle, server_name, user_id, transfer_id, ssl_check):
"""Gives a JSON object of a transfer.
:arg stream output_handle: Open writeable handle to a file.
:arg str server_name: Name of the transfer server.
:arg str user_id: User ID.
:arg str transfer_id: Transfer ID.
:arg bool ssl_check: Check server SSL certificate.
"""
_write(output_handle, TransferClient(server_name, ssl_check).status(
user_id, transfer_id))
def update(
output_handle, server_name, user_id, transfer_id, status, ssl_check):
"""Updates a transfer.
:arg stream output_handle: Open writeable handle to a file.
:arg str server_name: Name of the transfer server.
:arg str user_id: User ID.
:arg str transfer_id: Transfer ID.
:arg str status: Transfer status.
:arg bool ssl_check: Check server SSL certificate.
"""
_write(output_handle, TransferClient(server_name, ssl_check).update(
user_id, transfer_id, status))