Commit dc09935c authored by Jeroen F.J. Laros's avatar Jeroen F.J. Laros

First directory transfer test.

parent 36585d3a
......@@ -72,6 +72,7 @@ setup(
license=license,
platforms=['any'],
packages=[package],
package_data={'{}'.format(package): ['unpack.md']},
install_requires=dependencies,
tests_require=develop_dependencies,
entry_points={
......
from __future__ import unicode_literals
from os.path import basename
import StringIO
import argparse
import datetime
import getpass
import gzip
import hashlib
import json
import os
import socket
import sys
import tarfile
from jsonschema.exceptions import ValidationError
import jsonschema
......@@ -17,6 +21,121 @@ from . import doc_split, usage, version
from .transfer_client import TransferClient
class StreamMeta(object):
def __init__(self):
self._chunk_size = 102400
self._hash_sum = hashlib.md5()
self._len = 0
self._index = 0
self.info = []
def write(self, data):
self._len += len(data)
if self._len >= self._chunk_size:
overhang = len(data) - (self._len - self._chunk_size)
self._hash_sum.update(data[:overhang])
self.info.append({
'filename': 'file_{:04d}'.format(self._index),
'md5': self._hash_sum.hexdigest(),
'size': self._chunk_size})
self._hash_sum = hashlib.md5()
self._hash_sum.update(data[overhang:])
self._len -= self._chunk_size
self._index += 1
else:
self._hash_sum.update(data)
def flush(self):
self.info.append({
'filename': 'file_{:04d}'.format(self._index),
'md5': self._hash_sum.hexdigest(),
'size': self._len})
class Pipe(object):
def __init__(self):
r, w = os.pipe()
self._read = os.fdopen(r)
self._write = os.fdopen(w, 'w')
self.len = 0
def read(self, size=-1):
send_size = min(self.len, size)
self.len = max(0, self.len - size)
return self._read.read(send_size)
def write(self, data):
self._write.write(data)
def flush(self):
self._write.flush()
def set_size(self, size):
self.len = size
def tgz_stream(fileobj, directory):
"""
"""
gzip_stream = gzip.GzipFile(mode='w', fileobj=fileobj, mtime=0)
tar_stream = tarfile.open(mode='w|', fileobj=gzip_stream)
tar_stream.add(directory, arcname='data')
tar_stream.close()
gzip_stream.close()
def _make_stream_metadata(log_handle, directory, title):
"""
"""
readme = StringIO.StringIO(open(os.path.join(
os.path.dirname(__file__), 'unpack.md')).read())
readme.name = 'README.md'
metadata = _minimal_metadata(title)
metadata['tags'] = {
'directory': True,
'note': 'Unpack with: `cat file_* | tar -xzv`'}
metadata['files'].append({
'filename': 'README.md',
'md5': hashlib.md5(readme.getvalue()).hexdigest()})
stream_metadata = StreamMeta()
tgz_stream(stream_metadata, directory)
stream_metadata.flush()
metadata['files'] += stream_metadata.info
return metadata
def transfer_dir(log_handle, directory, server_name, user_id, title, ssl_check):
"""
"""
metadata = _make_stream_metadata(log_handle, directory, title)
pipe = Pipe()
processid = os.fork()
if not processid:
tgz_stream(pipe, directory)
pipe.flush()
os._exit(0)
transfer_client = TransferClient(server_name, verify=ssl_check)
transfer_id = transfer_client.transfers(
user_id, metadata, 'metadata.json')['id']
readme = StringIO.StringIO(open(os.path.join(
os.path.dirname(__file__), 'unpack.md')).read())
readme.name = 'README.md'
transfer_client.uploads(user_id, transfer_id, readme)
for item in metadata['files'][1:]:
pipe.name = item['filename']
pipe.set_size(item['size'])
transfer_client.uploads(user_id, transfer_id, pipe)
def _write(output_handle, data):
output_handle.write('{}\n'.format(json.dumps(data, indent=4)))
......@@ -473,6 +592,15 @@ def main():
description=doc_split(transfer))
subparser.set_defaults(func=transfer)
subparser = subparsers.add_parser(
'transfer_dir',
parents=[default_verbose_parser, title_parser],
description=doc_split(transfer_dir))
subparser.add_argument(
'directory', metavar='DIR', type=str,
help='directory to be transferred')
subparser.set_defaults(func=transfer_dir)
subparser = subparsers.add_parser(
'resume', parents=[default_verbose_parser],
description=doc_split(resume))
......
# Unpacking instructions
This transfer ...
To unpack, use the following command:
```bash
cat file_* | tar -xzv
```
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment