# -*- coding: utf-8 -*-
"""Storj HTTP module."""
import os
import logging
import json
import requests
import storj
import time
from base64 import b64encode
from binascii import b2a_hex
from ecdsa import SigningKey
from hashlib import sha256
from io import BytesIO
from six.moves.urllib.parse import urlencode, urljoin
try:
from json.decoder import JSONDecodeError
except ImportError:
# Python 2
JSONDecodeError = ValueError
from . import model
from .api import ecdsa_to_hex
from .exception import StorjBridgeApiError
from storj import web_socket
[docs]class Client(object):
"""
Attributes:
api_url (str): the Storj API endpoint.
session ():
email (str): user email address.
password (str): user password.
private_key ():
public_key ():
public_key_hex ():
"""
logger = logging.getLogger('%s.Client' % __name__)
def __init__(self, email, password):
self.api_url = 'https://api.storj.io/'
self.session = requests.Session()
self.email = email
self.password = password
self.private_key = None
self.public_key = None
self.public_key_hex = None
@property
def password(self):
"""(str): user password"""
return self._password
@password.setter
def password(self, value):
self._password = sha256(value.encode('ascii')).hexdigest()
[docs] def authenticate(self, ecdsa_private_key=None):
self.logger.debug('authenticate')
if isinstance(ecdsa_private_key, SigningKey):
self.private_key = ecdsa_private_key
self.public_key = self.private_key.get_verifying_key()
self.public_key_hex = ecdsa_to_hex(self.public_key)
def _add_basic_auth(self, request_kwargs):
self.logger.debug('using basic auth')
request_kwargs['headers'].update({
'Authorization': b'Basic ' + b64encode(
('%s:%s' % (self.email, self.password)).encode('ascii')
),
})
def _add_ecdsa_signature(self, request_kwargs):
method = request_kwargs.get('method', 'GET')
if method in ('GET', 'DELETE'):
request_kwargs.setdefault('params', {})
request_kwargs['params']['__nonce'] = int(time.time())
data = urlencode(request_kwargs['params'])
else:
request_kwargs.setdefault('json', {})
request_kwargs['json']['__nonce'] = int(time.time())
data = json.dumps(request_kwargs['json'])
contract = '\n'.join(
(method, request_kwargs['path'], data)).encode('utf-8')
signature_bytes = self.private_key.sign(
contract, sigencode=sigencode_der, hashfunc=sha256)
signature = b2a_hex(signature_bytes).decode('ascii')
request_kwargs['headers'].update(
{
'x-signature': signature,
'x-pubkey': ecdsa_to_hex(self.public_key),
})
def _prepare_request(self, **kwargs):
"""Prepares a HTTP request.
Args:
kwargs (dict): keyword arguments for the authentication function
(``_add_ecdsa_signature()`` or ``_add_basic_auth()``) and
:py:class:`requests.Request` class.
Raises:
AssertionError: in case ``kwargs['path']`` doesn't start with ``/``.
"""
kwargs.setdefault('headers', {})
# Add appropriate authentication headers
if isinstance(self.private_key, SigningKey):
self._add_ecdsa_signature(kwargs)
elif self.email and self.password:
self._add_basic_auth(kwargs)
# Generate URL from path
path = kwargs.pop('path')
assert path.startswith('/')
kwargs['url'] = urljoin(self.api_url, path)
return requests.Request(**kwargs).prepare()
def _request(self, **kwargs):
"""Perform HTTP request.
Args:
kwargs (dict): keyword arguments.
Raises:
:py:class:`StorjBridgeApiError`: in case::
- internal server error
- error attribute is present in the JSON response
- HTTP response JSON decoding failed
"""
response = self.session.send(self._prepare_request(**kwargs))
self.logger.debug('_request response %s', response.text)
try:
response.raise_for_status()
except requests.exceptions.RequestException as e:
self.logger.error(e)
self.logger.debug('response.text=%s', response.text)
raise StorjBridgeApiError(response.text)
# Raise any errors as exceptions
try:
if response.text != '':
response_json = response.json()
else:
return {}
if 'error' in response_json:
raise StorjBridgeApiError(response_json['error'])
return response_json
except JSONDecodeError as e:
self.logger.error(e)
self.logger.error('_request body %s', response.text)
raise StorjBridgeApiError('Could not decode response.')
[docs] def bucket_create(self, name, storage=None, transfer=None):
"""Create storage bucket.
See `API buckets: POST /buckets
<https://storj.github.io/bridge/#!/buckets/post_buckets>`_
Args:
name (str): name.
storage (int): storage limit (in GB).
transfer (int): transfer limit (in GB).
Returns:
(:py:class:`model.Bucket`): bucket.
"""
self.logger.info('bucket_create(%s, %s, %s)', name, storage, transfer)
data = {'name': name}
if storage:
data['storage'] = storage
if transfer:
data['transfer'] = transfer
return model.Bucket(**self._request(method='POST', path='/buckets', data=data))
[docs] def bucket_delete(self, bucket_id):
"""Destroy a storage bucket.
See `API buckets: DELETE /buckets/{id}
<https://storj.github.io/bridge/#!/buckets/delete_buckets_id>`_
Args:
bucket_id (string): unique identifier.
"""
self.logger.info('bucket_delete(%s)', bucket_id)
self._request(method='DELETE', path='/buckets/%s' % bucket_id)
[docs] def bucket_files(self, bucket_id):
"""List all the file metadata stored in the bucket.
See `API buckets: GET /buckets/{id}/files
<https://storj.github.io/bridge/#!/buckets/get_buckets_id_files>`_
Args:
bucket_id (string): unique identifier.
Returns:
(dict): to be changed to model in the future.
"""
self.logger.info('bucket_files(%s)', bucket_id)
return self._request(
method='GET',
path='/buckets/%s/files/' % (bucket_id),)
[docs] def bucket_get(self, bucket_id):
"""Return the bucket object.
See `API buckets: GET /buckets
<https://storj.github.io/bridge/#!/buckets/get_buckets_id>`_
Args:
bucket_id (str): bucket unique identifier.
Returns:
(:py:class:`model.Bucket`): bucket.
"""
self.logger.info('bucket_get(%s)', bucket_id)
try:
return model.Bucket(**self._request(
method='GET',
path='/buckets/%s' % bucket_id))
except requests.HTTPError as e:
if e.response.status_code == requests.codes.not_found:
return None
else:
self.logger.error('bucket_get() error=%s', e)
raise StorjBridgeApiError()
[docs] def bucket_list(self):
"""List all of the buckets belonging to the user.
See `API buckets: GET /buckets
<https://storj.github.io/bridge/#!/buckets/get_buckets>`_
Returns:
(generator[:py:class:`model.Bucket`]): buckets.
"""
self.logger.info('bucket_list()')
response = self._request(method='GET', path='/buckets')
if response is not None:
for element in response:
yield model.Bucket(**element)
else:
raise StopIteration
[docs] def bucket_set_keys(self, bucket_id, bucket_name, keys):
"""Update the bucket with the given public keys.
See `API buckets: PATCH /buckets/{bucket_id}
<https://storj.github.io/bridge/#!/buckets/patch_buckets_id>`_
Args:
bucket_id (str): bucket unique identifier.
bucket_name (str): bucket name.
keys (list[str]): public keys.
Returns:
(:py:class:`storj.model.Bucket`): updated bucket information.
"""
self.logger.info('bucket_set_keys(%s, %s)', bucket_name, keys)
return model.Bucket(**self._request(
method='PATCH',
path='/buckets/%s' % bucket_id,
data={
'name': bucket_name,
'pubkeys': keys}))
[docs] def bucket_set_mirrors(self, bucket_id, file_id, redundancy):
"""Establishes a series of mirrors for the given file.
See `API buckets: POST /buckets/{id}/mirrors
<https://storj.github.io/bridge/#!/buckets/post_buckets_id_mirrors>`_
Args:
bucket_id (str): bucket unique identifier.
file_id (str): file unique identitifer.
redundancy (int): number of replicas.
Returns:
(:py:class:`storj.model.Mirror`): the mirror settings.
"""
self.logger.info('bucket_set_mirrors(%s, %s, %s)', bucket_id, file_id, redundancy)
return model.Mirror(**self._request(
method='POST',
path='/buckets/%s/mirrors' % bucket_id,
data={
'file': file_id,
'redundancy': redundancy
}))
[docs] def file_pointers(self, bucket_id, file_id, skip=None, limit=None):
"""Get list of pointers associated with a file.
See `API buckets: GET /buckets/{id}/files/{file_id}
<https://storj.github.io/bridge/#!/buckets/get_buckets_id_files_file_id>`_
Args:
bucket_id (str): bucket unique identifier.
file_id (str): file unique identifier.
skip (str): pointer index to start the file slice.
limit (str): number of pointers to resolve tokens for.
Returns:
(generator[:py:class:`storj.model.FilePointer`]): file pointers.
"""
self.logger.info('bucket_files(%s, %s)', bucket_id, file_id)
pull_token = self.token_create(bucket_id, operation='PULL')
response = self._request(
method='GET',
path='/buckets/%s/files/%s/' % (bucket_id, file_id),
headers={'x-token': pull_token.id})
if response is not None:
for kwargs in response:
yield model.FilePointer(**kwargs)
else:
raise StopIteration
[docs] def file_download(self, bucket_id, file_id):
self.logger.info('file_pointers(%s, %s)', bucket_id, file_id)
pointers = self.file_pointers(
bucket_id=bucket_id, file_id=file_id)
file_contents = BytesIO()
for pointer in pointers:
ws = web_socket.Client(
pointer=pointer, file_contents=file_contents)
ws.connect()
ws.run_forever()
return file_contents
[docs] def file_upload(self, bucket_id, file, frame):
"""Upload file.
See `API buckets: POST /buckets/{id}/files
<https://storj.github.io/bridge/#!/buckets/post_buckets_id_files>`_
Args:
bucket_id (str): bucket unique identifier.
file (:py:class:`storj.model.File`): file to be uploaded.
frame (:py:class:`storj.model.Frame`): frame used to stage file.
"""
self.logger.info('file_upload(%s, %s, %s)', bucket_id, file, frame)
def get_size(file_like_object):
return os.stat(file_like_object.name).st_size
file_size = get_size(file)
# TODO:
# encrypt file
# shard file
push_token = self.token_create(bucket_id, 'PUSH')
self.logger.debug('file_upload() push_token=%s', push_token)
# upload shards to frame
# delete encrypted file
self._request(
method='POST', path='/buckets/%s/files' % bucket_id,
# files={'file' : file},
headers={
# 'x-token': push_token.id,
# 'x-filesize': str(file_size)}
'frame': frame.id,
'mimetype': file.mimetype,
'filename': file.filename,
})
[docs] def file_remove(self, bucket_id, file_id):
"""Delete a file pointer from a specified bucket.
See `API buckets: DELETE /buckets/{id}/files/{file_id}
<https://storj.github.io/bridge/#!/buckets/delete_buckets_id_files_file_id>`_
Args:
bucket_id (str): bucket unique identifier.
file_id (str): file unique identifier.
"""
self.logger.info('file_remove(%s, %s)', bucket_id, file_id)
self._request(
method='DELETE',
path='/buckets/%s/files/%s' % (bucket_id, file_id))
[docs] def frame_add_shard(self, shard, frame_id):
"""Adds a shard item to the staging frame and negotiates a storage contract.
See `API frames: PUT /frames/{frame_id}
<https://storj.github.io/bridge/#!/frames/put_frames_frame_id>`_
Args:
shard (:py:class:`storj.models.Shard`): the shard.
frame_id (str): the frame unique identifier.
"""
self.logger.info('frame_add_shard(%s, %s)', shard, frame_id)
data = {
'hash': shard.hash,
'size': shard.size,
'index': shard.index,
'challenges': shard.challenges,
'tree': shard.tree,
}
response = self._request(
method='PUT',
path='/frames/%s' % frame_id,
json=data)
if response is not None:
return response
[docs] def frame_create(self):
"""Creates a file staging frame.
See `API frames: POST /frames
<https://storj.github.io/bridge/#!/frames/post_frames>`_
Returns:
(:py:class:`storj.model.Frame`): the frame.
"""
self.logger.info('frame_create()')
response = self._request(
method='POST',
path='/frames')
if response is not None:
return model.Frame(**response)
[docs] def frame_delete(self, frame_id):
"""Destroys the file staging frame by it's unique ID.
See `API frames: DELETE /frames/{frame_id}
<https://storj.github.io/bridge/#!/frames/delete_frames_frame_id>`_
Args:
frame_id (str): unique identifier.
"""
self.logger.info('frame_delete(%s)', frame_id)
self._request(
method='DELETE',
path='/frames/%s' % frame_id,
data={'frame_id': frame_id})
[docs] def frame_get(self, frame_id):
"""Fetches the file staging frame by it's unique ID.
See `API frame: GET /frames/{frame_id}
<https://storj.github.io/bridge/#!/frames/get_frames_frame_id>`_
Args:
frame_id (str): unique identifier.
Returns:
(:py:class:`storj.model.Frame`): a frame.
"""
self.logger.info('frame_get(%s)', frame_id)
response = self._request(
method='GET',
path='/frames/%s' % frame_id,
data={'frame_id': frame_id})
if response is not None:
return model.Frame(**response)
[docs] def frame_list(self):
"""Returns all open file staging frames.
See `API frame: GET /frames
< https://storj.github.io/bridge/#!/frames/get_frames>`_
Returns:
(generator[:py:class:`storj.model.Frame`]): all open file staging frames.
"""
self.logger.info('frame_list()')
response = self._request(
method='GET',
path='/frames')
if response is not None:
for kwargs in response:
print "\n\n\n" + str(kwargs) + "\n\n"
yield model.Frame(**kwargs)
else:
raise StopIteration
[docs] def key_delete(self, public_key):
"""Removes a public ECDSA keys.
See `API keys: DELETE /keys/{pubkey}
<https://storj.github.io/bridge/#!/keys/delete_keys_pubkey>`_
Args:
public_key (str): key to be removed.
"""
self.logger.info('key_delete(%s)', public_key)
self._request(
method='DELETE',
path='/keys/%s' % public_key)
[docs] def key_dump(self):
self.logger.info('key_dump()')
if self.private_key is not None and \
self.public_key is not None:
print('Local Private Key: %s' % self.private_key
+ '\nLocal Public Key: %s' % self.public_key)
keys = self.key_list()
if not keys:
print('No keys associated with this account.')
else:
print('Public keys for this account: '
+ str([key['id'] for key in keys]))
[docs] def key_export(self):
self.logger.info('key_export()')
print('Writing your public key to file...')
with open('public.pem', 'wb') as keyfile:
keyfile.write(self.public_key.to_pem())
print('Writing private key to file... Keep this secret!')
with open('private.pem', 'wb') as keyfile:
keyfile.write(self.private_key.to_pem())
print('Wrote keyfiles to dir: %s' % os.getcwd())
[docs] def key_generate(self):
self.logger.info('key_generate()')
print("This will replace your public and private keys in 3 seconds...")
time.sleep(3)
(self.private_key, self.public_key) = storj.generate_new_key_pair()
s = raw_input('Export keys to file for later use? [Y/N]')
if 'Y' in s.upper():
self.key_export()
self.key_register(self.public_key)
[docs] def key_import(self, private_keyfile_path, public_keyfile_path):
self.logger.info(
'key_import(%s, %s)',
private_keyfile_path,
public_keyfile_path)
with open(public_keyfile_path, 'r') as f:
self.public_key = VerifyingKey.from_pem(f.read())
with open(private_keyfile_path, 'r') as f:
self.private_key = SigningKey.from_pem(f.read())
self.key_register(self.public_key)
[docs] def key_list(self):
"""Lists the public ECDSA keys associated with the user.
See `API keys: GET /keys
<https://storj.github.io/bridge/#!/keys/get_keys>`_
Returns:
(list[str]): public keys.
"""
self.logger.info('key_list()')
return [kwargs['key'] for kwargs in self._request(
method='GET',
path='/keys'
)]
[docs] def key_register(self, public_key):
"""Register an ECDSA public key.
See `API keys: POST /keys
<https://storj.github.io/bridge/#!/keys/post_keys>`_
Returns:
(list[:py:class:`storj.model.Key`]): public keys.
"""
self.logger.info('key_register(%s)', public_key)
self._request(
method='POST',
path='/keys',
data={'key': ecdsa_to_hex(public_key)})
[docs] def token_create(self, bucket_id, operation):
"""Creates a token for the specified operation.
See `API buckets: POST /buckets/{id}/tokens
<https://storj.github.io/bridge/#!/buckets/post_buckets_id_tokens>`_
Args:
bucket_id (str): bucket unique identifier.
operation (str): operation.
Returns:
(dict): ...
"""
self.logger.info('token_create(%s, %s)', bucket_id, operation)
return model.Token(**self._request(
method='POST',
path='/buckets/%s/tokens' % bucket_id,
data={'operation': operation}))
[docs] def user_activate(self, token):
"""Activate user.
See `API users: GET /activations/{token}
<https://storj.github.io/bridge/#!/users/get_activations_token>`_
Args:
token (str): activation token.
"""
self.logger.info('user_activate(%s)', token)
self._request(
method='GET',
path='/activations/%s' % token)
[docs] def user_activation_email(self, email, token):
"""Send user activation email.
See `API users: POST /activations/{token}
<https://storj.github.io/bridge/#!/users/post_activations_token>`_
Args:
email (str): user's email address.
token (str): activation token.
"""
self.logger.info('user_activation_email(%s, %s)', email, token)
self._request(
method='GET',
path='/activations/%s' % token,
data={
'email': email,
})
[docs] def user_create(self, email, password):
"""Create a new user with Storj bridge.
See `API users: POST /users
<https://storj.github.io/bridge/#!/users/post_users>`_
Args:
email (str): user's email address.
password (str): user's password.
"""
self.logger.info('user_create(%s, %s)', email, password)
password = sha256(password).hexdigest()
self._request(
method='POST',
path='/users',
data={
'email': email,
'password': password
})
self.authenticate(email=email, password=password)
[docs] def user_deactivate(self, token):
"""Discard activation token.
See `API users: GET /activations/{token}
<https://storj.github.io/bridge/#!/users/get_deactivations_token>`_
Args:
token (str): activation token.
"""
self.logger.info('user_deactivate(%s)', token)
self._request(
method='DELETE',
path='/activations/%s' % token)
[docs] def user_delete(self, email):
"""Delete user account.
See `API users: DELETE /users/{email}
<https://storj.github.io/bridge/#!/users/post_users>`_
Args:
email (str): user's email address.
"""
self.logger.info('user_delete(%s)', email)
self._request(
method='DELETE',
path='/users/%s' % email)
[docs] def user_reset_password(self, email):
"""Request a password reset.
See `API users: PATCH /users/{email}
<https://storj.github.io/bridge/#!/users/patch_users_email>`_
Args:
email (str): user's email address.
"""
self.logger.info('user_reset_password(%s)', email)
self._request(
method='PATCH',
path='/users/%s' % email)
[docs] def user_reset_password_confirmation(self, token):
"""Confirm a password reset request.
See `API users: GET /resets/{token}
<https://storj.github.io/bridge/#!/users/get_resets_token>`_
Args:
token (str): password reset token.
"""
self.logger.info('user_reset_password_confirmation(%s)', token)
self._request(
method='GET',
path='/resets/%s' % token)