#!/usr/bin/env python

# -*- coding: utf-8 -*-

## --------------------------------------------------------------------

## s3cmd - S3 client

##

## Authors : Michal Ludvig and contributors

## Copyright : TGRMN Software - http://www.tgrmn.com - and contributors

## Website : http://s3tools.org

## License : GPL Version 2

## --------------------------------------------------------------------

## This program is free software; you can redistribute it and/or modify

## it under the terms of the GNU General Public License as published by

## the Free Software Foundation; either version 2 of the License, or

## (at your option) any later version.

## This program is distributed in the hope that it will be useful,

## but WITHOUT ANY WARRANTY; without even the implied warranty of

## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

## GNU General Public License for more details.

## --------------------------------------------------------------------

from __future__ import absolute_import, print_function, division

import sys

if sys.version_info < (2, 6):

sys.stderr.write(u"ERROR: Python 2.6 or higher required, sorry.\n")

# 72 == EX_OSFILE

sys.exit(72)

PY3 = (sys.version_info >= (3, 0))

import codecs

import errno

import glob

import io

import locale

import logging

import os

import re

import shutil

import socket

import subprocess

import tempfile

import time

import traceback

from copy import copy

from optparse import OptionParser, Option, OptionValueError, IndentedHelpFormatter

from logging import debug, info, warning, error

try:

import htmlentitydefs

except Exception:

# python 3 support

import html.entities as htmlentitydefs

try:

unicode

except NameError:

# python 3 support

# In python 3, unicode -> str, and str -> bytes

unicode = str

try:

unichr

except NameError:

# python 3 support

# In python 3, unichr was removed as chr can now do the job

unichr = chr

try:

from shutil import which

except ImportError:

# python2 fallback code

from distutils.spawn import find_executable as which

def output(message):

sys.stdout.write(message + "\n")

sys.stdout.flush()

def check_args_type(args, type, verbose_type):

"""NOTE: This function looks like to not be used."""

for arg in args:

if S3Uri(arg).type != type:

raise ParameterError("Expecting %s instead of '%s'" % (verbose_type, arg))

def cmd_du(args):

s3 = S3(Config())

if len(args) > 0:

uri = S3Uri(args[0])

if uri.type == "s3" and uri.has_bucket():

subcmd_bucket_usage(s3, uri)

return EX_OK

subcmd_bucket_usage_all(s3)

return EX_OK

def subcmd_bucket_usage_all(s3):

"""

Returns: sum of bucket sizes as integer

Raises: S3Error

"""

cfg = Config()

response = s3.list_all_buckets()

buckets_size = 0

for bucket in response["list"]:

size = subcmd_bucket_usage(s3, S3Uri("s3://" + bucket["Name"]))

if size != None:

buckets_size += size

total_size, size_coeff = formatSize(buckets_size, cfg.human_readable_sizes)

total_size_str = str(total_size) + size_coeff

output(u"".rjust(12, "-"))

output(u"%s Total" % (total_size_str.ljust(12)))

return size

def subcmd_bucket_usage(s3, uri):

"""

Returns: bucket size as integer

Raises: S3Error

"""

bucket_size = 0

object_count = 0

extra_info = u''

bucket = uri.bucket()

prefix = uri.object()

try:

for _, _, objects in s3.bucket_list_streaming(bucket, prefix=prefix, recursive=True):

for obj in objects:

bucket_size += int(obj["Size"])

object_count += 1

except S3Error as e:

if e.info["Code"] in S3.codes:

error(S3.codes[e.info["Code"]] % bucket)

raise

except KeyboardInterrupt as e:

extra_info = u' [interrupted]'

total_size_str = u"%d%s" % formatSize(bucket_size,

Config().human_readable_sizes)

if Config().human_readable_sizes:

total_size_str = total_size_str.rjust(5)

else:

total_size_str = total_size_str.rjust(12)

output(u"%s %7s objects %s%s" % (total_size_str, object_count, uri,

extra_info))

return bucket_size

def cmd_ls(args):

cfg = Config()

s3 = S3(cfg)

if len(args) > 0:

uri = S3Uri(args[0])

if uri.type == "s3" and uri.has_bucket():

subcmd_bucket_list(s3, uri, cfg.limit)

return EX_OK

# If not a s3 type uri or no bucket was provided, list all the buckets

subcmd_all_buckets_list(s3)

return EX_OK

def subcmd_all_buckets_list(s3):

response = s3.list_all_buckets()

for bucket in sorted(response["list"], key=lambda b:b["Name"]):

output(u"%s s3://%s" % (formatDateTime(bucket["CreationDate"]),

bucket["Name"]))

def cmd_all_buckets_list_all_content(args):

cfg = Config()

s3 = S3(cfg)

response = s3.list_all_buckets()

for bucket in response["list"]:

subcmd_bucket_list(s3, S3Uri("s3://" + bucket["Name"]), cfg.limit)

output(u"")

return EX_OK

def subcmd_bucket_list(s3, uri, limit):

cfg = Config()

bucket = uri.bucket()

prefix = uri.object()

debug(u"Bucket 's3://%s':" % bucket)

if prefix.endswith('*'):

prefix = prefix[:-1]

try:

response = s3.bucket_list(bucket, prefix = prefix, limit = limit)

except S3Error as e:

if e.info["Code"] in S3.codes:

error(S3.codes[e.info["Code"]] % bucket)

raise

# md5 are 32 char long, but for multipart there could be a suffix

if Config().human_readable_sizes:

# %(size)5s%(coeff)1s

format_size = u"%5d%1s"

dir_str = u"DIR".rjust(6)

else:

format_size = u"%12d%s"

dir_str = u"DIR".rjust(12)

if cfg.long_listing:

format_string = u"%(timestamp)16s %(size)s %(md5)-35s %(storageclass)-11s %(uri)s"

elif cfg.list_md5:

format_string = u"%(timestamp)16s %(size)s %(md5)-35s %(uri)s"

else:

format_string = u"%(timestamp)16s %(size)s %(uri)s"

for prefix in response['common_prefixes']:

output(format_string % {

"timestamp": "",

"size": dir_str,

"md5": "",

"storageclass": "",

"uri": uri.compose_uri(bucket, prefix["Prefix"])})

for object in response["list"]:

md5 = object.get('ETag', '').strip('"\'')

storageclass = object.get('StorageClass','')

if cfg.list_md5:

if '-' in md5: # need to get md5 from the object

object_uri = uri.compose_uri(bucket, object["Key"])

info_response = s3.object_info(S3Uri(object_uri))

try:

md5 = info_response['s3cmd-attrs']['md5']

except KeyError:

pass

size_and_coeff = formatSize(object["Size"],

Config().human_readable_sizes)

output(format_string % {

"timestamp": formatDateTime(object["LastModified"]),

"size" : format_size % size_and_coeff,

"md5" : md5,

"storageclass" : storageclass,

"uri": uri.compose_uri(bucket, object["Key"]),

})

if response["truncated"]:

warning(u"The list is truncated because the settings limit was reached.")

def cmd_bucket_create(args):

cfg = Config()

s3 = S3(cfg)

for arg in args:

uri = S3Uri(arg)

if not uri.type == "s3" or not uri.has_bucket() or uri.has_object():

raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg)

try:

response = s3.bucket_create(uri.bucket(), cfg.bucket_location, cfg.extra_headers)

output(u"Bucket '%s' created" % uri.uri())

except S3Error as e:

if e.info["Code"] in S3.codes:

error(S3.codes[e.info["Code"]] % uri.bucket())

raise

return EX_OK

def cmd_website_info(args):

cfg = Config()

s3 = S3(cfg)

for arg in args:

uri = S3Uri(arg)

if not uri.type == "s3" or not uri.has_bucket() or uri.has_object():

raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg)

try:

response = s3.website_info(uri, cfg.bucket_location)

if response:

output(u"Bucket %s: Website configuration" % uri.uri())

output(u"Website endpoint: %s" % response['website_endpoint'])

output(u"Index document: %s" % response['index_document'])

output(u"Error document: %s" % response['error_document'])

else:

output(u"Bucket %s: Unable to receive website configuration." % (uri.uri()))

except S3Error as e:

if e.info["Code"] in S3.codes:

error(S3.codes[e.info["Code"]] % uri.bucket())

raise

return EX_OK

def cmd_website_create(args):

cfg = Config()

s3 = S3(cfg)

for arg in args:

uri = S3Uri(arg)

if not uri.type == "s3" or not uri.has_bucket() or uri.has_object():

raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg)

try:

response = s3.website_create(uri, cfg.bucket_location)

output(u"Bucket '%s': website configuration created." % (uri.uri()))

except S3Error as e:

if e.info["Code"] in S3.codes:

error(S3.codes[e.info["Code"]] % uri.bucket())

raise

return EX_OK

def cmd_website_delete(args):

cfg = Config()

s3 = S3(cfg)

for arg in args:

uri = S3Uri(arg)

if not uri.type == "s3" or not uri.has_bucket() or uri.has_object():

raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg)

try:

response = s3.website_delete(uri, cfg.bucket_location)

output(u"Bucket '%s': website configuration deleted." % (uri.uri()))

except S3Error as e:

if e.info["Code"] in S3.codes:

error(S3.codes[e.info["Code"]] % uri.bucket())

raise

return EX_OK

def cmd_expiration_set(args):

cfg = Config()

s3 = S3(cfg)

for arg in args:

uri = S3Uri(arg)

if not uri.type == "s3" or not uri.has_bucket() or uri.has_object():

raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg)

try:

response = s3.expiration_set(uri, cfg.bucket_location)

if response["status"] == 200:

output(u"Bucket '%s': expiration configuration is set." % (uri.uri()))

elif response["status"] == 204:

output(u"Bucket '%s': expiration configuration is deleted." % (uri.uri()))

except S3Error as e:

if e.info["Code"] in S3.codes:

error(S3.codes[e.info["Code"]] % uri.bucket())

raise

return EX_OK

def cmd_bucket_delete(args):

cfg = Config()

s3 = S3(cfg)

def _bucket_delete_one(uri, retry=True):

try:

response = s3.bucket_delete(uri.bucket())

output(u"Bucket '%s' removed" % uri.uri())

except S3Error as e:

if e.info['Code'] == 'NoSuchBucket':

if cfg.force:

return EX_OK

else:

raise

if e.info['Code'] == 'BucketNotEmpty' and retry and (cfg.force or cfg.recursive):

warning(u"Bucket is not empty. Removing all the objects from it first. This may take some time...")

rc = subcmd_batch_del(uri_str = uri.uri())

if rc == EX_OK:

return _bucket_delete_one(uri, False)

else:

output(u"Bucket was not removed")

elif e.info["Code"] in S3.codes:

error(S3.codes[e.info["Code"]] % uri.bucket())

raise

return EX_OK

for arg in args:

uri = S3Uri(arg)

if not uri.type == "s3" or not uri.has_bucket() or uri.has_object():

raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg)

rc = _bucket_delete_one(uri)

if rc != EX_OK:

return rc

return EX_OK

def cmd_object_put(args):

cfg = Config()

s3 = S3(cfg)

if len(args) == 0:

raise ParameterError("Nothing to upload. Expecting a local file or directory and a S3 URI destination.")

## Normalize URI to convert s3://bkt to s3://bkt/ (trailing slash)

destination_base_uri = S3Uri(args.pop())

if destination_base_uri.type != 's3':

raise ParameterError("Destination must be S3Uri. Got: %s" % destination_base_uri)

destination_base = destination_base_uri.uri()

if len(args) == 0:

raise ParameterError("Nothing to upload. Expecting a local file or directory.")

local_list, single_file_local, exclude_list, total_size_local = fetch_local_list(args, is_src = True)

local_count = len(local_list)

info(u"Summary: %d local files to upload" % local_count)

if local_count == 0:

raise ParameterError("Nothing to upload.")

if local_count > 0:

if not single_file_local and '-' in local_list.keys():

raise ParameterError("Cannot specify multiple local files if uploading from '-' (ie stdin)")

elif single_file_local and local_list.keys()[0] == "-" and destination_base.endswith("/"):

raise ParameterError("Destination S3 URI must not end with '/' when uploading from stdin.")

elif not destination_base.endswith("/"):

if not single_file_local:

raise ParameterError("Destination S3 URI must end with '/' (ie must refer to a directory on the remote side).")

local_list[local_list.keys()[0]]['remote_uri'] = destination_base

else:

for key in local_list:

local_list[key]['remote_uri'] = destination_base + key

if cfg.dry_run:

for key in exclude_list:

output(u"exclude: %s" % key)

for key in local_list:

if key != "-":

nicekey = local_list[key]['full_name']

else:

nicekey = "<stdin>"

output(u"upload: '%s' -> '%s'" % (nicekey, local_list[key]['remote_uri']))

warning(u"Exiting now because of --dry-run")

return EX_OK

seq = 0

ret = EX_OK

for key in local_list:

seq += 1

uri_final = S3Uri(local_list[key]['remote_uri'])

try:

src_md5 = local_list.get_md5(key)

except IOError:

src_md5 = None

extra_headers = copy(cfg.extra_headers)

full_name_orig = local_list[key]['full_name']

full_name = full_name_orig

seq_label = "[%d of %d]" % (seq, local_count)

if Config().encrypt:

gpg_exitcode, full_name, extra_headers["x-amz-meta-s3tools-gpgenc"] = gpg_encrypt(full_name_orig)

attr_header = _build_attr_header(local_list[key], key, src_md5)

debug(u"attr_header: %s" % attr_header)

extra_headers.update(attr_header)

try:

response = s3.object_put(full_name, uri_final, extra_headers, extra_label = seq_label)

except S3UploadError as exc:

error(u"Upload of '%s' failed too many times (Last reason: %s)" % (full_name_orig, exc))

if cfg.stop_on_error:

ret = EX_DATAERR

error(u"Exiting now because of --stop-on-error")

break

ret = EX_PARTIAL

continue

except InvalidFileError as exc:

error(u"Upload of '%s' is not possible (Reason: %s)" % (full_name_orig, exc))

ret = EX_PARTIAL

if cfg.stop_on_error:

ret = EX_OSFILE

error(u"Exiting now because of --stop-on-error")

break

continue

if response is not None:

speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True)

if not Config().progress_meter:

if full_name_orig != "-":

nicekey = full_name_orig

else:

nicekey = "<stdin>"

output(u"upload: '%s' -> '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" %

(nicekey, uri_final, response["size"], response["elapsed"],

speed_fmt[0], speed_fmt[1], seq_label))

if Config().acl_public:

output(u"Public URL of the object is: %s" %

(uri_final.public_url()))

if Config().encrypt and full_name != full_name_orig:

debug(u"Removing temporary encrypted file: %s" % full_name)

os.remove(deunicodise(full_name))

return ret

def cmd_object_get(args):

cfg = Config()

s3 = S3(cfg)

## Check arguments:

## if not --recursive:

## - first N arguments must be S3Uri

## - if the last one is S3 make current dir the destination_base

## - if the last one is a directory:

## - take all 'basenames' of the remote objects and

## make the destination name be 'destination_base'+'basename'

## - if the last one is a file or not existing:

## - if the number of sources (N, above) == 1 treat it

## as a filename and save the object there.

## - if there's more sources -> Error

## if --recursive:

## - first N arguments must be S3Uri

## - for each Uri get a list of remote objects with that Uri as a prefix

## - apply exclude/include rules

## - each list item will have MD5sum, Timestamp and pointer to S3Uri

## used as a prefix.

## - the last arg may be '-' (stdout)

## - the last arg may be a local directory - destination_base

## - if the last one is S3 make current dir the destination_base

## - if the last one doesn't exist check remote list:

## - if there is only one item and its_prefix==its_name

## download that item to the name given in last arg.

## - if there are more remote items use the last arg as a destination_base

## and try to create the directory (incl. all parents).

##

## In both cases we end up with a list mapping remote object names (keys) to local file names.

## Each item will be a dict with the following attributes

# {'remote_uri', 'local_filename'}

download_list = []

if len(args) == 0:

raise ParameterError("Nothing to download. Expecting S3 URI.")

if S3Uri(args[-1]).type == 'file':

destination_base = args.pop()

else:

destination_base = "."

if len(args) == 0:

raise ParameterError("Nothing to download. Expecting S3 URI.")

remote_list, exclude_list, remote_total_size = fetch_remote_list(args, require_attribs = False)

remote_count = len(remote_list)

info(u"Summary: %d remote files to download" % remote_count)

if remote_count > 0:

if destination_base == "-":

## stdout is ok for multiple remote files!

for key in remote_list:

remote_list[key]['local_filename'] = "-"

elif not os.path.isdir(deunicodise(destination_base)):

## We were either given a file name (existing or not)

if remote_count > 1:

raise ParameterError("Destination must be a directory or stdout when downloading multiple sources.")

remote_list[remote_list.keys()[0]]['local_filename'] = destination_base

else:

if destination_base[-1] != os.path.sep:

destination_base += os.path.sep

for key in remote_list:

local_filename = destination_base + key

if os.path.sep != "/":

local_filename = os.path.sep.join(local_filename.split("/"))

remote_list[key]['local_filename'] = local_filename

if cfg.dry_run:

for key in exclude_list:

output(u"exclude: %s" % key)

for key in remote_list:

output(u"download: '%s' -> '%s'" % (remote_list[key]['object_uri_str'], remote_list[key]['local_filename']))

warning(u"Exiting now because of --dry-run")

return EX_OK

seq = 0

ret = EX_OK

for key in remote_list:

seq += 1

item = remote_list[key]

uri = S3Uri(item['object_uri_str'])

## Encode / Decode destination with "replace" to make sure it's compatible with current encoding

destination = unicodise_safe(item['local_filename'])

seq_label = "[%d of %d]" % (seq, remote_count)

start_position = 0

if destination == "-":

## stdout

dst_stream = io.open(sys.__stdout__.fileno(), mode='wb', closefd=False)

dst_stream.stream_name = u'<stdout>'

file_exists = True

else:

## File

try:

file_exists = os.path.exists(deunicodise(destination))

try:

dst_stream = io.open(deunicodise(destination), mode='ab')

dst_stream.stream_name = destination

except IOError as e:

if e.errno == errno.ENOENT:

basename = destination[:destination.rindex(os.path.sep)]

info(u"Creating directory: %s" % basename)

os.makedirs(deunicodise(basename))

dst_stream = io.open(deunicodise(destination), mode='ab')

dst_stream.stream_name = destination

else:

raise

if file_exists:

if Config().get_continue:

start_position = dst_stream.tell()

elif Config().force:

start_position = 0

dst_stream.seek(0)

dst_stream.truncate()

elif Config().skip_existing:

info(u"Skipping over existing file: %s" % (destination))

continue

else:

dst_stream.close()

raise ParameterError(u"File %s already exists. Use either of --force / --continue / --skip-existing or give it a new name." % destination)

except IOError as e:

error(u"Creation of file '%s' failed (Reason: %s)"

% (destination, e.strerror))

if cfg.stop_on_error:

error(u"Exiting now because of --stop-on-error")

raise

ret = EX_PARTIAL

continue

try:

try:

response = s3.object_get(uri, dst_stream, destination, start_position = start_position, extra_label = seq_label)

finally:

dst_stream.close()

except S3DownloadError as e:

error(u"Download of '%s' failed (Reason: %s)" % (destination, e))

# Delete, only if file didn't exist before!

if not file_exists:

debug(u"object_get failed for '%s', deleting..." % (destination,))

os.unlink(deunicodise(destination))

if cfg.stop_on_error:

error(u"Exiting now because of --stop-on-error")

raise

ret = EX_PARTIAL

continue

except S3Error as e:

error(u"Download of '%s' failed (Reason: %s)" % (destination, e))

if not file_exists: # Delete, only if file didn't exist before!

debug(u"object_get failed for '%s', deleting..." % (destination,))

os.unlink(deunicodise(destination))

raise

if "x-amz-meta-s3tools-gpgenc" in response["headers"]:

gpg_decrypt(destination, response["headers"]["x-amz-meta-s3tools-gpgenc"])

response["size"] = os.stat(deunicodise(destination))[6]

if "last-modified" in response["headers"] and destination != "-":

last_modified = time.mktime(time.strptime(response["headers"]["last-modified"], "%a, %d %b %Y %H:%M:%S GMT"))

os.utime(deunicodise(destination), (last_modified, last_modified))

debug("set mtime to %s" % last_modified)

if not Config().progress_meter and destination != "-":

speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True)

output(u"download: '%s' -> '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s)" %

(uri, destination, response["size"], response["elapsed"], speed_fmt[0], speed_fmt[1]))

if Config().delete_after_fetch:

s3.object_delete(uri)

output(u"File '%s' removed after fetch" % (uri))

return ret

def cmd_object_del(args):

cfg = Config()

recursive = cfg.recursive

for uri_str in args:

uri = S3Uri(uri_str)

if uri.type != "s3":

raise ParameterError("Expecting S3 URI instead of '%s'" % uri_str)

if not uri.has_object():

if recursive and not cfg.force:

raise ParameterError("Please use --force to delete ALL contents of %s" % uri_str)

elif not recursive:

raise ParameterError("File name required, not only the bucket name. Alternatively use --recursive")

if not recursive:

rc = subcmd_object_del_uri(uri_str)

elif cfg.exclude or cfg.include or cfg.max_delete > 0:

# subcmd_batch_del_iterative does not support file exclusion and can't

# accurately know how many total files will be deleted, so revert to batch delete.

rc = subcmd_batch_del(uri_str = uri_str)

else:

rc = subcmd_batch_del_iterative(uri_str = uri_str)

if not rc:

return rc

return EX_OK

def subcmd_batch_del_iterative(uri_str = None, bucket = None):

""" Streaming version of batch deletion (doesn't realize whole list in memory before deleting).

Differences from subcmd_batch_del:

- Does not obey --exclude directives or obey cfg.max_delete (use subcmd_batch_del in those cases)

"""

if bucket and uri_str:

raise ValueError("Pass only one of uri_str or bucket")

if bucket: # bucket specified

uri_str = "s3://%s" % bucket

cfg = Config()

s3 = S3(cfg)

uri = S3Uri(uri_str)

bucket = uri.bucket()

deleted_bytes = deleted_count = 0

for _, _, to_delete in s3.bucket_list_streaming(bucket, prefix=uri.object(), recursive=True):

if not to_delete:

continue

if not cfg.dry_run:

response = s3.object_batch_delete_uri_strs([uri.compose_uri(bucket, item['Key']) for item in to_delete])

deleted_bytes += sum(int(item["Size"]) for item in to_delete)

deleted_count += len(to_delete)

output(u'\n'.join(u"delete: '%s'" % uri.compose_uri(bucket, p['Key']) for p in to_delete))

if deleted_count:

# display summary data of deleted files

if cfg.stats:

stats_info = StatsInfo()

stats_info.files_deleted = deleted_count

stats_info.size_deleted = deleted_bytes

output(stats_info.format_output())

else:

total_size, size_coeff = formatSize(deleted_bytes, Config().human_readable_sizes)

total_size_str = str(total_size) + size_coeff

info(u"Deleted %s objects (%s) from %s" % (deleted_count, total_size_str, uri))

else:

warning(u"Remote list is empty.")

return EX_OK

def subcmd_batch_del(uri_str = None, bucket = None, remote_list = None):

"""

Returns: EX_OK

Raises: ValueError

"""

cfg = Config()

s3 = S3(cfg)

def _batch_del(remote_list):

to_delete = remote_list[:1000]

remote_list = remote_list[1000:]

while len(to_delete):

debug(u"Batch delete %d, remaining %d" % (len(to_delete), len(remote_list)))

if not cfg.dry_run:

response = s3.object_batch_delete(to_delete)

output(u'\n'.join((u"delete: '%s'" % to_delete[p]['object_uri_str']) for p in to_delete))

to_delete = remote_list[:1000]

remote_list = remote_list[1000:]

if remote_list is not None and len(remote_list) == 0:

return False

if len([item for item in [uri_str, bucket, remote_list] if item]) != 1:

raise ValueError("One and only one of 'uri_str', 'bucket', 'remote_list' can be specified.")

if bucket: # bucket specified

uri_str = "s3://%s" % bucket

if remote_list is None: # uri_str specified

remote_list, exclude_list, remote_total_size = fetch_remote_list(uri_str, require_attribs = False)

if len(remote_list) == 0:

warning(u"Remote list is empty.")

return EX_OK

if cfg.max_delete > 0 and len(remote_list) > cfg.max_delete:

warning(u"delete: maximum requested number of deletes would be exceeded, none performed.")

return EX_OK

_batch_del(remote_list)

if cfg.dry_run:

warning(u"Exiting now because of --dry-run")

return EX_OK

def subcmd_object_del_uri(uri_str, recursive = None):

"""

Returns: True if XXX, False if XXX

Raises: ValueError

"""

cfg = Config()

s3 = S3(cfg)

if recursive is None:

recursive = cfg.recursive

remote_list, exclude_list, remote_total_size = fetch_remote_list(uri_str, require_attribs = False, recursive = recursive)

remote_count = len(remote_list)

info(u"Summary: %d remote files to delete" % remote_count)

if cfg.max_delete > 0 and remote_count > cfg.max_delete:

warning(u"delete: maximum requested number of deletes would be exceeded, none performed.")

return False

if cfg.dry_run:

for key in exclude_list:

output(u"exclude: %s" % key)

for key in remote_list:

output(u"delete: %s" % remote_list[key]['object_uri_str'])

warning(u"Exiting now because of --dry-run")

return True

for key in remote_list:

item = remote_list[key]

response = s3.object_delete(S3Uri(item['object_uri_str']))

output(u"delete: '%s'" % item['object_uri_str'])

return True

def cmd_object_restore(args):

cfg = Config()

s3 = S3(cfg)

if cfg.restore_days < 1:

raise ParameterError("You must restore a file for 1 or more days")

# accept case-insensitive argument but fix it to match S3 API

if cfg.restore_priority.title() not in ['Standard', 'Expedited', 'Bulk']:

raise ParameterError("Valid restoration priorities: bulk, standard, expedited")

else:

cfg.restore_priority = cfg.restore_priority.title()

remote_list, exclude_list, remote_total_size = fetch_remote_list(args, require_attribs = False, recursive = cfg.recursive)

remote_count = len(remote_list)

info(u"Summary: Restoring %d remote files for %d days at %s priority" % (remote_count, cfg.restore_days, cfg.restore_priority))

if cfg.dry_run:

for key in exclude_list:

output(u"exclude: %s" % key)

for key in remote_list:

output(u"restore: '%s'" % remote_list[key]['object_uri_str'])

warning(u"Exiting now because of --dry-run")

return EX_OK

for key in remote_list:

item = remote_list[key]

uri = S3Uri(item['object_uri_str'])

if not item['object_uri_str'].endswith("/"):

try:

response = s3.object_restore(S3Uri(item['object_uri_str']))

output(u"restore: '%s'" % item['object_uri_str'])

except S3Error as e:

if e.code == "RestoreAlreadyInProgress":

warning("%s: %s" % (e.message, item['object_uri_str']))

else:

raise e

else:

debug(u"Skipping directory since only files may be restored")

return EX_OK

def subcmd_cp_mv(args, process_fce, action_str, message):

cfg = Config()

if action_str == 'modify':

if len(args) < 1:

raise ParameterError("Expecting one or more S3 URIs for "

+ action_str)

destination_base = None

else:

if len(args) < 2:

raise ParameterError("Expecting two or more S3 URIs for "

+ action_str)

dst_base_uri = S3Uri(args.pop())

if dst_base_uri.type != "s3":

raise ParameterError("Destination must be S3 URI. To download a "

"file use 'get' or 'sync'.")

destination_base = dst_base_uri.uri()

scoreboard = ExitScoreboard()

remote_list, exclude_list, remote_total_size = \

fetch_remote_list(args, require_attribs=False)

remote_count = len(remote_list)

info(u"Summary: %d remote files to %s" % (remote_count, action_str))

if destination_base:

# Trying to mv dir1/ to dir2 will not pass a test in S3.FileLists,

# so we don't need to test for it here.

if not destination_base.endswith('/') \

and (len(remote_list) > 1 or cfg.recursive):

raise ParameterError("Destination must be a directory and end with"

" '/' when acting on a folder content or on "

"multiple sources.")

if cfg.recursive:

for key in remote_list:

remote_list[key]['dest_name'] = destination_base + key

else:

for key in remote_list:

if destination_base.endswith("/"):

remote_list[key]['dest_name'] = destination_base + key

else:

remote_list[key]['dest_name'] = destination_base

else:

for key in remote_list:

remote_list[key]['dest_name'] = remote_list[key]['object_uri_str']

if cfg.dry_run:

for key in exclude_list:

output(u"exclude: %s" % key)

for key in remote_list:

output(u"%s: '%s' -> '%s'" % (action_str,

remote_list[key]['object_uri_str'],

remote_list[key]['dest_name']))

warning(u"Exiting now because of --dry-run")

return EX_OK

seq = 0

for key in remote_list:

seq += 1

seq_label = "[%d of %d]" % (seq, remote_count)

item = remote_list[key]

src_uri = S3Uri(item['object_uri_str'])

dst_uri = S3Uri(item['dest_name'])

src_size = item.get('size')

extra_headers = copy(cfg.extra_headers)

try:

response = process_fce(src_uri, dst_uri, extra_headers,

src_size=src_size,

extra_label=seq_label)

output(message % {"src": src_uri, "dst": dst_uri,

"extra": seq_label})

if Config().acl_public:

info(u"Public URL is: %s" % dst_uri.public_url())

scoreboard.success()

except (S3Error, S3UploadError) as exc:

if isinstance(exc, S3Error) and exc.code == "NoSuchKey":

scoreboard.notfound()

warning(u"Key not found %s" % item['object_uri_str'])

else:

scoreboard.failed()

error(u"Copy failed for: '%s' (%s)", item['object_uri_str'],

exc)

if cfg.stop_on_error:

break

return scoreboard.rc()

def cmd_cp(args):

s3 = S3(Config())

return subcmd_cp_mv(args, s3.object_copy, "copy",

u"remote copy: '%(src)s' -> '%(dst)s' %(extra)s")

def cmd_modify(args):

s3 = S3(Config())

return subcmd_cp_mv(args, s3.object_modify, "modify",

u"modify: '%(src)s' %(extra)s")

def cmd_mv(args):

s3 = S3(Config())

return subcmd_cp_mv(args, s3.object_move, "move",

u"move: '%(src)s' -> '%(dst)s' %(extra)s")

def cmd_info(args):

cfg = Config()

s3 = S3(cfg)

while (len(args)):

uri_arg = args.pop(0)

uri = S3Uri(uri_arg)

if uri.type != "s3" or not uri.has_bucket():

raise ParameterError("Expecting S3 URI instead of '%s'" % uri_arg)

try:

if uri.has_object():

info = s3.object_info(uri)

output(u"%s (object):" % uri.uri())

output(u" File size: %s" % info['headers']['content-length'])

output(u" Last mod: %s" % info['headers']['last-modified'])

output(u" MIME type: %s" % info['headers'].get('content-type', 'none'))

output(u" Storage: %s" % info['headers'].get('x-amz-storage-class', 'STANDARD'))

md5 = info['headers'].get('etag', '').strip('"\'')

try:

md5 = info['s3cmd-attrs']['md5']

except KeyError:

pass

output(u" MD5 sum: %s" % md5)

if 'x-amz-server-side-encryption' in info['headers']:

output(u" SSE: %s" % info['headers']['x-amz-server-side-encryption'])

else:

output(u" SSE: none")

else:

info = s3.bucket_info(uri)

output(u"%s (bucket):" % uri.uri())

output(u" Location: %s" % (info['bucket-location']

or 'none'))

output(u" Payer: %s" % (info['requester-pays']

or 'none'))

expiration = s3.expiration_info(uri, cfg.bucket_location)

if expiration and expiration['prefix'] is not None:

expiration_desc = "Expiration Rule: "

if expiration['prefix'] == "":

expiration_desc += "all objects in this bucket "

elif expiration['prefix'] is not None:

expiration_desc += "objects with key prefix '" + expiration['prefix'] + "' "

expiration_desc += "will expire in '"

if expiration['days']: