Datacratic::S3Api Struct Reference

List of all members.

Classes
struct	Content
struct	MultiPartUpload
struct	MultiPartUploadPart
struct	ObjectInfo
struct	ObjectMetadata
struct	RequestParams
struct	Response
struct	SignedRequest
Public Types
enum	Redundancy { REDUNDANCY_STANDARD, REDUNDANCY_REDUCED, REDUNDANCY_GLACIER }
enum	ServerSideEncryption { SSE_NONE, SSE_AES256 }
enum	CheckMethod { CM_SIZE, CM_MD5_ETAG, CM_ASSUME_INVALID }
typedef std::vector< std::pair < std::string, std::string > >	StrPairVector
typedef std::function< void(const char *chunk, size_t size, int chunkIndex, uint64_t offset, uint64_t totalSize)	OnChunk )
typedef std::function< bool(const std::string &prefix, const std::string &objectName, const ObjectInfo &info, int depth)>	OnObject
typedef std::function< bool(const std::string &prefix, const std::string &dirName, int depth)>	OnSubdir
typedef std::function< bool(std::string bucket)>	OnBucket
Public Member Functions
	S3Api (const std::string &accessKeyId, const std::string &accessKey, double bandwidthToServiceMbps=defaultBandwidthToServiceMbps, const std::string &defaultProtocol="http", const std::string &serviceUri="s3.amazonaws.com")
void	init (const std::string &accessKeyId, const std::string &accessKey, double bandwidthToServiceMbps=defaultBandwidthToServiceMbps, const std::string &defaultProtocol="http", const std::string &serviceUri="s3.amazonaws.com")
std::string	signature (const RequestParams &request) const
SignedRequest	prepare (const RequestParams &request) const
Response	get (const std::string &bucket, const std::string &resource, uint64_t expectedBytesToTransfer, const std::string &subResource="", const StrPairVector &headers=StrPairVector(), const StrPairVector &queryParams=StrPairVector()) const
Response	post (const std::string &bucket, const std::string &resource, const std::string &subResource="", const StrPairVector &headers=StrPairVector(), const StrPairVector &queryParams=StrPairVector(), const Content &content=Content()) const
Response	put (const std::string &bucket, const std::string &resource, const std::string &subResource="", const StrPairVector &headers=StrPairVector(), const StrPairVector &queryParams=StrPairVector(), const Content &content=Content()) const
Response	erase (const std::string &bucket, const std::string &resource, const std::string &subResource="", const StrPairVector &headers=StrPairVector(), const StrPairVector &queryParams=StrPairVector(), const Content &content=Content()) const
std::string	upload (const char *data, size_t bytes, const std::string &bucket, const std::string &resource, CheckMethod check=CM_SIZE, const ObjectMetadata &md=ObjectMetadata(), int numInParallel=-1)
std::string	upload (const char *data, size_t bytes, const std::string &uri, CheckMethod check=CM_SIZE, const ObjectMetadata &md=ObjectMetadata(), int numInParallel=-1)
void	download (const std::string &bucket, const std::string &object, const OnChunk &onChunk, ssize_t startOffset=0, ssize_t endOffset=-1) const
void	download (const std::string &uri, const OnChunk &onChunk, ssize_t startOffset=0, ssize_t endOffset=-1) const
void	downloadToFile (const std::string &uri, const std::string &outfile, ssize_t endOffset=-1) const
std::auto_ptr< std::streambuf >	streamingDownload (const std::string &bucket, const std::string &object, ssize_t startOffset=0, ssize_t endOffset=-1, const OnChunk &onChunk=OnChunk()) const
std::auto_ptr< std::streambuf >	streamingDownload (const std::string &uri, ssize_t startOffset=0, ssize_t endOffset=-1, const OnChunk &onChunk=OnChunk()) const
std::auto_ptr< std::streambuf >	streamingUpload (const std::string &uri, const ObjectMetadata &md=ObjectMetadata()) const
std::auto_ptr< std::streambuf >	streamingUpload (const std::string &bucket, const std::string &object, const ObjectMetadata &md=ObjectMetadata()) const
void	forEachObject (const std::string &bucket, const std::string &prefix="", const OnObject &onObject=OnObject(), const OnSubdir &onSubdir=OnSubdir(), const std::string &delimiter="/", int depth=1) const
ObjectInfo	tryGetObjectInfo (const std::string &bucket, const std::string &object) const
ObjectInfo	tryGetObjectInfo (const std::string &uri) const
ObjectInfo	getObjectInfo (const std::string &bucket, const std::string &object) const
ObjectInfo	getObjectInfo (const std::string &uri) const
bool	forEachBucket (const OnBucket &bucket) const
MultiPartUpload	obtainMultiPartUpload (const std::string &bucket, const std::string &resource, const ObjectMetadata &metadata) const
std::string	finishMultiPartUpload (const std::string &bucket, const std::string &resource, const std::string &uploadId, const std::vector< std::string > &etags) const
void	uploadRecursive (std::string dirSrc, std::string bucketDest, bool includeDir)
void	setDefaultBandwidthToServiceMbps (double mpbs)
Static Public Member Functions
static std::string	sign (const std::string &stringToSign, const std::string &accessKey)
static OnChunk	writeToFile (const std::string &filename)
static std::string	getDigestMulti (const std::string &verb, const std::string &bucket, const std::string &resource, const std::string &subResource, const std::string &contentType, const std::string &contentMd5, const std::string &date, const std::vector< std::pair< std::string, std::string > > &headers)
static std::string	getDigest (const std::string &verb, const std::string &bucket, const std::string &resource, const std::string &subResource, const std::string &contentType, const std::string &contentMd5, const std::string &date, const std::map< std::string, std::string > &headers)
static std::pair< std::string, std::string >	parseUri (const std::string &uri)
Public Attributes
std::string	accessKeyId
std::string	accessKey
std::string	defaultProtocol
std::string	serviceUri
double	bandwidthToServiceMbps
Static Public Attributes
static double	defaultBandwidthToServiceMbps = 20.0

---

Detailed Description

Definition at line 21 of file s3.h.

---

Member Enumeration Documentation

enum Datacratic::S3Api::CheckMethod

Enumerator:

CM_SIZE	Check via the size of the content.
CM_MD5_ETAG	Check via the md5 of the content vs the etag.
CM_ASSUME_INVALID	Anything there is assumed invalid.

Definition at line 238 of file s3.h.

---

Constructor & Destructor Documentation

Datacratic::S3Api::S3Api	(	const std::string &	accessKeyId,
		const std::string &	accessKey,
		double	bandwidthToServiceMbps = defaultBandwidthToServiceMbps,
		const std::string &	defaultProtocol = "http",
		const std::string &	serviceUri = "s3.amazonaws.com"
	)

Set up the API to called with the given credentials.

Definition at line 58 of file s3.cc.

---

Member Function Documentation

void Datacratic::S3Api::download	(	const std::string &	bucket,
		const std::string &	object,
		const OnChunk &	onChunk,
		ssize_t	startOffset = 0,
		ssize_t	endOffset = -1
	)		const

Download the contents of a bucket. This will call the given output function for each chunk that is received. Note that there is no guarantee that the chunks will be received in order as the download happens in multiple parallel chunks.

Definition at line 1241 of file s3.cc.

void Datacratic::S3Api::downloadToFile	(	const std::string &	uri,
		const std::string &	outfile,
		ssize_t	endOffset = -1
	)		const

Downloads a file from s3 to a local file. If the maxSize is specified, only the first maxSize bytes will be downloaded.

Definition at line 1348 of file s3.cc.

S3Api::Response Datacratic::S3Api::erase	(	const std::string &	bucket,
		const std::string &	resource,
		const std::string &	subResource = "",
		const StrPairVector &	headers = StrPairVector(),
		const StrPairVector &	queryParams = StrPairVector(),
		const Content &	content = Content()
	)		const

Perform a DELETE request from end to end including data.

Definition at line 536 of file s3.cc.

bool Datacratic::S3Api::forEachBucket

(

const OnBucket &

bucket

)

const

Call the given callback on every bucket in the current account.

Definition at line 1965 of file s3.cc.

void Datacratic::S3Api::forEachObject	(	const std::string &	bucket,
		const std::string &	prefix = "",
		const OnObject &	onObject = OnObject(),
		const OnSubdir &	onSubdir = OnSubdir(),
		const std::string &	delimiter = "/",
		int	depth = 1
	)		const

For each file matching the given prefix in the given bucket, call the callback.

Definition at line 1056 of file s3.cc.

S3Api::Response Datacratic::S3Api::get	(	const std::string &	bucket,
		const std::string &	resource,
		uint64_t	expectedBytesToTransfer,
		const std::string &	subResource = "",
		const StrPairVector &	headers = StrPairVector(),
		const StrPairVector &	queryParams = StrPairVector()
	)		const

Perform a GET request from end to end.

Definition at line 470 of file s3.cc.

std::string Datacratic::S3Api::getDigest	(	const std::string &	verb,
		const std::string &	bucket,
		const std::string &	resource,
		const std::string &	subResource,
		const std::string &	contentType,
		const std::string &	contentMd5,
		const std::string &	date,
		const std::map< std::string, std::string > &	headers
	)		[static]

Get the digest string (the one that needs to be signed) from a set of s3 parameters. Directly implements the procedure in the s3 documentation.

This variant can only accept one of each kind of http header.

Definition at line 157 of file s3.cc.

std::string Datacratic::S3Api::getDigestMulti	(	const std::string &	verb,
		const std::string &	bucket,
		const std::string &	resource,
		const std::string &	subResource,
		const std::string &	contentType,
		const std::string &	contentMd5,
		const std::string &	date,
		const std::vector< std::pair< std::string, std::string > > &	headers
	)		[static]

Get the digest string (the one that needs to be signed) from a set of s3 parameters. Directly implements the procedure in the s3 documentation.

This variant can deal with multiple copies of each of the http headers.

Definition at line 100 of file s3.cc.

S3Api::ObjectInfo Datacratic::S3Api::getObjectInfo	(	const std::string &	bucket,
		const std::string &	object
	)		const

Return the ObjectInfo about the object. Throws an exception if it doesn't exist.

Definition at line 1139 of file s3.cc.

void Datacratic::S3Api::init	(	const std::string &	accessKeyId,
		const std::string &	accessKey,
		double	bandwidthToServiceMbps = defaultBandwidthToServiceMbps,
		const std::string &	defaultProtocol = "http",
		const std::string &	serviceUri = "s3.amazonaws.com"
	)

Set up the API to called with the given credentials.

Definition at line 73 of file s3.cc.

S3Api::MultiPartUpload Datacratic::S3Api::obtainMultiPartUpload	(	const std::string &	bucket,
		const std::string &	resource,
		const ObjectMetadata &	metadata
	)		const

Obtain a multipart upload, either in progress or a new one.

Definition at line 645 of file s3.cc.

std::pair< std::string, std::string > Datacratic::S3Api::parseUri

(

const std::string &

uri

)

[static]

Turn a s3:// uri string into a bucket name and object.

Definition at line 1890 of file s3.cc.

S3Api::Response Datacratic::S3Api::post	(	const std::string &	bucket,
		const std::string &	resource,
		const std::string &	subResource = "",
		const StrPairVector &	headers = StrPairVector(),
		const StrPairVector &	queryParams = StrPairVector(),
		const Content &	content = Content()
	)		const

Perform a POST request from end to end.

Definition at line 493 of file s3.cc.

S3Api::SignedRequest Datacratic::S3Api::prepare

(

const RequestParams &

request

)

const

Prepare a request to be executed.

Definition at line 423 of file s3.cc.

S3Api::Response Datacratic::S3Api::put	(	const std::string &	bucket,
		const std::string &	resource,
		const std::string &	subResource = "",
		const StrPairVector &	headers = StrPairVector(),
		const StrPairVector &	queryParams = StrPairVector(),
		const Content &	content = Content()
	)		const

Perform a PUT request from end to end including data.

Definition at line 515 of file s3.cc.

std::string Datacratic::S3Api::sign	(	const std::string &	stringToSign,
		const std::string &	accessKey
	)		[static]

Sign the given digest string with the given access key and return a base64 encoded signature.

Definition at line 199 of file s3.cc.

std::string Datacratic::S3Api::signature

(

const RequestParams &

request

)

const

Calculate the signature for a given request.

Definition at line 402 of file s3.cc.

std::auto_ptr< std::streambuf > Datacratic::S3Api::streamingDownload	(	const std::string &	bucket,
		const std::string &	object,
		ssize_t	startOffset = 0,
		ssize_t	endOffset = -1,
		const OnChunk &	onChunk = OnChunk()
	)		const

Get a streambuf that will allow a bucket to be streamed through. If an onChunk is provided, downloaded chunks will also be provided to that method.

Definition at line 1624 of file s3.cc.

std::auto_ptr< std::streambuf > Datacratic::S3Api::streamingDownload	(	const std::string &	uri,
		ssize_t	startOffset = 0,
		ssize_t	endOffset = -1,
		const OnChunk &	onChunk = OnChunk()
	)		const

Get a streambuf that will allow a bucket to be streamed through. If an onChunk is provided, downloaded chunks will also be provided to that method.

Definition at line 1906 of file s3.cc.

std::auto_ptr< std::streambuf > Datacratic::S3Api::streamingUpload	(	const std::string &	uri,
		const ObjectMetadata &	md = ObjectMetadata()
	)		const

Get a streambuf that will write to s3 when written to.

Definition at line 1867 of file s3.cc.

std::auto_ptr< std::streambuf > Datacratic::S3Api::streamingUpload	(	const std::string &	bucket,
		const std::string &	object,
		const ObjectMetadata &	md = ObjectMetadata()
	)		const

Get a streambuf that will write to s3 when written to.

Definition at line 1877 of file s3.cc.

S3Api::ObjectInfo Datacratic::S3Api::tryGetObjectInfo	(	const std::string &	bucket,
		const std::string &	object
	)		const

Does the object exist?

Definition at line 1177 of file s3.cc.

std::string Datacratic::S3Api::upload	(	const char *	data,
		size_t	bytes,
		const std::string &	bucket,
		const std::string &	resource,
		CheckMethod	check = CM_SIZE,
		const ObjectMetadata &	md = ObjectMetadata(),
		int	numInParallel = -1
	)

Upload a memory buffer into an s3 bucket. Uses a multi-part upload algorithm that can achieve 200MB/second for data already in memory.

If the resource already exists, then it will use the given method to determine whether it's OK or not.

Returns the etag field of the uploaded file.

Definition at line 802 of file s3.cc.

static OnChunk Datacratic::S3Api::writeToFile

(

const std::string &

filename

)

[static]

OnChunk function that writes to the given file.

---

Member Data Documentation

double Datacratic::S3Api::defaultBandwidthToServiceMbps = 20.0 [static]

Default value for bandwidth to service. In mega*bytes* per second. Default value is 20.0 MBPS for ec2 instances in the same availability zone.

Definition at line 32 of file s3.h.

---

The documentation for this struct was generated from the following files:

• soa/service/s3.h
• soa/service/s3.cc