soa/logger/cloud_output.h

/* cloud_output.h                                                  -*- C++ -*-
   Jeremy Barnes, 18 September 2012
   Copyright (c) 2012 Datacratic.  All rights reserved.

   Output that puts files into... the CLOUD!
*/

#include "file_output.h"


#include "logger.h"
#include "jml/arch/timers.h"
#include "soa/types/date.h"
#include <boost/thread/recursive_mutex.hpp>
#include "soa/types/periodic_utils.h"
#include "compressor.h"


namespace Datacratic {


/*****************************************************************************/
/* CLOUD SINK                                                                 */
/*****************************************************************************/

struct CloudSink : public CompressingOutput::Sink {
    CloudSink(const std::string & uri = "",
              bool append = true,
              bool disambiguate = true);

    virtual ~CloudSink();

    void open(const std::string & uri,
              bool append,
              bool disambiguate);

    virtual void close();

    virtual size_t write(const char * data, size_t size);

    virtual size_t flush(FileFlushLevel flushLevel);

    std::string currentUri_;
    std::string tmpFileDir_;

    ML::filter_ostream cloudStream;
    // we write to a temporary file on local disk which we delete when
    // the corresponding cloud stream is closed.
    ML::filter_ostream fileStream;

    //int fd;
};


/*****************************************************************************/
/* CLOUD OUTPUT                                                              */
/*****************************************************************************/

struct CloudOutput : public NamedOutput {

    CloudOutput(const std::string & uri = "",
               size_t ringBufferSize = 65536);

    virtual ~CloudOutput();

    virtual std::shared_ptr<Sink>
    createSink(const std::string & uri, bool append);
};


/*****************************************************************************/
/* ROTATING CLOUD OUTPUT                                                      */
/*****************************************************************************/

struct RotatingCloudOutput : public RotatingOutputAdaptor {

    RotatingCloudOutput();

    virtual ~RotatingCloudOutput();

    /* Open the cloud for rotation. */
    /* uriPattern: The URI of the s3 bucket
     * periodPattern: The frequency of rotation i.e "2s" means "rotate
     * every 2 seconds." The accepted symbols are
     * x: milliseconds
     * s: seconds
     * m: minutes
     * h: hours
     * d: days
     * w: weeks
     * M: months
     * y: years
     */
    void open(const std::string & uriPattern,
              const std::string & periodPattern,
              const std::string & compression = "",
              int level = -1);

private:
    CloudOutput * createFile(const std::string & filename);

    std::string compression;
    int level;
};

} // namespace Datacratic